universe/deploy/lux-testnet/luxd.yaml

# luxd — the lux-testnet validator set.
#
# Derived MECHANICALLY from the StatefulSet running in lux-testnet on
# do-sfo3-lux-k8s, not hand-written: every field below was read off the live
# object and 13 structural assertions were checked against it (replicas,
# serviceName, podManagementPolicy, PVC retention, volumeClaimTemplates, image
# repo+tag+digest, and the env / volumeMounts / volumes / initContainers / ports
# counts, plus the command verbatim).
#
# ⚠️ NOT RENDER-VERIFIED. devnet's sibling file was converted AND rendered back
# through the chart to prove the only diffs were intended ones. That check needs
# a pull from oci.hanzo.ai, which requires registry credentials this workstation
# does not have (the token endpoint answers 401). So the render diff is UNKNOWN
# here and must be read from CD before anyone syncs.
#
# Until now this StatefulSet was applied BY HAND: it carries no ownerReferences
# and no CD tracking-id, while every other workload in lux-testnet is CD-managed.
# That is the gap this file closes.
#
# ⛔ DO NOT SYNC BLIND. The generated Application (lux-lux-testnet-luxd) has no
# `automated` block, so adding this file changes nothing on its own — it makes CD
# COMPUTE the diff. Read that diff first. updateStrategy is OnDelete
# (strategy: Recreate below), so a spec change does not roll validators; rolling
# them stays a deliberate act. persistentVolumeClaimRetentionPolicy is
# Retain/Retain because the default, Delete, would take the ledger with the
# StatefulSet.
nameOverride: luxd
containerName: luxd
replicas: 5
workload:
  kind: StatefulSet
  serviceName: luxd-headless
  podManagementPolicy: Parallel
strategy: Recreate
persistentVolumeClaimRetentionPolicy:
  whenDeleted: Retain
  whenScaled: Retain
volumeClaimTemplates:
- apiVersion: v1
  kind: PersistentVolumeClaim
  metadata:
    name: data
  spec:
    accessModes:
    - ReadWriteOnce
    resources:
      requests:
        storage: 200Gi
    volumeMode: Filesystem
persistence:
  enabled: false
service:
  type: ClusterIP
  headless: false
image:
  # v1.36.94 asks a second bootstrap question: of the tips you named, which have
  # you accepted? This fleet is the reason. Five nodes sat at five different heights
  # and none converged, because the first question asks for ONE tip and requires a
  # stake quorum on that exact block — and the chain moves while the answers are in
  # flight, so honest peers name different blocks and no quorum ever forms.
  #
  # Acceptance does not split that way: peers at different heights have all accepted
  # the lower candidates, so the tie breaks on the first round trip. And bootstrap
  # fetches ancestry and accepts directly, needing no certificate, so a node recovers
  # across a range whose certificates were never persisted.
  #
  # An Ancestors reply is now matched to the peer it was asked of; correlating on the
  # request id alone let any connected peer answer a fetch we never made, and one
  # such reply abandons the pass.
  repository: ghcr.io/luxfi/node
  tag: v1.36.133
  pullPolicy: Always
  digest: sha256:50232bed0fbaba41644ce20392725597f3aa852f80e319ac05b1290045278b27
command:
- /bin/sh
- /scripts/startup.sh
env:
# GOMEMLIMIT is the half that actually prevents an OOM kill. It is a SOFT limit:
# the Go runtime GCs harder as it approaches, instead of sailing into the cgroup
# ceiling and being SIGKILLed. lux-mainnet has always set it (6GiB under a 12Gi
# limit) and does not OOM; lux-testnet carried NO env at all, and luxd-0 and
# luxd-4 died OOMKilled (exit 137) 160 and 149 times respectively.
#
# The trigger is being BEHIND, not being busy: luxd-1/luxd-2, caught up, sit at
# ~760-800Mi, while a node catching up (and serving ancestry to other catching-up
# peers) holds several times that. So the two nodes that most needed to finish
# syncing were the two the ceiling killed first — and each kill restarted the
# sync from a cold cache, which is why neither ever converged.
#
# Held below the hard limit on purpose. The LIMIT is what kills, so only the
# limit is raised; `requests` stays at 1Gi because it is the scheduler's
# RESERVATION and raising it can make a pod permanently Pending on a committed
# node pool without changing the OOM behaviour at all.
- name: GOMEMLIMIT
  value: 6GiB
- name: ZAP_RPC_LISTEN
  value: :9632
- name: LUX_NETWORK_ID
  value: '2'
- name: POD_NAME
  valueFrom:
    fieldRef:
      apiVersion: v1
      fieldPath: metadata.name
- name: LUX_CONSENSUS_VIEW_CHANGE
  value: 'true'
- name: VIEW_CHANGE
  value: 'true'
ports:
- containerPort: 9632
  name: zap-rpc
  protocol: TCP
- containerPort: 9640
  name: http
  protocol: TCP
- containerPort: 9641
  name: staking
  protocol: TCP
- containerPort: 9090
  name: metrics
  protocol: TCP
resources:
  limits:
    cpu: '2'
    memory: 8Gi
  requests:
    cpu: 500m
    memory: 1Gi
volumeMounts:
- mountPath: /data
  name: data
- mountPath: /scripts
  name: startup-script
- mountPath: /staking-keys
  name: staking-keys
  readOnly: true
livenessProbe:
  failureThreshold: 10
  httpGet:
    path: /v1/health/liveness
    port: 9640
    scheme: HTTP
  initialDelaySeconds: 60
  periodSeconds: 15
  successThreshold: 1
  timeoutSeconds: 5
readinessProbe:
  exec:
    command:
    - /bin/sh
    - -c
    - 'B=$(curl -sf -m 4 -X POST http://127.0.0.1:9640/v1/info -H ''Content-Type: application/json'' -d
      ''{"jsonrpc":"2.0","id":1,"method":"info.isBootstrapped","params":{"chain":"C"}}'' | grep -c ''"isBootstrapped":true'');
      [ "$B" = "1" ] || exit 1; curl -sf -m 4 -X POST http://127.0.0.1:9640/v1/chain/C/rpc -H ''Content-Type:
      application/json'' -d ''{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}'' | grep
      -q ''"result":"0x'' || exit 1'
  failureThreshold: 4
  initialDelaySeconds: 30
  periodSeconds: 15
  successThreshold: 1
  timeoutSeconds: 6
startupProbe:
  failureThreshold: 30
  initialDelaySeconds: 5
  periodSeconds: 10
  successThreshold: 1
  tcpSocket:
    port: 9640
  timeoutSeconds: 1
volumes:
- name: data
  persistentVolumeClaim:
    claimName: data-luxd-1
- configMap:
    defaultMode: 420
    name: luxd-config
  name: config
- configMap:
    defaultMode: 493
    name: luxd-startup
  name: startup-script
- name: staking-keys
  secret:
    defaultMode: 420
    secretName: luxd-staking
initContainers:
- args:
  - "set -eu\n# \u2500\u2500 Lux C-Chain RLP recovery init (incident-1082814) \u2500\u2500\n# Order: [arm-gate]\
    \ -> [sentinel-skip] -> [SHA-verify] -> [WIPE] -> [marker].\n# SHA-verify MUST precede the wipe; mismatch\
    \ aborts before any destructive op.\nDB=/data/db\nNET=network-2\nCBID=2dbTRWZrj65UZQe6yWruV1jEiq9oR9hawVF8SRwm9UtpGder6p\n\
    RLP=/data/lux-testnet-96368.rlp\nEXPECTED_SHA=2b9667bd68994533139b3a16e85e418df30967e7961bd32979fab770c52fd2d4\n\
    H=2b9667bd6899\nARM=\"$DB/.rlp-import-arm\"\nSENTINEL=\"$DB/.tenant-import-c-chain-$H\"\nMARKER=\"\
    $DB/.tenant-import-request-c-chain-$H.json\"\nif [ ! -f \"$ARM\" ]; then\n  echo \"[rlp-import-init]\
    \ node NOT armed (no $ARM); skipping (no wipe)\"\n  exit 0\nfi\nif [ -f \"$SENTINEL\" ]; then\n  echo\
    \ \"[rlp-import-init] sentinel present; already imported; clearing arm\"\n  rm -f \"$ARM\"\n  exit\
    \ 0\nfi\nif [ ! -f \"$RLP\" ]; then\n  echo \"[rlp-import-init] ERROR: RLP not found at $RLP\" >&2\n\
    \  exit 1\nfi\nGOT=$(sha256sum \"$RLP\" | awk '{print $1}')\nif [ \"$GOT\" != \"$EXPECTED_SHA\" ];\
    \ then\n  echo \"[rlp-import-init] SHA MISMATCH want=$EXPECTED_SHA got=$GOT -- ABORT before wipe\"\
    \ >&2\n  exit 1\nfi\necho \"[rlp-import-init] SHA OK ($GOT) -- proceeding to wipe C-Chain\"\nTARGET=\"\
    /data/chainData/$NET/$CBID\"\necho \"[rlp-import-init] wiping $TARGET\"\nrm -rf \"$TARGET\"\nprintf\
    \ '{\"tenant\":\"c-chain\",\"chainAlias\":\"C\",\"contentHash\":\"%s\",\"rlpPath\":\"%s\",\"sentinelPath\"\
    :\"%s\",\"blockchainID\":\"%s\"}' \"$H\" \"$RLP\" \"$SENTINEL\" \"$CBID\" > \"$MARKER.tmp\"\nmv \"\
    $MARKER.tmp\" \"$MARKER\"\nrm -f \"$ARM\"\necho \"[rlp-import-init] wipe done; marker=$MARKER; startup.sh\
    \ will admin_importChain on /v1/chain/C/rpc\"\n"
  command:
  - /bin/sh
  - -c
  env:
  - name: LUX_CONSENSUS_VIEW_CHANGE
    value: 'false'
  - name: VIEW_CHANGE
    value: 'false'
  image: ghcr.io/luxfi/node:v1.34.0
  imagePullPolicy: IfNotPresent
  name: rlp-import
  resources: {}
  terminationMessagePath: /dev/termination-log
  terminationMessagePolicy: File
  volumeMounts:
  - mountPath: /data
    name: data
- args:
  - "#!/bin/sh\nset -e\n\nPOD_INDEX=${HOSTNAME##*-}\necho \"[GATE] Startup gate for $HOSTNAME (luxd, network-id=2)\"\
    \nDNS_PREFIX=\"luxd\"\nDNS_SUFFIX=\"luxd-headless.lux-testnet.svc.cluster.local\"\nGATE_MIN_PEERS=2\n\
    GATE_TIMEOUT=60\nGATE_INTERVAL=5\nGATE_START=$(date +%s)\necho \"[GATE] Waiting for $GATE_MIN_PEERS\
    \ peers (timeout ${GATE_TIMEOUT}s)...\"\n\nwhile true; do\n  REACHABLE=0\n  if [ \"$POD_INDEX\" !=\
    \ \"0\" ]; then\n    if nc -z -w 2 \"${DNS_PREFIX}-0.${DNS_SUFFIX}\" 9641 2>/dev/null; then\n    \
    \  REACHABLE=$((REACHABLE + 1))\n    fi\n  fi\n  if [ \"$POD_INDEX\" != \"1\" ]; then\n    if nc -z\
    \ -w 2 \"${DNS_PREFIX}-1.${DNS_SUFFIX}\" 9641 2>/dev/null; then\n      REACHABLE=$((REACHABLE + 1))\n\
    \    fi\n  fi\n  if [ \"$POD_INDEX\" != \"2\" ]; then\n    if nc -z -w 2 \"${DNS_PREFIX}-2.${DNS_SUFFIX}\"\
    \ 9641 2>/dev/null; then\n      REACHABLE=$((REACHABLE + 1))\n    fi\n  fi\n  if [ \"$POD_INDEX\"\
    \ != \"3\" ]; then\n    if nc -z -w 2 \"${DNS_PREFIX}-3.${DNS_SUFFIX}\" 9641 2>/dev/null; then\n \
    \     REACHABLE=$((REACHABLE + 1))\n    fi\n  fi\n  if [ \"$POD_INDEX\" != \"4\" ]; then\n    if nc\
    \ -z -w 2 \"${DNS_PREFIX}-4.${DNS_SUFFIX}\" 9641 2>/dev/null; then\n      REACHABLE=$((REACHABLE +\
    \ 1))\n    fi\n  fi\n\n  if [ \"$REACHABLE\" -ge \"$GATE_MIN_PEERS\" ]; then\n    echo \"[GATE] $REACHABLE\
    \ peers reachable, proceeding with startup\"\n    break\n  fi\n\n  ELAPSED=$(( $(date +%s) - GATE_START\
    \ ))\n  if [ \"$ELAPSED\" -ge \"$GATE_TIMEOUT\" ]; then\n    echo \"[GATE] WARNING: Timeout after\
    \ ${ELAPSED}s with only $REACHABLE peers. Proceeding (onTimeout=StartAnyway).\"\n    break\n  fi\n\
    \n  echo \"[GATE] $REACHABLE/$GATE_MIN_PEERS peers reachable (${ELAPSED}/${GATE_TIMEOUT}s), waiting\
    \ ${GATE_INTERVAL}s...\"\n  sleep \"$GATE_INTERVAL\"\ndone\n\necho \"[GATE] Gate passed, luxd may\
    \ start.\"\n"
  command:
  - /bin/sh
  - -c
  env:
  - name: LUX_CONSENSUS_VIEW_CHANGE
    value: 'false'
  - name: VIEW_CHANGE
    value: 'false'
  image: ghcr.io/luxfi/node:v1.28.20
  imagePullPolicy: IfNotPresent
  name: startup-gate
  resources: {}
  terminationMessagePath: /dev/termination-log
  terminationMessagePolicy: File
# ⚠️ The key is `app.kubernetes.io/name`, NOT `app`. These pods carry the
# chart's standard labels and have no bare `app` label at all, so the previous
# `key: app` selector matched ZERO pods — and an anti-affinity that selects
# nothing is indistinguishable from one that is satisfied. It was inert for as
# long as it has existed: lux-testnet ran THREE of five validators on
# lux-nodes-374ju1 (losing that one node drops below alpha=4 and halts the
# chain) while reporting a `required` constraint, and lux-mainnet's clean
# five-node spread was luck rather than enforcement.
# IgnoredDuringExecution means correcting this moves nothing now; it binds the
# next time a pod is scheduled.
affinity:
  podAntiAffinity:
    requiredDuringSchedulingIgnoredDuringExecution:
    - labelSelector:
        matchExpressions:
        - key: app.kubernetes.io/name
          operator: In
          values:
          - luxd
      topologyKey: kubernetes.io/hostname
terminationGracePeriodSeconds: 120
fsGroup: 1000
podSecurityContext:
  runAsGroup: 1000
  runAsUser: 1000
podAnnotations: {}
topologySpread:
  enabled: false
partOf: lux
component: validator


# ⛔ Deliberately NOT using the chart's `pdb:` here. Its selector is
# selectorLabels (name+instance), which on this fleet ALSO matches the hourly
# luxd-export-c-chain job pods — 8 pods on lux-mainnet, not 5. A budget that
# counts finished Job pods as unhealthy drives disruptionsAllowed to 0 and
# blocks legitimate node drains, which is the failure this was meant to prevent.
# The validators are pinned by `app.kubernetes.io/component: validator`, a key
# the chart's PDB template cannot express, so `luxd-pdb` stays hand-applied
# until the chart grows an extra-selector option.
#
# The long-standing luxd-pdb was INERT: it demanded
# `app.kubernetes.io/managed-by: lux-operator`, a fossil from before this fleet
# moved to the chart (pods now say `Helm`), so it matched ZERO pods and reported
# expectedPods=0 for 162 days. Corrected 2026-08-06 to
# {name: luxd, component: validator} — verified selecting exactly 5.