352 lines
20 KiB
YAML
352 lines
20 KiB
YAML
# chain-traffic — the one continuous traffic generator for every Lux net.
|
|
#
|
|
# WHY THIS EXISTS
|
|
#
|
|
# On 2026-07-27 mainnet 96369 stopped producing for three days and nothing
|
|
# noticed. Not because monitoring was absent, but because the chains are
|
|
# post-Etna with etnaTimestamp=0: a block is sealed only when a transaction is
|
|
# pending, so a flat tip on an idle chain is CORRECT. "No blocks" and "wedged"
|
|
# are the same picture, and every liveness signal in the stack — kubelet Ready,
|
|
# luxd /v1/health, the LB rotation — happily reported green through it.
|
|
#
|
|
# Demand resolves the ambiguity. With traffic always flowing, a flat tip can
|
|
# only mean stalled. That is the whole purpose: this Deployment is not a load
|
|
# test, it is the signal that makes silence readable.
|
|
#
|
|
# It replaces two earlier attempts, and the ways they failed are the reasons
|
|
# for its three safety properties:
|
|
#
|
|
# lux-hb2 (python, mainnet) sent every 3s on a LOCALLY incremented nonce.
|
|
# Against a chain that stopped draining it built
|
|
# an unbounded backlog, and queue depth then read
|
|
# as activity. -> MAX_INFLIGHT.
|
|
# mainnet-heartbeat (v2) fixed that by capping in-flight at exactly one,
|
|
# which is safe but ceilings throughput at one tx
|
|
# per block interval. -> pipelined bursts inside
|
|
# a bounded window.
|
|
# neither ever asked whether the chain was moving before
|
|
# sending more. -> STALL_SECS pause.
|
|
#
|
|
# CONFIGURATION (all optional except RPC_URLS)
|
|
# RPC_URLS comma-separated PUBLISH targets. The mempool does not
|
|
# gossip on these nets — a tx known only to node 0 is
|
|
# buildable only by node 0 — so every tx is signed once and
|
|
# published to each node, and whoever proposes next can
|
|
# include it. This is why pods are addressed individually
|
|
# here while every other client uses a Service.
|
|
# READ_URL where tip/nonce/base-fee are read (default: first of
|
|
# RPC_URLS). Point it at a tip-gated Service where one
|
|
# exists, so state is never read from a node that is behind.
|
|
# TX_PER_BURST transactions per burst, nonces pipelined (2)
|
|
# BURST_SECS seconds between bursts; rate = burst/secs (2)
|
|
# MAX_INFLIGHT hard cap on pending-minus-latest nonce (8)
|
|
# STALL_SECS tip frozen this long -> stop sending (90)
|
|
# GAS_MULT gasPrice = GAS_MULT x base fee (2)
|
|
# KEY_IDX BIP44 m/44'/60'/0'/0/<idx> for the sender (11)
|
|
# SEED_IDX top up the sender from this idx at startup (-1 = off)
|
|
# TRAFFIC_KEY raw key; takes precedence over the mnemonic. Mainnet uses
|
|
# this, from KMS — no mnemonic ever reaches that pod.
|
|
#
|
|
# KEY POLICY: index 11 is the traffic sender and nothing else. Validators hold
|
|
# 0-4, chain control 5-7, the chain-heartbeat canary 10. The sender is never
|
|
# 0x9011 — a bot must not consume the deployer's nonce, and on mainnet 0x9011
|
|
# holds 0.02 LUX, which is 38 transactions.
|
|
#
|
|
# KILL SWITCH: kubectl -n <ns> scale deploy/chain-traffic --replicas=0
|
|
#
|
|
# RPC_URLS names all five pods on every net. A node whose C-Chain is down answers
|
|
# 404, the publish is backgrounded and its failure ignored, so listing it costs
|
|
# nothing — while pruning it bakes today's outage into a manifest that outlives it.
|
|
#
|
|
# Three voters is NOT enough, contrary to what this file claimed on 2026-07-28.
|
|
# luxd runs --consensus-sample-size=5 --consensus-quorum-size=4: a poll needs four
|
|
# agreeing validators, so a fleet with three live C-Chains cannot finalize at all.
|
|
# Measured 07:05Z that day: devnet luxd-0/1/2 bootstrapped and luxd-3/4 404, tip
|
|
# frozen at 1045 since 05:37:55Z while luxd-0 rebuilt the same block ~10x/second
|
|
# and 30 transactions sat pending. Traffic cannot fix a quorum shortfall; the
|
|
# feeder correctly PAUSEs and waits, and resumes by itself once the tip moves.
|
|
#
|
|
# Sender idx 11 = 0x1B475A4C983DfE4f32bbA4dE8DA8fd2c37f3A2A6, seeded from the 2T
|
|
# devnet treasury at idx 0.
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: chain-traffic-code
|
|
namespace: lux-devnet
|
|
labels: {app.kubernetes.io/name: chain-traffic, app.kubernetes.io/part-of: lux, app.kubernetes.io/component: traffic}
|
|
data:
|
|
chain-traffic.sh: |
|
|
set -u
|
|
# chain-traffic — continuous, rate-limited, backpressured, tip-aware EVM traffic.
|
|
#
|
|
# These C-Chains are post-Etna with etnaTimestamp=0: a block is sealed only when a
|
|
# transaction is pending. With no traffic a chain produces nothing, and "no blocks"
|
|
# is then indistinguishable from "wedged" — which is how a mainnet freeze ran unseen
|
|
# for three days. Continuous traffic removes the ambiguity: with demand always
|
|
# present, a flat tip means STALLED, full stop.
|
|
#
|
|
# Three properties make it safe to leave running unattended:
|
|
#
|
|
# RATE TX_PER_BURST transactions every BURST_SECS seconds. Nonces inside a
|
|
# burst are pipelined so a block can carry more than one tx.
|
|
# BOUNDED never more than MAX_INFLIGHT transactions between the latest and
|
|
# pending nonce. A chain that stops draining stops receiving. This is
|
|
# the property the first-generation python heartbeat lacked: it
|
|
# incremented a local nonce every 3s regardless, so against a wedged
|
|
# chain it built an unbounded backlog that read as "activity".
|
|
# PAUSED if the tip has not moved for STALL_SECS, sending stops entirely and
|
|
# says so, then resumes on its own when the tip moves. Never flood a
|
|
# chain that is not producing.
|
|
#
|
|
# Multi-broadcast: the mempool does not gossip on these nets, so a transaction sent
|
|
# to one node is only buildable by that node. Each tx is signed once and published
|
|
# to every endpoint in RPC_URLS; whichever node proposes next can include it.
|
|
#
|
|
# Cost is gas only — every transaction is a zero-value self-transfer.
|
|
|
|
RPC_URLS="${RPC_URLS:?RPC_URLS required (comma-separated)}"
|
|
TX_PER_BURST="${TX_PER_BURST:-2}"
|
|
BURST_SECS="${BURST_SECS:-2}"
|
|
MAX_INFLIGHT="${MAX_INFLIGHT:-8}"
|
|
STALL_SECS="${STALL_SECS:-90}"
|
|
GAS_MULT="${GAS_MULT:-2}"
|
|
KEY_IDX="${KEY_IDX:-11}"
|
|
SEED_IDX="${SEED_IDX:--1}"
|
|
SEED_FLOOR_WEI="${SEED_FLOOR_WEI:-100000000000000000}"
|
|
SEED_AMOUNT_WEI="${SEED_AMOUNT_WEI:-1000000000000000000}"
|
|
LOG_EVERY="${LOG_EVERY:-15}"
|
|
|
|
# Reads (tip, nonce, base fee) come from ONE endpoint; writes go to all of them.
|
|
# Where a tip-gated Service exists, point READ_URL at it so state is always read
|
|
# from a node known to be caught up.
|
|
READ="${READ_URL:-$(echo "$RPC_URLS" | cut -d, -f1)}"
|
|
URLS=$(echo "$RPC_URLS" | tr ',' ' ')
|
|
isnum() { case "$1" in ''|*[!0-9]*) return 1;; *) return 0;; esac; }
|
|
# The one lesson under every stall this bot has met: during a disagreement,
|
|
# never trust ONE node's view. Heads and nonces are read from EVERY endpoint
|
|
# and the MAX wins — a lagging read node must not put the bot into STALL, and
|
|
# a nonce from the lowest view is nonce-too-low on every node ahead of it.
|
|
# CANONICAL GROUP, not "whichever node is highest". Taking the max across all
|
|
# endpoints is wrong the moment ONE node is stranded, forked or isolated: it can
|
|
# sit ahead on a chain nobody else has and poison every read. That is not
|
|
# hypothetical here — RLP-imported nodes sit frozen at their import height while
|
|
# the fleet advances.
|
|
#
|
|
# Group endpoints by their FINALIZED block hash, take the largest group, and read
|
|
# only from it. A stranded node is a group of one and loses. Ties break toward the
|
|
# higher finalized height. Falls back to the plain max only when no node answers
|
|
# with a finalized hash at all (fresh chain), which cannot make things worse.
|
|
CANON=""; CANON_H=""
|
|
canon() {
|
|
CANON=""; CANON_H=""; _bh=""; _bn=""; _bc=0
|
|
for u in $URLS; do
|
|
_r=$(cast rpc eth_getBlockByNumber finalized false --rpc-url "$u" 2>/dev/null)
|
|
_h=$(printf '%s' "$_r" | sed -n 's/.*"hash":"\(0x[0-9a-f]\{64\}\)".*/\1/p')
|
|
_n=$(printf '%s' "$_r" | sed -n 's/.*"number":"\(0x[0-9a-f]*\)".*/\1/p')
|
|
[ -n "$_h" ] || continue
|
|
_n=$(printf '%d' "$_n" 2>/dev/null) || _n=0
|
|
_c=0; _grp=""
|
|
for v in $URLS; do
|
|
_r2=$(cast rpc eth_getBlockByNumber finalized false --rpc-url "$v" 2>/dev/null)
|
|
_h2=$(printf '%s' "$_r2" | sed -n 's/.*"hash":"\(0x[0-9a-f]\{64\}\)".*/\1/p')
|
|
[ "$_h2" = "$_h" ] && { _c=$((_c+1)); _grp="$_grp $v"; }
|
|
done
|
|
if [ "$_c" -gt "$_bc" ] || { [ "$_c" -eq "$_bc" ] && [ "$_n" -gt "${_bn:-0}" ]; }; then
|
|
_bc="$_c"; _bh="$_h"; _bn="$_n"; CANON="$_grp"
|
|
fi
|
|
done
|
|
CANON_H="$_bh"
|
|
[ -n "$CANON" ] || CANON="$URLS"
|
|
}
|
|
# Heads and nonces come ONLY from the canonical group.
|
|
maxhead() { mh=""; for u in $CANON; do x=$(cast block-number --rpc-url "$u" 2>/dev/null); isnum "$x" || continue; if [ -z "$mh" ] || [ "$x" -gt "$mh" ]; then mh="$x"; fi; done; }
|
|
maxnonce() { mn=""; mu=""; for u in $CANON; do x=$(cast nonce "$SELF" --rpc-url "$u" 2>/dev/null); isnum "$x" || continue; if [ -z "$mn" ] || [ "$x" -gt "$mn" ]; then mn="$x"; mu="$u"; fi; done; }
|
|
now() { date -u +%s; }
|
|
ts() { date -u +%H:%M:%S; }
|
|
|
|
if [ -n "${TRAFFIC_KEY:-}" ]; then
|
|
KEY="$TRAFFIC_KEY"; SRC="TRAFFIC_KEY"
|
|
elif [ -n "${LUX_MNEMONIC:-}" ]; then
|
|
KEY=$(cast wallet private-key --mnemonic "$LUX_MNEMONIC" --mnemonic-index "$KEY_IDX") || exit 1
|
|
SRC="mnemonic idx $KEY_IDX"
|
|
else
|
|
echo "FATAL: set TRAFFIC_KEY or LUX_MNEMONIC"; exit 1
|
|
fi
|
|
SELF=$(cast wallet address --private-key "$KEY") || exit 1
|
|
CHAIN=$(cast chain-id --rpc-url "$READ") || exit 1
|
|
isnum "$CHAIN" || { echo "FATAL: no chainId from $READ"; exit 1; }
|
|
|
|
echo "$(ts) chain-traffic chain=$CHAIN self=$SELF key=$SRC rate=${TX_PER_BURST}/${BURST_SECS}s maxInflight=$MAX_INFLIGHT stall=${STALL_SECS}s endpoints=$(echo "$URLS" | wc -w)"
|
|
|
|
# One-time top-up so a re-genesis'd net does not silently leave the sender broke.
|
|
if [ "$SEED_IDX" -ge 0 ] && [ -n "${LUX_MNEMONIC:-}" ]; then
|
|
BAL=$(cast balance "$SELF" --rpc-url "$READ" 2>/dev/null)
|
|
if isnum "$BAL" && [ "$(echo "$BAL $SEED_FLOOR_WEI" | awk '{print ($1<$2)?1:0}')" = "1" ]; then
|
|
SK=$(cast wallet private-key --mnemonic "$LUX_MNEMONIC" --mnemonic-index "$SEED_IDX")
|
|
SA=$(cast wallet address --private-key "$SK")
|
|
echo "$(ts) SEED $SA -> $SELF $SEED_AMOUNT_WEI wei (balance $BAL < floor $SEED_FLOOR_WEI)"
|
|
cast send "$SELF" --value "$SEED_AMOUNT_WEI" --private-key "$SK" --rpc-url "$READ" \
|
|
--legacy --gas-price $((25000000000 * GAS_MULT)) --gas-limit 21000 --async || echo "$(ts) SEED failed"
|
|
sleep 15
|
|
echo "$(ts) SEED done balance=$(cast balance "$SELF" --rpc-url "$READ" 2>/dev/null)"
|
|
fi
|
|
fi
|
|
|
|
last_head=""; last_move=$(now); paused=0; sent=0; beat=0; last_kick=0; kicks=0; ridx=1; localnext=0
|
|
while true; do
|
|
beat=$((beat + 1))
|
|
canon
|
|
maxhead; head="$mh"
|
|
if ! isnum "$head"; then
|
|
# The read node being down must not blind the whole bot (it once skipped
|
|
# 36k beats pinned to a dead endpoint). Rotate reads across RPC_URLS.
|
|
set -- $URLS
|
|
ridx=$(( (ridx % $#) + 1 ))
|
|
eval "READ=\${$ridx}"
|
|
echo "$(ts) WARN read endpoint down beat=$beat — rotating reads to $READ"
|
|
sleep "$BURST_SECS"; continue
|
|
fi
|
|
if [ "$head" != "$last_head" ]; then
|
|
last_head="$head"; last_move=$(now); kicks=0
|
|
if [ "$paused" = "1" ]; then
|
|
paused=0
|
|
echo "$(ts) RESUME tip moved to $head after stall"
|
|
fi
|
|
elif [ $(( $(now) - last_move )) -ge "$STALL_SECS" ]; then
|
|
# Tip stuck. These chains seal a block ONLY when a tx is pending (header
|
|
# above), so a sender that goes fully silent on stall can never see the tip
|
|
# move again: the old PAUSE was a stable deadlock with the empty-block
|
|
# builder — each side waited for the other. Keep suppressing the normal
|
|
# burst (never flood a stuck chain), but send ONE bounded KICK per stall
|
|
# window: a zero-value self-transfer at the LOWEST unconfirmed nonce with
|
|
# escalating price. On a healthy-but-empty chain it seals the next block;
|
|
# on a price-stuck mempool it is the replacement that clears the jam; on a
|
|
# truly wedged chain it costs one 21000-gas tx per STALL_SECS — noise.
|
|
if [ "$paused" = "0" ]; then
|
|
paused=1
|
|
echo "$(ts) STALL tip stuck at $head for >=${STALL_SECS}s — kicking once per ${STALL_SECS}s until it moves"
|
|
fi
|
|
if [ $(( $(now) - last_kick )) -ge "$STALL_SECS" ]; then
|
|
# Nonce: the MAX finalized nonce across ALL endpoints. Nodes disagree
|
|
# about the tip during exactly the stalls being broken here, and a kick
|
|
# built from the lowest view is nonce-too-low on every node ahead of it.
|
|
maxnonce; kn="$mn"
|
|
base=$(cast base-fee --rpc-url "$READ" 2>/dev/null); isnum "$base" || base=25000000000
|
|
kp=$((base * GAS_MULT)); j=0
|
|
while [ "$j" -lt "$kicks" ] && [ "$j" -lt 3 ]; do kp=$((kp * 2)); j=$((j + 1)); done
|
|
if isnum "$kn"; then
|
|
raw=$(cast mktx "$SELF" --value 0 --private-key "$KEY" --chain "$CHAIN" \
|
|
--legacy --nonce "$kn" --gas-price "$kp" --gas-limit 21000 2>/dev/null)
|
|
case "$raw" in
|
|
0x*) for u in $CANON; do cast publish "$raw" --rpc-url "$u" --async >/dev/null 2>&1 & done; wait
|
|
last_kick=$(now); kicks=$((kicks + 1))
|
|
echo "$(ts) KICK nonce=$kn price=$((kp / 1000000000))gwei kicks=$kicks head=$head" ;;
|
|
*) echo "$(ts) WARN kick mktx failed at nonce=$kn" ;;
|
|
esac
|
|
fi
|
|
fi
|
|
fi
|
|
[ "$paused" = "1" ] && { sleep "$BURST_SECS"; continue; }
|
|
|
|
maxnonce; latest="$mn"
|
|
isnum "$latest" || { sleep "$BURST_SECS"; continue; }
|
|
pending=$(cast nonce "$SELF" --block pending --rpc-url "$mu" 2>/dev/null)
|
|
isnum "$pending" || pending="$latest"
|
|
[ "$pending" -lt "$latest" ] && pending="$latest"
|
|
inflight=$((pending - latest))
|
|
budget=$((MAX_INFLIGHT - inflight))
|
|
if [ "$budget" -le 0 ]; then
|
|
echo "$(ts) BACKPRESSURE inflight=$inflight >= max=$MAX_INFLIGHT head=$head — skipping beat"
|
|
sleep "$BURST_SECS"; continue
|
|
fi
|
|
[ "$budget" -gt "$TX_PER_BURST" ] && budget="$TX_PER_BURST"
|
|
|
|
base=$(cast base-fee --rpc-url "$READ" 2>/dev/null)
|
|
isnum "$base" || base=25000000000
|
|
price=$((base * GAS_MULT))
|
|
|
|
# Never go backwards: a node that briefly reports a stale nonce must not make
|
|
# us reuse one we already broadcast.
|
|
[ "$pending" -lt "$localnext" ] && pending="$localnext"
|
|
n="$pending"; i=0
|
|
while [ "$i" -lt "$budget" ]; do
|
|
raw=$(cast mktx "$SELF" --value 0 --private-key "$KEY" --chain "$CHAIN" \
|
|
--legacy --nonce "$n" --gas-price "$price" --gas-limit 21000 2>/dev/null)
|
|
case "$raw" in
|
|
0x*) for u in $CANON; do cast publish "$raw" --rpc-url "$u" --async >/dev/null 2>&1 & done; wait
|
|
sent=$((sent + 1)) ;;
|
|
*) echo "$(ts) WARN mktx failed at nonce=$n" ;;
|
|
esac
|
|
n=$((n + 1)); i=$((i + 1))
|
|
[ "$n" -gt "$localnext" ] && localnext="$n"
|
|
done
|
|
|
|
if [ $((beat % LOG_EVERY)) -eq 1 ]; then
|
|
echo "$(ts) beat=$beat head=$head sent=$sent latest=$latest inflight=$inflight price=$((price / 1000000000))gwei bal=$(cast balance "$SELF" --rpc-url "$READ" 2>/dev/null)"
|
|
fi
|
|
sleep "$BURST_SECS"
|
|
done
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: chain-traffic
|
|
namespace: lux-devnet
|
|
labels: {app.kubernetes.io/name: chain-traffic, app.kubernetes.io/part-of: lux, app.kubernetes.io/component: traffic}
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: chain-traffic
|
|
template:
|
|
metadata:
|
|
labels: {app.kubernetes.io/name: chain-traffic, app.kubernetes.io/part-of: lux, app.kubernetes.io/component: traffic}
|
|
spec:
|
|
containers:
|
|
- name: traffic
|
|
image: ghcr.io/foundry-rs/foundry:latest
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["/bin/sh", "/app/chain-traffic.sh"]
|
|
env:
|
|
- name: RPC_URLS
|
|
value: "http://luxd-0.luxd-headless.lux-devnet:9650/v1/chain/C/rpc,http://luxd-1.luxd-headless.lux-devnet:9650/v1/chain/C/rpc,http://luxd-2.luxd-headless.lux-devnet:9650/v1/chain/C/rpc,http://luxd-3.luxd-headless.lux-devnet:9650/v1/chain/C/rpc,http://luxd-4.luxd-headless.lux-devnet:9650/v1/chain/C/rpc"
|
|
- name: TX_PER_BURST
|
|
value: "2"
|
|
- name: BURST_SECS
|
|
value: "2"
|
|
- name: MAX_INFLIGHT
|
|
value: "8"
|
|
- name: STALL_SECS
|
|
value: "90"
|
|
- name: GAS_MULT
|
|
value: "2"
|
|
- name: KEY_IDX
|
|
value: "11"
|
|
- name: SEED_IDX
|
|
value: "0"
|
|
- name: SEED_FLOOR_WEI
|
|
value: "10000000000000000000"
|
|
- name: SEED_AMOUNT_WEI
|
|
value: "100000000000000000000000"
|
|
- name: LUX_MNEMONIC
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: lux-deployer
|
|
key: LUX_MNEMONIC
|
|
resources:
|
|
limits: {cpu: 500m, memory: 256Mi}
|
|
requests: {cpu: 50m, memory: 64Mi}
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities: {drop: ["ALL"]}
|
|
readOnlyRootFilesystem: true
|
|
volumeMounts:
|
|
- {name: code, mountPath: /app, readOnly: true}
|
|
volumes:
|
|
- name: code
|
|
configMap: {name: chain-traffic-code, defaultMode: 0444}
|
|
terminationGracePeriodSeconds: 5
|