universe/scripts/roll_luxd.sh

#!/usr/bin/env bash
# Pin and roll the three production Lux validator fleets without dropping quorum.
#
# Usage: scripts/roll_luxd.sh X.Y.Z        (a leading v is accepted)
#
# The release is resolved to one immutable registry digest, committed to the
# three CD values files, then promoted in order: devnet, testnet, mainnet. Each
# five-validator fleet is OnDelete, so pods are replaced one at a time and the
# next pod is not touched until the replacement is Ready, bootstrapped, running
# the requested version and backed by the requested image digest.
#
# The tag is written in the spelling the values files already use (bare
# `1.36.181` today; `v1.36.181` if a file says so) and the digest is resolved
# for that spelling, so a pin can never name an image the registry does not
# have. One tag, one digest, three fleets, one commit: three fleets pinning the
# same tag under three digests is how a roll ships old bytes under a new name.
set -euo pipefail

version=${1:?usage: roll_luxd.sh X.Y.Z}
version=${version#v}
case "$version" in
	[0-9]*.[0-9]*.[0-9]*) ;;
	*) echo "release must be a semver such as 1.36.181" >&2; exit 2 ;;
esac

repo_root=$(git rev-parse --show-toplevel)
cd "$repo_root"
if [ -n "$(git status --porcelain)" ]; then
	echo "refusing to mix a production roll with uncommitted changes" >&2
	exit 1
fi

values=(
	deploy/lux-devnet/luxd.yaml
	deploy/lux-testnet/luxd.yaml
	deploy/lux-mainnet/luxd.yaml
)

# Read one key out of a values file's top-level image: block.
image_field() {
	awk -v key="$2" '
		/^image:/                   { in_image = 1; next }
		in_image && /^[^[:space:]]/ { in_image = 0 }
		in_image && $1 == key":"    { print $2; exit }
	' "$1"
}

# The spelling comes from the files, not the caller: pinning one convention
# writes tags that do not exist. Every fleet must already agree on it.
prefix=""
case "$(image_field deploy/lux-mainnet/luxd.yaml tag)" in v*) prefix=v ;; esac
tag="$prefix$version"
for f in "${values[@]}"; do
	current=$(image_field "$f" tag)
	case "$current" in
		"$prefix"[0-9]*) ;;
		*) echo "$f pins '$current'; every fleet must spell the tag the same way" >&2; exit 1 ;;
	esac
done

registry=ghcr.io/luxfi/node
registry_token=$(curl -fsS "https://ghcr.io/token?scope=repository:luxfi/node:pull&service=ghcr.io" |
	python3 -c 'import json,sys; print(json.load(sys.stdin)["token"])')
digest=$(curl -fsSI \
	-H "Authorization: Bearer $registry_token" \
	-H 'Accept: application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.manifest.v1+json' \
	"https://ghcr.io/v2/luxfi/node/manifests/$tag" |
	tr -d '\r' | awk 'tolower($1) == "docker-content-digest:" { print $2; exit }')
case "$digest" in sha256:[0-9a-f]*) ;; *) echo "no immutable image for $registry:$tag" >&2; exit 1 ;; esac
echo "$registry:$tag@$digest"

python3 - "$tag" "$digest" "${values[@]}" <<'PY'
import pathlib
import re
import sys

tag, digest, *paths = sys.argv[1:]
for name in paths:
    path = pathlib.Path(name)
    text = path.read_text()
    text, tag_count = re.subn(r'^(  tag:)\s+\S+', rf'\1 {tag}', text, count=1, flags=re.M)
    text, digest_count = re.subn(r'^(  digest:)\s+\S+', rf'\1 {digest}', text, count=1, flags=re.M)
    text = re.sub(
        r'^(\s+image:\s+ghcr\.io/luxfi/node:)\S+',
        rf'\1{tag}@{digest}',
        text,
        flags=re.M,
    )
    if tag_count != 1 or digest_count != 1:
        raise SystemExit(f'{name}: expected exactly one primary image tag and digest')
    path.write_text(text)
PY

# One image per pod: whatever stages the plugins must be what runs them. An init
# container pinned at another image writes a plugin from a stale source at every
# boot, and nothing about the artifact can show it.
python3 - "$tag" "$digest" "${values[@]}" <<'PY'
import pathlib
import re
import sys

tag, digest, *paths = sys.argv[1:]
for name in paths:
    text = pathlib.Path(name).read_text()
    tags = re.findall(r'^  tag:\s+(\S+)$', text, re.M)
    digests = re.findall(r'^  digest:\s+(\S+)$', text, re.M)
    repositories = re.findall(r'^  repository:\s+(\S+)$', text, re.M)
    if tags != [tag]:
        raise SystemExit(f'{name}: primary tag is not pinned')
    if digests != [digest]:
        raise SystemExit(f'{name}: primary digest is not pinned')
    if repositories != ['ghcr.io/luxfi/node']:
        raise SystemExit(f'{name}: expected exactly one primary node repository')
    for image in re.findall(r'^\s+image:\s+(ghcr\.io/luxfi/node:\S+)', text, re.M):
        if image != f'ghcr.io/luxfi/node:{tag}@{digest}':
            raise SystemExit(f'{name}: stale init-container pin: {image}')
PY

git diff --check
if ! git diff --quiet -- "${values[@]}"; then
	git add "${values[@]}"
	git commit -m "luxd: pin every fleet to $tag

$registry:$tag
$digest

One tag, one digest, three fleets. The pin is what the CD plane reconciles."
	git push "${GIT_REMOTE:-origin}" HEAD:main
else
	echo "values already pin $tag@$digest"
fi
revision=$(git rev-parse HEAD)

management_context=${MANAGEMENT_CONTEXT:-do-sfo3-hanzo-k8s}
runtime_context=${RUNTIME_CONTEXT:-do-sfo3-lux-k8s}

port_for() {
	case "$1" in
		lux-mainnet) echo 9630 ;;
		lux-testnet) echo 9640 ;;
		lux-devnet) echo 9650 ;;
	esac
}

rpc() {
	local namespace=$1 pod=$2 path=$3 body=$4 port
	port=$(port_for "$namespace")
	kubectl --context "$runtime_context" exec -n "$namespace" "$pod" -c luxd -- \
		curl -fsS -m 5 -X POST -H 'Content-Type: application/json' -d "$body" \
		"http://127.0.0.1:$port$path" 2>/dev/null
}

pod_verified() {
	local namespace=$1 pod=$2 boot
	pod_has_release "$namespace" "$pod" || return 1
	boot=$(rpc "$namespace" "$pod" /v1/info \
		'{"jsonrpc":"2.0","method":"info.isBootstrapped","params":{"chain":"C"},"id":1}' || true)
	[[ "$boot" == *'"isBootstrapped":true'* ]]
}

pod_has_release() {
	local namespace=$1 pod=$2 image_id info
	image_id=$(kubectl --context "$runtime_context" get pod -n "$namespace" "$pod" \
		-o jsonpath='{.status.containerStatuses[?(@.name=="luxd")].imageID}' 2>/dev/null || true)
	[[ "$image_id" == *"$digest"* ]] || return 1
	info=$(rpc "$namespace" "$pod" /v1/info \
		'{"jsonrpc":"2.0","method":"info.getNodeVersion","params":{},"id":1}' || true)
	[[ "$info" == *"luxd/$version"* ]] || return 1
}

# all_chains_bootstrapped proves the complete registered chain set on one
# validator, not just the C-Chain readiness probe. The P-Chain is the registry
# owner and is checked explicitly; platform.getBlockchains supplies every other
# chain ID actually registered on this network (so networks with no F/M chain do
# not invent one, while networks that have them cannot silently leave them down).
all_chains_bootstrapped() {
	local namespace=$1 pod=$2 chains chain boot
	chains=$(rpc "$namespace" "$pod" /v1/platform \
		'{"jsonrpc":"2.0","method":"platform.getBlockchains","params":{},"id":1}' |
		python3 -c 'import json,sys; print(" ".join(x["id"] for x in json.load(sys.stdin)["result"]["blockchains"]))') || return 1
	for chain in P $chains; do
		boot=$(rpc "$namespace" "$pod" /v1/info \
			"{\"jsonrpc\":\"2.0\",\"method\":\"info.isBootstrapped\",\"params\":{\"chain\":\"$chain\"},\"id\":1}" || true)
		[[ "$boot" == *'"isBootstrapped":true'* ]] || return 1
	done
}

evict_pod() {
	local namespace=$1 pod=$2
	kubectl --context "$runtime_context" create --raw \
		"/api/v1/namespaces/$namespace/pods/$pod/eviction" -f - >/dev/null <<EOF
{"apiVersion":"policy/v1","kind":"Eviction","metadata":{"name":"$pod","namespace":"$namespace"}}
EOF
}

height() {
	rpc "$1" "$2" /v1/chain/C/rpc \
		'{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' |
		python3 -c 'import json,sys; print(int(json.load(sys.stdin)["result"], 16))'
}

sync_application() {
	local app=$1
	kubectl --context "$management_context" -n hanzo-cd annotate application "$app" \
		argocd.argoproj.io/refresh=hard --overwrite >/dev/null
	kubectl --context "$management_context" -n hanzo-cd patch application "$app" --type merge \
		-p '{"operation":{"sync":{"prune":false}}}' >/dev/null
	for _ in $(seq 1 60); do
		local phase sync health revisions
		phase=$(kubectl --context "$management_context" -n hanzo-cd get application "$app" -o jsonpath='{.status.operationState.phase}')
		sync=$(kubectl --context "$management_context" -n hanzo-cd get application "$app" -o jsonpath='{.status.sync.status}')
		health=$(kubectl --context "$management_context" -n hanzo-cd get application "$app" -o jsonpath='{.status.health.status}')
		revisions=$(kubectl --context "$management_context" -n hanzo-cd get application "$app" -o jsonpath='{.status.sync.revisions[*]}')
		# OnDelete StatefulSets are expected to be Progressing after the template
		# sync and before their pods are deliberately replaced below.
		if [ "$phase" = Succeeded ] && [ "$sync" = Synced ] && [[ "$revisions" == *"$revision"* ]]; then
			return 0
		fi
		[ "$phase" != Failed ] && [ "$phase" != Error ] || return 1
		sleep 5
	done
	return 1
}

roll_fleet() {
	local namespace=$1 app=$2 desired ready pod target_ready before after
	local -a order unready
	echo "promoting $tag to $namespace"
	sync_application "$app"
	desired=$(kubectl --context "$runtime_context" get statefulset -n "$namespace" luxd \
		-o jsonpath='{.spec.template.spec.containers[?(@.name=="luxd")].image}')
	[ "$desired" = "$registry:$tag@$digest" ] || {
		echo "$namespace template is $desired, not the requested immutable image" >&2
		return 1
	}

	# If one pod is already unready, replacing a healthy pod first would reduce
	# the fleet below its four-node safety floor. Recover that exact unavailable
	# ordinal first, then resume the normal staggered order. More than one
	# unavailable validator is an operator incident, not an automated rollout.
	unready=()
	for ordinal in 0 1 2 3 4; do
		target_ready=$(kubectl --context "$runtime_context" get pod -n "$namespace" "luxd-$ordinal" \
			-o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
		[ "$target_ready" = True ] || unready+=("$ordinal")
	done
	[ "${#unready[@]}" -le 1 ] || {
		echo "$namespace has ${#unready[@]} unavailable validators; refusing an automated roll" >&2
		return 1
	}
	order=()
	[ "${#unready[@]}" -eq 0 ] || order+=("${unready[0]}")
	for ordinal in 2 4 1 3 0; do
		[ "${#unready[@]}" -eq 0 ] || [ "$ordinal" != "${unready[0]}" ] || continue
		order+=("$ordinal")
	done

	for ordinal in "${order[@]}"; do
		ready=$(kubectl --context "$runtime_context" get statefulset -n "$namespace" luxd -o jsonpath='{.status.readyReplicas}')
		pod=luxd-$ordinal
		target_ready=$(kubectl --context "$runtime_context" get pod -n "$namespace" "$pod" \
			-o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
		if [ "$target_ready" = True ]; then
			[ "$ready" = 5 ] || { echo "$namespace has only ${ready:-0}/5 Ready; refusing to reduce quorum" >&2; return 1; }
		else
			[ "${ready:-0}" -ge 4 ] || { echo "$namespace has only ${ready:-0}/5 Ready; refusing recovery below the safety floor" >&2; return 1; }
		fi
		if [ "$target_ready" != True ] && pod_has_release "$namespace" "$pod"; then
			echo "$namespace/$pod already runs $tag@$digest; continuing its in-place recovery"
		else
			# Use the eviction subresource so the cluster's disruption budget is an
			# independent safety gate. The CD service account intentionally cannot
			# delete pods directly, but may request an eviction that the API server
			# rejects whenever replacing this validator would violate availability.
			evict_pod "$namespace" "$pod"
		fi
		# Historical recovery may have to execute thousands of canonical EVM
		# blocks. Keep the one-validator gate in place for up to four hours rather
		# than restarting a healthy, forward-progressing database on a short
		# operational timeout.
		for _ in $(seq 1 960); do
			if kubectl --context "$runtime_context" wait -n "$namespace" --for=condition=Ready "pod/$pod" --timeout=5s >/dev/null 2>&1 && pod_verified "$namespace" "$pod"; then
				echo "$namespace/$pod is Ready, bootstrapped and verified"
				break
			fi
			sleep 10
		done
		pod_verified "$namespace" "$pod" || return 1
	done

	before=$(height "$namespace" luxd-0)
	for _ in $(seq 1 12); do
		sleep 10
		after=$(height "$namespace" luxd-0)
		if [ "$after" -gt "$before" ]; then
			for _ in $(seq 1 30); do
				local sync health
				sync=$(kubectl --context "$management_context" -n hanzo-cd get application "$app" -o jsonpath='{.status.sync.status}')
				health=$(kubectl --context "$management_context" -n hanzo-cd get application "$app" -o jsonpath='{.status.health.status}')
				if [ "$sync" = Synced ] && [ "$health" = Healthy ]; then
					local all_ready=true
					for ordinal in 0 1 2 3 4; do
						if ! all_chains_bootstrapped "$namespace" "luxd-$ordinal"; then
							all_ready=false
							break
						fi
					done
					if [ "$all_ready" = true ]; then
						echo "$namespace produced blocks: $before -> $after; every registered chain is bootstrapped on 5/5 validators; CD is Synced/Healthy"
						return 0
					fi
				fi
				sleep 5
			done
			echo "$app did not reach Synced/Healthy with every registered chain bootstrapped on 5/5 validators" >&2
			return 1
		fi
	done
	echo "$namespace did not produce a C-Chain block after the roll" >&2
	return 1
}

# The estate after the roll: every validator's C-Chain height and restart count,
# one line per fleet, so a pod that came back on a lower tip or through a crash
# loop is visible without a second command.
report() {
	local namespace ordinal h
	for namespace in lux-devnet lux-testnet lux-mainnet; do
		printf '%-12s' "$namespace"
		for ordinal in 0 1 2 3 4; do
			h=$(height "$namespace" "luxd-$ordinal" 2>/dev/null || true)
			printf '%10s' "${h:-x}"
		done
		printf '  restarts:'
		kubectl --context "$runtime_context" get pods -n "$namespace" --no-headers 2>/dev/null |
			awk '/^luxd-/ { printf " %s", $4 }'
		echo
	done
}

roll_fleet lux-devnet lux-devnet-luxd
roll_fleet lux-testnet lux-testnet-luxd
roll_fleet lux-mainnet lux-mainnet-luxd
report