universe/k8s/lux-testnet/rpc-tip-gate.yaml

# rpc-tip-gate for lux-testnet.
#
# Mainnet has had one since 2026-07-28; testnet has not, and that asymmetry is
# what let two validators frozen 580 blocks behind keep answering
# api.lux-test.network -- they were Ready, and Ready was the only thing between
# a stale pod and a public read.
#
# Readiness cannot carry freshness: it is a property of ONE pod asked about
# itself, and "am I behind" is only answerable against the others. This gate
# polls every luxd pod, compares heights, and owns the lux.network/rpc=fresh
# label; luxd-rpc selects on it, so a node that falls behind leaves the serving
# set without anyone deciding to remove it.
#
# The script is byte-identical to the mainnet gate's and reads its whole
# configuration from env -- only RPC_PORT differs (9640 here, 9630 there).
apiVersion: v1
kind: ServiceAccount
metadata:
  name: rpc-tip-gate
  namespace: lux-testnet
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: rpc-tip-gate
  namespace: lux-testnet
rules:
# patch is for the label and nothing else; the gate never deletes or evicts.
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["get", "list", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: rpc-tip-gate
  namespace: lux-testnet
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: rpc-tip-gate
subjects:
- kind: ServiceAccount
  name: rpc-tip-gate
  namespace: lux-testnet
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: rpc-tip-gate-code
  namespace: lux-testnet
data:
  tip_gate.py: |
    """Label luxd pods that are keeping up; unlabel the ones that are not.

    Stdlib only, so it runs on a stock python image with no build step and no
    registry artifact of its own.
    """
    import json
    import os
    import ssl
    import time
    import urllib.request
    from urllib.parse import quote

    NS = os.environ["NAMESPACE"]
    SELECTOR = os.environ["POD_SELECTOR"]
    RPC_PORT = os.environ["RPC_PORT"]
    RPC_PATH = os.environ["RPC_PATH"]
    LABEL_KEY = os.environ["LABEL_KEY"]
    LABEL_VALUE = os.environ["LABEL_VALUE"]
    MAX_LAG = int(os.environ["MAX_LAG_BLOCKS"])
    INTERVAL = float(os.environ["INTERVAL_SECONDS"])
    TIMEOUT = float(os.environ["PROBE_TIMEOUT_SECONDS"])

    API = "https://kubernetes.default.svc"
    SA = "/var/run/secrets/kubernetes.io/serviceaccount"
    _TLS = None


    def tls():
        """Built on first use, not at import, so a missing or not-yet-mounted
        CA surfaces inside the retry loop instead of killing the process
        before the loop exists."""
        global _TLS
        if _TLS is None:
            _TLS = ssl.create_default_context(cafile=SA + "/ca.crt")
        return _TLS


    def kube(method, path, body=None):
        """One call to the apiserver. The token is re-read every time because
        projected serviceaccount tokens are rotated under a running pod."""
        with open(SA + "/token") as fh:
            token = fh.read().strip()
        req = urllib.request.Request(API + path, method=method)
        req.add_header("Authorization", "Bearer " + token)
        data = None
        if body is not None:
            data = json.dumps(body).encode()
            req.add_header("Content-Type", "application/merge-patch+json")
        with urllib.request.urlopen(req, data=data, timeout=TIMEOUT, context=tls()) as r:
            return json.load(r)


    def tip(ip):
        """Accepted C-Chain head, or None if this node will not answer.

        None is the honest answer for every failure mode that matters -- a
        connection refused, a bootstrapping node that 404s the chain handler,
        a body that is not JSON -- and all of them mean the same thing: do not
        send it public traffic."""
        body = json.dumps(
            {"jsonrpc": "2.0", "id": 1, "method": "eth_blockNumber", "params": []}
        ).encode()
        req = urllib.request.Request(
            "http://%s:%s%s" % (ip, RPC_PORT, RPC_PATH), data=body, method="POST"
        )
        req.add_header("Content-Type", "application/json")
        try:
            with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
                return int(json.load(r)["result"], 16)
        except Exception:
            return None


    def reconcile():
        pods = kube(
            "GET",
            "/api/v1/namespaces/%s/pods?labelSelector=%s" % (NS, quote(SELECTOR)),
        )["items"]

        tips = {}
        for p in pods:
            status = p.get("status", {})
            ip = status.get("podIP")
            tips[p["metadata"]["name"]] = (
                tip(ip) if ip and status.get("phase") == "Running" else None
            )

        answered = [t for t in tips.values() if t is not None]
        if not answered:
            # Fail static. Losing sight of the fleet is not evidence that the
            # fleet is unfit, and emptying the pool would take the endpoint
            # down outright.
            print("no node answered; labels unchanged", flush=True)
            return
        top = max(answered)

        for p in pods:
            name = p["metadata"]["name"]
            t = tips[name]
            want = t is not None and top - t <= MAX_LAG
            has = p["metadata"].get("labels", {}).get(LABEL_KEY) == LABEL_VALUE
            if want != has:
                kube(
                    "PATCH",
                    "/api/v1/namespaces/%s/pods/%s" % (NS, name),
                    {"metadata": {"labels": {LABEL_KEY: LABEL_VALUE if want else None}}},
                )
            print(
                "%s tip=%s lag=%s serving=%s%s"
                % (
                    name,
                    t,
                    "-" if t is None else top - t,
                    want,
                    " (changed)" if want != has else "",
                ),
                flush=True,
            )


    def main():
        while True:
            try:
                reconcile()
            except Exception as err:
                # Never exit the loop. A crash-looping gate stops reconciling,
                # and the labels it leaves behind freeze whatever it last
                # decided.
                print("reconcile failed: %r" % (err,), flush=True)
            time.sleep(INTERVAL)


    if __name__ == "__main__":
        main()

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: rpc-tip-gate
  namespace: lux-testnet
spec:
  replicas: 1
  selector:
    matchLabels:
      app: rpc-tip-gate
  template:
    metadata:
      labels:
        app: rpc-tip-gate
    spec:
      serviceAccountName: rpc-tip-gate
      containers:
      - name: gate
        image: python:3.12-alpine
        command: ["python", "-u", "/app/tip_gate.py"]
        env:
        - name: NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: POD_SELECTOR
          value: app.kubernetes.io/name=luxd,!job-name
        # testnet serves the C-Chain on 9640, not 9630.
        - name: RPC_PORT
          value: "9640"
        - name: RPC_PATH
          value: /v1/chain/C/rpc
        - name: LABEL_KEY
          value: lux.network/rpc
        - name: LABEL_VALUE
          value: fresh
        - name: MAX_LAG_BLOCKS
          value: "1"
        - name: INTERVAL_SECONDS
          value: "10"
        - name: PROBE_TIMEOUT_SECONDS
          value: "5"
        volumeMounts:
        - name: code
          mountPath: /app
        resources:
          requests: {cpu: 10m, memory: 32Mi}
          limits: {cpu: 200m, memory: 128Mi}
      volumes:
      - name: code
        configMap:
          name: rpc-tip-gate-code
---
apiVersion: v1
kind: Service
metadata:
  name: luxd-rpc
  namespace: lux-testnet
spec:
  type: ClusterIP
  # The freshness label is part of the SELECTOR, so falling behind removes a pod
  # from this Service without any controller acting on the Service itself.
  selector:
    app.kubernetes.io/instance: luxd
    app.kubernetes.io/name: luxd
    lux.network/rpc: fresh
  ports:
  - name: http
    port: 9640
    targetPort: 9640