universe/deploy/lux-system/otel-gateway.yaml

# Lux's cluster-scoped o11y collector — the singleton half, as in hanzo-k8s.
#
# Two things cannot be answered per-node, and they are what this replaces:
#
#   kube-state-metrics  ->  the k8s_cluster receiver. One process reading the
#                           API server, so there is no exporter to deploy and
#                           no scrape job to maintain.
#   the lux-validators  ->  a prometheus receiver. lux-validator-exporter
#   scrape job              publishes Prometheus text on :9101 and nothing
#                           will push it, so something must scrape; one
#                           scraper per cluster is the right number.
#
# Between this and the DaemonSet beside it, lux-validator-vmagent has nothing
# left to do, which is the point: the remote-write sender was the last thing in
# this cluster speaking a telemetry protocol that was not OTLZ.
#
# ONE replica, and not for cost: k8s_cluster reports the whole cluster from
# whichever pod runs it, so a second replica is a second copy of every series.
replicas: 1
strategy: Recreate
selectorLabels:
  app: otel-gateway
containerName: otel-gateway
image:
  repository: ghcr.io/hanzoai/otel-collector
  #
  # ⚠️ THIS SHA IS A PIN, NOT A WARNING SIGN — and the next bump is wanted.
  # otel-collector main CONTAINS this build (`git merge-base --is-ancestor
  # e321e94 origin/main` is true), and this build CARRIES the restart fix:
  # plane needs: nodeID gained a per-process boot nonce (otel-collector e321e94),
  # so a collector that restarts inside its pod arrives as a NEW ZAP peer instead
  # of reclaiming an identity the receiver still holds and being refused with EOF
  # forever. Four incidents on 2026-08-08 each ended with a manual delete of the
  # pod; that is exactly what the nonce removes. Build main and move these pins.
  #
  # (An earlier revision of this comment said main had diverged onto an OTLP proto
  # wire the receiver could not read, and called it measured. It was measured
  # against a STALE LOCAL CHECKOUT, not the remote — origin/main has the
  # MetricBatch wire and luxfi/zap, same as this image. Check origin, never a
  # local main.)
  tag: sha-e321e94
  pullPolicy: IfNotPresent
imagePullSecrets: []
# EMPTY ON PURPOSE — the chart defaults this to `ghcr-secret`, and on THIS
# cluster that credential is user Darkhorse7stars, whose token the ghcr token
# endpoint answers 403. ghcr.io/hanzoai/otel-collector is public: an
# anonymous pull of this exact tag returns 200, and a credential-free pod on
# a lux node pulls it in seconds. Sending a rejected credential turns a public
# image into ImagePullBackOff, which is how all nine agents and the gateway
# sat unschedulable while the image was freely available the whole time.
#
# (hanzo-k8s has a secret of the SAME NAME holding hanzo-dev, which does
# authenticate — so the chart default is right there and wrong here. lux's
# ghcr-secret being dead is a separate problem: nothing new can be pulled on
# this cluster from a private hanzoai package, and everything still running
# is running from node cache.)
partOf: o11y
component: observability
args:
  - --config=/etc/otel/otel-gateway.yaml
# This gateway reads NOTHING from the apiserver. It scrapes two static
# targets -- the validator exporter and kube-state-metrics -- and forwards over
# OTLZ. It previously carried a ServiceAccount bound to cluster-wide get/list/
# watch on events, nodes, pods, services, endpoints, deployments, jobs and HPAs,
# left from when the pipeline had a k8s_cluster receiver. Scraping
# kube-state-metrics replaced that receiver (same data, one INSERT per scrape
# instead of one per object), and the grant outlived its reader.
#
# An unused ServiceAccount is not neutral: it is a token mounted in a pod,
# holding cluster-wide read, that no one has reason to look at again. So the
# grant goes, and the token with it.
automountServiceAccountToken: false
ports:
  - containerPort: 13133
    name: health
resources:
  requests:
    cpu: 100m
    memory: 256Mi
    ephemeral-storage: 256Mi
  limits:
    cpu: 1000m
    memory: 1Gi
readinessProbe:
  httpGet:
    path: /
    port: 13133
  initialDelaySeconds: 5
  periodSeconds: 15
volumeMounts:
  - name: config
    mountPath: /etc/otel
    readOnly: true
volumes:
  - name: config
    configMap:
      name: otel-gateway-config
configMaps:
  - name: otel-gateway-config
    data:
      otel-gateway.yaml: |
        receivers:

          prometheus:
            config:
              scrape_configs:
                # The validator exporter, carried over from vmagent's
                # `lux-validators` job. It is the one genuine scrape left in
                # this cluster: chain state that no SDK emits.
                - job_name: lux-validators
                  scrape_interval: 30s
                  static_configs:
                    - targets: ['lux-validator-exporter.lux-system.svc:9101']
                # Object state — kube-state-metrics, SCRAPED, not k8s_cluster.
                #
                # k8s_cluster works: measured 35 metric names and 14,720 datapoints in
                # isolation, exactly the families kube-state-metrics publishes. It is
                # unusable HERE for one reason — it models every object as its own
                # RESOURCE, and the OTLZ exporter sends one message per resource which
                # cloud turns into one INSERT. 8,105 resources per collection is ~16,000
                # inserts a minute; 410 a minute is what took the datastore down today.
                #
                # Scraping the exporter is the same data at one resource per scrape, so
                # one INSERT per 30s instead of 8,105. The keep-list is carried over from
                # the vmagent config that used to do this, and it is load-bearing:
                # kube-state-metrics publishes far more than anyone reads, and unbounded
                # cardinality is the other way to hurt this datastore.
                - job_name: kube-state-metrics
                  honor_labels: true
                  scrape_interval: 30s
                  static_configs:
                    - targets: ['kube-state-metrics.lux-system.svc:8080']
                      labels:
                        cluster: lux-k8s
                  metric_relabel_configs:
                    - source_labels: [__name__]
                      action: keep
                      regex: (kube_pod_status_phase|kube_pod_status_reason|kube_pod_info|kube_pod_container_status_restarts_total|kube_pod_container_status_waiting_reason|kube_pod_container_status_last_terminated_reason|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kube_node_info|kube_node_status_condition|kube_node_status_capacity|kube_node_status_allocatable|kube_deployment_spec_replicas|kube_deployment_status_replicas|kube_deployment_status_replicas_available|kube_statefulset_status_replicas|kube_statefulset_status_replicas_ready|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_ready|kube_job_status_failed|kube_job_status_succeeded)

        processors:
          memory_limiter:
            check_interval: 5s
            limit_mib: 800
            spike_limit_mib: 160
          resourcedetection:
            detectors: [env, system]
            timeout: 2s
            override: false
          resource:
            attributes:
              - key: k8s.cluster.name
                value: lux-k8s
                action: upsert
              - key: service.namespace
                value: lux
                action: upsert
              - key: deployment.environment
                value: production
                action: upsert
          batch:
            timeout: 2s
            send_batch_size: 2048
            send_batch_max_size: 4096
          # Metrics get their OWN batch settings, far coarser than logs.
          #
          # Every insert into event.metric fans out through the fill_metric_5m
          # and fill_metric_30m materialized views, so ONE insert becomes THREE
          # parts. At batch/timeout 2s across 42 collectors that is ~60 inserts
          # a second and ~180 parts a second, and ClickHouse merges cannot keep
          # up: event.metric_30m reached 3,015 parts averaging 3.91 KiB and the
          # server began answering code 252 "Too many parts" to EVERY metric
          # write — killing app telemetry that had been fine for months, not
          # just the new host metrics.
          #
          # Fewer, larger inserts is the fix. 60s/16384 cuts the part rate by
          # ~30x and costs a minute of latency on data whose finest rollup is
          # five minutes wide.
          batch/metrics:
            timeout: 60s
            send_batch_size: 8192
            send_batch_max_size: 16384

        exporters:
          # The `otlz` internal LoadBalancer in hanzo-k8s (10.124.0.71). Both
          # clusters sit in one DigitalOcean VPC, so this is a private address
          # reachable from here and from nothing outside the region.
          zap/metrics:
            endpoint: 10.124.0.71:4319
            retry_on_failure:
              enabled: true

        extensions:
          health_check:
            endpoint: 0.0.0.0:13133

        service:
          extensions: [health_check]
          pipelines:
            metrics:
              receivers: [prometheus]
              processors: [memory_limiter, resourcedetection, resource, batch/metrics]
              exporters: [zap/metrics]
          telemetry:
            logs:
              level: info
              encoding: json
cd:
  automated: false

# One amd64 manifest, so evo is the only node that can execute this. spark
# registers qemu-x86_64 binfmt, which is worse than a refusal: the pod starts,
# runs emulated, and faults at random — 12 restarts, never ready. Stated rather than left
# to the scheduler, the same way trade.yaml states it. The toleration keeps the
# pin working if `hanzo.ai/no-ci` goes back on evo.
nodeSelector:
  kubernetes.io/arch: amd64
tolerations:
- key: hanzo.ai/no-ci
  operator: Exists
  effect: NoSchedule