zeekay
0730def03a
Every file that opted in described the DigitalOcean estate, so CD applying it put a pod in front of static sites the edge already serves from disk. Automation stays per service; a file re-opts in once what it declares has been shown to run on this cluster.
307 lines
11 KiB
YAML
307 lines
11 KiB
YAML
# Lux's node-local o11y collector — the same one hanzo-k8s runs.
|
|
#
|
|
# This replaces node-exporter (9 pods) and the cadvisor scrape that
|
|
# lux-validator-vmagent performed. Lux now produces OTLZ like everything else
|
|
# and writes it to the o11y plane directly, so there is one telemetry standard
|
|
# in the estate rather than one per cluster.
|
|
#
|
|
# The ONLY difference from hanzo's agent is the endpoint: cloud's ZAP sockets
|
|
# are a ClusterIP over there, so from here they are reached through the `otlz`
|
|
# internal LoadBalancer on the shared DigitalOcean VPC. Everything else —
|
|
# image, receivers, processors, pipelines — is deliberately identical. If this
|
|
# file ever needs a Lux-specific receiver or a different wire, that is the
|
|
# signal that the plane has grown a second way of doing something.
|
|
replicas: 1
|
|
workload:
|
|
kind: DaemonSet
|
|
selectorLabels:
|
|
app: otel-agent
|
|
containerName: otel-agent
|
|
image:
|
|
# Pinned to the same build hanzo runs, and it must stay in step: this SHA is
|
|
# the first with per-pod ZAP node identity (aeff4d2) and lowercase span kind
|
|
# (9aab666). ghcr.io/hanzoai/otel-collector publishes no resolvable semver,
|
|
# so a tag would not pin anything.
|
|
repository: ghcr.io/hanzoai/otel-collector
|
|
#
|
|
# ⚠️ THIS SHA IS A PIN, NOT A WARNING SIGN — and the next bump is wanted.
|
|
# otel-collector main CONTAINS this build (`git merge-base --is-ancestor
|
|
# e321e94 origin/main` is true), and this build CARRIES the restart fix:
|
|
# plane needs: nodeID gained a per-process boot nonce (otel-collector e321e94),
|
|
# so a collector that restarts inside its pod arrives as a NEW ZAP peer instead
|
|
# of reclaiming an identity the receiver still holds and being refused with EOF
|
|
# forever. Four incidents on 2026-08-08 each ended with a manual delete of the
|
|
# pod; that is exactly what the nonce removes. Build main and move these pins.
|
|
#
|
|
# (An earlier revision of this comment said main had diverged onto an OTLP proto
|
|
# wire the receiver could not read, and called it measured. It was measured
|
|
# against a STALE LOCAL CHECKOUT, not the remote — origin/main has the
|
|
# MetricBatch wire and luxfi/zap, same as this image. Check origin, never a
|
|
# local main.)
|
|
tag: sha-e321e94
|
|
pullPolicy: IfNotPresent
|
|
imagePullSecrets: []
|
|
# EMPTY ON PURPOSE — the chart defaults this to `ghcr-secret`, and on THIS
|
|
# cluster that credential is user Darkhorse7stars, whose token the ghcr token
|
|
# endpoint answers 403. ghcr.io/hanzoai/otel-collector is public: an
|
|
# anonymous pull of this exact tag returns 200, and a credential-free pod on
|
|
# a lux node pulls it in seconds. Sending a rejected credential turns a public
|
|
# image into ImagePullBackOff, which is how all nine agents and the gateway
|
|
# sat unschedulable while the image was freely available the whole time.
|
|
#
|
|
# (hanzo-k8s has a secret of the SAME NAME holding hanzo-dev, which does
|
|
# authenticate — so the chart default is right there and wrong here. lux's
|
|
# ghcr-secret being dead is a separate problem: nothing new can be pulled on
|
|
# this cluster from a private hanzoai package, and everything still running
|
|
# is running from node cache.)
|
|
partOf: o11y
|
|
component: observability
|
|
args:
|
|
- --config=/etc/otel/otel-agent.yaml
|
|
env:
|
|
- name: K8S_NODE_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
# kubeletstats dials the node's ADDRESS. Node names are not DNS records on
|
|
# DOKS, so ${K8S_NODE_NAME} there fails every interval with "no such host"
|
|
# while the collector otherwise looks healthy.
|
|
- name: K8S_NODE_IP
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: status.hostIP
|
|
- name: OTEL_RESOURCE_ATTRIBUTES
|
|
value: k8s.node.name=$(K8S_NODE_NAME)
|
|
rbac:
|
|
create: true
|
|
serviceAccountName: otel-agent
|
|
clusterRules:
|
|
# k8sattributes: resolve pod/namespace/workload metadata.
|
|
- apiGroups: ['']
|
|
resources: ['pods', 'namespaces', 'nodes']
|
|
verbs: ['get', 'list', 'watch']
|
|
- apiGroups: ['apps']
|
|
resources: ['replicasets', 'deployments', 'daemonsets', 'statefulsets']
|
|
verbs: ['get', 'list', 'watch']
|
|
# kubeletstats: this node's kubelet summary API.
|
|
- apiGroups: ['']
|
|
resources: ['nodes/stats', 'nodes/proxy']
|
|
verbs: ['get']
|
|
tolerations:
|
|
- operator: Exists
|
|
ports:
|
|
- containerPort: 13133
|
|
name: health
|
|
- containerPort: 4317
|
|
name: otlp-grpc
|
|
- containerPort: 4318
|
|
name: otlp-http
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 192Mi
|
|
ephemeral-storage: 512Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 1Gi
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /
|
|
port: 13133
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 15
|
|
volumeMounts:
|
|
- name: config
|
|
mountPath: /etc/otel
|
|
readOnly: true
|
|
- name: varlogpods
|
|
mountPath: /var/log/pods
|
|
readOnly: true
|
|
- name: checkpoint
|
|
mountPath: /var/lib/otel-agent
|
|
- name: hostfs
|
|
mountPath: /hostfs
|
|
readOnly: true
|
|
mountPropagation: HostToContainer
|
|
volumes:
|
|
- name: config
|
|
configMap:
|
|
name: otel-agent-config
|
|
- name: varlogpods
|
|
hostPath:
|
|
path: /var/log/pods
|
|
type: Directory
|
|
- name: checkpoint
|
|
hostPath:
|
|
path: /var/lib/otel-agent
|
|
type: DirectoryOrCreate
|
|
- name: hostfs
|
|
hostPath:
|
|
path: /
|
|
type: Directory
|
|
configMaps:
|
|
- name: otel-agent-config
|
|
data:
|
|
otel-agent.yaml: |
|
|
receivers:
|
|
otlp:
|
|
protocols:
|
|
grpc:
|
|
endpoint: 0.0.0.0:4317
|
|
http:
|
|
endpoint: 0.0.0.0:4318
|
|
|
|
filelog:
|
|
include:
|
|
- /var/log/pods/*/*/*.log
|
|
exclude:
|
|
- /var/log/pods/lux-system_otel-agent-*/**/*.log
|
|
start_at: end
|
|
include_file_path: true
|
|
include_file_name: false
|
|
retry_on_failure:
|
|
enabled: true
|
|
operators:
|
|
- type: container
|
|
add_metadata_from_filepath: true
|
|
|
|
|
|
# Host metrics — what node-exporter used to answer, at a rate this
|
|
# datastore can carry. 300s, not 30s: the OTLZ exporter emits one
|
|
# message per resource and cloud turns each into one INSERT, fanning
|
|
# out through seven tables. At 30s across 45 agents that helped push
|
|
# event.metric_30m over its part ceiling and the server rejected every
|
|
# metric write for an hour. Five-minute resolution is the right trade —
|
|
# the finest rollup in this store is five minutes wide.
|
|
hostmetrics:
|
|
collection_interval: 300s
|
|
root_path: /hostfs
|
|
scrapers:
|
|
cpu:
|
|
load:
|
|
memory:
|
|
paging:
|
|
disk:
|
|
filesystem:
|
|
network:
|
|
|
|
# [node] ONLY: pod and container each model as their own resource,
|
|
# which is one INSERT each.
|
|
kubeletstats:
|
|
collection_interval: 300s
|
|
auth_type: serviceAccount
|
|
# The node's IP, not its name — node names are not DNS on DOKS.
|
|
endpoint: https://${env:K8S_NODE_IP}:10250
|
|
insecure_skip_verify: true
|
|
metric_groups: [node]
|
|
|
|
processors:
|
|
memory_limiter:
|
|
check_interval: 5s
|
|
limit_mib: 800
|
|
spike_limit_mib: 160
|
|
k8sattributes:
|
|
auth_type: serviceAccount
|
|
passthrough: false
|
|
extract:
|
|
metadata:
|
|
- k8s.namespace.name
|
|
- k8s.pod.name
|
|
- k8s.pod.uid
|
|
- k8s.deployment.name
|
|
- k8s.statefulset.name
|
|
- k8s.daemonset.name
|
|
- k8s.node.name
|
|
- k8s.container.name
|
|
pod_association:
|
|
- sources:
|
|
- from: resource_attribute
|
|
name: k8s.pod.uid
|
|
- sources:
|
|
- from: connection
|
|
resourcedetection:
|
|
detectors: [env, system]
|
|
timeout: 2s
|
|
override: false
|
|
resource:
|
|
attributes:
|
|
# Both clusters write to ONE store, so the cluster has to be on
|
|
# the series or a lux node and a hanzo node are indistinguishable.
|
|
# vmagent carried this as -remoteWrite.label=cluster=lux-k8s; this
|
|
# is the same fact under its OpenTelemetry name.
|
|
- key: k8s.cluster.name
|
|
value: lux-k8s
|
|
action: upsert
|
|
- key: service.namespace
|
|
value: lux
|
|
action: upsert
|
|
- key: deployment.environment
|
|
value: production
|
|
action: upsert
|
|
batch:
|
|
timeout: 2s
|
|
send_batch_size: 2048
|
|
send_batch_max_size: 4096
|
|
# Metrics get their OWN batch settings, far coarser than logs.
|
|
#
|
|
# Every insert into event.metric fans out through the fill_metric_5m
|
|
# and fill_metric_30m materialized views, so ONE insert becomes THREE
|
|
# parts. At batch/timeout 2s across 42 collectors that is ~60 inserts
|
|
# a second and ~180 parts a second, and ClickHouse merges cannot keep
|
|
# up: event.metric_30m reached 3,015 parts averaging 3.91 KiB and the
|
|
# server began answering code 252 "Too many parts" to EVERY metric
|
|
# write — killing app telemetry that had been fine for months, not
|
|
# just the new host metrics.
|
|
#
|
|
# Fewer, larger inserts is the fix. 60s/16384 cuts the part rate by
|
|
# ~30x and costs a minute of latency on data whose finest rollup is
|
|
# five minutes wide.
|
|
batch/metrics:
|
|
timeout: 60s
|
|
send_batch_size: 8192
|
|
send_batch_max_size: 16384
|
|
|
|
exporters:
|
|
# One exporter per signal: cloud binds one socket per signal, and the
|
|
# ZAP node identity derives from the component ID, so two exporters
|
|
# sharing a name would be two nodes claiming one identity and the
|
|
# second would be refused.
|
|
zap/logs:
|
|
endpoint: 10.124.0.71:4318
|
|
retry_on_failure:
|
|
enabled: true
|
|
zap/traces:
|
|
endpoint: 10.124.0.71:4317
|
|
retry_on_failure:
|
|
enabled: true
|
|
zap/metrics:
|
|
endpoint: 10.124.0.71:4319
|
|
retry_on_failure:
|
|
enabled: true
|
|
|
|
extensions:
|
|
health_check:
|
|
endpoint: 0.0.0.0:13133
|
|
|
|
service:
|
|
extensions: [health_check]
|
|
pipelines:
|
|
logs:
|
|
receivers: [filelog, otlp]
|
|
processors: [memory_limiter, k8sattributes, resourcedetection, resource, batch]
|
|
exporters: [zap/logs]
|
|
traces:
|
|
receivers: [otlp]
|
|
processors: [memory_limiter, k8sattributes, resourcedetection, resource, batch]
|
|
exporters: [zap/traces]
|
|
metrics:
|
|
receivers: [hostmetrics, kubeletstats]
|
|
processors: [memory_limiter, k8sattributes, resourcedetection, resource, batch/metrics]
|
|
exporters: [zap/metrics]
|
|
telemetry:
|
|
logs:
|
|
level: info
|
|
encoding: json
|
|
cd:
|
|
automated: false
|