mirror of
https://github.com/luxfi/monitoring.git
synced 2026-10-30 15:24:48 +00:00
zeekayandHanzo Dev
7e35c626c1
ghcr.io/luxfi/monitoring is an `internal` package with NO linked repository, so it is not pullable anonymously and not every org credential is authorised for it. The Deployment declared no imagePullSecrets at all, so the pod never started: 8,394 backoff events over 31h, during which the validator-health alerting this repo exists to provide was blind. It must be `ghcr-secret`, NOT `ghcr-luxfi`. Verified against the registry from inside the cluster — ghcr-secret gets 200 on the v0.1.1 manifest, ghcr-luxfi gets 403. ghcr-luxfi is used by 53 other deployments in that cluster and works for luxfi/node, so "it pulls other luxfi images" would have been the wrong inference; the package's own ACL is what matters. Verified after applying: pod 1/1 Running, /metrics 200 serving 211 lux_ series. Context for whoever reads this next: the same 31h window contained a REAL incident the alerting correctly detected and nobody received — the Hanzo mainnet chain is halted at height 1218 with 3 of 5 validators against a 4-of-5 quorum. Separately, alertmanager's SLACK_WEBHOOK_URL points at http://alert-sink.hanzo.svc (the internal logging pod), so every page is delivered to a log line rather than a human. Co-authored-by: Hanzo Dev <[email protected]>
105 lines
4.1 KiB
YAML
105 lines
4.1 KiB
YAML
# Lux validator-health exporter — runs in the lux cluster (lux-system) and polls
|
|
# the luxd validators in lux-mainnet/testnet/devnet via JSON-RPC + the k8s API.
|
|
# Code lives in cmd/validator-exporter (Go); the image is ghcr.io/luxfi/monitoring.
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: lux-validator-exporter
|
|
namespace: lux-system
|
|
labels: {app: lux-validator-exporter, brand: lux}
|
|
---
|
|
# Read-only pod + pod/log access, granted only in the validator namespaces.
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRole
|
|
metadata:
|
|
name: lux-validator-exporter
|
|
labels: {app: lux-validator-exporter, brand: lux}
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: [pods]
|
|
verbs: [get, list, watch]
|
|
- apiGroups: [""]
|
|
resources: [pods/log]
|
|
verbs: [get]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata: {name: lux-validator-exporter, namespace: lux-mainnet, labels: {brand: lux}}
|
|
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: lux-validator-exporter}
|
|
subjects: [{kind: ServiceAccount, name: lux-validator-exporter, namespace: lux-system}]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata: {name: lux-validator-exporter, namespace: lux-testnet, labels: {brand: lux}}
|
|
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: lux-validator-exporter}
|
|
subjects: [{kind: ServiceAccount, name: lux-validator-exporter, namespace: lux-system}]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata: {name: lux-validator-exporter, namespace: lux-devnet, labels: {brand: lux}}
|
|
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: lux-validator-exporter}
|
|
subjects: [{kind: ServiceAccount, name: lux-validator-exporter, namespace: lux-system}]
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: lux-validator-exporter
|
|
namespace: lux-system
|
|
labels: {app: lux-validator-exporter, brand: lux}
|
|
spec:
|
|
replicas: 1
|
|
selector: {matchLabels: {app: lux-validator-exporter}}
|
|
template:
|
|
metadata:
|
|
labels: {app: lux-validator-exporter, brand: lux}
|
|
spec:
|
|
serviceAccountName: lux-validator-exporter
|
|
# ghcr.io/luxfi/monitoring is an `internal` package with NO linked
|
|
# repository, so it is not pullable anonymously and not every org
|
|
# credential is authorised for it. Without this the pod sits in
|
|
# ImagePullBackOff — observed 31h / 8394 backoff events, during which the
|
|
# validator-health alerting this repo exists to provide was blind.
|
|
#
|
|
# It must be `ghcr-secret`, NOT `ghcr-luxfi`: verified against the registry
|
|
# from inside the cluster, ghcr-secret gets 200 on the v0.1.1 manifest and
|
|
# ghcr-luxfi gets 403 (that credential works for luxfi/node, so "it pulls
|
|
# other luxfi images" is not evidence it can pull this one).
|
|
imagePullSecrets:
|
|
- name: ghcr-secret
|
|
securityContext: {runAsNonRoot: true, runAsUser: 65532, runAsGroup: 65532, fsGroup: 65532}
|
|
containers:
|
|
- name: exporter
|
|
image: ghcr.io/luxfi/monitoring:v0.1.1
|
|
command: ["/validator-exporter"]
|
|
env:
|
|
- {name: POLL_INTERVAL, value: "15"}
|
|
- {name: LISTEN_PORT, value: "9101"}
|
|
- {name: REPLICAS, value: "5"}
|
|
ports:
|
|
- {name: metrics, containerPort: 9101}
|
|
readinessProbe: {httpGet: {path: /health, port: 9101}, initialDelaySeconds: 5, periodSeconds: 10}
|
|
livenessProbe: {httpGet: {path: /health, port: 9101}, initialDelaySeconds: 10, periodSeconds: 30}
|
|
resources:
|
|
requests: {cpu: 50m, memory: 64Mi}
|
|
limits: {cpu: 300m, memory: 128Mi}
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities: {drop: [ALL]}
|
|
volumeMounts:
|
|
- {name: tmp, mountPath: /tmp}
|
|
volumes:
|
|
- name: tmp
|
|
emptyDir: {}
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: lux-validator-exporter
|
|
namespace: lux-system
|
|
labels: {app: lux-validator-exporter, brand: lux}
|
|
spec:
|
|
selector: {app: lux-validator-exporter}
|
|
ports:
|
|
- {name: metrics, port: 9101, targetPort: 9101}
|