mirror of
https://github.com/luxfi/monitoring.git
synced 2026-10-30 15:24:48 +00:00
zeekayandHanzo Dev
3e48e928f9
luxd v1.34 serves the JSON-RPC/info/health APIs under /v1/* not /ext/*, so the validator-health exporter was polling dead paths and every mainnet validator falsely reported down. Repoint exporter.py to /v1/*. Add the incident-critical node/pod memory pipeline on lux-k8s (the owner is debugging luxd OOM-evictions): - node-exporter DaemonSet (per-node host memory, all nodes incl luxd nodes) - kube-state-metrics (pod phase/restarts/OOM+Evicted reasons, node conditions) - vmagent RBAC + scrape jobs for node-exporter, kubelet cAdvisor (pod mem, drop-filtered), kube-state-metrics; heavy pods pinned to roomy non-luxd nodes - Grafana dashboard 'Lux — Cluster Nodes & OOM' (uid lux-nodes-oom) Read-only w.r.t. luxd/ns lux-mainnet. Ships via the existing vmagent -> metrics.hanzo.ai hub. Co-authored-by: Hanzo Dev <[email protected]>
78 lines
3.0 KiB
YAML
78 lines
3.0 KiB
YAML
# node-exporter DaemonSet — per-node host memory/CPU/disk on the lux cluster.
|
|
# Runs on EVERY node (incl. the luxd nodes) BECAUSE the owner is debugging OOM
|
|
# evictions on those exact nodes — you cannot see luxd-node memory without an
|
|
# agent on the luxd node. Footprint is deliberately tiny (24Mi req / 64Mi lim),
|
|
# hostNetwork, read-only host mounts — it does NOT touch ns lux-mainnet or churn
|
|
# any luxd pod. Scraped by lux-validator-vmagent, shipped to the Hanzo o11y hub.
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: node-exporter
|
|
namespace: lux-system
|
|
labels: {app: node-exporter, brand: lux}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata:
|
|
name: node-exporter
|
|
namespace: lux-system
|
|
labels: {app: node-exporter, brand: lux}
|
|
spec:
|
|
selector: {matchLabels: {app: node-exporter}}
|
|
updateStrategy:
|
|
type: RollingUpdate
|
|
rollingUpdate: {maxUnavailable: 1}
|
|
template:
|
|
metadata:
|
|
labels: {app: node-exporter, brand: lux}
|
|
spec:
|
|
serviceAccountName: node-exporter
|
|
hostNetwork: true
|
|
hostPID: true
|
|
priorityClassName: system-node-critical
|
|
# Cover every node (incl. tainted/luxd) so per-node memory is complete.
|
|
tolerations:
|
|
- operator: Exists
|
|
securityContext: {runAsNonRoot: true, runAsUser: 65534, runAsGroup: 65534}
|
|
containers:
|
|
- name: node-exporter
|
|
image: prom/node-exporter:v1.8.2
|
|
args:
|
|
- --path.rootfs=/host/root
|
|
- --path.procfs=/host/proc
|
|
- --path.sysfs=/host/sys
|
|
- --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/)
|
|
- --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$
|
|
- --no-collector.wifi
|
|
- --no-collector.hwmon
|
|
- --web.listen-address=:9100
|
|
ports:
|
|
- {name: metrics, containerPort: 9100, hostPort: 9100, protocol: TCP}
|
|
resources:
|
|
requests: {cpu: 10m, memory: 24Mi}
|
|
limits: {cpu: 100m, memory: 64Mi}
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities: {drop: [ALL]}
|
|
volumeMounts:
|
|
- {name: proc, mountPath: /host/proc, readOnly: true}
|
|
- {name: sys, mountPath: /host/sys, readOnly: true}
|
|
- {name: root, mountPath: /host/root, mountPropagation: HostToContainer, readOnly: true}
|
|
volumes:
|
|
- {name: proc, hostPath: {path: /proc}}
|
|
- {name: sys, hostPath: {path: /sys}}
|
|
- {name: root, hostPath: {path: /}}
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: node-exporter
|
|
namespace: lux-system
|
|
labels: {app: node-exporter, brand: lux}
|
|
spec:
|
|
clusterIP: None
|
|
selector: {app: node-exporter}
|
|
ports:
|
|
- {name: metrics, port: 9100, targetPort: 9100}
|