mirror of
https://github.com/luxfi/monitoring.git
synced 2026-10-30 15:24:48 +00:00
zeekayandHanzo Dev
3e48e928f9
luxd v1.34 serves the JSON-RPC/info/health APIs under /v1/* not /ext/*, so the validator-health exporter was polling dead paths and every mainnet validator falsely reported down. Repoint exporter.py to /v1/*. Add the incident-critical node/pod memory pipeline on lux-k8s (the owner is debugging luxd OOM-evictions): - node-exporter DaemonSet (per-node host memory, all nodes incl luxd nodes) - kube-state-metrics (pod phase/restarts/OOM+Evicted reasons, node conditions) - vmagent RBAC + scrape jobs for node-exporter, kubelet cAdvisor (pod mem, drop-filtered), kube-state-metrics; heavy pods pinned to roomy non-luxd nodes - Grafana dashboard 'Lux — Cluster Nodes & OOM' (uid lux-nodes-oom) Read-only w.r.t. luxd/ns lux-mainnet. Ships via the existing vmagent -> metrics.hanzo.ai hub. Co-authored-by: Hanzo Dev <[email protected]>
99 lines
6.7 KiB
JSON
99 lines
6.7 KiB
JSON
{
|
|
"annotations": { "list": [] },
|
|
"editable": true,
|
|
"graphTooltip": 1,
|
|
"time": { "from": "now-3h", "to": "now" },
|
|
"refresh": "30s",
|
|
"schemaVersion": 39,
|
|
"tags": ["lux", "nodes", "oom", "incident"],
|
|
"title": "Lux — Cluster Nodes & OOM (lux-k8s)",
|
|
"uid": "lux-nodes-oom",
|
|
"templating": { "list": [] },
|
|
"panels": [
|
|
{
|
|
"type": "stat",
|
|
"title": "Max node memory %",
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
|
|
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 75 }, { "color": "red", "value": 85 } ] } } },
|
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "area" },
|
|
"targets": [ { "refId": "A", "expr": "max(100*(1 - sum by (node)(node_memory_MemAvailable_bytes{cluster=\"lux-k8s\"}) / sum by (node)(node_memory_MemTotal_bytes{cluster=\"lux-k8s\"})))" } ]
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"title": "Mainnet validators UP (of 5)",
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
|
|
"fieldConfig": { "defaults": { "unit": "short", "min": 0, "max": 5, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "orange", "value": 4 }, { "color": "green", "value": 5 } ] } } },
|
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" },
|
|
"targets": [ { "refId": "A", "expr": "lux_network_validators_up{network=\"mainnet\"}" } ]
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"title": "Pods not Running",
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
|
|
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 } ] } } },
|
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" },
|
|
"targets": [ { "refId": "A", "expr": "sum(kube_pod_status_phase{cluster=\"lux-k8s\",phase=~\"Pending|Failed|Unknown\"} > 0)" } ]
|
|
},
|
|
{
|
|
"type": "stat",
|
|
"title": "OOMKilled containers (now)",
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
|
|
"fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } } },
|
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" },
|
|
"targets": [ { "refId": "A", "expr": "sum(kube_pod_container_status_last_terminated_reason{cluster=\"lux-k8s\",reason=\"OOMKilled\"} > 0) or vector(0)" } ]
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Per-node memory % (node-exporter) — which node is about to OOM",
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"gridPos": { "h": 9, "w": 24, "x": 0, "y": 4 },
|
|
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "custom": { "drawStyle": "line", "fillOpacity": 8, "showPoints": "never" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 75 }, { "color": "red", "value": 85 } ] } } },
|
|
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull", "max"] }, "tooltip": { "mode": "multi", "sort": "desc" } },
|
|
"targets": [ { "refId": "A", "legendFormat": "{{node}}", "expr": "100*(1 - sum by (node)(node_memory_MemAvailable_bytes{cluster=\"lux-k8s\"}) / sum by (node)(node_memory_MemTotal_bytes{cluster=\"lux-k8s\"}))" } ]
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Top pods by memory working set (which pod is eating the node)",
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 13 },
|
|
"fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "fillOpacity": 8, "showPoints": "never" } } },
|
|
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull", "max"] }, "tooltip": { "mode": "multi", "sort": "desc" } },
|
|
"targets": [ { "refId": "A", "legendFormat": "{{namespace}}/{{pod}}", "expr": "topk(12, sum by (namespace,pod)(container_memory_working_set_bytes{cluster=\"lux-k8s\",pod!=\"\"}))" } ]
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Container restarts (increase 1h) — crash-loop / OOM churn",
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 13 },
|
|
"fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "bars", "fillOpacity": 40, "showPoints": "never" } } },
|
|
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull", "max"] }, "tooltip": { "mode": "multi", "sort": "desc" } },
|
|
"targets": [ { "refId": "A", "legendFormat": "{{namespace}}/{{pod}}", "expr": "topk(12, increase(kube_pod_container_status_restarts_total{cluster=\"lux-k8s\"}[1h])) > 0" } ]
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "luxd validators — C-Chain height (mainnet)",
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 },
|
|
"fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "showPoints": "never" } } },
|
|
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull"] }, "tooltip": { "mode": "multi" } },
|
|
"targets": [ { "refId": "A", "legendFormat": "{{instance}}", "expr": "lux_validator_c_block_height{network=\"mainnet\"}" } ]
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "luxd validators — up / peers (mainnet)",
|
|
"datasource": { "type": "prometheus", "uid": "victoriametrics" },
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 },
|
|
"fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "fillOpacity": 10, "showPoints": "always" } } },
|
|
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull"] }, "tooltip": { "mode": "multi" } },
|
|
"targets": [
|
|
{ "refId": "A", "legendFormat": "up {{instance}}", "expr": "lux_validator_up{network=\"mainnet\"}" },
|
|
{ "refId": "B", "legendFormat": "peers {{instance}}", "expr": "lux_validator_peers{network=\"mainnet\"}" }
|
|
]
|
|
}
|
|
]
|
|
}
|