mirror of
https://github.com/luxfi/monitoring.git
synced 2026-11-02 14:36:33 +00:00
zeekayandHanzo Dev
3e48e928f9
luxd v1.34 serves the JSON-RPC/info/health APIs under /v1/* not /ext/*, so the validator-health exporter was polling dead paths and every mainnet validator falsely reported down. Repoint exporter.py to /v1/*. Add the incident-critical node/pod memory pipeline on lux-k8s (the owner is debugging luxd OOM-evictions): - node-exporter DaemonSet (per-node host memory, all nodes incl luxd nodes) - kube-state-metrics (pod phase/restarts/OOM+Evicted reasons, node conditions) - vmagent RBAC + scrape jobs for node-exporter, kubelet cAdvisor (pod mem, drop-filtered), kube-state-metrics; heavy pods pinned to roomy non-luxd nodes - Grafana dashboard 'Lux — Cluster Nodes & OOM' (uid lux-nodes-oom) Read-only w.r.t. luxd/ns lux-mainnet. Ships via the existing vmagent -> metrics.hanzo.ai hub. Co-authored-by: Hanzo Dev <[email protected]>
96 lines
3.3 KiB
YAML
96 lines
3.3 KiB
YAML
# kube-state-metrics — pod phase, restart counts, OOMKilled/Evicted reasons,
|
|
# per-node conditions (MemoryPressure), resource requests/limits. The "which pod
|
|
# and why" half of the OOM picture (node-exporter is the "which node" half).
|
|
# Single replica, pinned by preference to the ROOMY non-luxd nodes.
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: kube-state-metrics
|
|
namespace: lux-system
|
|
labels: {app: kube-state-metrics, brand: lux}
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRole
|
|
metadata:
|
|
name: lux-kube-state-metrics
|
|
labels: {brand: lux}
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: [nodes, pods, services, resourcequotas, replicationcontrollers, limitranges, persistentvolumeclaims, persistentvolumes, namespaces, endpoints]
|
|
verbs: [list, watch]
|
|
- apiGroups: [apps]
|
|
resources: [statefulsets, daemonsets, deployments, replicasets]
|
|
verbs: [list, watch]
|
|
- apiGroups: [batch]
|
|
resources: [cronjobs, jobs]
|
|
verbs: [list, watch]
|
|
- apiGroups: [autoscaling]
|
|
resources: [horizontalpodautoscalers]
|
|
verbs: [list, watch]
|
|
- apiGroups: [policy]
|
|
resources: [poddisruptionbudgets]
|
|
verbs: [list, watch]
|
|
- apiGroups: [storage.k8s.io]
|
|
resources: [storageclasses, volumeattachments]
|
|
verbs: [list, watch]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRoleBinding
|
|
metadata:
|
|
name: lux-kube-state-metrics
|
|
labels: {brand: lux}
|
|
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: lux-kube-state-metrics}
|
|
subjects:
|
|
- {kind: ServiceAccount, name: kube-state-metrics, namespace: lux-system}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: kube-state-metrics
|
|
namespace: lux-system
|
|
labels: {app: kube-state-metrics, brand: lux}
|
|
spec:
|
|
replicas: 1
|
|
selector: {matchLabels: {app: kube-state-metrics}}
|
|
template:
|
|
metadata:
|
|
labels: {app: kube-state-metrics, brand: lux}
|
|
spec:
|
|
serviceAccountName: kube-state-metrics
|
|
# Prefer the roomy non-luxd nodes; never add pressure to kzit5/kzith/luxd.
|
|
affinity:
|
|
nodeAffinity:
|
|
preferredDuringSchedulingIgnoredDuringExecution:
|
|
- weight: 100
|
|
preference:
|
|
matchExpressions:
|
|
- {key: kubernetes.io/hostname, operator: In, values: [lux-nodes-3cgkgx, lux-nodes-3cr32q, lux-nodes-3csr33]}
|
|
securityContext: {runAsNonRoot: true, runAsUser: 65534, runAsGroup: 65534}
|
|
containers:
|
|
- name: kube-state-metrics
|
|
image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.13.0
|
|
args:
|
|
- --port=8080
|
|
- --resources=nodes,pods,deployments,daemonsets,statefulsets,replicasets,namespaces,jobs,cronjobs,persistentvolumeclaims
|
|
ports:
|
|
- {name: http, containerPort: 8080}
|
|
resources:
|
|
requests: {cpu: 20m, memory: 48Mi}
|
|
limits: {cpu: 200m, memory: 160Mi}
|
|
readinessProbe: {httpGet: {path: /healthz, port: 8080}, initialDelaySeconds: 5, periodSeconds: 15}
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities: {drop: [ALL]}
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: kube-state-metrics
|
|
namespace: lux-system
|
|
labels: {app: kube-state-metrics, brand: lux}
|
|
spec:
|
|
selector: {app: kube-state-metrics}
|
|
ports:
|
|
- {name: http, port: 8080, targetPort: 8080}
|