dex/k8s/production/monitoring.yaml

apiVersion: v1
kind: ServiceMonitor
metadata:
  name: lxdex-metrics
  namespace: lxdex-production
  labels:
    app: lxdex
    prometheus: kube-prometheus
spec:
  selector:
    matchLabels:
      app: lxdex
      component: node
  endpoints:
  - port: metrics
    interval: 30s
    path: /metrics
    scheme: http
    relabelings:
    - sourceLabels: [__meta_kubernetes_pod_name]
      targetLabel: pod
    - sourceLabels: [__meta_kubernetes_pod_node_name]
      targetLabel: node
---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: lxdex-alerts
  namespace: lxdex-production
  labels:
    prometheus: kube-prometheus
    role: alert-rules
spec:
  groups:
  - name: lxdex.rules
    interval: 30s
    rules:
    - alert: LXDexHighLatency
      expr: histogram_quantile(0.99, lx_matching_latency_seconds_bucket) > 0.001
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "High order matching latency on {{ $labels.pod }}"
        description: "99th percentile latency is {{ $value }}s (threshold: 1ms)"
    
    - alert: LXDexLowThroughput
      expr: rate(lx_orders_processed_total[5m]) < 1000000
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: "Low order throughput on {{ $labels.pod }}"
        description: "Processing only {{ $value }} orders/sec (target: 1M+)"
    
    - alert: LXDexNodeDown
      expr: up{job="lxdex"} == 0
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: "LXDex node {{ $labels.instance }} is down"
        description: "Node has been unreachable for more than 1 minute"
    
    - alert: LXDexConsensusFailure
      expr: lx_consensus_failures_total > 0
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: "Consensus failures detected"
        description: "{{ $value }} consensus failures in the last minute"
    
    - alert: LXDexHighMemoryUsage
      expr: container_memory_usage_bytes{pod=~"lxdex-node-.*"} / container_spec_memory_limit_bytes > 0.9
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "High memory usage on {{ $labels.pod }}"
        description: "Memory usage is {{ $value | humanizePercentage }} of limit"
    
    - alert: LXDexDiskSpaceLow
      expr: kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"data-lxdex-node-.*"} / kubelet_volume_stats_capacity_bytes < 0.1
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "Low disk space on {{ $labels.persistentvolumeclaim }}"
        description: "Only {{ $value | humanizePercentage }} disk space remaining"