monitoring/internal/fleet/render.go

package fleet

import (
	"fmt"
	"sort"
	"strings"
	"time"
)

// metrics is the exposition's declaration order and its help text. The order is
// part of the contract only in the weak sense that a scraper does not care —
// but a human diffing two scrapes does, so it is fixed and the series inside
// each family are sorted. Determinism costs nothing and makes a diff mean
// something.
var metrics = []struct{ Name, Type, Help string }{
	{"lux_validator_up", "gauge", "1 if C-Chain RPC reachable (process up)."},
	{"lux_validator_healthy", "gauge", "1 if /v1/health reports healthy."},
	{"lux_validator_bootstrapped", "gauge", "1 if C-Chain bootstrapped."},
	{"lux_validator_peers", "gauge", "Connected peer count."},
	{"lux_validator_c_block_height", "gauge", "Accepted C-Chain tip height."},
	{"lux_validator_c_block_timestamp", "gauge", "Unix timestamp carried by the accepted C-Chain tip block."},
	{"lux_validator_c_finalized_height", "gauge", "Finalized C-Chain height."},
	{"lux_validator_health_check_failed", "gauge", "1 per named luxd health check currently failing (the reason behind healthy=0)."},
	{"lux_validator_ready", "gauge", "1 if pod Ready condition is True."},
	{"lux_validator_restarts_total", "counter", "Cumulative container restarts."},
	{"lux_validator_log_matches_total", "counter", "Cumulative luxd log lines matching a pattern."},
	{"lux_network_validators_total", "gauge", "Expected validators in the set."},
	{"lux_network_validators_up", "gauge", "Validators with RPC reachable."},
	{"lux_network_validators_healthy", "gauge", "Validators reporting healthy."},
	{"lux_network_validators_ready", "gauge", "Validators with pod Ready."},
	{"lux_network_finalized_fork", "gauge", "1 if validators report different finalized hashes at the same height."},
	{"lux_network_preferred_fork", "gauge", "1 if validators report different latest hashes at the same height."},
	{"lux_network_finalized_height_spread", "gauge", "max-min finalized height."},
	{"lux_network_block_height_spread", "gauge", "max-min accepted height."},
	{"lux_network_c_tip_age_seconds", "gauge", "Age in seconds of the freshest C-Chain tip in the network (wall clock minus the tip block's own timestamp). Grows without bound on a frozen fleet."},
	{"lux_network_tip_hash_variants", "gauge", "Largest number of DISTINCT accepted block hashes reported at any one height. >1 is a fork."},
	{"lux_network_ready_but_rpc_dead", "gauge", "Validators whose pod reports Ready while their C-Chain RPC does not answer — the readiness-lies failure class."},
}

// render turns one poll's samples into the exposition text.
func render(ss []sample, dur time.Duration, replicas int) string {
	fam := map[string][]string{}
	add := func(name, series string) { fam[name] = append(fam[name], series) }

	for _, s := range ss {
		net := Networks[s.Net].Name
		g := fmt.Sprintf(`{network=%q,instance="luxd-%d",brand=%q,chain="C"}`, net, s.Idx, Brand)
		add("lux_validator_up", fmt.Sprintf("%s %d", g, b2i(s.Up)))
		add("lux_validator_healthy", fmt.Sprintf("%s %d", g, b2i(s.Healthy)))
		add("lux_validator_bootstrapped", fmt.Sprintf("%s %d", g, b2i(s.Boot)))
		if s.Peers != nil {
			add("lux_validator_peers", fmt.Sprintf("%s %d", g, *s.Peers))
		}
		if s.AccH != nil {
			add("lux_validator_c_block_height", fmt.Sprintf("%s %d", g, *s.AccH))
		}
		if s.AccTS != nil {
			add("lux_validator_c_block_timestamp", fmt.Sprintf("%s %d", g, *s.AccTS))
		}
		if s.FinH != nil {
			add("lux_validator_c_finalized_height", fmt.Sprintf("%s %d", g, *s.FinH))
		}
		// luxd exposes a small fixed set of named checks, so one series per
		// failing check is bounded cardinality and turns "unhealthy" into a
		// diagnosis.
		for _, chk := range s.Failing {
			add("lux_validator_health_check_failed", fmt.Sprintf(
				`{network=%q,instance="luxd-%d",brand=%q,check=%q} 1`, net, s.Idx, Brand, chk))
		}
		if s.Ready != nil {
			add("lux_validator_ready", fmt.Sprintf("%s %d", g, b2i(*s.Ready)))
		}
		if s.Restarts != nil {
			add("lux_validator_restarts_total", fmt.Sprintf("%s %d", g, *s.Restarts))
		}
		for _, pat := range sortedKeys(s.Logs) {
			add("lux_validator_log_matches_total", fmt.Sprintf(
				`{network=%q,instance="luxd-%d",brand=%q,pattern=%q} %d`, net, s.Idx, Brand, pat, s.Logs[pat]))
		}
	}

	// Per-network aggregates, in the declared network order so two scrapes of a
	// steady fleet are textually identical.
	byNet := map[int][]sample{}
	for _, s := range ss {
		byNet[s.Net] = append(byNet[s.Net], s)
	}
	for ni := range Networks {
		ds, ok := byNet[ni]
		if !ok {
			continue
		}
		nl := fmt.Sprintf(`{network=%q,brand=%q}`, Networks[ni].Name, Brand)
		var ups []sample
		healthy, ready, readyDead := 0, 0, 0
		for _, d := range ds {
			if d.Up {
				ups = append(ups, d)
			}
			if d.Healthy {
				healthy++
			}
			if d.Ready != nil && *d.Ready {
				ready++
				if !d.Up {
					readyDead++
				}
			}
		}
		add("lux_network_validators_total", fmt.Sprintf("%s %d", nl, replicas))
		add("lux_network_validators_up", fmt.Sprintf("%s %d", nl, len(ups)))
		add("lux_network_validators_healthy", fmt.Sprintf("%s %d", nl, healthy))
		add("lux_network_validators_ready", fmt.Sprintf("%s %d", nl, ready))
		add("lux_network_finalized_fork", fmt.Sprintf("%s %d", nl, fork(ups, finKey)))
		add("lux_network_preferred_fork", fmt.Sprintf("%s %d", nl, fork(ups, accKey)))
		add("lux_network_finalized_height_spread", fmt.Sprintf("%s %d", nl, spread(ups, finH)))
		add("lux_network_block_height_spread", fmt.Sprintf("%s %d", nl, spread(ups, accH)))
		// The freshest tip in the set, aged against OUR clock — not against
		// another validator. Emitted only when at least one node answered, so a
		// total RPC outage leaves the series absent (alerted separately) rather
		// than reporting a fake age of zero.
		if newest, ok := newestTip(ups); ok {
			age := time.Now().Unix() - int64(newest)
			if age < 0 {
				age = 0
			}
			add("lux_network_c_tip_age_seconds", fmt.Sprintf("%s %d", nl, age))
		}
		add("lux_network_tip_hash_variants", fmt.Sprintf("%s %d", nl, hashVariants(ups)))
		add("lux_network_ready_but_rpc_dead", fmt.Sprintf("%s %d", nl, readyDead))
	}

	var b strings.Builder
	b.WriteString("# HELP lux_validator_exporter_up Exporter liveness.\n")
	b.WriteString("# TYPE lux_validator_exporter_up gauge\n")
	b.WriteString("lux_validator_exporter_up 1\n")
	b.WriteString("# HELP lux_validator_poll_duration_seconds Time to poll all validators.\n")
	b.WriteString("# TYPE lux_validator_poll_duration_seconds gauge\n")
	fmt.Fprintf(&b, "lux_validator_poll_duration_seconds %.3f\n", dur.Seconds())
	for _, m := range metrics {
		series := fam[m.Name]
		if len(series) == 0 {
			continue
		}
		sort.Strings(series)
		fmt.Fprintf(&b, "# HELP %s %s\n# TYPE %s %s\n", m.Name, m.Help, m.Name, m.Type)
		for _, s := range series {
			b.WriteString(m.Name)
			b.WriteString(s)
			b.WriteByte('\n')
		}
	}
	return b.String()
}

func b2i(v bool) int {
	if v {
		return 1
	}
	return 0
}

func sortedKeys(m map[string]int) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}

type keyFn func(sample) (uint64, string, bool)

func accKey(s sample) (uint64, string, bool) {
	if s.AccH == nil || s.AccHash == "" {
		return 0, "", false
	}
	return *s.AccH, s.AccHash, true
}

func finKey(s sample) (uint64, string, bool) {
	if s.FinH == nil || s.FinHash == "" {
		return 0, "", false
	}
	return *s.FinH, s.FinHash, true
}

func accH(s sample) (uint64, bool) { return deref(s.AccH) }
func finH(s sample) (uint64, bool) { return deref(s.FinH) }

func deref(p *uint64) (uint64, bool) {
	if p == nil {
		return 0, false
	}
	return *p, true
}

// fork is 1 when two validators accepted DIFFERENT blocks at the SAME height.
func fork(ups []sample, key keyFn) int {
	if hashesPerHeight(ups, key).max() > 1 {
		return 1
	}
	return 0
}

// hashVariants is the width of the split, not just its existence, so the alert
// can say how bad it is.
func hashVariants(ups []sample) int { return hashesPerHeight(ups, accKey).max() }

type heightHashes map[uint64]map[string]struct{}

func (h heightHashes) max() int {
	n := 0
	for _, s := range h {
		if len(s) > n {
			n = len(s)
		}
	}
	return n
}

func hashesPerHeight(ups []sample, key keyFn) heightHashes {
	out := heightHashes{}
	for _, s := range ups {
		h, hash, ok := key(s)
		if !ok {
			continue
		}
		if out[h] == nil {
			out[h] = map[string]struct{}{}
		}
		out[h][hash] = struct{}{}
	}
	return out
}

func spread(ups []sample, f func(sample) (uint64, bool)) uint64 {
	var lo, hi uint64
	seen := false
	for _, s := range ups {
		v, ok := f(s)
		if !ok {
			continue
		}
		if !seen {
			lo, hi, seen = v, v, true
			continue
		}
		if v < lo {
			lo = v
		}
		if v > hi {
			hi = v
		}
	}
	if !seen {
		return 0
	}
	return hi - lo
}

func newestTip(ups []sample) (uint64, bool) {
	var newest uint64
	seen := false
	for _, s := range ups {
		if s.AccTS == nil {
			continue
		}
		if !seen || *s.AccTS > newest {
			newest, seen = *s.AccTS, true
		}
	}
	return newest, seen
}