monitoring/internal/fleet/fleet_test.go

package fleet

import (
	"context"
	"fmt"
	"os"
	"regexp"
	"strings"
	"testing"
	"time"
)

// ── render: the exposition contract, no network ───────────────────────────────

func u(v uint64) *uint64 { return &v }
func i(v int) *int       { return &v }
func b(v bool) *bool     { return &v }

// One mainnet-shaped poll with every interesting condition present at once: a
// fork at the tip, a lagging node with two failing health checks, a node that
// never answered, and a pod that says Ready while its RPC is dead.
func mixedSamples() []sample {
	return []sample{
		{Net: 0, Idx: 0, Up: true, Healthy: true, Boot: true, Peers: i(4),
			AccH: u(100), AccHash: "0xaaa", AccTS: u(1753500000), FinH: u(100), FinHash: "0xaaa",
			Restarts: i(0), Ready: b(true), Logs: map[string]int{"fatal": 0, "equivocation": 0, "cert_rejected": 0}},
		{Net: 0, Idx: 1, Up: true, Healthy: true, Boot: true, Peers: i(4),
			AccH: u(100), AccHash: "0xbbb", AccTS: u(1753500000), FinH: u(100), FinHash: "0xbbb",
			Restarts: i(2), Ready: b(true), Logs: map[string]int{"fatal": 1, "equivocation": 1, "cert_rejected": 2}},
		{Net: 0, Idx: 2, Up: true, Healthy: false, Boot: true, Peers: i(3), Failing: []string{"C", "P"},
			AccH: u(98), AccHash: "0xccc", AccTS: u(1753499990), FinH: u(97), FinHash: "0xddd",
			Restarts: i(1), Ready: b(true), Logs: map[string]int{"fatal": 0, "equivocation": 0, "cert_rejected": 0}},
		{Net: 0, Idx: 3, Ready: b(true), Restarts: i(0),
			Logs: map[string]int{"fatal": 0, "equivocation": 0, "cert_rejected": 0}},
		{Net: 0, Idx: 4, Logs: map[string]int{"fatal": 0, "equivocation": 0, "cert_rejected": 0}},
	}
}

func TestExpositionIsTheDeclaredContract(t *testing.T) {
	got := normalise(render(mixedSamples(), 250*time.Millisecond, 5))
	want := `# HELP lux_validator_exporter_up Exporter liveness.
# TYPE lux_validator_exporter_up gauge
lux_validator_exporter_up 1
# HELP lux_validator_poll_duration_seconds Time to poll all validators.
# TYPE lux_validator_poll_duration_seconds gauge
lux_validator_poll_duration_seconds <DUR>
# HELP lux_validator_up 1 if C-Chain RPC reachable (process up).
# TYPE lux_validator_up gauge
lux_validator_up{network="mainnet",instance="luxd-0",brand="lux",chain="C"} 1
lux_validator_up{network="mainnet",instance="luxd-1",brand="lux",chain="C"} 1
lux_validator_up{network="mainnet",instance="luxd-2",brand="lux",chain="C"} 1
lux_validator_up{network="mainnet",instance="luxd-3",brand="lux",chain="C"} 0
lux_validator_up{network="mainnet",instance="luxd-4",brand="lux",chain="C"} 0
# HELP lux_validator_healthy 1 if /v1/health reports healthy.
# TYPE lux_validator_healthy gauge
lux_validator_healthy{network="mainnet",instance="luxd-0",brand="lux",chain="C"} 1
lux_validator_healthy{network="mainnet",instance="luxd-1",brand="lux",chain="C"} 1
lux_validator_healthy{network="mainnet",instance="luxd-2",brand="lux",chain="C"} 0
lux_validator_healthy{network="mainnet",instance="luxd-3",brand="lux",chain="C"} 0
lux_validator_healthy{network="mainnet",instance="luxd-4",brand="lux",chain="C"} 0
# HELP lux_validator_bootstrapped 1 if C-Chain bootstrapped.
# TYPE lux_validator_bootstrapped gauge
lux_validator_bootstrapped{network="mainnet",instance="luxd-0",brand="lux",chain="C"} 1
lux_validator_bootstrapped{network="mainnet",instance="luxd-1",brand="lux",chain="C"} 1
lux_validator_bootstrapped{network="mainnet",instance="luxd-2",brand="lux",chain="C"} 1
lux_validator_bootstrapped{network="mainnet",instance="luxd-3",brand="lux",chain="C"} 0
lux_validator_bootstrapped{network="mainnet",instance="luxd-4",brand="lux",chain="C"} 0
# HELP lux_validator_peers Connected peer count.
# TYPE lux_validator_peers gauge
lux_validator_peers{network="mainnet",instance="luxd-0",brand="lux",chain="C"} 4
lux_validator_peers{network="mainnet",instance="luxd-1",brand="lux",chain="C"} 4
lux_validator_peers{network="mainnet",instance="luxd-2",brand="lux",chain="C"} 3
# HELP lux_validator_c_block_height Accepted C-Chain tip height.
# TYPE lux_validator_c_block_height gauge
lux_validator_c_block_height{network="mainnet",instance="luxd-0",brand="lux",chain="C"} 100
lux_validator_c_block_height{network="mainnet",instance="luxd-1",brand="lux",chain="C"} 100
lux_validator_c_block_height{network="mainnet",instance="luxd-2",brand="lux",chain="C"} 98
# HELP lux_validator_c_block_timestamp Unix timestamp carried by the accepted C-Chain tip block.
# TYPE lux_validator_c_block_timestamp gauge
lux_validator_c_block_timestamp{network="mainnet",instance="luxd-0",brand="lux",chain="C"} 1753500000
lux_validator_c_block_timestamp{network="mainnet",instance="luxd-1",brand="lux",chain="C"} 1753500000
lux_validator_c_block_timestamp{network="mainnet",instance="luxd-2",brand="lux",chain="C"} 1753499990
# HELP lux_validator_c_finalized_height Finalized C-Chain height.
# TYPE lux_validator_c_finalized_height gauge
lux_validator_c_finalized_height{network="mainnet",instance="luxd-0",brand="lux",chain="C"} 100
lux_validator_c_finalized_height{network="mainnet",instance="luxd-1",brand="lux",chain="C"} 100
lux_validator_c_finalized_height{network="mainnet",instance="luxd-2",brand="lux",chain="C"} 97
# HELP lux_validator_health_check_failed 1 per named luxd health check currently failing (the reason behind healthy=0).
# TYPE lux_validator_health_check_failed gauge
lux_validator_health_check_failed{network="mainnet",instance="luxd-2",brand="lux",check="C"} 1
lux_validator_health_check_failed{network="mainnet",instance="luxd-2",brand="lux",check="P"} 1
# HELP lux_validator_ready 1 if pod Ready condition is True.
# TYPE lux_validator_ready gauge
lux_validator_ready{network="mainnet",instance="luxd-0",brand="lux",chain="C"} 1
lux_validator_ready{network="mainnet",instance="luxd-1",brand="lux",chain="C"} 1
lux_validator_ready{network="mainnet",instance="luxd-2",brand="lux",chain="C"} 1
lux_validator_ready{network="mainnet",instance="luxd-3",brand="lux",chain="C"} 1
# HELP lux_validator_restarts_total Cumulative container restarts.
# TYPE lux_validator_restarts_total counter
lux_validator_restarts_total{network="mainnet",instance="luxd-0",brand="lux",chain="C"} 0
lux_validator_restarts_total{network="mainnet",instance="luxd-1",brand="lux",chain="C"} 2
lux_validator_restarts_total{network="mainnet",instance="luxd-2",brand="lux",chain="C"} 1
lux_validator_restarts_total{network="mainnet",instance="luxd-3",brand="lux",chain="C"} 0
# HELP lux_validator_log_matches_total Cumulative luxd log lines matching a pattern.
# TYPE lux_validator_log_matches_total counter
lux_validator_log_matches_total{network="mainnet",instance="luxd-0",brand="lux",pattern="cert_rejected"} 0
lux_validator_log_matches_total{network="mainnet",instance="luxd-0",brand="lux",pattern="equivocation"} 0
lux_validator_log_matches_total{network="mainnet",instance="luxd-0",brand="lux",pattern="fatal"} 0
lux_validator_log_matches_total{network="mainnet",instance="luxd-1",brand="lux",pattern="cert_rejected"} 2
lux_validator_log_matches_total{network="mainnet",instance="luxd-1",brand="lux",pattern="equivocation"} 1
lux_validator_log_matches_total{network="mainnet",instance="luxd-1",brand="lux",pattern="fatal"} 1
lux_validator_log_matches_total{network="mainnet",instance="luxd-2",brand="lux",pattern="cert_rejected"} 0
lux_validator_log_matches_total{network="mainnet",instance="luxd-2",brand="lux",pattern="equivocation"} 0
lux_validator_log_matches_total{network="mainnet",instance="luxd-2",brand="lux",pattern="fatal"} 0
lux_validator_log_matches_total{network="mainnet",instance="luxd-3",brand="lux",pattern="cert_rejected"} 0
lux_validator_log_matches_total{network="mainnet",instance="luxd-3",brand="lux",pattern="equivocation"} 0
lux_validator_log_matches_total{network="mainnet",instance="luxd-3",brand="lux",pattern="fatal"} 0
lux_validator_log_matches_total{network="mainnet",instance="luxd-4",brand="lux",pattern="cert_rejected"} 0
lux_validator_log_matches_total{network="mainnet",instance="luxd-4",brand="lux",pattern="equivocation"} 0
lux_validator_log_matches_total{network="mainnet",instance="luxd-4",brand="lux",pattern="fatal"} 0
# HELP lux_network_validators_total Expected validators in the set.
# TYPE lux_network_validators_total gauge
lux_network_validators_total{network="mainnet",brand="lux"} 5
# HELP lux_network_validators_up Validators with RPC reachable.
# TYPE lux_network_validators_up gauge
lux_network_validators_up{network="mainnet",brand="lux"} 3
# HELP lux_network_validators_healthy Validators reporting healthy.
# TYPE lux_network_validators_healthy gauge
lux_network_validators_healthy{network="mainnet",brand="lux"} 2
# HELP lux_network_validators_ready Validators with pod Ready.
# TYPE lux_network_validators_ready gauge
lux_network_validators_ready{network="mainnet",brand="lux"} 4
# HELP lux_network_finalized_fork 1 if validators report different finalized hashes at the same height.
# TYPE lux_network_finalized_fork gauge
lux_network_finalized_fork{network="mainnet",brand="lux"} 1
# HELP lux_network_preferred_fork 1 if validators report different latest hashes at the same height.
# TYPE lux_network_preferred_fork gauge
lux_network_preferred_fork{network="mainnet",brand="lux"} 1
# HELP lux_network_finalized_height_spread max-min finalized height.
# TYPE lux_network_finalized_height_spread gauge
lux_network_finalized_height_spread{network="mainnet",brand="lux"} 3
# HELP lux_network_block_height_spread max-min accepted height.
# TYPE lux_network_block_height_spread gauge
lux_network_block_height_spread{network="mainnet",brand="lux"} 2
# HELP lux_network_c_tip_age_seconds Age in seconds of the freshest C-Chain tip in the network (wall clock minus the tip block's own timestamp). Grows without bound on a frozen fleet.
# TYPE lux_network_c_tip_age_seconds gauge
lux_network_c_tip_age_seconds{network="mainnet",brand="lux"} <AGE>
# HELP lux_network_tip_hash_variants Largest number of DISTINCT accepted block hashes reported at any one height. >1 is a fork.
# TYPE lux_network_tip_hash_variants gauge
lux_network_tip_hash_variants{network="mainnet",brand="lux"} 2
# HELP lux_network_ready_but_rpc_dead Validators whose pod reports Ready while their C-Chain RPC does not answer — the readiness-lies failure class.
# TYPE lux_network_ready_but_rpc_dead gauge
lux_network_ready_but_rpc_dead{network="mainnet",brand="lux"} 1
`
	if got != want {
		t.Fatalf("exposition drift:\n--- got ---\n%s\n--- want ---\n%s", got, want)
	}
}

// A network where nothing answered must leave tip age ABSENT, not zero: a fake
// age of zero reads as a perfectly fresh chain, which is the exact lie this
// metric exists to prevent.
func TestTotalOutageLeavesTipAgeAbsent(t *testing.T) {
	out := render([]sample{{Net: 2, Idx: 0}, {Net: 2, Idx: 1}}, time.Second, 5)
	if strings.Contains(out, "lux_network_c_tip_age_seconds") {
		t.Fatalf("tip age must be absent when no node answered:\n%s", out)
	}
	if !strings.Contains(out, `lux_network_validators_up{network="devnet",brand="lux"} 0`) {
		t.Fatalf("validators_up must still report zero:\n%s", out)
	}
}

// An agreeing fleet is not a fork, however many validators there are.
func TestAgreementIsNotAFork(t *testing.T) {
	ss := []sample{
		{Net: 0, Idx: 0, Up: true, AccH: u(7), AccHash: "0xa", FinH: u(7), FinHash: "0xa"},
		{Net: 0, Idx: 1, Up: true, AccH: u(7), AccHash: "0xa", FinH: u(7), FinHash: "0xa"},
	}
	out := render(ss, time.Second, 2)
	for _, want := range []string{
		`lux_network_preferred_fork{network="mainnet",brand="lux"} 0`,
		`lux_network_finalized_fork{network="mainnet",brand="lux"} 0`,
		`lux_network_tip_hash_variants{network="mainnet",brand="lux"} 1`,
		`lux_network_block_height_spread{network="mainnet",brand="lux"} 0`,
	} {
		if !strings.Contains(out, want) {
			t.Fatalf("missing %q in:\n%s", want, out)
		}
	}
}

// ── log patterns: the same lines must score the same counts ───────────────────

func TestLogPatternsScoreTheLinesTheyWereTunedFor(t *testing.T) {
	for _, tc := range []struct {
		line string
		want map[string]bool
	}{
		{"fatal EQUIVOCATION: conflicting finality cert at height 7",
			map[string]bool{"fatal": true, "equivocation": true, "cert_rejected": true}},
		{"build block failure, retrying",
			map[string]bool{"fatal": false, "equivocation": false, "cert_rejected": false}},
		{"certificate rejected by peer",
			map[string]bool{"fatal": false, "equivocation": false, "cert_rejected": true}},
		{"panic: nil map write",
			map[string]bool{"fatal": true, "equivocation": false, "cert_rejected": false}},
		{"everything is fine",
			map[string]bool{"fatal": false, "equivocation": false, "cert_rejected": false}},
	} {
		for _, p := range logPatterns {
			if got := p.RE.MatchString(tc.line); got != tc.want[p.Name] {
				t.Errorf("%s on %q: got %v want %v", p.Name, tc.line, got, tc.want[p.Name])
			}
		}
	}
}

// ── differential: same fake cluster, both exporters ───────────────────────────

// TestExpositionAgainstFakeCluster drives the real poll path against the shared
// fake cluster the Python exporter is also pointed at, and prints the result so
// the two can be diffed. Skipped unless FLEET_FAKE names that cluster.
func TestExpositionAgainstFakeCluster(t *testing.T) {
	base := os.Getenv("FLEET_FAKE")
	if base == "" {
		t.Skip("set FLEET_FAKE=http://host:port to run the differential")
	}
	e := NewExporter(Config{
		Replicas: 5,
		KubeBase: base,
		NodeBase: func(net, ns, pod string, port int) string {
			return fmt.Sprintf("%s/fake/%s/%s", base, net, strings.TrimPrefix(pod, "luxd-"))
		},
	})
	out := e.Once(context.Background())
	if err := os.WriteFile(os.Getenv("FLEET_OUT"), []byte(normalise(out)), 0o644); err != nil {
		t.Fatal(err)
	}
}

// normalise blanks the two values that are a function of wall-clock time. Tip
// age and poll duration are asserted for PRESENCE and label set; their value
// cannot be compared across two processes that ran a second apart.
var (
	durRE = regexp.MustCompile(`(?m)^lux_validator_poll_duration_seconds .*$`)
	ageRE = regexp.MustCompile(`(?m)^(lux_network_c_tip_age_seconds\{[^}]*\}) \d+$`)
)

func normalise(s string) string {
	s = durRE.ReplaceAllString(s, "lux_validator_poll_duration_seconds <DUR>")
	return ageRE.ReplaceAllString(s, "$1 <AGE>")
}