mirror of
https://github.com/hanzoai/bootnode.git
synced 2026-09-30 20:46:32 +00:00
The lux.cloud dashboard was blank because the collector could not reach a
single validator. Two independent breaks, either one fatal:
1. It called /ext. luxd 1.36 serves /v1 only — /ext/info is 404 — and the
collector's _rpc() maps any non-200 to None, so every field came back
null with no error anywhere.
2. It dialled 15 hardcoded public LoadBalancer IPs. There are now zero
LoadBalancer Services in lux-mainnet/testnet/devnet; those IPs do not
answer at all (connect timeout, not 404).
Both are the same root defect: the observer was outside the thing it observes
and had to be told where to look. So the observer moves in. Running in lux-k8s,
a validator's address is its pod IP, read from the pod list on every
collection — there is nothing left to configure and nothing left to go stale.
Chain facts come from the node itself over /v1: info.getNodeVersion,
getNodeID, getNodeIP, peers, isBootstrapped per chain, eth_blockNumber at
/v1/bc/C/rpc (the bc/ segment is load-bearing; /v1/C/rpc is not a route) and
platform.getHeight at /v1/bc/P, which quotes its integer.
That also collapses the MODE A / MODE B split: there is one path, not two.
Two corrections to what the old collector reported:
version is now what the fleet RUNS, not what the CR declares. The mainnet
LuxNetwork CR asks for ghcr.io/luxfi/node:v1.34.0, a tag that does not
exist; the nodes are on luxd/1.36.2. The dashboard must show the running
truth, so observation wins and image keeps the declared value.
external_ip becomes ip, matching console/lib/fleet.ts, and carries the
address the node advertises for itself (info.getNodeIP) rather than an
address someone wrote down.
Degraded is reported as degraded. A node that does not answer keeps every
field null and counts against nodes_rpc_ok; it is never smoothed over.
Placement: this stays in hanzoai/bootnode rather than folding into
hanzoai/cloud. cloud is not deployed on lux-k8s, and standing the whole
monolith up inside the chain cluster to run a read-only collector is a far
larger change than the collector itself. The surface is registered with zip's
Group; the route set is unchanged.
Deploy: Deployment/Service/RBAC in lux-k8s (lux-fleet.yaml, read-only —
list/get on pods and luxnetworks, no pods/proxy, no exec). The public URL
resolves to bootnode-k8s, so that cluster forwards just /v1/fleets/lux across
(lux-fleet-bridge.yaml); no other path on the host is touched.
Tests: a luxd stub that 404s anything outside /v1 and quotes platform height;
assertions that every probe path is under /v1, that no wire key is ever
omitted (an absent key reads as undefined and blanks a cell), that a down node
stays null, and that /v1/fleets/lux/zzzz is 404 next to a 200 on /overview.
Co-authored-by: Hanzo Dev <[email protected]>
335 lines
11 KiB
Go
335 lines
11 KiB
Go
package fleet
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
)
|
|
|
|
// fakeCluster stands in for the Kubernetes API server.
|
|
type fakeCluster struct {
|
|
pods map[string][]Pod
|
|
declared map[string]Declared
|
|
}
|
|
|
|
func (f fakeCluster) Pods(_ context.Context, ns string) ([]Pod, error) { return f.pods[ns], nil }
|
|
func (f fakeCluster) Declared(_ context.Context, ns string) (Declared, error) {
|
|
d, ok := f.declared[ns]
|
|
if !ok {
|
|
return Declared{}, http.ErrNoLocation
|
|
}
|
|
return d, nil
|
|
}
|
|
|
|
// luxdStub speaks exactly what luxd 1.36 speaks: /v1 answers, /ext is gone.
|
|
// paths records every path asked for, so a test can prove /ext is never tried.
|
|
type luxdStub struct {
|
|
mu sync.Mutex
|
|
paths []string
|
|
cHeight string
|
|
boot map[string]bool
|
|
peers int
|
|
}
|
|
|
|
func newLuxd() *luxdStub {
|
|
return &luxdStub{cHeight: "0x10c1cf", peers: 2,
|
|
boot: map[string]bool{"C": true, "P": true, "X": true, "D": false}}
|
|
}
|
|
|
|
func (s *luxdStub) handler() http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
s.mu.Lock()
|
|
s.paths = append(s.paths, r.URL.Path)
|
|
s.mu.Unlock()
|
|
|
|
if !strings.HasPrefix(r.URL.Path, "/v1") {
|
|
http.NotFound(w, r) // /ext was removed in luxd 1.36
|
|
return
|
|
}
|
|
if r.URL.Path == "/v1/health" {
|
|
write(w, `{"healthy":true,"error":"health reply encode failed"}`)
|
|
return
|
|
}
|
|
var req struct {
|
|
Method string `json:"method"`
|
|
Params json.RawMessage `json:"params"`
|
|
}
|
|
_ = json.NewDecoder(r.Body).Decode(&req)
|
|
switch req.Method {
|
|
case "info.getNodeVersion":
|
|
write(w, `{"jsonrpc":"2.0","id":1,"result":{"version":"luxd/1.36.2","databaseVersion":"v1.4.5","gitCommit":"132fa688"}}`)
|
|
case "info.getNodeID":
|
|
write(w, `{"jsonrpc":"2.0","id":1,"result":{"nodeID":"NodeID-Mf3JfSY91oDwfBqf7rCLmhg4NDtDghw1f"}}`)
|
|
case "info.getNodeIP":
|
|
write(w, `{"jsonrpc":"2.0","id":1,"result":{"ip":"129.212.164.46:9631"}}`)
|
|
case "info.peers":
|
|
write(w, `{"jsonrpc":"2.0","id":1,"result":{"numPeers":"2","peers":[{"nodeID":"a"},{"nodeID":"b"}]}}`)
|
|
case "info.isBootstrapped":
|
|
var p struct{ Chain string }
|
|
_ = json.Unmarshal(req.Params, &p)
|
|
write(w, `{"jsonrpc":"2.0","id":1,"result":{"isBootstrapped":`+boolJSON(s.boot[p.Chain])+`}}`)
|
|
case "eth_blockNumber":
|
|
write(w, `{"jsonrpc":"2.0","id":1,"result":"`+s.cHeight+`"}`)
|
|
case "platform.getHeight":
|
|
// luxd quotes this integer; a decoder that only accepts numbers drops it.
|
|
write(w, `{"jsonrpc":"2.0","id":1,"result":{"height":"1098191"}}`)
|
|
default:
|
|
http.Error(w, "unknown method", http.StatusNotFound)
|
|
}
|
|
})
|
|
}
|
|
|
|
func write(w http.ResponseWriter, s string) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = w.Write([]byte(s))
|
|
}
|
|
|
|
func boolJSON(b bool) string {
|
|
if b {
|
|
return "true"
|
|
}
|
|
return "false"
|
|
}
|
|
|
|
// collectOne runs the collector against one stub node standing in for luxd-0.
|
|
func collectOne(t *testing.T, stub *luxdStub, extra ...Pod) NetworkFleet {
|
|
t.Helper()
|
|
srv := httptest.NewServer(stub.handler())
|
|
t.Cleanup(srv.Close)
|
|
host := strings.TrimPrefix(srv.URL, "http://")
|
|
|
|
pods := append([]Pod{{Name: "luxd-0", Index: 0, Phase: "Running", Ready: true, IP: host}}, extra...)
|
|
c := &Collector{
|
|
Cluster: fakeCluster{
|
|
pods: map[string][]Pod{"lux-mainnet": pods},
|
|
declared: map[string]Declared{"lux-mainnet": {Phase: "Unknown", Validators: 5, Image: "ghcr.io/luxfi/node:v1.34.0", PluginSet: "s3://lux-plugins/v1.1.0/", VMPlugins: []string{"dexvm"}}},
|
|
},
|
|
Prober: portless{NewRPC()},
|
|
}
|
|
ov := c.Overview(context.Background(), []string{"mainnet"})
|
|
if len(ov.Networks) != 1 {
|
|
t.Fatalf("want 1 network, got %d", len(ov.Networks))
|
|
}
|
|
return ov.Networks[0]
|
|
}
|
|
|
|
// portless dials the stub's host:port verbatim instead of appending luxd's port.
|
|
type portless struct{ p Prober }
|
|
|
|
func (d portless) Probe(ctx context.Context, base string, n *Node) {
|
|
d.p.Probe(ctx, base[:strings.LastIndex(base, ":")], n)
|
|
}
|
|
|
|
func TestOverviewReportsLiveNode(t *testing.T) {
|
|
f := collectOne(t, newLuxd())
|
|
|
|
if f.NodesRPCOK != 1 {
|
|
t.Errorf("nodes_rpc_ok = %d, want 1", f.NodesRPCOK)
|
|
}
|
|
if f.NodesReady != 1 || f.NodesTotal != 1 {
|
|
t.Errorf("nodes_ready/total = %d/%d, want 1/1", f.NodesReady, f.NodesTotal)
|
|
}
|
|
if f.TotalPeers != 2 {
|
|
t.Errorf("total_peers = %d, want 2", f.TotalPeers)
|
|
}
|
|
// The CR declares v1.34.0; the fleet runs 1.36.2. Observation wins.
|
|
if f.Version == nil || *f.Version != "luxd/1.36.2" {
|
|
t.Errorf("version = %v, want luxd/1.36.2", deref(f.Version))
|
|
}
|
|
if f.Image == nil || *f.Image != "ghcr.io/luxfi/node:v1.34.0" {
|
|
t.Errorf("image = %v, want the declared CR image", deref(f.Image))
|
|
}
|
|
if f.Phase != "Running" {
|
|
t.Errorf("phase = %q, want Running", f.Phase)
|
|
}
|
|
|
|
n := f.Nodes[0]
|
|
if !n.RPCReachable {
|
|
t.Fatal("rpc_reachable = false")
|
|
}
|
|
for field, got := range map[string]*string{
|
|
"node_id": n.NodeID,
|
|
"ip": n.IP,
|
|
"git_commit": n.GitCommit,
|
|
"db_version": n.DBVersion,
|
|
} {
|
|
if got == nil {
|
|
t.Errorf("%s is null", field)
|
|
}
|
|
}
|
|
if n.IP != nil && *n.IP != "129.212.164.46" {
|
|
t.Errorf("ip = %q, want the advertised host without luxd's staking port", *n.IP)
|
|
}
|
|
heights := map[string]int64{"C": 0x10c1cf, "P": 1098191}
|
|
for _, ch := range n.Chains {
|
|
want, tracked := heights[ch.Chain]
|
|
switch {
|
|
case tracked && (ch.Height == nil || *ch.Height != want):
|
|
t.Errorf("%s height = %v, want %d", ch.Chain, ch.Height, want)
|
|
case !tracked && ch.Height != nil:
|
|
t.Errorf("%s height = %v, want null", ch.Chain, *ch.Height)
|
|
}
|
|
if ch.Bootstrapped == nil {
|
|
t.Errorf("%s bootstrapped is null", ch.Chain)
|
|
}
|
|
}
|
|
if f.Dex.DChainBootstrapped == nil || *f.Dex.DChainBootstrapped {
|
|
t.Errorf("d_chain_bootstrapped = %v, want false", f.Dex.DChainBootstrapped)
|
|
}
|
|
if f.Dex.Note == nil {
|
|
t.Error("dex note missing for an unbootstrapped D-Chain")
|
|
}
|
|
}
|
|
|
|
// The outage: the collector called /ext, luxd 1.36 answered 404, and every
|
|
// field came back null. Nothing may ask for /ext again.
|
|
func TestNeverCallsExt(t *testing.T) {
|
|
stub := newLuxd()
|
|
f := collectOne(t, stub)
|
|
if f.NodesRPCOK == 0 {
|
|
t.Fatal("no node answered: the /v1 surface is not being used")
|
|
}
|
|
stub.mu.Lock()
|
|
defer stub.mu.Unlock()
|
|
for _, p := range stub.paths {
|
|
if !strings.HasPrefix(p, "/v1/") {
|
|
t.Errorf("asked for %q; luxd only serves /v1", p)
|
|
}
|
|
if strings.HasPrefix(p, "/v1/C/") {
|
|
t.Errorf("asked for %q; chain RPC is /v1/bc/C/rpc", p)
|
|
}
|
|
}
|
|
if !contains(stub.paths, "/v1/bc/C/rpc") {
|
|
t.Error("C-Chain height was never requested at /v1/bc/C/rpc")
|
|
}
|
|
}
|
|
|
|
// A node that is down must show as down, not drag the network's truth with it.
|
|
func TestDownNodeStaysNull(t *testing.T) {
|
|
f := collectOne(t, newLuxd(), Pod{Name: "luxd-1", Index: 1, Phase: "Running", Ready: false, IP: ""})
|
|
if f.NodesTotal != 2 || f.NodesRPCOK != 1 || f.NodesReady != 1 {
|
|
t.Fatalf("total/rpc_ok/ready = %d/%d/%d, want 2/1/1", f.NodesTotal, f.NodesRPCOK, f.NodesReady)
|
|
}
|
|
down := f.Nodes[1]
|
|
if down.RPCReachable || down.Version != nil || down.IP != nil || down.Peers != nil {
|
|
t.Errorf("unreachable node reported data: %+v", down)
|
|
}
|
|
if len(down.Chains) != len(TrackedChains) {
|
|
t.Errorf("chains = %d, want %d placeholders", len(down.Chains), len(TrackedChains))
|
|
}
|
|
}
|
|
|
|
func TestTotalsAndEnvelope(t *testing.T) {
|
|
stub := newLuxd()
|
|
srv := httptest.NewServer(stub.handler())
|
|
defer srv.Close()
|
|
host := strings.TrimPrefix(srv.URL, "http://")
|
|
c := &Collector{
|
|
Cluster: fakeCluster{pods: map[string][]Pod{
|
|
"lux-mainnet": {{Name: "luxd-0", Index: 0, Phase: "Running", Ready: true, IP: host}},
|
|
"lux-testnet": {{Name: "luxd-0", Index: 0, Phase: "Running", Ready: true, IP: host}},
|
|
}},
|
|
Prober: portless{NewRPC()},
|
|
}
|
|
ov := c.Overview(context.Background(), nil)
|
|
if len(ov.Networks) != 3 {
|
|
t.Fatalf("networks = %d, want all 3", len(ov.Networks))
|
|
}
|
|
if ov.Networks[0].Network != "mainnet" || ov.Networks[2].Network != "devnet" {
|
|
t.Errorf("network order = %s..%s, want mainnet..devnet", ov.Networks[0].Network, ov.Networks[2].Network)
|
|
}
|
|
for k, want := range map[string]int{"networks": 3, "nodes_total": 2, "nodes_ready": 2, "nodes_rpc_ok": 2, "total_peers": 4, "validators": 2} {
|
|
if ov.Totals[k] != want {
|
|
t.Errorf("totals[%s] = %d, want %d", k, ov.Totals[k], want)
|
|
}
|
|
}
|
|
if ov.Cluster != "lux-k8s" || ov.GeneratedAt <= 0 {
|
|
t.Errorf("envelope = %q/%v", ov.Cluster, ov.GeneratedAt)
|
|
}
|
|
// A network with no pods is an empty fleet with an error, never a missing key.
|
|
dev := ov.Networks[2]
|
|
if dev.Error == nil || dev.Nodes == nil || dev.VMPlugins == nil {
|
|
t.Errorf("empty network must still carry error + empty lists: %+v", dev)
|
|
}
|
|
}
|
|
|
|
// Every field the dashboard's TypeScript declares must be present in the JSON,
|
|
// as null rather than absent — an absent key reads as undefined and blanks a cell.
|
|
func TestWireKeysAreNeverOmitted(t *testing.T) {
|
|
f := collectOne(t, newLuxd(), Pod{Name: "luxd-1", Index: 1, Phase: "Pending", IP: ""})
|
|
blob, err := json.Marshal(Overview{Cluster: "lux-k8s", Networks: []NetworkFleet{f}, Totals: map[string]int{}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var doc map[string]any
|
|
if err := json.Unmarshal(blob, &doc); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
net := doc["networks"].([]any)[0].(map[string]any)
|
|
for _, k := range []string{"network", "namespace", "network_id", "chain_id", "phase", "image",
|
|
"version", "plugin_set", "vm_plugins", "validators", "nodes_total", "nodes_ready",
|
|
"nodes_rpc_ok", "total_peers", "dex", "nodes", "error"} {
|
|
if _, ok := net[k]; !ok {
|
|
t.Errorf("networks[0].%s is absent", k)
|
|
}
|
|
}
|
|
node := net["nodes"].([]any)[1].(map[string]any)
|
|
for _, k := range []string{"pod_name", "pod_index", "phase", "ready", "ip",
|
|
"rpc_reachable", "node_id", "version", "git_commit", "db_version", "peers", "healthy", "chains"} {
|
|
if _, ok := node[k]; !ok {
|
|
t.Errorf("nodes[1].%s is absent", k)
|
|
}
|
|
}
|
|
dex := net["dex"].(map[string]any)
|
|
for _, k := range []string{"d_chain_bootstrapped", "venue", "markets", "note"} {
|
|
if _, ok := dex[k]; !ok {
|
|
t.Errorf("dex.%s is absent", k)
|
|
}
|
|
}
|
|
ch := node["chains"].([]any)[0].(map[string]any)
|
|
for _, k := range []string{"chain", "bootstrapped", "height"} {
|
|
if _, ok := ch[k]; !ok {
|
|
t.Errorf("chains[0].%s is absent", k)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNumericAcceptsQuotedAndBare(t *testing.T) {
|
|
for _, in := range []string{`"1098191"`, `1098191`} {
|
|
var n numeric
|
|
if err := n.UnmarshalJSON([]byte(in)); err != nil || n != 1098191 {
|
|
t.Errorf("%s -> %d, %v", in, n, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSelectNetworks(t *testing.T) {
|
|
if got := selectNetworks(nil); len(got) != 3 {
|
|
t.Errorf("no filter -> %d networks, want 3", len(got))
|
|
}
|
|
got := selectNetworks([]string{"devnet", "nope"})
|
|
if len(got) != 1 || got[0].Name != "devnet" {
|
|
t.Errorf("filter -> %+v, want devnet only", got)
|
|
}
|
|
}
|
|
|
|
func contains(ss []string, want string) bool {
|
|
for _, s := range ss {
|
|
if s == want {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func deref(s *string) string {
|
|
if s == nil {
|
|
return "<nil>"
|
|
}
|
|
return *s
|
|
}
|