test(engine/chain): async-gossip (WAN jitter/reorder) idle-storm variant

Adds an opt-in per-message delivery latency to the sim bus (nil default =
unchanged instant delivery) so a test can model async WAN gossip: heavy-tail
jitter where ~10% of messages arrive after the RoundTO/2 settle window, so
prevotes reach different nodes in different orders. Plus the async+zero-margin+
idle storm scenario.

FINDING: PASSES at HEAD (v1.35.16), like the naive and zero-margin idle storms.
THREE in-process variants (sync idle, zero-margin idle, async-gossip idle) all
finalize cleanly — the in-process consensus harness robustly cannot reproduce
the live VC stall. The trigger is at the node-integration/live layer (real
proposervm + real transport), not the consensus engine in isolation. The
epoch-churn hypothesis is separately refuted by static analysis: epochHeightLocked
returns pChainHeight (0 on these nets, stable), not the churning epoch number, so
the moving epoch does not affect vote pubkey/stake resolution.

Existing storm tests remain green (nil-default opt-in). The bisect + forward-fix
must run on a live net.

Co-authored-by: Hanzo Dev <[email protected]>

engine/chain/idle_storm_test.go

@@ -19,6 +19,8 @@
 package chain
 
 import (
+	"math/rand"
+	"sync"
 	"testing"
 	"time"
 
@@ -135,3 +137,63 @@ func TestStorm_IdleWindows_ZeroMargin(t *testing.T) {
 	}
 	t.Logf("ZERO-MARGIN IDLE-STORM PASS: α=4 exact quorum, %d heights across %d idle windows, no stall", total, cycles)
 }
+
+// TestStorm_IdleWindows_AsyncGossip_ZeroMargin is the async-gossip variant — the
+// closest in-process analogue of the live WAN condition. It gives the bus a heavy-tail
+// delivery latency (most messages fast, ~10% arrive AFTER the RoundTO/2 settle window),
+// so prevotes arrive in DIFFERENT orders at different nodes and each round-view advances
+// on a different arrival order — the p99-exceeds-settle regime the live fleet stalled in.
+// Combined with exact α=4 (zero margin) + idle windows. If the round-skip/POL machinery
+// has a live-only convergence gap, this reproduces it as a per-height stall.
+func TestStorm_IdleWindows_AsyncGossip_ZeroMargin(t *testing.T) {
+	if testing.Short() {
+		t.Skip("async idle storm is timing-heavy; skipped in -short")
+	}
+	params := stormParams5VC()
+	net := newSimNet(t, 5, params)
+
+	rng := rand.New(rand.NewSource(1)) // seeded for reproducibility
+	var rngMu sync.Mutex
+	net.bus.setDelay(func() time.Duration {
+		rngMu.Lock()
+		defer rngMu.Unlock()
+		if rng.Intn(10) == 0 {
+			return time.Duration(300+rng.Intn(600)) * time.Millisecond // slow tail 300-900ms (> 500ms settle)
+		}
+		return time.Duration(rng.Intn(50)) * time.Millisecond // fast 0-50ms
+	})
+
+	net.down(4) // exact α=4 — zero margin (the mainnet condition)
+
+	const cycles = 8
+	const heightsPerBurst = 3
+	idle := 3 * params.RoundTO
+
+	parentID := ids.Empty
+	parentStateRoot := simGenesisRoot()
+	var h uint64
+	for c := 0; c < cycles; c++ {
+		for b := 0; b < heightsPerBurst; b++ {
+			h++
+			blocks := make(map[ids.ID]*simBlock)
+			for i := 0; i < 5; i++ {
+				if !net.nodes[i].reachable() {
+					continue
+				}
+				blk := newHonestBlock(parentID, parentStateRoot, h, "async-"+itoa(i)+"-h"+itoa(int(h%10)))
+				blocks[blk.ID()] = blk
+				net.build(i, blk)
+			}
+			head := stormAwaitSingleHead(t, net, h)
+			won, ok := blocks[head]
+			if !ok {
+				t.Fatalf("cycle %d height %d: head %s not among siblings", c, h, head)
+			}
+			parentID = head
+			parentStateRoot = won.stateRoot
+		}
+		t.Logf("async cycle %d done at height %d, idling %s", c, h, idle)
+		time.Sleep(idle)
+	}
+	t.Logf("ASYNC IDLE-STORM PASS: %d heights, WAN heavy-tail jitter + idle + zero-margin, no stall", h)
+}

engine/chain/multinode_harness_test.go

@@ -298,6 +298,12 @@ type simBus struct {
 	// α-of-K vote across siblings — the production down-proposer + WAN-latency condition
 	// the broadcast bus (uniform, instant delivery) cannot express.
 	link func(from, to ids.NodeID, kind busMsgKind) bool
+	// delay, when non-nil, returns a per-message-per-target delivery latency. It turns
+	// the uniform instant bus into an ASYNC bus with jitter/reorder — the WAN condition
+	// where node A sees B's prevote before C's and every node's round-view advances on a
+	// DIFFERENT message-arrival order. This is the async-gossip regime the live fleet
+	// stalls in and the synchronous bus (0/200) cannot express. nil ⇒ instant (default).
+	delay func() time.Duration
 }
 
 func newSimBus() *simBus { return &simBus{nodes: map[ids.NodeID]*simNode{}} }
@@ -315,6 +321,13 @@ func (bus *simBus) setLink(f func(from, to ids.NodeID, kind busMsgKind) bool) {
 	bus.mu.Unlock()
 }
 
+// setDelay installs (or clears, with nil) the per-message async delivery latency.
+func (bus *simBus) setDelay(f func() time.Duration) {
+	bus.mu.Lock()
+	bus.delay = f
+	bus.mu.Unlock()
+}
+
 // deliver enqueues msg to every node except `from` that is currently reachable AND
 // whose link from `from` is up (per bus.link, if set).
 func (bus *simBus) deliver(from ids.NodeID, m busMsg) int {
@@ -331,6 +344,7 @@ func (bus *simBus) deliver(from ids.NodeID, m busMsg) int {
 		targets = append(targets, target{id: id, n: n})
 	}
 	link := bus.link
+	delay := bus.delay
 	bus.mu.Unlock()
 	sent := 0
 	for _, t := range targets {
@@ -338,7 +352,19 @@ func (bus *simBus) deliver(from ids.NodeID, m busMsg) int {
 			continue // partitioned link — drop this message toward this target
 		}
 		if t.n.reachable() {
-			t.n.enqueue(m)
+			if delay != nil {
+				// ASYNC delivery: each target receives after its own jittered latency, so
+				// messages arrive in DIFFERENT orders at different nodes (the WAN reorder).
+				tn, mm, d := t.n, m, delay()
+				go func() {
+					time.Sleep(d)
+					if tn.reachable() {
+						tn.enqueue(mm)
+					}
+				}()
+			} else {
+				t.n.enqueue(m)
+			}
 			sent++
 		}
 	}