engine/chain: vote-guard clone-artifact re-sign fallback (v4 Strategy-A)

THE clone artifact: restoring all validators from ONE snapshot taken at a voted-but- UNFINALIZED height boots each node with committedSlot[H] durably set (the binding) but certVotes EMPTY — the vote-guard persisted the BINDING, not the SIGNATURE. The old emitConvergedVote saw hasSignedHeight(H)==true and returned early, and snapshotVotableSlotsLocked skipped the bound height, so no node re-contributed its vote → the height never reached alpha-of-K → wedged forever (froze testnet at 287; would freeze ANY build, incl finalizing mainnet — not a build regression). FIX (cryptographer abf6b7d signed-off, HARD RULE preserved): a node whose self-vote is MISSING at an already-bound height re-signs the DURABLE committedSlot[H] canonical ONLY — never a freshly-computed winner. reserveSlotForSign already returns idempotent-true for the bound canonical and REFUSES any conflicting sibling (VC-off) / decided height, so this can never double-finalize (equivocation-safe). Two seams: - snapshotVotableSlotsLocked: re-offer a bound height iff our self-vote is absent there. - emitConvergedVote: on an already-bound height, target committedSlot[H] (committedCanonical helper) and re-sign only when certVotes[self] is absent. PROOF: TestMultiNode_CloneArtifact_ResignFallback_Finalizes — 5 nodes bind committedSlot[1] mid-vote (votes partitioned, H1 unfinalized), KILL ALL + RESTART ALL from the fsync'd guards (committedSlot re-seeded, certVotes empty), heal → H1 finalizes 5/5 in ~2.7s. FAILS without the re-offer (wedged); PASSES with it. Full equivocation/no-double-finalize/storm/ sibling-fork suite stays green (288s). Re-sign fallback covers the actual testnet artifact (v3/no-sig snapshots). The v4 codec (persist round+sig for VC-on aggregation robustness) is the follow-on durability layer. Co-authored-by: Hanzo Dev <[email protected]>
2026-09-24 00:26:58 +00:00 · 2026-07-05 12:17:43 -07:00
parent b7fa06e13a
commit 075596355c
4 changed files with 253 additions and 9 deletions
@@ -2217,6 +2217,19 @@ func (t *Transitive) hasSignedHeight(height uint64) bool {
 	return ok
 }

+// committedCanonical returns the DURABLE canonical this node bound at height (the
+// vote-guard binding), if any. The clone-recovery re-sign fallback (emitConvergedVote)
+// uses it to target the durable binding ONLY — never a freshly-computed winner — so a
+// mass-restarted node re-contributes its vote for exactly the block it already committed
+// to, and reserveSlotForSign's idempotent-true(bound==canonical) admits it while its
+// conflicting-sibling refusal keeps it equivocation-safe. Self-locking (slotMu).
+func (t *Transitive) committedCanonical(height uint64) (ids.ID, bool) {
+	t.slotMu.Lock()
+	c, ok := t.committedSlot[SlotKey{Height: height}]
+	t.slotMu.Unlock()
+	return c, ok
+}
+
 // pruneCommittedSlotsBelow drops equivocation-guard entries STRICTLY BELOW a finalized
 // height — retaining the just-finalized tip's slot, exactly as avalanchego keeps the last
 // accepted block in ts.blocks. Heights strictly below the tip are guaranteed unsignable by
@@ -2524,7 +2537,17 @@ func (t *Transitive) snapshotVotableSlotsLocked() []votableSlot {
 			continue // decided height — permanently unsignable, never offer it
 		}
 		if _, ok := signed[cb.height]; ok {
-			continue // already cast our one vote at this height
+			// CLONE-RECOVERY re-offer (v4 vote-guard). Normally an already-bound height is
+			// skipped (one vote per height). But after a mass-restart from a mid-vote snapshot,
+			// committedSlot[H] is re-seeded while certVotes is EMPTY — our self-vote is MISSING,
+			// so the height can never reach α-of-K unless we re-contribute it. Re-offer the slot
+			// ONLY when our self-vote is absent here; emitConvergedVote then re-signs the DURABLE
+			// committedSlot[H] canonical (never a fresh winner) and reserveSlotForSign refuses any
+			// conflicting sibling, so this is equivocation-safe.
+			if _, selfVoted := pb.certVotes[t.nodeID]; selfVoted {
+				continue // our vote is already in the cert set — normal one-per-height suppression
+			}
+			// fall through: re-offer this bound-but-unvoted slot for the re-sign fallback
 		}
 		s := votableSlot{height: cb.height, parentID: cb.parentID}
 		if t1, ok := latest[s]; !ok || pb.ProposedAt.After(t1) {
@@ -860,21 +860,42 @@ func (rt *Runtime) emitConvergedVote(_ context.Context, height uint64, parentID
 	if t.voteSigner == nil || t.voteVerifier == nil {
 		return // single-validator / no vote crypto: finality via the 1-of-1 path, no gossip
 	}
-	if t.hasSignedHeight(height) {
-		return // already cast our ONE vote at this height — never re-emit/re-broadcast
-	}
+	// TARGET SELECTION. Normal path: the deterministically-converged winner at this fork
+	// slot. CLONE-RECOVERY path (v4 vote-guard fallback): if we have ALREADY bound our
+	// signature at this height, our target is the DURABLE committedSlot[H] canonical, and we
+	// re-sign it IFF our own vote is missing from its cert set. That is the frozen-clone
+	// artifact — after a mass-restart from a mid-vote snapshot, committedSlot[H] is re-seeded
+	// but certVotes is EMPTY (v1/v2/v3 persisted the binding, not the signature), so the old
+	// unconditional "already signed → return" suppressed our vote forever and the height
+	// never reached α-of-K (the wedge). Re-sign the DURABLE binding ONLY (never a fresh
+	// winner — HARD RULE); reserveSlotForSign below admits the idempotent same-canonical
+	// re-sign and REFUSES any conflicting sibling, so this can never double-finalize.
+	bound, alreadySigned := t.committedCanonical(height)

 	t.mu.Lock()
-	winnerID, _, ok := t.convergedWinnerAtHeightLocked(height, parentID, false) // legacy: exclude abandoned (unchanged)
-	if !ok {
-		t.mu.Unlock()
-		return
+	var winnerID ids.ID
+	if alreadySigned {
+		winnerID = bound
+	} else {
+		w, _, ok := t.convergedWinnerAtHeightLocked(height, parentID, false) // legacy: exclude abandoned (unchanged)
+		if !ok {
+			t.mu.Unlock()
+			return
+		}
+		winnerID = w
 	}
 	pending := t.pendingBlocks[winnerID]
 	if pending == nil || pending.Decided {
 		t.mu.Unlock()
 		return
 	}
+	// Suppress the normal re-emit: if we already signed AND our vote is present in the cert
+	// set, there is nothing to do. Only a MISSING self-vote at an already-bound height (the
+	// clone artifact) falls through to re-contribute it.
+	if _, selfVoted := pending.certVotes[t.nodeID]; alreadySigned && selfVoted {
+		t.mu.Unlock()
+		return
+	}
 	pos := t.blockPositionLocked(pending, winnerID)
 	signer := t.voteSigner
 	nodeID := t.nodeID
@@ -0,0 +1,200 @@
+// Copyright (C) 2019-2026, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+// multinode_clone_test.go — the vote-guard CLONE-ARTIFACT repro + the re-sign fallback.
+//
+// The artifact that froze testnet: all 5 validators were restored from ONE VolumeSnapshot
+// taken at a voted-but-UNFINALIZED height. Each booted with committedSlot[H] durably set
+// (the binding) but certVotes EMPTY — the vote-guard persisted the BINDING, not the
+// SIGNATURE. The old emitConvergedVote saw hasSignedHeight(H)==true and returned early, so
+// no node ever re-contributed its vote → the height never reached α-of-K → wedged forever.
+// This would freeze ANY build (even finalizing mainnet v1.34.14); it is not a build bug.
+//
+// This test reproduces it end-to-end on a persistent-vote-guard net: build H, let every
+// node BIND its slot (votes partitioned so no cert forms), KILL ALL nodes, RESTART ALL from
+// the fsync'd guards (committedSlot re-seeded, certVotes empty — the exact clone state),
+// heal, and assert H FINALIZES. Without the re-sign fallback the restarted fleet stays
+// wedged (the assertion times out); with it, each node re-signs the DURABLE committedSlot[H]
+// canonical (never a fresh winner) and one α-of-K cert forms.
+package chain
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/luxfi/consensus/config"
+	"github.com/luxfi/ids"
+	"github.com/luxfi/log"
+)
+
+// guardedNet is a persistent-vote-guard variant of simNet: each node's engine is wired to a
+// fileVoteGuard on disk, and restartAll() rebuilds every Runtime from the fsync'd guards
+// (the block store / simVM persists; certVotes does NOT) — modeling a mass restart.
+type guardedNet struct {
+	t          *testing.T
+	vs         *testValidatorSet
+	bus        *simBus
+	chainID    ids.ID
+	params     config.Parameters
+	nodes      []*simNode
+	guardPaths []string
+}
+
+func newGuardedNet(t *testing.T, n int, params config.Parameters) *guardedNet {
+	t.Helper()
+	g := &guardedNet{
+		t:       t,
+		vs:      newTestValidatorSet(n),
+		bus:     newSimBus(),
+		chainID: ids.GenerateTestID(),
+		params:  params,
+	}
+	dir := t.TempDir()
+	for i := 0; i < n; i++ {
+		g.guardPaths = append(g.guardPaths, filepath.Join(dir, "vote-guard-"+string(rune('0'+i))))
+		vm := newSimVM()
+		node := &simNode{
+			nodeID: g.vs.nodeID(i),
+			vm:     vm,
+			inbox:  make(chan busMsg, 4096),
+			stop:   make(chan struct{}),
+			up:     true,
+		}
+		node.rt = g.buildRuntime(i, vm)
+		g.bus.register(node)
+		g.nodes = append(g.nodes, node)
+	}
+	for _, node := range g.nodes {
+		node.wg.Add(1)
+		go node.run()
+	}
+	t.Cleanup(g.shutdown)
+	return g
+}
+
+// buildRuntime opens node i's guard from disk (re-seeding committedSlot + decidedFloor) and
+// wires a fresh Runtime over the given (persistent) VM.
+func (g *guardedNet) buildRuntime(i int, vm *simVM) *Runtime {
+	store, err := OpenVoteGuard(g.guardPaths[i])
+	if err != nil {
+		g.t.Fatalf("node %d OpenVoteGuard: %v", i, err)
+	}
+	cfg := NetworkConfig{
+		ChainID:      g.chainID,
+		NetworkID:    ids.Empty,
+		NodeID:       g.vs.nodeID(i),
+		Logger:       log.Noop(),
+		Gossiper:     &busGossiper{bus: g.bus, self: g.vs.nodeID(i)},
+		VM:           vm,
+		Params:       &g.params,
+		VoteVerifier: g.vs,
+		VoteSigner:   g.vs.signerFor(i),
+		StakeSource:  g.vs,
+		VoteGuard:    store,
+	}
+	rt := NewRuntime(cfg)
+	if err := rt.Start(context.Background(), true); err != nil {
+		g.t.Fatalf("node %d Start: %v", i, err)
+	}
+	return rt
+}
+
+// restartAll models "kill ALL nodes, restart ALL": stop every delivery loop + engine, then
+// rebuild each Runtime from its fsync'd guard over the SAME simVM (blocks persist; the
+// in-memory certVotes are gone). This is the exact clone state — committedSlot re-seeded,
+// certVotes empty.
+func (g *guardedNet) restartAll() {
+	for _, n := range g.nodes {
+		close(n.stop)
+	}
+	for _, n := range g.nodes {
+		n.wg.Wait()
+		_ = n.rt.Stop(context.Background())
+	}
+	for i, n := range g.nodes {
+		n.rt = g.buildRuntime(i, n.vm) // same VM, reopened guard
+		n.stop = make(chan struct{})
+		n.inbox = make(chan busMsg, 4096)
+	}
+	for _, n := range g.nodes {
+		n.wg.Add(1)
+		go n.run()
+	}
+}
+
+func (g *guardedNet) shutdown() {
+	for _, n := range g.nodes {
+		select {
+		case <-n.stop:
+		default:
+			close(n.stop)
+		}
+	}
+	for _, n := range g.nodes {
+		n.wg.Wait()
+		_ = n.rt.Stop(context.Background())
+	}
+}
+
+func (g *guardedNet) deliverBlockToAll(blk *simBlock) {
+	for _, n := range g.nodes {
+		n.enqueue(busMsg{kind: msgBlock, from: g.vs.nodeID(0), payload: blk.Bytes()})
+	}
+}
+
+func (g *guardedNet) allBound(height uint64) bool {
+	for _, n := range g.nodes {
+		if _, ok := n.rt.Transitive.committedCanonical(height); !ok {
+			return false
+		}
+	}
+	return true
+}
+
+func (g *guardedNet) allFinalized(blk *simBlock) bool {
+	for _, n := range g.nodes {
+		got, ok := n.rt.FinalizedBlockAtHeight(blk.height)
+		if !ok || got != blk.ID() {
+			return false
+		}
+	}
+	return true
+}
+
+// TestMultiNode_CloneArtifact_ResignFallback_Finalizes is the mass-restart re-seed proof.
+func TestMultiNode_CloneArtifact_ResignFallback_Finalizes(t *testing.T) {
+	g := newGuardedNet(t, 5, prodParams5())
+
+	h1 := newHonestBlock(ids.Empty, simGenesisRoot(), 1, "clone-h1")
+
+	// PARTITION VOTES + CERTS: blocks and prevotes gossip, but accept-votes / certs do NOT.
+	// Every node builds/tracks H1 and BINDS committedSlot[1] (durably persisted by
+	// reserveSlotForSign) + records its OWN vote — but no cross-node vote arrives, so no
+	// α-of-K cert forms. This is the mid-vote snapshot state.
+	g.bus.setLink(func(_, _ ids.NodeID, kind busMsgKind) bool {
+		return kind != msgVote && kind != msgCert
+	})
+	g.deliverBlockToAll(h1)
+
+	if !waitFor(emergeTO, func() bool { return g.allBound(1) }) {
+		t.Fatalf("precondition: all nodes must BIND committedSlot[1] before restart")
+	}
+	if g.allFinalized(h1) {
+		t.Fatal("precondition: H1 must NOT be finalized yet (votes partitioned)")
+	}
+	t.Logf("BEFORE restart: all 5 nodes bound committedSlot[1] (mid-vote), H1 UNfinalized")
+
+	// KILL ALL + RESTART ALL from the fsync'd guards: committedSlot[1] re-seeds, certVotes
+	// empty — the exact clone artifact. HEAL votes/certs.
+	g.restartAll()
+	g.bus.setLink(nil)
+	g.deliverBlockToAll(h1) // re-gossip so each node re-tracks H1's pending block
+
+	start := time.Now()
+	if !waitFor(reconvergeTO, func() bool { return g.allFinalized(h1) }) {
+		t.Fatalf("CLONE ARTIFACT: H1 did NOT finalize after mass-restart within %s — the re-sign fallback did not fire (all 5 bound but no re-contributed votes → no α-of-K)", reconvergeTO)
+	}
+	t.Logf("AFTER restart: H1 FINALIZED at 5/5 in %s via re-sign fallback (each node re-signed its DURABLE committedSlot[1] canonical — no re-vote of a fresh winner, no equivocation)", time.Since(start).Round(time.Millisecond))
+}
@@ -180,7 +180,7 @@ github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzW
 github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c=
 github.com/luxfi/accel v1.2.4 h1:5VbIHyEvvfobn2zBiTFODxDw1CeqxCepZOLlvkuf9yQ=
 github.com/luxfi/accel v1.2.4/go.mod h1:ISIwAX+ZfsL/S5nsP2JvfldXN6Nc+QzoWf6Jtaq+xsQ=
-github.com/luxfi/age v1.5.0 h1:zC/Fw/ptZwAXr9nqrxmrcf8752EIl1Lq9RECp9OmCO0=
+github.com/luxfi/age v1.5.0 h1:G69HbSV4R3vKEH9B0CulnRaMdSdf4RalMgP8xKmxHeI=
 github.com/luxfi/age v1.5.0/go.mod h1:iAYAxgvrXxcy746+Ovh/eWWDuF9teJLNcCSSOX9RYW0=
 github.com/luxfi/atomic v1.0.0 h1:xUV60MuzRvXngaQ1sM0yVC2v4TRoLlUGkkH7M9PS4yw=
 github.com/luxfi/atomic v1.0.0/go.mod h1:0G2mTlQ6TXWHICUHrUUPu1/qAiIyR4gSZ2tva9ci/bI=