mirror of
https://github.com/luxfi/consensus.git
synced 2026-09-24 00:26:58 +00:00
chain: end the frontier walk at the tip, and pace cert gossip
Two defects that together keep a chain from advancing under load. pathFromTip walks parents from a cert toward the finalized tip. When an outer block has a local canonical wrapper the walk rebases onto it, assigning cur mid-body without re-testing the loop condition. If that wrapper IS the tip the walk has arrived, but execution falls through to the below-frontier check and refuses a cert that extends the tip by exactly one block — naming the tip itself as the losing branch, so the message has the same id on both sides. Nothing recovers from it: the self-heal path fires only on ErrAncestorNotTracked. TryAccept gossips an assembled cert before finalizing, and is re-entered on every arriving vote and every poll tick. A block whose finalize keeps failing stays pending and re-broadcasts the same cert on each entry; each broadcast lands on every peer as a finalize attempt whose votes come back and re-enter TryAccept. A cert is deterministic evidence about one height, so re-sending it tells a peer nothing the first send did, and it is now paced on the same schedule as the re-poll backstop. The claim is made under the lock that holds the pending block, so concurrent entries cannot each decide they are due. Both tests fail without their fix and pass with it, and the frontier test keeps a negative control: a rebase onto a non-tip block is still refused. Co-authored-by: Hanzo Dev <[email protected]>
This commit is contained in:
Hanzo Dev
@@ -0,0 +1,87 @@
|
||||
// Copyright (C) 2019-2026, Lux Industries, Inc. All rights reserved.
|
||||
// See the file LICENSE for licensing terms.
|
||||
|
||||
package chain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/luxfi/ids"
|
||||
"github.com/luxfi/log"
|
||||
)
|
||||
|
||||
// TestConsensus_CertGossipPacedWhileFinalizeKeepsFailing pins the pacing on the
|
||||
// finality-cert broadcast.
|
||||
//
|
||||
// TryAccept assembles the cert, hands it to the gossiper, and only THEN finalizes.
|
||||
// When the finalize fails — the VM refuses to apply the block, or its ancestor is
|
||||
// not tracked yet — the block correctly stays pending, so the very next arriving
|
||||
// vote or poll tick re-enters TryAccept and re-broadcasts the identical cert. Every
|
||||
// peer answers that broadcast with a finalize attempt of its own, whose votes come
|
||||
// back here: the loop feeds itself and the volume is bounded only by CPU.
|
||||
//
|
||||
// This is what lux-mainnet's C-Chain was doing while stuck at outer height
|
||||
// 1,159,288 — 7,392 cert gossips in 60 seconds across 145 pending blocks from one
|
||||
// node, ~8,280 inbound certs per node per minute, on a chain that had not advanced
|
||||
// in 11 hours.
|
||||
//
|
||||
// A cert is deterministic evidence about one decided height; the second copy tells
|
||||
// a peer nothing the first did not. So it goes out once and is then paced on the
|
||||
// same schedule as the re-poll backstop.
|
||||
func TestConsensus_CertGossipPacedWhileFinalizeKeepsFailing(t *testing.T) {
|
||||
vs := newTestValidatorSet(5)
|
||||
rec := &recordingGossiper{}
|
||||
e, chainID := newQuorumEngine(t, params5(), vs, 0, rec)
|
||||
rt := &Runtime{Transitive: e, config: NetworkConfig{ChainID: chainID, Logger: log.Noop()}}
|
||||
|
||||
blk := &trackingMockBlock{
|
||||
id: ids.GenerateTestID(),
|
||||
parentID: ids.Empty,
|
||||
height: 1,
|
||||
timestamp: time.Now(),
|
||||
bytes: []byte("stuck"),
|
||||
acceptErr: errors.New("vm refuses to apply"),
|
||||
}
|
||||
trackMockBlock(rt, blk, 0)
|
||||
pos := VotePosition{ChainID: chainID, Height: 1, Round: 0, BlockID: blk.id, ParentID: ids.Empty}
|
||||
|
||||
// Full 5-of-5 signed accepts: the quorum is unambiguously present, so the cert
|
||||
// assembles. The finalize behind it fails on VM.Accept.
|
||||
for i := 0; i < 5; i++ {
|
||||
e.ReceiveVote(vs.signedVote(i, pos))
|
||||
}
|
||||
|
||||
certCount := func() int {
|
||||
rec.mu.Lock()
|
||||
defer rec.mu.Unlock()
|
||||
return len(rec.certs)
|
||||
}
|
||||
if !waitFor(2*time.Second, func() bool { return certCount() >= 1 }) {
|
||||
t.Fatal("harness never reached the quorum: no cert was assembled, so this test proves nothing")
|
||||
}
|
||||
e.mu.RLock()
|
||||
pending, retained := e.pendingBlocks[blk.id]
|
||||
decided := retained && pending.Decided
|
||||
e.mu.RUnlock()
|
||||
if !retained || decided {
|
||||
t.Fatal("harness precondition: the block must stay pending and undecided (the VM.Accept error path)")
|
||||
}
|
||||
|
||||
before := certCount()
|
||||
|
||||
// Every one of these stands for an arriving vote or a poll tick on a block that
|
||||
// cannot finalize.
|
||||
const retries = 50
|
||||
for i := 0; i < retries; i++ {
|
||||
_ = e.TryAccept(context.Background(), blk.id)
|
||||
}
|
||||
|
||||
if extra := certCount() - before; extra != 0 {
|
||||
t.Fatalf("the same cert was re-broadcast %d times across %d finalize retries — "+
|
||||
"one stuck block turns into an unbounded gossip storm (mainnet: 51 sends per block "+
|
||||
"per minute); the broadcast must be paced, not fired on every entry", extra, retries)
|
||||
}
|
||||
}
|
||||
@@ -189,6 +189,27 @@ type PendingBlock struct {
|
||||
// request and so a follower's accept is gated on holding it.
|
||||
cert *QuorumCert
|
||||
|
||||
// lastCertGossip is when this block's assembled cert was last put on the wire
|
||||
// (zero until the first send). A cert is deterministic evidence about one
|
||||
// height: re-sending it tells a peer nothing the first send did not, so it is
|
||||
// paced on the SAME schedule as the re-poll backstop rather than fired on
|
||||
// every entry.
|
||||
//
|
||||
// TryAccept is re-entered on every arriving vote and every poll tick, and it
|
||||
// gossips BEFORE the finalize — so a block whose finalize keeps failing (the
|
||||
// VM refuses to apply it, or its ancestor is not tracked yet) stays pending
|
||||
// and re-broadcasts the same cert on every entry. Each broadcast lands on every
|
||||
// peer as a HandleIncomingCert → finalize attempt, whose votes come back and
|
||||
// re-enter TryAccept: the loop feeds itself. Measured on lux-mainnet C-Chain,
|
||||
// stuck at outer height 1,159,288: one node emitted 7,392 cert gossips in 60s
|
||||
// across 145 pending blocks (51 per block per minute) while every peer took
|
||||
// ~8,280 inbound certs in the same window, on a chain that had not advanced in
|
||||
// 11 hours.
|
||||
//
|
||||
// Pacing keeps the liveness the broadcast exists for — a follower that missed
|
||||
// the send still gets the proof on a later pass — and takes the amplifier away.
|
||||
lastCertGossip time.Time
|
||||
|
||||
// lastRePoll is when the re-poll loop last re-solicited votes for this block
|
||||
// (zero until the first re-poll). The re-poll loop re-drives a block at most
|
||||
// once per its CURRENT backoff window (rePollBackoff), so a stuck block
|
||||
@@ -3320,6 +3341,16 @@ func (t *Transitive) TryAccept(ctx context.Context, blockID ids.ID) error {
|
||||
if b, err := cert.Cert().MarshalBinary(); err == nil {
|
||||
certBytes = b
|
||||
}
|
||||
// PACE IT. Claim the broadcast under the same lock that holds the pending
|
||||
// block, so concurrent entries for one block cannot each decide they are due.
|
||||
if certBytes != nil {
|
||||
if now := time.Now(); pending.lastCertGossip.IsZero() ||
|
||||
now.Sub(pending.lastCertGossip) >= maxRePollBackoff {
|
||||
pending.lastCertGossip = now
|
||||
} else {
|
||||
certBytes = nil
|
||||
}
|
||||
}
|
||||
chainID := t.chainID
|
||||
gossiper := t.certGossiper
|
||||
t.mu.Unlock()
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
// Copyright (C) 2019-2026, Lux Industries, Inc. All rights reserved.
|
||||
// See the file LICENSE for licensing terms.
|
||||
|
||||
// frontier_rebase_test.go — the ancestry walk must re-test the frontier AFTER a
|
||||
// canonical rebase.
|
||||
//
|
||||
// pathFromTip walks parent links from the certified block down to the finalized tip,
|
||||
// looping while cur != led.tip. When the exact outer envelope a cert names is untracked,
|
||||
// the walk stands a LOCAL canonical-equivalent wrapper in its place and assigns cur =
|
||||
// localID. That assignment moves cur mid-body, so the loop condition no longer holds for
|
||||
// the value being tested by the checks below it. When the local wrapper IS the finalized
|
||||
// tip, execution reached `curHeight <= led.height` with cur == led.tip and returned
|
||||
// ErrConflictsWithFinalizedBranch — refusing a cert that extends the frontier by exactly
|
||||
// one block, and naming the tip itself as the "losing/pruned sibling branch" it descends
|
||||
// from.
|
||||
//
|
||||
// Observed on lux-testnet luxd-0, which refused every incoming cert at a static tip:
|
||||
//
|
||||
// incoming cert: REFUSED by finality guard (no VM.Accept)
|
||||
// error="chain: cert-selected block does not extend the finalized frontier (it descends
|
||||
// from a losing/pruned sibling branch): tH96v3tgug2SqQS1uPK6BRo2nUNV6r32SP5RZitUtsQwSHb7Q
|
||||
// ancestry reaches oYkSNtSbAaYHfQECmHS1P5VZcrcobANbVLLzcKj2xMheztvPU (height 36060)
|
||||
// not finalized tip oYkSNtSbAaYHfQECmHS1P5VZcrcobANbVLLzcKj2xMheztvPU"
|
||||
//
|
||||
// The reached ancestor and the finalized tip are the same id. The refusal is self-
|
||||
// contradictory, and it is self-sustaining: the tip never moves, so the next cert takes
|
||||
// the same path, so the validator is stranded permanently.
|
||||
|
||||
package chain
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/luxfi/ids"
|
||||
)
|
||||
|
||||
// TestPathFromTip_RebaseOntoTip_Finalizes pins the walk: when the canonical rebase lands
|
||||
// on the finalized tip itself, the path is complete and the cert finalizes. The pre-fix
|
||||
// walk returned ErrConflictsWithFinalizedBranch here.
|
||||
func TestPathFromTip_RebaseOntoTip_Finalizes(t *testing.T) {
|
||||
const tipHeight = 36060
|
||||
|
||||
tipInner := ids.GenerateTestID() // the tip's inner execution commitment
|
||||
tip := ids.GenerateTestID() // the tip envelope this node holds
|
||||
tipParent := ids.GenerateTestID() // one below the frontier
|
||||
remoteTip := ids.GenerateTestID() // a sibling envelope of the SAME inner block, unheld
|
||||
certBlock := ids.GenerateTestID() // the certified block at tipHeight+1
|
||||
|
||||
led := seedLedger(tip, tipInner, tipHeight)
|
||||
|
||||
// This node tracks its own tip envelope, never the remote sibling wrapper the cert
|
||||
// names as parent. Both wrap tipInner at tipHeight, so they are interchangeable.
|
||||
dag := mapAncestry{
|
||||
tip: {parent: tipParent, height: tipHeight, canonical: tipInner},
|
||||
certBlock: {
|
||||
parent: remoteTip, height: tipHeight + 1,
|
||||
canonical: certBlock, parentCanonical: tipInner,
|
||||
},
|
||||
}
|
||||
|
||||
cert := Cert{
|
||||
Block: certBlock, Parent: remoteTip, Height: tipHeight + 1,
|
||||
Canonical: certBlock, ParentCanonical: tipInner,
|
||||
}
|
||||
|
||||
next, _, err := Finalize(led, cert, dag)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrConflictsWithFinalizedBranch) {
|
||||
t.Fatalf("a cert extending the tip by one block was refused as a losing "+
|
||||
"branch — the walk rebased onto the tip and did not re-test the "+
|
||||
"frontier: %v", err)
|
||||
}
|
||||
t.Fatalf("finalize must succeed; got %v", err)
|
||||
}
|
||||
if next.height != tipHeight+1 {
|
||||
t.Fatalf("finalized height = %d, want %d", next.height, tipHeight+1)
|
||||
}
|
||||
if next.tip != certBlock {
|
||||
t.Fatalf("finalized tip = %s, want the certified block %s", next.tip, certBlock)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPathFromTip_RebaseOntoNonTip_StillRefuses pins the other side: a rebase that lands
|
||||
// BELOW the frontier on a block that is genuinely not the tip is still a conflict. The
|
||||
// re-test must not soften the guard into accepting real sibling branches.
|
||||
func TestPathFromTip_RebaseOntoNonTip_StillRefuses(t *testing.T) {
|
||||
const tipHeight = 36060
|
||||
|
||||
tip := ids.GenerateTestID()
|
||||
tipInner := ids.GenerateTestID()
|
||||
|
||||
loserInner := ids.GenerateTestID() // a DIFFERENT inner block at the same height
|
||||
loser := ids.GenerateTestID() // the losing sibling this node happens to track
|
||||
loserParent := ids.GenerateTestID()
|
||||
remoteLoser := ids.GenerateTestID() // unheld envelope of the losing block
|
||||
certBlock := ids.GenerateTestID()
|
||||
|
||||
led := seedLedger(tip, tipInner, tipHeight)
|
||||
|
||||
dag := mapAncestry{
|
||||
tip: {parent: ids.GenerateTestID(), height: tipHeight, canonical: tipInner},
|
||||
loser: {parent: loserParent, height: tipHeight, canonical: loserInner},
|
||||
certBlock: {
|
||||
parent: remoteLoser, height: tipHeight + 1,
|
||||
canonical: certBlock, parentCanonical: loserInner,
|
||||
},
|
||||
}
|
||||
|
||||
cert := Cert{
|
||||
Block: certBlock, Parent: remoteLoser, Height: tipHeight + 1,
|
||||
Canonical: certBlock, ParentCanonical: loserInner,
|
||||
}
|
||||
|
||||
if _, _, err := Finalize(led, cert, dag); !errors.Is(err, ErrConflictsWithFinalizedBranch) {
|
||||
t.Fatalf("a cert descending from a losing sibling at the frontier height must "+
|
||||
"still be refused; got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -443,6 +443,15 @@ func pathFromTip(led FinalityLedger, cert Cert, dag Ancestry) ([]step, error) {
|
||||
// wrapper the VM actually holds, never the unheld outer alias.
|
||||
steps[len(steps)-1].parentID = localID
|
||||
cur = localID
|
||||
// The rebase moves cur, so the loop condition must be re-tested
|
||||
// before the below-frontier check below. When the local wrapper IS
|
||||
// the finalized tip, the walk has arrived at the frontier and the
|
||||
// path is complete; falling through would reach `curHeight <=
|
||||
// led.height` with cur == led.tip and refuse a cert that extends
|
||||
// the tip exactly.
|
||||
if cur == led.tip {
|
||||
break
|
||||
}
|
||||
parent, curHeight, curCanonical, curParentCanon, ok = dag.Parent(cur)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user