Files
node/chains/bootstrap_sync.go
zeekay 7cf87b230b node-unified is the node
The module already called itself github.com/luxfi/node while containing no node
in Go: cmd/luxd was twenty-one lines that printed the size of a few types, and
the Go column of the matrix measured a VM plugin. A repo that fetches the thing
it claims to be has not replaced it, so the Go sources move in — 1515 files,
the daemon and everything under it — and the build reaches outside this module
for nothing.

Rust and C++ sit beside it unchanged and are still measured against it by the
differential. That is the point of one repo: the reference implementation and
the two ports are the same checkout, so parity is a thing you can run rather
than a thing you assert.

Verified: builds, and the closure carries no lux-private — checked at build time
rather than believed. The daemon starts twelve chains on a local network
(P X C Q Z A B G K M F, plus D through its plugin) and reports healthy.
2026-09-07 16:15:25 -07:00

1733 lines
88 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Copyright (C) 2019-2026, Lux Industries, Inc. All rights reserved.
// See the file LICENSE for licensing terms.
// bootstrap_sync.go — the node-side wiring for INITIAL SYNC. The blockHandler is both
// the fetch transport (chainbootstrap.BlockSource) and the execute sink
// (chainbootstrap.Chain) the engine/chain/bootstrap loop drives, so an EMPTY or BEHIND
// node converges from its local last-accepted to the network frontier by fetch+execute.
// On a quorum chain, every fetched block must carry the same signed certificate used
// by live catch-up; only the sole-validator topology uses the legacy cert-less path.
// When the loop reaches the frontier the driver ends the engine's bootstrap phase and
// flips bootstrapDone, the REAL ready
// signal monitorBootstrap gates the chain on. Decomplected from the live cert/vote
// path: when bsActive is false every method below is inert and the handler behaves
// exactly as it did before initial sync existed.
package chains
import (
"context"
"encoding/binary"
"errors"
"time"
cblock "github.com/luxfi/consensus/engine/chain/block"
chainbootstrap "github.com/luxfi/consensus/engine/chain/bootstrap"
"github.com/luxfi/ids"
"github.com/luxfi/log"
"github.com/luxfi/math/set"
"github.com/luxfi/node/proto/p2p"
)
// The blockHandler IS the bootstrap loop's fetch transport AND execute sink. If it
// ever stops satisfying either, the chain cannot initial-sync — catch it at compile
// time, not in production (the same discipline as the ancestorRequester assertion). It is
// also the BootstrapPolicy's AncestrySource (the content-addressed ancestry the responder-
// agreement tally walks), keeping the trust DECISION (bootstrap_trust.go) separate from the
// transport that feeds it.
var (
_ chainbootstrap.BlockSource = (*blockHandler)(nil)
_ chainbootstrap.Chain = (*blockHandler)(nil)
_ AncestrySource = (*blockHandler)(nil)
_ AcceptanceSource = (*blockHandler)(nil)
)
// Keep the certificate attached to its block between the wire decoder and the
// node's bootstrap adapter. The consensus bootstrap interface is intentionally
// kept source-compatible; Ancestors caches the proof by content-addressed block ID
// before returning bare block bytes to that interface.
type bootstrapFetchedBlock struct {
Bytes []byte
Certificate []byte
}
type catchupCertificateVerifier interface {
VerifyCatchupCertificate(context.Context, []byte, []byte) error
}
var errBootstrapCertificateRequired = errors.New("chains: signed quorum certificate required for multi-validator bootstrap block")
const (
// bootstrapFrontierWindow bounds how long FrontierTip collects weighted beacon
// replies before tallying the ⅔-by-stake quorum. A beacon answering later just
// misses this round (the driver re-samples) — it is not abandoned.
bootstrapFrontierWindow = 3 * time.Second
// bootstrapAncestorsTimeout bounds how long Ancestors waits for a peer to serve ONE
// batch. It is a PER-REQUEST bound and must stay well below the per-attempt budget
// (bootstrapNamingTimeout), because the ancestor-tolerant descent issues MANY sequential
// requests inside one attempt: a child request that can consume the whole parent budget
// starves every later chunk.
//
// A per-request bound above its parent budget is worse than no chunking: the attempt
// spends itself inside the FIRST chunk and never reaches a second, and a chunked walk
// that never completes a walk reports FrontierNoQuorum. Nothing hangs — the ctx.Done()
// case in the fetch loop keeps a request inside its attempt — but the descent gets
// nowhere.
//
// Kept in step with the GetAncestors WIRE deadline below: asking a peer for more time than
// we will wait just wastes its work.
bootstrapAncestorsTimeout = 3 * time.Second
// bootstrapAcceptedCandidates bounds the SECOND naming question in both directions: an asker
// names at most this many candidate tips, and a responder refuses a larger question rather
// than walking a list of ids someone else chose. Honest beacons report one tip each and
// cluster on a handful, so the bound is never why a frontier is not named; it is what stops
// one message from turning into an unbounded number of block lookups on the answering node.
bootstrapAcceptedCandidates = 64
// bootstrapAncestorSample is how many beacons one Ancestors round asks. The sample
// ROTATES (bsRotor) so no single beacon monopolizes the descent or can repeatedly
// stall it (M1). Content addressing in the loop makes the fetched ancestry safe
// regardless of WHICH beacon serves it.
bootstrapAncestorSample = 4
// bootstrapMinAgreeingBeacons floors the DISTINCT beacons that must name the same
// tip, so a single (even >⅔-stake) validator cannot alone determine the frontier.
// Capped at the beacon-set size for tiny networks.
bootstrapMinAgreeingBeacons = 2
// bootstrapNamingWindow is the PER-FETCH ancestry chunk of the ANCESTOR-TOLERANT frontier
// tally. Ancestry returns FULL BLOCKS, so one fetch must stay inside a network message —
// this is a TRANSPORT bound, matching the consensus descent's per-round fetch size.
//
// It was previously ALSO doing two other jobs: the total ancestry budget, and a HEALTH
// HEURISTIC ("a ⅔-common height more than this far below the highest tip is not a healthy
// bleeding-edge split"). That heuristic is true for a LIVE chain and FALSE for a HALTED
// one, where the skew between a straggler and a node that ran on alone is arbitrarily large
// and perfectly healthy — no fork, just a stopped chain. Conflating the three WEDGES exactly
// the split a halt produces: responders divide between a high tip and a low one that IS an
// ancestor of it, the ⅔-of-responder floor would name the low block, but when the gap exceeds
// one fetch the high tip never reaches down far enough to vouch for it and the tally names
// nothing on every retry, forever. Recovery from a halt is exactly when this path matters
// most, and it was disabled precisely then.
// The total budget is now maxNamingDepth; the health heuristic is gone.
bootstrapNamingWindow = 256
// maxNamingAnchors bounds how many DISTINCT reported tips the ancestor-tolerant tally will
// fetch ancestry for in one round. Honest beacons cluster on a handful of adjacent tips, so
// this is never reached in practice; it caps the work a Byzantine swarm reporting many
// distinct forged tips could induce. Anchors are tried most-stake-first (honest tips hold ⅔
// of the stake, so they always fall within the cap) and a tip already seen on a
// previously-fetched chain is skipped, so the common split costs ONE or TWO fetches.
maxNamingAnchors = 8
// bootstrapNamingTimeout TOTAL-bounds the ancestor-tolerant resolution (all anchor fetches
// combined) so a genuine eclipse/partition — beacons that answer the frontier query but do
// not serve ancestry — cannot make FrontierTip hang. When it elapses the resolution returns
// what it found (or nothing → FrontierNoQuorum), and the loop's bounded retry (F2) tries
// again next round against a fresh, rotated beacon sample. Honest beacons that JUST answered
// the frontier serve the small skew ancestry near-instantly, well within this window.
//
// PER-ATTEMPT, and it must exceed bootstrapAncestorsTimeout by enough to fit the SEQUENTIAL
// chunk fetches a deep descent needs — at 3s per request this is 10 chunks, i.e. 2560 blocks
// of skew resolved in one attempt, with the bounded retry covering more. It was itself 3s,
// equal to a single request, so the chunked descent could never advance past its first
// fetch. The invariant is simply: a parent operation must be able to outlive its children.
bootstrapNamingTimeout = 30 * time.Second
)
// beaconWeights returns the beacon set's nodeID→stake map and total stake under this
// chain's validation network, or ok=false when no beacon set is configured (single-node
// / --skip-bootstrap). This is the TRUST ANCHOR for initial sync: only a node in this
// set, weighted by its stake, can contribute to naming the frontier — the C1
// forged-chain gate. An arbitrary connected peer is not in this map and is ignored.
func (b *blockHandler) beaconWeights() (weights map[ids.NodeID]uint64, total uint64, ok bool) {
if b.beacons == nil {
return nil, 0, false
}
vmap := b.beacons.GetMap(b.networkID)
if len(vmap) == 0 {
return nil, 0, false
}
weights = make(map[ids.NodeID]uint64, len(vmap))
for id, v := range vmap {
w := v.Weight
if w == 0 {
w = v.Light // Weight is an alias for Light
}
weights[id] = w
total += w
}
if total == 0 {
return nil, 0, false
}
return weights, total, true
}
// connectedBeacons returns the beacons from `weights` currently CONNECTED and tracking
// this chain's validation NETWORK. The frontier quorum is tallied over these, but the FULL
// beacon stake (the `total` from beaconWeights) stays the quorum DENOMINATOR — so a node
// must hear a ⅔-stake supermajority. An eclipse that reaches < ⅔ of beacon stake cannot
// name a frontier. Returns nil when none are connected (or there is no transport).
//
// The tracking filter matches against b.networkID (the NET the beacon set is anchored to —
// constants.PrimaryNetworkID for native chains, the L1 net ID for sovereign chains), NOT
// b.chainID. The filter must match what a peer actually advertises: a peer names the NETS it
// tracks in its handshake (network/peer/peer.go adds constants.PrimaryNetworkID plus every
// tracked L1 net ID to peer.Info.TrackedChains) — it NEVER advertises an individual native chain ID like
// the C-Chain's. Filtering on b.chainID therefore matched ZERO real peers, so connectedStake
// stayed 0, FrontierTip reported FrontierConnecting forever, and a healthy stale node failed
// safe at the connect deadline instead of converging. Matching on b.networkID counts exactly
// the beacons participating in this chain's validation network. C1 is untouched: a peer is
// still admitted ONLY if it is in `weights` (the staked validator set) — a non-staked peer
// that tracks the network is still ignored.
func (b *blockHandler) connectedBeacons(weights map[ids.NodeID]uint64) []ids.NodeID {
if b.net == nil || len(weights) == 0 {
return nil
}
beaconIDs := make([]ids.NodeID, 0, len(weights))
for id := range weights {
beaconIDs = append(beaconIDs, id)
}
var connected []ids.NodeID
var beaconPeers []ids.NodeID
for _, p := range b.net.PeerInfo(beaconIDs) {
if _, isBeacon := weights[p.ID]; !isBeacon {
continue
}
// Connected AND staked — this peer counts unless the tracked-nets filter
// below can positively exclude it.
beaconPeers = append(beaconPeers, p.ID)
if p.TrackedChains.Contains(b.networkID) {
connected = append(connected, p.ID)
}
}
// THE FILTER IS AN OPTIMISATION, NOT A SAFETY PROPERTY, so it must never be the
// thing that decides a node cannot bootstrap. There is no other such decision: the
// startup gate accumulates ConnectedWeight from the network's own Connected()
// callbacks and LATCHES (`shouldStart = shouldStart || weight >= threshold`), so
// once enough stake has connected the node starts and never un-starts. Ours
// re-derives the answer from PeerInfo on every poll, which means a predicate that
// matches nothing produces zero connected stake forever, however many peers are
// genuinely there.
//
// That is not hypothetical: the b.chainID case above (see the note there) is exactly
// it. A validator sits frozen at its own height reporting a healthy peer count while
// FrontierTip answers FrontierConnecting on every pass, because the beacon set is
// populated and the filter still admits nobody.
//
// So: if the filter admits NO ONE while staked beacons are demonstrably connected,
// trust the connection. C1 is preserved either way — a peer is admitted only if it
// is in `weights`, the staked validator set, so a non-staked peer never counts.
// The filter ORDERS; it does not EXCLUDE. Rescuing only the all-or-nothing case
// leaves the partial one, which is worse because it looks like it worked: some
// beacons pass, `connected` is non-empty, the rescue never fires, and a peer that
// holds exactly the ancestry we are missing is dropped for the life of the
// process. A node connected to a peer at the tip then samples only peers behind it
// and reports that no peer served the missing ancestry, without ever once asking
// the one that could.
//
// So: peers that advertise the chain come first, and the remaining staked beacons
// follow. Asking a peer that turns out not to serve costs one empty reply; not
// asking the only peer that can costs the bootstrap.
//
// C1 is unchanged — every id here came from `weights`, the staked validator set,
// so a non-staked peer is still never admitted.
if len(connected) < len(beaconPeers) {
if len(connected) == 0 {
b.logger.Warn("tracked-nets filter admitted no beacon; asking the connected staked beacons instead",
log.Stringer("networkID", b.networkID),
log.Int("connectedStakedBeacons", len(beaconPeers)))
}
seen := set.NewSet[ids.NodeID](len(beaconPeers))
seen.Add(connected...)
for _, id := range beaconPeers {
if !seen.Contains(id) {
connected = append(connected, id)
}
}
}
return connected
}
// sampleAncestorBeacons returns a small ROTATED sample of connected beacons to fetch
// ancestry from (M1 — bsRotor advances each call so the sample walks the beacon set and
// no single beacon monopolizes the descent). Safety is independent of WHICH beacon
// serves: the loop's content-addressed descent rejects any block off the agreed
// frontier's parent chain, so a withholding/forging beacon costs only a re-sample.
func (b *blockHandler) sampleAncestorBeacons() (set.Set[ids.NodeID], bool) {
weights, _, ok := b.beaconWeights()
if !ok {
return nil, false
}
connected := b.connectedBeacons(weights)
if len(connected) == 0 {
return nil, false
}
// Prefer beacons that reported an ahead tip in the last frontier round — they HOLD the
// ancestry the descent needs, so asking them (rather than a rotated peer that may be at
// genesis) is what lets a re-bootstrapping node obtain ancestry even when ≥2 peers are still
// at genesis. This is ava's PeerTracker "ask a prover" bias sourced from the frontier replies
// we already have (no separate tracker). Fill any remaining slots from the rotated full set so
// the sample never shrinks below what blind rotation would pick (defense against a stale/empty
// ahead-set). Empty ahead-set ⇒ pure rotation, identical to prior behavior.
b.bsMu.Lock()
ahead := b.bsAheadBeacons
b.bsMu.Unlock()
sample := set.NewSet[ids.NodeID](bootstrapAncestorSample)
for _, id := range connected {
if sample.Len() >= bootstrapAncestorSample {
break
}
if ahead != nil && ahead.Contains(id) {
sample.Add(id)
}
}
start := int(b.bsRotor.Add(1)-1) % len(connected)
for i := 0; i < len(connected) && sample.Len() < bootstrapAncestorSample; i++ {
sample.Add(connected[(start+i)%len(connected)])
}
return sample, sample.Len() > 0
}
// fullyConnectedBeacons reports whether the node has reached its ENTIRE EXTERNAL beacon set — every
// beacon OTHER than itself is connected and tracking this chain's network. This is the eclipse-free
// condition that makes the FrontierTip SELF-VOTE safe: when the full set is reached no beacon can be
// suppressed, so a unanimous "nobody ahead" from the full set PLUS the node itself is a TRUE caught-up
// — an eclipse hiding any ahead-producer would drop that beacon from `connected` and break full-set,
// falling back to the partition-capture floor (fail safe). `connected` is connectedBeacons(weights);
// the external-set size is the beacon set minus the node itself (a node cannot, and need not, connect
// to itself). Returns false for an empty external set (a degenerate single-beacon / self-only set, so
// the self-vote is never the SOLE basis for caught-up).
func (b *blockHandler) fullyConnectedBeacons(weights map[ids.NodeID]uint64, connected []ids.NodeID) bool {
external := len(weights)
if _, selfIsBeacon := weights[b.selfNodeID]; selfIsBeacon {
external-- // the node is in its own beacon set but is never one of its own peers
}
return external > 0 && len(connected) >= external
}
// hasExternalBeacons reports whether the staked beacon set contains any validator OTHER than this
// node. False for a self-only (single-validator) set — there is then no peer to sync from, so the
// node's own tip IS the frontier (FrontierTip returns FrontierNoBeacons). The set is read from
// P-chain STATE, not connectivity, so this is a TOPOLOGY fact (a genuine single-validator net),
// never an eclipse artifact (an eclipse drops peers from `connected`, never from `weights`).
func (b *blockHandler) hasExternalBeacons(weights map[ids.NodeID]uint64) bool {
external := len(weights)
if _, selfIsBeacon := weights[b.selfNodeID]; selfIsBeacon {
external--
}
return external > 0
}
// withSelfVote returns `replies` plus the node's OWN accepted frontier as a beacon reply — the
// SELF-VOTE. The node is itself a beacon (selfNodeID in `weights`, the trust anchor) and knows its
// own accepted tip (lastID) with certainty, so it vouches for it exactly as a connected peer's reply
// would (deduped by NodeID in the policy's tally; collectFrontierReplies samples only PEERS, never
// self, so there is no double-count). Returned UNCHANGED when the node is not in the beacon set
// (degenerate / single-node / a P-chain whose CustomBeacons omit self) or has no accepted tip — the
// self-vote is then inert. self only ever vouches for the block IT HAS ACCEPTED, so it can never name
// a forged tip (C1 untouched); the SOLE caller gates its use on FULL connectivity, so it can never
// tip a partial (eclipsed) view into caught-up.
func (b *blockHandler) withSelfVote(replies []BeaconReply, weights map[ids.NodeID]uint64, lastID ids.ID) []BeaconReply {
w, selfIsBeacon := weights[b.selfNodeID]
if !selfIsBeacon || lastID == ids.Empty {
return replies
}
out := make([]BeaconReply, 0, len(replies)+1)
out = append(out, replies...)
out = append(out, BeaconReply{NodeID: b.selfNodeID, Tip: lastID, Weight: w})
return out
}
// repliedCovers reports whether EVERY connected beacon answered THIS frontier
// round — set containment {reply.NodeID} ⊇ connected, NOT a count (frontier
// replies are not yet round/connected-filtered, so a stale/non-connected reply
// could pad a length check). The self-vote caught-up shortcut requires this:
// without it the gate keys on CONNECTIVITY (fullyConnectedBeacons) while CaughtUp
// tallies REPLIES, and the two diverge for a beacon that is CONNECTED but SILENT
// — TCP up, its frontier reply delayed or withheld. That is a capability strictly
// WEAKER than a full eclipse (no TCP drop needed) and it occurs NATURALLY when an
// ahead beacon replays state on a mass co-restart. Such a beacon counts as "fully
// connected" yet is absent from the tally, so self's (heavy) weight backfills the
// floor and the node goes live at a forged STALE height — the precise failure the
// frontier-quorum gate exists to prevent. Requiring every connected
// beacon to have replied means a silent ahead-beacon blocks caught-up → the node
// waits / fails safe, while a genuine fresh net (every connected beacon answers
// genesis) still completes.
func repliedCovers(replies []BeaconReply, connected []ids.NodeID) bool {
replied := make(map[ids.NodeID]struct{}, len(replies))
for _, r := range replies {
replied[r.NodeID] = struct{}{}
}
for _, id := range connected {
if _, ok := replied[id]; !ok {
return false
}
}
return true
}
// FrontierTip implements chainbootstrap.BlockSource. It returns a FrontierStatus that
// decomplects the THREE reasons a tip may not be named. A single ok=false meant both "no
// beacon quorum reachable yet" and "nothing ahead — done", so a freshly-booted STALE node
// asking the frontier BEFORE any beacon had connected concluded "caught up" at its local
// stale height:
//
// - FrontierNoBeacons — no beacon set configured (single-node / dev). Nothing to sync to.
// - FrontierConnecting — beacons configured but not enough stake CONNECTED yet to even
// FORM a ⅔ quorum (the boot race). The node cannot ASK — the loop must WAIT, never
// conclude caught-up here.
// - FrontierNoQuorum — enough beacon stake IS connected to ask, but no ⅔-by-stake
// agreement is reachable (genuine eclipse / partition). The loop fails SAFE.
// - FrontierNamed — a ⅔-by-stake SUPERMAJORITY of the configured beacons agreed on
// the SAME accepted tip (the C1 forged-chain gate, unchanged). The tip is meaningful
// ONLY in this case.
//
// A non-beacon peer, a single peer, or any sub-⅔-stake set can NEVER reach FrontierNamed,
// so a forged chain can never be the thing an empty/behind node syncs to.
func (b *blockHandler) FrontierTip(ctx context.Context) (ids.ID, chainbootstrap.FrontierStatus) {
// A measurement from the previous round says nothing about this one, and a round that
// never reaches the floor check (no beacon set yet, no transport, P-chain still loading)
// measures nothing at all. Dropping it here is what keeps runInitialSync from reporting a
// count that some earlier round took.
b.shortfall.Store(nil)
// Every wait says what it is waiting for. The status this returns carries a
// state and no numbers, and the wait an operator reads is assembled from
// whatever was recorded here — so a round that returns without recording
// leaves the ambiguous form, which reads the same whether the whole beacon
// set answered or none of it did.
waiting := func(why string) { b.shortfall.Store(&why) }
weights, _, haveBeacons := b.beaconWeights()
if !haveBeacons {
if b.expectsStakedBeacons {
// This chain syncs against the STAKED primary-network validator set, which the
// already-bootstrapped P-chain populates — an empty set here means it is not yet
// loaded (sequencing) or misconfigured, NOT a single-node network. WAIT (bounded by
// the loop's ConnectDeadline → fail safe). Critically we do NOT conclude caught-up
// at the local stale height, and we do NOT fall back to unweighted / endpoint trust:
// a connected peer can name the frontier ONLY through the ⅔-by-stake quorum below,
// which is empty here. The sequencing case resolves (initValidatorSets populates the
// set) and the node converges; a genuine empty/misconfigured set fails safe.
waiting("the validator set is empty; no beacon can be asked yet")
return ids.Empty, chainbootstrap.FrontierConnecting
}
// No beacon set expected (single-node / dev / --skip-bootstrap / P-chain endpoint-only
// bootstrappers). Nothing to sync to.
return ids.Empty, chainbootstrap.FrontierNoBeacons
}
if b.net == nil || b.msgCreator == nil {
// Beacons configured but no transport to ask them — treat as still connecting
// (bounded by the loop's connect deadline). In practice runBootstrapThenPoll
// short-circuits a transport-less handler before Run is ever called.
waiting("beacons are configured but there is no transport to ask them over")
return ids.Empty, chainbootstrap.FrontierConnecting
}
// THE PARTIAL-SET GATE. A native chain's beacon set is the STAKED primary-network
// validator set the P-chain populates as it replays its blocks (genesis stakers + every later
// AddValidatorTx). This bootstrap goroutine starts right after the chain's VM.Initialize — NOT
// after the P-chain has finished syncing — so without this gate FrontierTip can run while that
// set is PARTIAL. A partial set is a NON-REPRESENTATIVE stake denominator: the MinResponseWeight
// stake-majority floor (total/2+1) is fail-secure only when `total` is the TRUE full-set stake,
// and under a partial denominator a degenerate set of at-or-below responders (the genuinely-ahead
// producers not yet loaded / not yet connected) clears the floor and the decision below
// false-completes the node at its stale local height, leaving a validator serving and voting
// behind the network with nothing left converging it. WAIT (→ the loop's ConnectDeadline →
// fail safe) until the staked set is fully loaded, so every branch below
// judges the TRUE full validator set. Deadlock-free: the P-chain converges independently from its
// own configured CustomBeacons; a native chain only waits the bounded extra moment for it.
if b.expectsStakedBeacons && b.primaryNetworkReady != nil && !b.primaryNetworkReady() {
b.logger.Debug("bootstrap frontier: staked validator set not yet fully loaded (P-chain still syncing) — waiting, NOT concluding caught-up",
log.Int("partialBeaconCount", len(weights)))
return ids.Empty, chainbootstrap.FrontierConnecting
}
// SELF-ONLY staked set: the configured validator set is exactly THIS node — a genuine
// single-validator network (a sybil-protected devnet, or the sole validator of an L1). There
// is no OTHER beacon to sync from, so the node's own accepted tip IS the network frontier →
// immediate start (FrontierNoBeacons). This is NOT an eclipse: the staked set is read from
// P-chain STATE, not connectivity, so a hidden peer would still appear in `weights`; only a
// genuine single-validator topology reaches here. Placed AFTER the P-ready gate so a boot-race
// partial set (transiently self-only mid-P-replay) WAITS above rather than false-completing here.
// This is the sybil-protected single-validator counterpart to the empty-set FrontierNoBeacons
// path: a multi-validator staked set (the ≥2 case) still runs the ⅔-by-stake quorum below.
if !b.hasExternalBeacons(weights) {
b.logger.Debug("bootstrap frontier: self-only staked set (single validator) — NoBeacons, nothing to sync to")
return ids.Empty, chainbootstrap.FrontierNoBeacons
}
// THE MASS-RECOVERY FIX. The acceptance decision is the BootstrapPolicy (a SEPARATE object
// with a SEPARATE threat model — bootstrap_trust.go), NOT the ⅔-of-CURRENT-total-stake
// consensus floor. The prior code required a ⅔-by-stake quorum of the WHOLE validator set to
// be CONNECTED before naming a frontier; when the recovery targets are themselves validators
// (a crashed node IS one of the 5), that floor is mathematically unsatisfiable during a mass
// outage — the deadlock. The policy instead requires MinResponses authenticated CONFIGURED
// beacons to respond and a ⅔-of-RESPONDERS agreement, so 3 of 5 reachable beacons all agreeing
// recover the network even though 3 of 5 STAKE is not a finalizing supermajority. Finality is
// untouched (it lives in consensus and still needs > ⅔ of current stake).
connected := b.connectedBeacons(weights)
policy := b.bootstrapPolicy(weights)
replies := b.collectFrontierReplies(ctx, connected, weights)
// INSTRUMENTATION (ground truth for the decision below). Per attempt, record the FULL
// beacon-set size + total stake (the floor DENOMINATOR — a partial value here is the smoking
// gun), the connected count, and every reply's tip + LOCALLY-resolved height + held-or-not.
// Debug level so production is silent; raise the chain's log level to Debug to capture it.
b.logFrontierInputs(weights, connected, replies)
lastID, lastH, lastErr := b.LastAccepted(ctx)
haveLast := lastErr == nil && lastID != ids.Empty
// Full-coverage judgement for the own-tip frontier exemption: every configured beacon connected
// AND replied this round (the same pair of guards the self-vote shortcut uses), so no ahead
// beacon can be hidden. Reusing both is deliberate — connectivity alone admits a CONNECTED but
// SILENT beacon, which is a strictly weaker capability than an eclipse.
if haveLast {
policy.Tip = lastID
policy.Covered = b.fullyConnectedBeacons(weights, connected) && repliedCovers(replies, connected)
}
// A signed quorum certificate outranks the unsigned responder tally, because a
// tally counts peers and a certificate carries proof. A majority naming an
// older tip does not make it the tip: a minority holding newer finalized
// blocks can prove it, and those blocks carry portable certs that say so.
//
// Probe every distinct ahead claim and choose the highest one whose certificate
// independently verifies against our validator set. An arbitrary peer can now
// advertise any ID it likes, but it cannot make that ID a frontier without the
// same signed proof required by live finality. Invalid/unserved claims are ignored
// and fall through to the bounded bootstrap policy below.
if b.signedVotesRequired {
localHeight := uint64(0)
if haveLast {
localHeight = lastH
}
if certifiedID, certifiedHeight, ok := b.highestCertifiedFrontier(ctx, replies, localHeight); ok {
b.logger.Info("bootstrap frontier: selected cryptographically certified tip over unsigned responder tally",
log.Stringer("tip", certifiedID),
log.Uint64("height", certifiedHeight),
log.Uint64("localHeight", localHeight))
b.namingProgress = nil
return certifiedID, chainbootstrap.FrontierNamed
}
}
frontier, err := policy.AcceptsFrontier(ctx, replies)
switch {
case err == nil:
// A configured-beacon quorum named a safe sync anchor (NOT a finality cert — the loop
// re-executes the descent and re-enters consensus, where FinalityQuorum alone governs).
// With the P-ready gate above, this judgement is over the TRUE FULL staked set, so a tip the
// quorum actively names is a real frontier (when it equals our own held tip, a ⅔-of-responders
// supermajority is AT our height, so we ARE at the network frontier — the loop's Accepted()-shortcut
// then correctly concludes caught-up). A Byzantine minority reporting an unheld forged sibling
// cannot reach the ⅔ agreement, so it is never named (C1) and — crucially — never BLOCKS the
// honest named tip either (we do NOT require holding every reported tip to accept the named
// one; that stricter rule belongs to the dual CaughtUp path below, which decides the
// no-quorum/own-height case).
b.logger.Debug("bootstrap frontier: NAMED",
log.Stringer("tip", frontier.ID),
log.Uint64("namedHeight", frontier.Height),
log.Int("responders", frontier.Responders),
// ACCEPTED, not merely held: a gossiped-but-unaccepted frontier logs
// accepted=false here, so the loop's Accepted()-shortcut DESCENDS instead of completing
// at the stale tip. The diagnostic now distinguishes "in the store" from "finalized".
log.Bool("accepted", b.Accepted(ctx, frontier.ID)))
// The descent is over, so its retained refs are released.
b.namingProgress = nil
return frontier.ID, chainbootstrap.FrontierNamed
case errors.Is(err, ErrInsufficientBootstrapResponses):
// Below the PEER-ONLY response floor. Before WAITING, apply the SELF-VOTE under FULL
// connectivity — THE FRESH-NET FIX. The node is itself a beacon and knows its OWN accepted
// tip with certainty. When it has reached its ENTIRE beacon set (fullyConnectedBeacons — so no
// eclipse can be hiding an ahead-tip) and that full set PLUS itself unanimously hold a tip the
// node has ALREADY ACCEPTED, the node IS at the network frontier — even though the peer-only
// responder weight could not reach the stake-majority NAMING floor. That floor is unsatisfiable
// by PEERS ALONE precisely when the node's own stake is large relative to the rest (a heavy
// validator, a small beacon set like the P-chain's CustomBeacons, or skewed stake): peers =
// total − self, so peers < total/2+1 ⟺ self > total/2−1. On a fresh net every validator holds
// only genesis, so such a node would otherwise hang in FrontierConnecting forever with the
// WHOLE set connected. SAFE: self is counted ONLY under full connectivity, so an eclipse that
// suppresses ANY beacon breaks full-set and we fall through to the partition-capture floor
// (FrontierConnecting → fail safe) — self can never tip a PARTIAL (eclipsed) view into a false
// caught-up. self also only ever vouches for its OWN accepted tip, so C1 (forged-frontier
// naming) is untouched, and a genuinely behind node has an ahead peer in the full set → CaughtUp
// is false → it keeps waiting/syncing.
// EXECUTED, not merely DECIDED. The acceptance oracle answers from the consensus
// ledger, which is the right source for "has this block been accepted" and the
// wrong one for "may this node go live": the ledger advances when a quorum
// decides, and the VM advances when this node runs the block. They diverge, and
// on the divergence the ledger is HIGHER — so a node whose VM is behind reads as
// caught up, goes live at its stale height, and then serves and votes there
// while never converging. That is the failure this whole gate exists to make
// impossible, arriving through the one input nobody checked.
//
// A validator can therefore report itself bootstrapped with its ledger at the
// network's height and its VM hundreds of blocks below it — live, polling
// normally, and behind the whole time.
//
// So going live additionally requires the VM to have EXECUTED to the head we are
// claiming. This can only ever REFUSE, never permit: a node that has genuinely
// executed to its head is unaffected, and a node that has not stays syncing,
// which is the safe side of a go-live decision. Both caught-up branches consult
// the same rule — there is one way to be allowed live.
if haveLast && b.executedTo(ctx, lastH) && b.fullyConnectedBeacons(weights, connected) &&
repliedCovers(replies, connected) &&
policy.CaughtUp(b.withSelfVote(replies, weights, lastID), lastH, b.acceptedHeight) {
b.logger.Debug("bootstrap frontier: CAUGHT UP at own tip (full beacon set + self all at/below it, peer-only weight below the naming floor)",
log.Stringer("tip", lastID),
log.Uint64("height", lastH),
log.Int("beaconSetSize", len(weights)),
log.Int("connected", len(connected)))
return lastID, chainbootstrap.FrontierCaughtUp
}
// Fewer than MinResponses configured beacons have RESPONDED (and not provably caught-up under
// full connectivity) — not a partition-capture-safe quorum yet. More may connect: WAIT (bounded
// by the loop's ConnectDeadline, then fail safe). Never false-complete; never trust the few.
//
// This is the only place the shortfall is known: the status this returns carries a state and
// no numbers, and the loop above it fails safe on a bare sentinel. So record it for
// runInitialSync, which is where the wait an operator asks about is declared.
responders, responderWeight, _, _ := policy.tallyResponders(replies)
short := policy.shortfall(responders, responderWeight)
waiting(short)
b.logger.Debug("bootstrap frontier: CONNECTING (below the MinResponses floor)",
log.String("shortfall", short),
log.Int("beaconSetSize", len(weights)),
log.Int("connected", len(connected)),
log.Int("replies", len(replies)))
return ids.Empty, chainbootstrap.FrontierConnecting
default:
// ErrNoBootstrapQuorum: enough beacons responded (the FLOOR is met) but no ⅔-of-responders
// agreement named a common committed block this round. Before failing safe, check whether we
// are CAUGHT UP. A TIP-HOLDER on a mixed-height co-restart sees the responders SPLIT below ⅔
// (the tip-holders are only half), so NOTHING is named, yet it is plainly not behind — and
// the ancestor-tolerant tally cannot name the ⅔-backed COMMON ancestor either, because that
// ancestor is AT OR BELOW the node's own last-accepted (MinFrontierHeight names only STRICTLY
// ABOVE it). That own-height exclusion is also the M1 eclipse-stale fix: a block at the node's
// height that accrues ⅔ purely as the shared ANCESTOR of ahead-tips an eclipse throttled below
// the naming threshold is NOT named — so this same CaughtUp gate, not nameFrontier, decides the
// at-own-height case, and it REFUSES when any responder is genuinely ahead (the node lacks that
// tip). Without this the tip-holder fails safe DOWN at its own tip — the opposite of the
// stale-go-live bug.
//
// policy.CaughtUp is true ONLY when the floor is met AND no responder reports an accepted tip
// above our height AND we hold every reported tip (its full safety argument lives on the
// method). When it holds, the network frontier IS our own accepted tip: return it NAMED so
// the loop's existing Accepted()-shortcut (bootstrapper "named a tip we have ACCEPTED → synced")
// concludes caught-up and we go Ready at our OWN height — never below it, never at a stale
// height. A genuinely stale node has an honest responder ahead (or lacks its tip) → CaughtUp
// is false → it keeps syncing; an eclipse hiding the ahead-nodes drops below the floor →
// CaughtUp is false → it fails safe. Distinct from the transient finality-skew retry the loop
// runs on a bare FrontierNoQuorum ("connected but momentarily split" — keep polling): this is
// "split AND I am at the top of every responder" — provably DONE, not waiting.
if haveLast && b.executedTo(ctx, lastH) && policy.CaughtUp(replies, lastH, b.acceptedHeight) {
b.logger.Debug("bootstrap frontier: CAUGHT UP at own tip (floor met, no responder ahead, hold every reported tip)",
log.Stringer("tip", lastID),
log.Uint64("height", lastH))
return lastID, chainbootstrap.FrontierNamed
}
// WHY nothing was named decides whether retrying can ever succeed, so the two
// reasons are logged as the different facts they are. ErrNoBootstrapQuorum is a
// statement about the network — the responders agree on nothing we may name —
// and retrying polls a fresh sample. ErrNamingIncomplete is a statement about
// our own budget: the descent ran out mid-walk, its cursors are saved, and the
// next round resumes DEEPER. Reported as no-quorum, a wide gap looks like a
// permanent disagreement and the descent restarts at the tip every round, so a
// gap wider than one attempt's budget is never crossed.
if errors.Is(err, ErrNamingIncomplete) {
deepest := 0
for _, prog := range b.namingProgress {
if prog.Traversed > deepest {
deepest = prog.Traversed
}
}
b.logger.Debug("bootstrap frontier: INCOMPLETE (attempt budget spent, not a disagreement) — resuming deeper next round",
log.Uint64("lastAccepted", lastH),
log.Int("anchorsInProgress", len(b.namingProgress)),
log.Int("deepestTraversed", deepest))
return ids.Empty, chainbootstrap.FrontierNoQuorum
}
b.logger.Debug("bootstrap frontier: NO QUORUM (responders split, not caught up) — retry/fail safe",
log.Uint64("lastAccepted", lastH),
log.Int("replies", len(replies)))
return ids.Empty, chainbootstrap.FrontierNoQuorum
}
}
// highestCertifiedFrontier returns the highest strict-ahead responder tip backed
// by a locally verified quorum certificate. It is read-only: certificate checking
// neither tracks nor finalizes the block, so ordered execution remains the
// bootstrapper's responsibility.
func (b *blockHandler) highestCertifiedFrontier(ctx context.Context, replies []BeaconReply, localHeight uint64) (ids.ID, uint64, bool) {
if b.engine == nil || b.vm == nil {
return ids.Empty, 0, false
}
verifier, ok := any(b.engine).(catchupCertificateVerifier)
if !ok {
return ids.Empty, 0, false
}
return selectHighestCertifiedFrontier(replies, localHeight, func(tip ids.ID) (uint64, bool) {
if b.Accepted(ctx, tip) {
return 0, false
}
fetched, err := b.fetchBootstrapAncestors(ctx, tip, 1)
if err != nil || len(fetched) == 0 {
return 0, false
}
candidate := fetched[len(fetched)-1]
blk, err := b.vm.ParseBlock(ctx, candidate.Bytes)
if err != nil || blk.ID() != tip {
return 0, false
}
if err := verifier.VerifyCatchupCertificate(ctx, candidate.Bytes, candidate.Certificate); err != nil {
b.logger.Debug("bootstrap frontier: responder tip has no valid quorum certificate",
log.Stringer("tip", tip),
log.Uint64("height", blk.Height()),
log.Err(err))
return 0, false
}
return blk.Height(), true
})
}
// selectHighestCertifiedFrontier is the deterministic policy core: responder
// multiplicity is irrelevant once a tip supplies its own quorum proof. probe must
// return true only after cryptographic verification.
func selectHighestCertifiedFrontier(replies []BeaconReply, localHeight uint64, probe func(ids.ID) (uint64, bool)) (ids.ID, uint64, bool) {
seen := make(map[ids.ID]struct{}, len(replies))
var bestID ids.ID
var bestHeight uint64
for _, reply := range replies {
if reply.Tip == ids.Empty {
continue
}
if _, duplicate := seen[reply.Tip]; duplicate {
continue
}
seen[reply.Tip] = struct{}{}
height, verified := probe(reply.Tip)
if !verified || height <= localHeight {
continue
}
if bestID == ids.Empty || height > bestHeight || (height == bestHeight && reply.Tip.Compare(bestID) < 0) {
bestID = reply.Tip
bestHeight = height
}
}
return bestID, bestHeight, bestID != ids.Empty
}
// logFrontierInputs records, at Debug level, the raw inputs to one FrontierTip decision — the
// ground truth for it. The FULL beacon-set size + total stake is the floor's
// DENOMINATOR: if it is a PARTIAL value (the staked set still loading), the stake-majority floor
// is non-representative and a degenerate at-or-below responder subset can false-complete the node
// — so logging the denominator alongside every reply's tip + locally-resolved height + held-or-not
// is exactly what distinguishes "caught up" from "tricked by a partial set". Silent in production
// (Debug); raise the chain's log level to Debug to capture it.
func (b *blockHandler) logFrontierInputs(weights map[ids.NodeID]uint64, connected []ids.NodeID, replies []BeaconReply) {
var total uint64
for _, w := range weights {
total += w
}
for _, r := range replies {
h, accepted := b.acceptedHeight(r.Tip)
b.logger.Debug("bootstrap frontier reply",
log.Stringer("from", r.NodeID),
log.Stringer("reportedTip", r.Tip),
log.Uint64("beaconWeight", r.Weight),
// ACCEPTED (on our finalized chain), not merely present in the store: a beacon tip we
// hold-but-have-not-accepted logs accepted=false with resolvedHeight 0 — the smoking gun
// that distinguishes a genuine catch-up from a stored-but-unaccepted tip.
log.Bool("accepted", accepted),
log.Uint64("resolvedHeight", h))
}
b.logger.Debug("bootstrap frontier inputs",
log.Int("beaconSetSize", len(weights)),
log.Uint64("beaconSetTotalStake", total),
log.Int("connectedBeacons", len(connected)),
log.Int("replies", len(replies)))
}
// bootstrapPolicy builds the BootstrapTrust DECISION object for this round from the configured
// beacon set (the trust anchor — INVARIANT 1: configured/checkpoint/genesis, never peer
// self-report). MinResponses defaults to a MAJORITY of the configured beacons (the largest floor
// that still lets the node recover when a minority of validators is down); the agreement is ⅔ of
// the RESPONDERS; MinFrontierHeight is the node's last-accepted height (the ancestor-tolerant path
// names only a block STRICTLY ABOVE it — never at or beneath where the node already is, so an
// eclipse cannot make the node's own height a false frontier; that case routes to CaughtUp — fail
// safe, not false-complete). All are operator-overridable via the blockHandler fields; zero ⇒ the
// documented default.
func (b *blockHandler) bootstrapPolicy(weights map[ids.NodeID]uint64) *BootstrapPolicy {
minResp := b.bootstrapMinResponses
if minResp <= 0 {
minResp = len(weights)/2 + 1 // MAJORITY of the configured beacon set
}
if minResp > len(weights) {
minResp = len(weights)
}
var lastH uint64
if _, h, err := b.LastAccepted(context.Background()); err == nil {
lastH = h
}
// STAKE-MAJORITY FLOOR (closes the skewed-weight partition-capture forgery). MinResponses is a
// COUNT of beacons; AgreementThreshold is over responder WEIGHT. Under SKEWED validator stake
// these diverge: an attacker who eclipses the HEAVY honest beacons but lets enough LIGHT honest
// beacons through to satisfy the count floor can shrink the responder-weight denominator until
// his < ⅓-of-total Byzantine stake clears the ⅔-OF-RESPONDERS agreement and names a forged
// frontier. Requiring the responders to also carry > ½ of TOTAL configured beacon stake bounds
// that denominator from below, so a < ⅓-stake adversary can never reach a responder-weight ⅔
// majority. Proven safe AND deadlock-free: equal-weight recovery needs only > ½ (e.g. 3/5 =
// 0.6·total ≥ 0.5), NOT the > ⅔ that caused the original mass-recovery deadlock. Disabled only
// when no weights are known (degenerate single-node / pre-P-chain).
var minRespWeight uint64
var total uint64
for _, w := range weights {
total += w
}
if total > 0 {
minRespWeight = total/2 + 1 // ⌈total/2⌉ stake-majority floor
}
return &BootstrapPolicy{
TrustedBeacons: weights,
AgreementThreshold: b.bootstrapAgreement, // zero ⇒ policy default ⅔
MinResponses: minResp,
MinResponseWeight: minRespWeight, // stake-majority floor (anti skewed-weight partition-capture)
MinResponders: bootstrapMinAgreeingBeacons,
MinFrontierHeight: lastH,
Checkpoint: b.bootstrapCheckpoint,
NamingWindow: bootstrapNamingWindow,
MaxAnchors: maxNamingAnchors,
Source: b,
Acceptance: b,
// The descent state is the HANDLER's and outlives this policy, so an attempt
// that runs out of budget resumes deeper next round instead of restarting at
// the tip. Budget fields default in namingBudget().
Progress: b.ancestryProgress(),
}
}
// collectFrontierReplies sends GetAcceptedFrontier to the connected beacons and gathers ONE
// reply per beacon within the window, returning them as []BeaconReply for the BootstrapPolicy to
// JUDGE. This is pure TRANSPORT — it does not decide. The DECISION (which beacons count, the
// MinResponses floor, the ⅔-of-responders agreement, the ancestor-tolerant tally) is the
// policy's (bootstrap_trust.go), keeping the trust object separate from the wire.
//
// It early-returns the instant every connected beacon has answered (no need to wait out the
// window on the common fully-connected path), bounding a slow/silent minority by the window.
// Non-beacon and empty replies are dropped here too (the policy re-filters — defense in depth).
func (b *blockHandler) collectFrontierReplies(ctx context.Context, connected []ids.NodeID, weights map[ids.NodeID]uint64) []BeaconReply {
if len(connected) == 0 || b.net == nil || b.msgCreator == nil {
return nil
}
ch := make(chan bsFrontierReply, len(connected))
b.bsMu.Lock()
b.bsFrontierCh = ch
b.bsMu.Unlock()
defer func() {
b.bsMu.Lock()
b.bsFrontierCh = nil
b.bsMu.Unlock()
}()
b.contextRequestMu.Lock()
b.requestIDCounter++
requestID := b.requestIDCounter
b.contextRequestMu.Unlock()
msg, err := b.msgCreator.GetAcceptedFrontier(b.chainID, requestID, 10*time.Second)
if err != nil {
return nil
}
sample := set.NewSet[ids.NodeID](len(connected))
sample.Add(connected...)
b.net.Send(msg, sample, b.networkID, 0)
replies := make([]BeaconReply, 0, len(connected))
seen := make(map[ids.NodeID]struct{}, len(connected))
// ahead = beacons reporting a tip this node has NOT accepted (they hold blocks we lack, so
// they can SERVE the ancestry the descent needs). sampleAncestorBeacons prefers them so the
// GetAncestors sample targets peers that can serve — never a peer still at genesis (our own
// tip). Recorded per round; publishes into b.bsAheadBeacons before returning.
ahead := set.NewSet[ids.NodeID](len(connected))
record := func() []BeaconReply {
b.bsMu.Lock()
b.bsAheadBeacons = ahead
b.bsMu.Unlock()
return replies
}
deadline := time.After(bootstrapFrontierWindow)
for {
select {
case rep := <-ch:
w, isBeacon := weights[rep.nodeID]
if !isBeacon || rep.tip == ids.Empty {
continue // only a configured beacon's non-empty reply counts
}
if _, dup := seen[rep.nodeID]; dup {
continue // one reply per beacon
}
seen[rep.nodeID] = struct{}{}
replies = append(replies, BeaconReply{NodeID: rep.nodeID, Tip: rep.tip, Weight: w})
if _, accepted := b.acceptedHeight(rep.tip); !accepted {
ahead.Add(rep.nodeID) // reported a tip we have not accepted → genuinely ahead
}
if len(seen) >= len(connected) {
return record() // every connected beacon answered — resolve now, do not wait the window
}
case <-deadline:
return record()
case <-ctx.Done():
return record()
}
}
}
// Acceptance implements AcceptanceSource: ask `from` which of `candidates` they have ACCEPTED and
// gather ONE answer per beacon within the reply window. Pure TRANSPORT — which answers count and
// which candidate is named is the policy's (bootstrap_trust.go), the same split collectFrontierReplies
// keeps for the first question. It reuses that question's window: it is the same round trip to the
// same peers, moments later.
//
// A candidate this node has ALREADY ACCEPTED is dropped before asking. That is what keeps the second
// question from ever moving the node sideways or backwards: every block it can name is one this node
// lacks, so a named block always makes the loop DESCEND and fetch it, and can never be read as
// "already synced" at a height the node is stuck on. A node at the top of every reported tip
// therefore asks NOTHING and sends no message at all.
func (b *blockHandler) Acceptance(ctx context.Context, candidates []ids.ID, from []ids.NodeID) []AcceptedReply {
if b.net == nil || b.msgCreator == nil || len(from) == 0 {
return nil
}
want := make([]ids.ID, 0, len(candidates))
for _, id := range candidates {
if _, accepted := b.acceptedHeightCtx(ctx, id); !accepted {
want = append(want, id)
}
}
if len(want) == 0 {
return nil
}
b.contextRequestMu.Lock()
b.requestIDCounter++
requestID := b.requestIDCounter
b.contextRequestMu.Unlock()
sample := set.NewSet[ids.NodeID](len(from))
sample.Add(from...)
round := bsAcceptedRound{asked: sample, ch: make(chan bsAcceptedReply, len(from))}
b.bsMu.Lock()
// Self-arming, as the serve limiter is: a handler assembled as a struct literal rather than
// through newBlockHandler must not nil-map-panic on a path that only recovery reaches.
if b.bsAccepted == nil {
b.bsAccepted = make(map[uint32]bsAcceptedRound)
}
b.bsAccepted[requestID] = round
b.bsMu.Unlock()
defer func() {
b.bsMu.Lock()
delete(b.bsAccepted, requestID)
b.bsMu.Unlock()
}()
msg, err := b.msgCreator.GetAccepted(b.chainID, requestID, bootstrapFrontierWindow, want)
if err != nil {
return nil
}
b.net.Send(msg, sample, b.networkID, 0)
answers := make([]AcceptedReply, 0, len(from))
seen := make(map[ids.NodeID]struct{}, len(from))
deadline := time.After(bootstrapFrontierWindow)
for {
select {
case rep := <-round.ch:
if _, dup := seen[rep.nodeID]; dup {
continue // one answer per beacon
}
seen[rep.nodeID] = struct{}{}
answers = append(answers, AcceptedReply{NodeID: rep.nodeID, Accepted: rep.accepted})
if len(seen) >= len(from) {
return answers // everyone asked has answered — resolve now, do not wait the window
}
case <-deadline:
return answers
case <-ctx.Done():
return answers
}
}
}
// Ancestors implements chainbootstrap.BlockSource: fetch up to maxBlocks blocks ending
// at blockID, OLDEST-FIRST, from a ROTATED sample of beacons (wire: GetAncestors ->
// Ancestors). An empty result (no error) means the sampled beacon did not serve — the
// loop re-samples. The fetched ancestry is made safe by the loop's content-addressed
// descent (off-path blocks ignored), not by trusting the serving peer.
func (b *blockHandler) Ancestors(ctx context.Context, blockID ids.ID, maxBlocks int) ([][]byte, error) {
fetched, err := b.fetchBootstrapAncestors(ctx, blockID, maxBlocks)
if err != nil || len(fetched) == 0 {
return nil, err
}
raw := make([][]byte, 0, len(fetched))
certs := make(map[ids.ID][]byte, len(fetched))
for _, entry := range fetched {
raw = append(raw, entry.Bytes)
if len(entry.Certificate) == 0 || b.vm == nil {
continue
}
blk, parseErr := b.vm.ParseBlock(ctx, entry.Bytes)
if parseErr != nil {
continue
}
certs[blk.ID()] = append([]byte(nil), entry.Certificate...)
}
if len(certs) > 0 {
b.bsMu.Lock()
if b.bsCertificates == nil {
b.bsCertificates = make(map[ids.ID][]byte)
}
for id, cert := range certs {
b.bsCertificates[id] = cert
}
b.bsMu.Unlock()
}
return raw, nil
}
func (b *blockHandler) fetchBootstrapAncestors(ctx context.Context, blockID ids.ID, maxBlocks int) ([]bootstrapFetchedBlock, error) {
if b.net == nil || b.msgCreator == nil {
return nil, nil
}
sample, ok := b.sampleAncestorBeacons()
if !ok {
return nil, nil
}
b.contextRequestMu.Lock()
b.requestIDCounter++
requestID := b.requestIDCounter
b.contextRequestMu.Unlock()
// Buffer the whole sample so EVERY sampled beacon's reply can queue — the loop
// then skips the EMPTY ones (a beacon that lacks the requested block, e.g. a peer
// still at genesis) and returns the first NON-EMPTY batch. With a size-1 channel a
// fast empty reply won the race and starved a slower peer that actually held the
// ancestry, so a re-bootstrapping node with ≥2 peers at genesis could keep drawing
// empties and stall. The contract is the settled one: an empty Ancestors
// reply means "this peer can't serve — take another's," never "done" (getter serves an
// EXPLICIT empty batch when it lacks the block; see GetContext).
ch := make(chan []bootstrapFetchedBlock, bootstrapAncestorSample)
b.bsMu.Lock()
b.bsAncestorCh[requestID] = ch
// Remember WHO was asked. A request id is ours, not a secret: any connected peer
// can send a reply carrying it. Correlating on the id alone lets an unsampled peer
// answer a fetch we never made of it, and one such reply abandons the pass — a
// stranger can stall a node's initial sync indefinitely by answering promptly.
if b.bsAncestorPeers == nil {
b.bsAncestorPeers = make(map[uint32]set.Set[ids.NodeID])
}
b.bsAncestorPeers[requestID] = sample
b.bsMu.Unlock()
defer func() {
b.bsMu.Lock()
delete(b.bsAncestorCh, requestID)
delete(b.bsAncestorPeers, requestID)
b.bsMu.Unlock()
}()
// The WIRE deadline we grant the peer is the SAME bound we will actually wait
// (bootstrapAncestorsTimeout) — one number, so a peer is never asked to spend longer
// building a batch than we are prepared to accept it for.
msg, err := b.msgCreator.GetAncestors(b.chainID, requestID, bootstrapAncestorsTimeout, blockID, p2p.EngineType_ENGINE_TYPE_CHAIN)
if err != nil {
return nil, err
}
b.net.Send(msg, sample, b.networkID, 0)
deadline := time.After(bootstrapAncestorsTimeout)
for {
select {
case blocks := <-ch:
if len(blocks) == 0 {
continue // this beacon can't serve the block — wait for a peer that can
}
return blocks, nil
case <-deadline:
return nil, nil // no beacon in the sample served — the loop re-samples (rotated)
case <-ctx.Done():
return nil, ctx.Err()
}
}
}
// Ancestry implements the BootstrapPolicy's AncestrySource: fetch a tip's ancestry over the wire
// and parse each block to its CONTENT-ADDRESSED (id, height, parent). It reuses the SAME Ancestors
// transport + ParseBlock the sync loop trusts, so a forging peer cannot fake the parent linkage
// the responder-agreement tally walks — and the trust DECISION (bootstrap_trust.go) stays free of
// any VM/block dependency, taking only []BlockRef. An empty fetch (peer did not serve) yields nil
// so that anchor simply contributes nothing; a malformed served block fails the whole anchor (it
// is not safe to trust a chain we cannot parse).
func (b *blockHandler) Ancestry(ctx context.Context, tip ids.ID, max int) ([]BlockRef, error) {
raw, err := b.Ancestors(ctx, tip, max)
if err != nil || len(raw) == 0 {
return nil, err
}
refs := make([]BlockRef, 0, len(raw))
for _, bz := range raw {
blk, perr := b.vm.ParseBlock(ctx, bz)
if perr != nil {
return nil, perr
}
refs = append(refs, BlockRef{ID: blk.ID(), Height: blk.Height(), Parent: blk.ParentID()})
}
return refs, nil
}
// ParseBlock implements chainbootstrap.Chain: decode block bytes through the SAME
// builder the engine parses through (identity-preserving), so the loop reads each
// block's height + parent for ordering and the descent.
func (b *blockHandler) ParseBlock(ctx context.Context, raw []byte) (cblock.Block, error) {
return b.vm.ParseBlock(ctx, raw)
}
// LastAccepted implements chainbootstrap.Chain: the node's ACCEPTED tip id + height, read
// from the IN-PROCESS consensus finalized ledger — the SINGLE advancing source of truth the
// loop drives convergence off (THE one-source decomplect). It is NOT read from the VM cache.
//
// THE FREEZE-STALL: the descent DOES accept the run-up (the consensus finalized
// ledger AND the EVM's on-disk store both advance), but the ZAP VM client caches its
// LastAccepted at Initialize and a fire-and-forget Accept never refreshes it (client.go) —
// so VM.LastAccepted is FROZEN for the process life. Reading lastH off that frozen cache made
// every pass after the first re-descend from the stale height while AcceptBootstrapBlock
// no-oped each block (height ≤ the ADVANCED finalizedHeight) → advanced=false → ErrStalled:
// "unfreezes but stalls". AcceptBootstrapBlock's own contiguity guard already trusts the
// in-process ledger (consensus.GetFinalizedHeight); driving lastH/the caught-up oracle off the
// SAME ledger is the decomplect — the loop trusts the ledger it is building, not a VM cache.
// executedTo reports whether this node's VM has actually RUN the chain up to h.
//
// The acceptance oracle answers from the consensus ledger — what a quorum decided —
// which is right for "has this block been accepted" and wrong for "may this node go
// live". The two diverge, and on the divergence the ledger is HIGHER: a quorum
// decides a block before this node executes it, and if execution then fails or lags
// the ledger keeps the higher number. A gate reading only the ledger therefore lets
// a node whose VM is far below the head declare itself at the frontier, go live, and
// serve and vote at a stale height while nothing is left trying to converge it.
//
// A VM that cannot be read is not a VM that has executed: an unproven measurement is
// not a passing one, so it refuses. This can only ever REFUSE go-live, never permit
// it, which is the safe side of the decision.
func (b *blockHandler) executedTo(ctx context.Context, h uint64) bool {
if b.vm == nil {
return false
}
applied, err := b.appliedHeight(ctx)
if err != nil {
return false // cannot prove execution ⇒ do not go live
}
return applied >= h
}
// innerLastAccepted is the head of the VM that actually executes blocks.
// appliedHeight is how far this node has actually RUN the chain — a different
// question from LastAccepted, which names the chain's head.
//
// On a wrapped chain the wrapper is committed before its inner block is accepted,
// so it leads during every accept and keeps leading if the inner never lands. A
// caller that asks the wrapper how far the node has run is told yes for blocks
// the inner VM never got: it lowers its position onto a height it does not hold,
// skips exactly the band it must fetch, and asks forever while reporting full
// batches landed. A VM that cannot speak for an inner self is its own inner self,
// and answers with its head.
func (b *blockHandler) appliedHeight(ctx context.Context) (uint64, error) {
type innerHeight interface {
InnerLastAcceptedHeight(context.Context) (uint64, error)
}
if inner, ok := b.vm.(innerHeight); ok {
return inner.InnerLastAcceptedHeight(ctx)
}
_, h, err := b.vmLastAccepted(ctx)
return h, err
}
func (b *blockHandler) LastAccepted(ctx context.Context) (ids.ID, uint64, error) {
return b.finalizedTip(ctx)
}
// finalizedTip returns the in-process consensus finalized ledger position (the ADVANCING
// source), falling back to the VM's last-accepted ONLY when the ledger is unset — the
// empty-genesis boot window before the first finalize (consensus SyncState seeds the ledger
// from a NON-empty last-accepted; an empty node leaves it unset until block 1 finalizes), or a
// degenerate/test handler with no engine. The VM fallback is correct there: at genesis the VM
// last-accepted IS the anchor and has not yet had a chance to go stale (it only diverges from
// the ledger once blocks finalize, at which point the ledger takes over). This mirrors the
// engine's own M2 first-block anchor (bootstrap_accept.go localLastAccepted), keeping ONE rule.
func (b *blockHandler) finalizedTip(ctx context.Context) (ids.ID, uint64, error) {
if b.engine != nil {
if tip, h, set := b.engine.FinalizedLedger(); set {
// Position is the LOWER of the two heads. The ledger records what a quorum
// decided; the VM records what this node executed, and finalization can fold
// the ledger across blocks the VM never ran. A loop positioned at the ledger
// then believes it stands past blocks it has not executed, skips exactly the
// band it must fetch, and re-asks forever while reporting full batches landed.
//
// Reading the VM here rests on one contract: the ZAP client refreshes
// LastAccepted on every successful Accept (vms/rpcchainvm/zap/client.go),
// so the applied head it reports is live rather than a value fixed at
// Initialize. A VM naming NO block reports nothing — absence is not a
// reading of zero — so only a VM that names a block lowers the position.
if id, _, ierr := b.vmLastAccepted(ctx); ierr == nil && id != ids.Empty {
if applied, aerr := b.appliedHeight(ctx); aerr == nil && applied < h {
return id, applied, nil
}
}
return tip, h, nil
}
}
return b.vmLastAccepted(ctx)
}
// vmLastAccepted reads the VM's last-accepted id + height. It is the genesis-anchor fallback
// for finalizedTip (used only before the consensus ledger is seeded), NOT a convergence
// signal — a converging node's height MUST come from the finalized ledger, never this cache.
func (b *blockHandler) vmLastAccepted(ctx context.Context) (ids.ID, uint64, error) {
if b.vm == nil {
return ids.Empty, 0, nil
}
id, err := b.vm.LastAccepted(ctx)
if err != nil {
return ids.Empty, 0, err
}
if id == ids.Empty {
return ids.Empty, 0, nil
}
blk, err := b.vm.GetBlock(ctx, id)
if err != nil {
return id, 0, nil // id known, height unknown — treat as 0 (genesis-ish)
}
return id, blk.Height(), nil
}
// Accepted implements chainbootstrap.Chain: reports whether id is on the node's ACCEPTED chain
// (finalized) — NOT merely PRESENT in the block store. This is the loop's caught-up predicate.
// THE FREEZE was a store-vs-acceptance conflation: the prior Has() returned true for a frontier
// GOSSIPED into the store but UNACCEPTED (height ABOVE last-accepted), short-circuiting the loop
// to caught-up at the stale last-accepted and never descending to accept it. A stored-but-
// unaccepted block returns false here, so the loop descends and DRIVES its acceptance.
func (b *blockHandler) Accepted(ctx context.Context, id ids.ID) bool {
_, ok := b.acceptedHeightCtx(ctx, id)
return ok
}
// acceptedHeight is the ACCEPTANCE oracle injected into BootstrapPolicy.CaughtUp (as heightOf):
// it returns id's height and TRUE iff the node has ACCEPTED id (id is on the node's finalized
// chain), and (0,false) when id is merely PRESENT IN THE STORE but NOT accepted — the store-vs-
// acceptance distinction the caught-up decision hinges on. CaughtUp uses it to (c) require the
// node to have ACCEPTED every reported tip and (b) read those tips' heights to confirm none is above the
// node's accepted height; a stored-but-unaccepted tip now makes CaughtUp FALSE (the node is behind
// in ACCEPTANCE → it syncs). It NEVER fetches over the network — an unaccepted/unheld tip simply
// makes the node not-caught-up, the safe direction.
func (b *blockHandler) acceptedHeight(id ids.ID) (uint64, bool) {
return b.acceptedHeightCtx(context.Background(), id)
}
// acceptedHeightCtx is the ctx-honoring core of the acceptance oracle (shared by Accepted and
// acceptedHeight). Authoritative and VM-internals-light: the accepted head id+height come from
// the IN-PROCESS consensus finalized ledger (LastAccepted → finalizedTip — the ADVANCING source,
// never the frozen VM cache), the height-bound (id ABOVE finalizedHeight ⇒ not yet accepted —
// the gossiped-ahead freeze case, regardless of store presence) is the primary anchor, and the
// in-process per-height ledger is the fork-sibling oracle for a block at/below the finalized head.
func (b *blockHandler) acceptedHeightCtx(ctx context.Context, id ids.ID) (uint64, bool) {
if id == ids.Empty || b.vm == nil {
return 0, false
}
lastID, lastH, err := b.LastAccepted(ctx)
if err != nil {
return 0, false
}
if id == lastID {
return lastH, true // the finalized head itself — the authoritative anchor
}
blk, err := b.vm.GetBlock(ctx, id)
if err != nil {
return 0, false // not even in the store → certainly not accepted
}
h := blk.Height()
if h > lastH {
// ABOVE our finalized head: a block GOSSIPED into the store ahead of acceptance. THE FREEZE
// — present, but NOT accepted. NOT caught up; the node must descend and accept it.
return 0, false
}
// h <= lastH and id != lastID: id is at a height we have finalized PAST. It is accepted IFF it
// is the block our per-height ledger finalized at h. Ask the IN-PROCESS finalized ledger — the
// same source LastAccepted reads — rather than the VM's height index, because finality is
// decided in one place: the ledger is what finality means here, while a VM's index answers what
// that VM has accepted, and the two can disagree while a decision is in flight. The ledger
// answers authoritatively for every height finalized THIS session.
if canonical, ok := b.finalizedBlockAtHeight(h); ok {
if canonical != id {
return 0, false // a stored sibling/fork at a finalized height — NOT on the accepted chain
}
return h, true
}
// The per-height ledger does not know h: it is a height BELOW the boot seed (consensus SyncState
// seeds only the boot height + advances upward; older heights are never re-seeded). We have
// finalized PAST h and we hold id, so on a BFT-final chain (one finalized block per height) id is
// on our accepted chain. A node CAN gossip-hold a non-finalized sibling at such a height with NO
// safety break (it received a losing fork via gossip) — but that costs nothing HERE: this oracle
// only feeds the caught-up decision, which acts on the ⅔-NAMED frontier, and C1 (⅔-by-stake
// frontier naming) guarantees a forged/losing sibling is NEVER the named frontier; the descent's
// content-addressing + the per-height accept guard reject any off-chain block during execution.
// So treating a held sub-boot-seed block as accepted is the safe over-approximation.
return h, true
}
// finalizedBlockAtHeight returns the block the IN-PROCESS consensus ledger finalized at height h
// (ok=false when the node has not finalized h this session — a height below the boot seed — or has
// no engine). It is the authoritative fork-sibling oracle that replaces the EVM's dead height
// index; degrading to ok=false simply routes acceptedHeightCtx to its height-bound anchor.
func (b *blockHandler) finalizedBlockAtHeight(h uint64) (ids.ID, bool) {
if b.engine == nil {
return ids.Empty, false
}
return b.engine.FinalizedBlockAtHeight(h)
}
// AcceptBootstrapBlock implements chainbootstrap.Chain. A quorum chain accepts a
// fetched block only through the same signed-certificate predicate as live catch-up;
// a peer/beacon majority is discovery input, not finality authority. The legacy
// frontier-trust path remains available only when no signed votes are required (the
// sole-validator/dev topology, where no external quorum certificate can exist).
func (b *blockHandler) AcceptBootstrapBlock(ctx context.Context, raw []byte) error {
if !b.signedVotesRequired {
return b.engine.AcceptBootstrapBlock(ctx, raw)
}
if b.engine == nil || b.vm == nil {
return errBootstrapCertificateRequired
}
blk, err := b.vm.ParseBlock(ctx, raw)
if err != nil {
return err
}
b.bsMu.Lock()
cert := b.bsCertificates[blk.ID()]
delete(b.bsCertificates, blk.ID())
b.bsMu.Unlock()
if len(cert) == 0 {
return errBootstrapCertificateRequired
}
return b.engine.AcceptCatchupBlock(ctx, raw, cert)
}
// BootstrapComplete reports whether initial sync has reached the frontier. This is the
// REAL bootstrap-ready signal monitorBootstrap gates the chain on — replacing the
// premature engine.IsBootstrapped() poll that returned true at the local last-accepted
// height (the bug: an empty node declared itself bootstrapped at genesis and never
// synced).
func (b *blockHandler) BootstrapComplete() bool { return b.bootstrapDone.Load() }
// bsFailure records the fail-safe reason initial sync returned (ErrBeaconsUnreachable /
// ErrNoBeaconQuorum / ErrGapTooLarge / ...). Stored behind an atomic pointer so the
// monitorBootstrap goroutine reads it race-free — the precise diagnostic for the health check.
type bsFailure struct{ err error }
// BootstrapFailed reports whether initial sync ended in a fail-SAFE error (eclipse / partition
// / deep gap) rather than reaching the frontier (F5). monitorBootstrap polls this so a fail-safe
// return STOPS the chain promptly — it does not sit in the dead window between Run returning and
// the 5-min no-progress watchdog. Distinct from BootstrapComplete: a chain is ready XOR failed
// XOR still-syncing.
func (b *blockHandler) BootstrapFailed() bool { return b.bootstrapFailed.Load() != nil }
// BootstrapFailure returns the fail-safe reason once BootstrapFailed (else nil) — the precise
// cause (eclipsed/partitioned/too-far-behind) the operator's health check surfaces.
func (b *blockHandler) BootstrapFailure() error {
if f := b.bootstrapFailed.Load(); f != nil {
return f.err
}
return nil
}
// BootstrapWait reports WHY initial sync is in its bounded connectivity-RETRY wait — a transient
// beacon-quorum-unreachable fail-safe it is re-attempting (the SELF-HEAL path) — and empty when it
// is not waiting. The node is correctly failing safe DOWN (VM in Bootstrapping, serving nothing as
// head) and waiting for the quorum to return; the network cannot make progress without it.
// monitorBootstrap's no-progress watchdog polls this so it does NOT force-STOP a node that is
// deliberately waiting — which, given the K8s probes only poll the always-green
// /v1/health/ops/liveness, would be a permanent brick. It is the discriminator between "stuck on a
// served gap" (a real stall → stop) and "waiting for the quorum" (self-heal → keep waiting).
// Distinct from BootstrapFailed (a terminal/structural fail).
//
// Whether the node is waiting and why it is waiting are one question, so they are one value:
// asking twice admits an answer where the two halves disagree, and the reason is what the
// operator came for anyway.
func (b *blockHandler) BootstrapWait() string {
if w := b.bootstrapWait.Load(); w != nil {
return *w
}
return ""
}
// BootstrapHeight reports the node's current ACCEPTED height — the PROGRESS signal
// monitorBootstrap uses (H2) to tell a slow-but-advancing sync (reset the stall timer) from a
// genuine no-progress stall (fail). It reads the IN-PROCESS finalized ledger (finalizedTip), the
// SAME advancing source as the convergence oracle — NOT the VM cache, which a fire-and-forget ZAP
// Accept leaves FROZEN: reading the frozen cache here made monitorBootstrap's progress probe see
// zero advance even while the descent finalized block after block, so a deep but healthy sync
// looked like a stall. Best-effort: 0 if unknown.
func (b *blockHandler) BootstrapHeight() uint64 {
_, h, err := b.finalizedTip(context.Background())
if err != nil {
return 0
}
return h
}
// deliverBootstrapFrontier routes a frontier reply (TAGGED with the responding beacon)
// to the waiting FrontierTip when the bootstrap loop is driving. Returns true iff
// consumed (the caller must then NOT run the live auto-fetch). The nodeID is what lets
// FrontierTip weight the reply by the responder's stake — the heart of the α-quorum.
// Non-blocking: if the buffered channel is momentarily full the reply is dropped (the
// window/loop re-samples).
func (b *blockHandler) deliverBootstrapFrontier(nodeID ids.NodeID, containerID ids.ID) bool {
if !b.bsActive.Load() {
return false
}
b.bsMu.Lock()
ch := b.bsFrontierCh
b.bsMu.Unlock()
if ch != nil {
select {
case ch <- bsFrontierReply{nodeID: nodeID, tip: containerID}:
default:
}
}
return true
}
// deliverBootstrapAccepted routes an Accepted reply — a beacon's answer to the second naming
// question — to the waiting Acceptance call. Returns true iff this lane TOOK it.
//
// An answer is taken only if this lane asked THAT peer under THAT request id. Both halves matter:
// the id keeps an answer out of a round it was not part of, and the sender keeps a peer nobody
// asked from occupying a slot in the round's buffer — which would deny it to a beacon whose answer
// the tally needs. Non-blocking, so a full buffer drops the answer rather than holding the message
// goroutine.
func (b *blockHandler) deliverBootstrapAccepted(requestID uint32, nodeID ids.NodeID, accepted []ids.ID) bool {
if !b.bsActive.Load() {
return false
}
b.bsMu.Lock()
round, open := b.bsAccepted[requestID]
b.bsMu.Unlock()
if !open || !round.asked.Contains(nodeID) {
return false
}
select {
case round.ch <- bsAcceptedReply{nodeID: nodeID, accepted: accepted}:
return true
default:
return false
}
}
// deliverBootstrapAncestors routes an Ancestors reply (framed block batch) to the
// waiting Ancestors call when the bootstrap loop is driving. Returns true iff this
// lane actually TOOK the reply — the caller hands anything else to the live path.
//
// bsActive means "initial sync is running", NOT "initial sync owns every Ancestors
// reply on this chain". The live cert catch-up path issues its own requests
// concurrently — that is how a node which is merely behind fetches the block a
// vote or cert referenced. Claiming a reply this lane never asked for strands such
// a node permanently: it fetches the right blocks and applies none of them, logging
// that the catch-up response was consumed by initial sync over full replies it had
// just successfully fetched, while it stays thousands of blocks behind its peers.
// A reply belongs to this lane only if this lane asked for it.
func (b *blockHandler) deliverBootstrapAncestors(nodeID ids.NodeID, requestID uint32, data []byte) bool {
if !b.bsActive.Load() {
return false
}
b.bsMu.Lock()
ch := b.bsAncestorCh[requestID]
asked := b.bsAncestorPeers[requestID]
b.bsMu.Unlock()
if ch == nil {
return false
}
// A reply belongs to this fetch only if we asked THIS peer for it.
if !asked.Contains(nodeID) {
return false
}
select {
case ch <- decodeContextBlocks(data):
return true
default:
// The waiter exists but is not receiving (it timed out and moved on). The
// send is non-blocking, so claiming the reply would drop the payload AND
// deny it to the live path — losing the blocks twice. Let the live path try.
return false
}
}
// runBootstrapThenPoll is the chain's startup sync driver (the goroutine buildChain
// launches). It runs INITIAL SYNC to the network frontier with the VM's normal-operation
// transition GATED on that completion (runInitialSync), then — ONLY if the chain actually
// went live — hands off to the live frontier poller (runtime cert-carry catch-up). On
// fail-safe runInitialSync returns false and the poller is NOT started: the chain stays
// not-ready for monitorBootstrap to surface, and the VM stays in Bootstrapping (it never
// serves a stale height). Exits when ctx is done (shutdown). The gating logic lives in
// runInitialSync so it is unit-testable without the blocking poller hand-off.
func (b *blockHandler) runBootstrapThenPoll(ctx context.Context) {
if b.runInitialSync(ctx) {
b.runFrontierPoller(ctx)
}
}
// runInitialSync drives the fetch+execute bootstrap loop to the network frontier and, ON
// SUCCESS, transitions the VM to normal operation (transitionVMReady → vm.Ready), ENDS the
// engine's bootstrap phase (FinishBootstrap — only the α-of-K cert-gate finalizes
// thereafter), and flips bootstrapDone — IN THAT ORDER, so "VM serves / builds" and "engine
// cert-gates finality" go live TOGETHER at the named frontier. Returns true iff the chain
// went live.
//
// THE ORDERING IS THE FIX. Previously buildChain put the VM into normal operation
// UNCONDITIONALLY right after Initialize — at the LOCAL last-accepted height — and ran this
// sync afterward as a detached, non-gating goroutine. A restarted STALE validator therefore
// went live (block building, mempool, validator dispatch via the EVM's
// onNormalOperationsStarted) at its stale height and never converged to the finalized
// frontier. Gating the normal-op transition on reaching the frontier makes "VM live at a
// stale height" structurally impossible: the VM is in Bootstrapping until it has converged.
//
// GO-LIVE includes the CAUGHT-UP path. A tip-holding producer on a mixed-height co-restart names
// no frontier (the responders split below ⅔) yet is at the top of every responder — FrontierTip
// returns its OWN held tip NAMED (BootstrapPolicy.CaughtUp), the loop's Accepted()-shortcut concludes
// caught-up, and bs.Run returns nil → it goes Ready at its own tip. Without that path the producer
// would fail safe DOWN at its own tip (the regression this round fixes), the opposite of the
// stale-go-live bug.
//
// FAIL-SAFE + SELF-HEAL (eclipse / isolation / majority outage). Each bs.Run attempt is internally
// BOUNDED: it WAITS for beacon connectivity (re-sampling every RetryInterval up to ConnectDeadline)
// then returns rather than hanging. A TRANSIENT connectivity fail-safe (ErrBeaconsUnreachable /
// ErrNoBeaconQuorum — the quorum was not reachable this attempt) is RE-ATTEMPTED (bootstrapMaxAttempts
// ≤ 0 ⇒ until the quorum returns or shutdown), with the wait reason published so monitorBootstrap's
// no-progress watchdog treats it as a deliberate WAIT, not a stall: the node stays in Bootstrapping
// (serving nothing as head, NEVER live at the stale height) and CONVERGES the instant the quorum
// returns — the in-process self-heal the K8s probes do NOT provide (they poll the always-green
// /v1/health/ops/liveness, so a fail-safe-DOWN node is never restarted). A STRUCTURAL failure (deep gap
// → state-sync) or an exhausted attempt bound returns false WITHOUT going Ready, bootstrapFailed
// recording the reason so monitorBootstrap surfaces it. The node NEVER false-completes at its stale
// height, and a transient outage NEVER becomes a permanent brick.
func (b *blockHandler) runInitialSync(ctx context.Context) bool {
if b.engine == nil || b.net == nil || b.msgCreator == nil {
// Degenerate handler (no transport/engine to drive sync): nothing to converge to.
// Go live immediately so a single-node / transport-less chain is not pinned
// unbootstrapped — the same fast-path the no-beacon-set case takes.
if b.engine != nil {
b.engine.FinishBootstrap()
}
if err := b.transitionVMReady(ctx); err != nil {
b.logger.Error("degenerate chain: VM SetState(Ready) failed — NOT marking bootstrapped", log.Err(err))
b.bootstrapFailed.Store(&bsFailure{err: err})
return false
}
b.bootstrapDone.Store(true)
return true
}
// Ancestors bridges a cert-carrying wire format through a source-compatible
// bootstrap interface by caching proofs under the block ID. Executed blocks
// consume their entries immediately; frontier probes and abandoned retries can
// leave entries behind. Keep the cache scoped to this initial-sync run.
defer func() {
b.bsMu.Lock()
clear(b.bsCertificates)
b.bsMu.Unlock()
}()
bs := chainbootstrap.New(chainbootstrap.Config{
Source: b,
Chain: b,
Log: b.logger,
// Bounded beacon-connect WAIT + re-sample pause (zero ⇒ library defaults 3m / 1s).
// ConnectDeadline is the IN-ATTEMPT retry: bs.Run re-samples the beacon set every
// RetryInterval up to this deadline, converging the instant the beacons return.
ConnectDeadline: b.bootstrapConnectDeadline,
RetryInterval: b.bootstrapRetryInterval,
// Optional operator-pinned weak-subjectivity anchor: the α-agreed frontier must
// descend from this id at this height (defense-in-depth for empty-genesis). Zero
// ⇒ disabled (the beacon + ⅔-stake quorum is the primary anchor).
WeakSubjectivityID: b.wsCheckpointID,
WeakSubjectivityHeight: b.wsCheckpointHeight,
})
// OUTER SELF-HEAL RETRY. A bs.Run attempt that fails because the beacon quorum was UNREACHABLE
// (eclipse / partition / a majority co-restart still in flight) is RE-ATTEMPTED — the node stays
// in Bootstrapping (engine alive, VM serving nothing as head, never live at the stale height)
// and CONVERGES the instant the quorum returns. This is the recovery the K8s probes do NOT
// provide (they all poll the always-green /v1/health/ops/liveness, so a fail-safe-DOWN node is
// never restarted). bootstrapMaxAttempts ≤ 0 ⇒ retry until the quorum returns or shutdown; a
// test pins it to 1 to assert the single-attempt terminal fail-safe. A STRUCTURAL failure (deep
// gap → state-sync) is NOT retried — a retry cannot fix it; it is surfaced for the operator.
// ADOPTION FIRST. A node whose gap is deeper than the descent can close has no
// way home through the descent, however many times it retries — asking for a
// summary is the other door, and it has to be tried before the walk, because
// adopting one moves the floor the walk starts from. A chain whose VM cannot
// sync state, or whose network has nothing to offer, falls straight through to
// the descent it would have run anyway.
b.adoptSummary(ctx)
var lastErr error
for attempt := 1; ; attempt++ {
b.bsActive.Store(true)
err := bs.Run(ctx)
b.bsActive.Store(false)
if err == nil {
// Reached the frontier (or no beacon set / already at / above the tip — caught up): go
// live. Close the bootstrap phase FIRST, then transition the VM to normal operation.
// Quorum-chain bootstrap acceptance is already certificate-gated by this adapter;
// ending the phase also closes the legacy cert-less engine entry point. Ordering
// them this way
// (matching the degenerate path above) leaves NO window where the live VM is
// building/serving while the cert-less accept path is still open. FinishBootstrap is
// idempotent and void; if the VM transition then fails we fail safe (record the reason,
// return false). A VM that REFUSES normal-op is a real failure: do NOT mark ready.
b.bootstrapWait.Store(nil)
b.engine.FinishBootstrap()
if verr := b.transitionVMReady(ctx); verr != nil {
b.logger.Error("chain reached the frontier but VM SetState(Ready) failed — NOT marking bootstrapped", log.Err(verr))
b.bootstrapFailed.Store(&bsFailure{err: verr})
return false
}
b.bootstrapDone.Store(true)
b.logger.Info("chain initial sync complete — VM live (normal operation) at the network frontier")
return true
}
if ctx.Err() != nil {
b.bootstrapWait.Store(nil)
return false // shutdown — not a bootstrap failure
}
lastErr = err
// Decide whether to RE-ATTEMPT. A transient connectivity fail-safe self-heals when the
// quorum returns (retry); a structural failure or an exhausted attempt bound does not (break
// → surface). While we wait between attempts, publish the reason so the monitor's
// no-progress watchdog treats this as a deliberate quorum WAIT, not a stall — a node failing
// safe DOWN and waiting must not be force-stopped (that is the brick this avoids).
boundReached := b.bootstrapMaxAttempts > 0 && attempt >= b.bootstrapMaxAttempts
if boundReached || !isRetryableBootstrapFailure(err) {
break
}
// The wait is declared here because this is where it is decided. bs.Run fails safe on a
// bare sentinel, so the counts come from the last round that measured them; without them
// this says only that a quorum is missing, which reads the same whether five beacons
// answered or none did.
wait := "waiting for the beacon quorum"
if s := b.shortfall.Load(); s != nil {
wait += ": " + *s
}
b.bootstrapWait.Store(&wait)
if perr := b.pauseBootstrapRetry(ctx); perr != nil {
b.bootstrapWait.Store(nil)
return false // shutdown during the inter-attempt backoff
}
}
// Exhausted the attempt bound, or a structural failure (deep gap / disjoint peer). DO NOT
// transition to normal operation — leaving the VM in Bootstrapping is the fail-safe: it does not
// serve / build at the stale local height (that was the freeze defect). Record the reason so
// monitorBootstrap surfaces it (F5).
b.bootstrapWait.Store(nil)
b.logger.Warn("chain initial sync did not reach the frontier — VM stays bootstrapping (NOT serving normal-op), failing safe",
log.Err(lastErr))
b.bootstrapFailed.Store(&bsFailure{err: lastErr})
return false
}
// isRetryableBootstrapFailure reports whether a bs.Run fail-safe is a TRANSIENT connectivity
// failure — the beacon quorum was not reachable this attempt — which RECOVERS when the quorum
// returns, so re-attempting bootstrap (the node staying in Bootstrapping meanwhile) self-heals it.
// A STRUCTURAL failure (ErrGapTooLarge needs state-sync; ErrCannotConnect / ErrStalled is a serving
// peer naming a disjoint chain or withholding ancestry) is NOT retried here — a retry cannot fix it;
// it is surfaced (bootstrapFailed → monitorBootstrap) for operator action.
func isRetryableBootstrapFailure(err error) bool {
return errors.Is(err, chainbootstrap.ErrBeaconsUnreachable) ||
errors.Is(err, chainbootstrap.ErrNoBeaconQuorum) ||
// ErrStalled says NO PEER SERVED THE ANCESTRY THIS ROUND. That is a statement
// about one pass over a sampled set of peers, not about the chain: the peers
// that could serve may be busy, mid-restart, or simply not the ones this round
// happened to draw. Treating it as structural ends initial sync for the life of
// the process, and the node then has only the live catch-up path — which cannot
// close a gap wider than a served window, so it re-asks forever and the chain
// never rejoins. The only exit was a restart, which draws a new sample and can
// stall again on the next unlucky round.
//
// Retrying costs one backoff interval and re-samples. Not retrying costs the
// bootstrap. The attempt bound still terminates it, and a genuinely structural
// failure — a gap too deep for the window — has its own error and is still not
// retried, because a retry cannot fix that one.
errors.Is(err, chainbootstrap.ErrStalled)
}
// pauseBootstrapRetry backs off one RetryInterval between bootstrap re-attempts (never a hot loop),
// or returns ctx.Err() on shutdown. Mirrors the bootstrapper's own pause so the OUTER self-heal
// retry has the same bounded, shutdown-terminable backoff as the inner connect wait.
func (b *blockHandler) pauseBootstrapRetry(ctx context.Context) error {
d := b.bootstrapRetryInterval
if d <= 0 {
d = time.Second
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(d):
return nil
}
}
// transitionVMReady moves the VM to NORMAL OPERATION (vm.Ready) through the gated callback
// buildChain wired from the VM's SetState. It is the SINGLE place the VM goes live, called
// by runInitialSync ONLY after initial sync has reached the named frontier. No-op when no
// SetState VM is wired (degenerate / test handlers — nothing to transition). Bounded by a 30s
// timeout DERIVED FROM the sync/shutdown ctx (not context.Background): a wedged VM transition
// cannot hang the goroutine past 30s, AND a shutdown that cancels ctx cancels the transition
// immediately instead of blocking the chain teardown for up to 30s.
func (b *blockHandler) transitionVMReady(ctx context.Context) error {
if b.vmReady == nil {
return nil
}
sctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
return b.vmReady(sctx)
}
// decodeContextBlocks extracts blocks and their finality certificates (oldest-first) from a framed
// Ancestors payload — the inverse of GetContext's framing, shared by the bootstrap
// fetch path. Each outer [entryLen:4][entry] entry is either a v2 cert-carrying frame
// or a legacy raw block (the entry IS the block and Certificate is empty). Strict: a
// malformed length stops the walk (returns what parsed cleanly so far).
func decodeContextBlocks(data []byte) []bootstrapFetchedBlock {
var out []bootstrapFetchedBlock
remaining := data
for len(remaining) >= 4 {
entryLen := int(binary.BigEndian.Uint32(remaining[:4]))
remaining = remaining[4:]
if entryLen <= 0 || entryLen > len(remaining) {
break
}
entry := remaining[:entryLen]
remaining = remaining[entryLen:]
if blockBytes, certBytes, isV2 := decodeCatchupEntry(entry); isV2 {
out = append(out, bootstrapFetchedBlock{Bytes: blockBytes, Certificate: certBytes})
} else {
out = append(out, bootstrapFetchedBlock{Bytes: entry})
}
}
return out
}
// ancestryProgress returns the handler's cross-attempt descent state, allocating on
// first use. The map is the recovery state: the per-attempt budget resets each round
// and these cursors do not, which is what turns a gap wider than one attempt into
// progress rather than repetition.
func (b *blockHandler) ancestryProgress() map[ids.ID]*NamingProgress {
if b.namingProgress == nil {
b.namingProgress = make(map[ids.ID]*NamingProgress)
}
return b.namingProgress
}