fix(evm): bounded-wake backstop so an idle demand-driven chain mines the first tx in seconds

Pre-existing idle-builder wake race (not caused by QCv2): on a demand-driven
C-Chain with no empty blocks, a tx arriving while no builder is parked in
waitForNeedToBuild broadcasts pendingSignal to no waiter and is lost; the tx is
then only caught on the NEXT needToBuild() poll, which — depending on the
proposervm slot loop and engine re-call cadence — could leave the first tx after
idle pending for a long/indefinite time (the mainnet treasury-sweep-stuck and
v1.32.1 devnet DEXFill-didn't-complete symptoms).

Fix: startPendingTxPoll — a node-lifetime ticker (builderPollInterval = 500ms)
that periodically broadcasts pendingSignal so any parked builder re-evaluates
needToBuild() against the live mempool. It is decomplected from the tx-submit
subscription (awaitSubmittedTxs remains the primary, immediate wake) so the two
wake sources are independent. It NEVER forces an empty block — a wake only
produces a build when needToBuild() is true — so demand-driven chains stay
empty-block-free, and it changes wake TIMING only, never block CONTENTS, so
validators still build byte-identical blocks (deterministic). needToBuild now
routes through an injectable pendingTxs predicate so the wake path is testable
without a live txpool.

Regression: TestBuilderWakesOnPendingTxAfterIdle reproduces the node bridge's
exact waitForEvent call on an idle builder, delivers a pending tx with NO submit
broadcast, and asserts it returns PendingTxs within a bounded deadline. It FAILS
on the pre-fix code (times out — the lost wakeup) and passes after.
TestBuilderPollBackstopStopsOnShutdown asserts the ticker exits cleanly. Builder
tests green under -race.

Co-authored-by: Hanzo Dev <[email protected]>

plugin/evm/block_builder.go

@@ -20,6 +20,20 @@ const (
 	// pacing is to the target block rate (see waitForEvent); this only bounds
 	// the retry cadence when the target-rate time has already passed.
 	minBlockBuildingRetryDelay = 100 * time.Millisecond
+
+	// builderPollInterval bounds how long a parked waitForNeedToBuild can sleep
+	// before it re-polls the mempool, independent of the tx-submit subscription.
+	// The subscription (awaitSubmittedTxs) is the primary wake; this periodic
+	// re-poll is the backstop that closes the subscribe-gap window — a tx that
+	// arrives while no builder is parked (between two WaitForEvent calls, or
+	// while the proposervm is sleeping out its slot) broadcasts to no waiter and
+	// is otherwise only caught on the next needToBuild() poll. With this backstop
+	// a tx-after-idle ALWAYS wakes a parked builder within builderPollInterval,
+	// so a demand-driven chain mines the first tx within a bounded time with no
+	// keep-warm hack. It only ever produces a build when needToBuild() is true,
+	// so idle chains stay empty-block-free; it changes wake TIMING only, never
+	// block CONTENTS, so validators still build identical blocks.
+	builderPollInterval = 500 * time.Millisecond
 )
 
 type blockBuilder struct {
@@ -38,6 +52,12 @@ type blockBuilder struct {
 	// This is used to ensure that we don't build blocks too frequently,
 	// but at least after a minimum delay of minBlockBuildingRetryDelay.
 	lastBuildTime time.Time
+
+	// pendingTxs reports whether the mempool holds at least one buildable
+	// transaction. It is the single source of truth for needToBuild and is
+	// injectable so the wake path can be tested without a live txpool. Set by
+	// NewBlockBuilder to poll the real pool.
+	pendingTxs func() bool
 }
 
 func (vm *VM) NewBlockBuilder() *blockBuilder {
@@ -47,6 +67,12 @@ func (vm *VM) NewBlockBuilder() *blockBuilder {
 		shutdownChan: vm.shutdownChan,
 		shutdownWg:   &vm.shutdownWg,
 	}
+	// Use empty filter to check if ANY pending transactions exist. The miner
+	// applies proper fee filters when building; a MinTip filter here would
+	// reject valid legacy transactions (no GasTipCap) and stall production.
+	b.pendingTxs = func() bool {
+		return b.txPool.PendingSize(txpool.PendingFilter{}) > 0
+	}
 	b.pendingSignal = sync.NewCond(&b.buildBlockLock)
 	return b
 }
@@ -61,12 +87,7 @@ func (b *blockBuilder) handleGenerateBlock() {
 // needToBuild returns true if there are outstanding transactions to be issued
 // into a block.
 func (b *blockBuilder) needToBuild() bool {
-	// Use empty filter to check if ANY pending transactions exist
-	// The miner will apply proper fee filters when building the block
-	// Using MinTip filter here rejects valid legacy transactions that
-	// don't have GasTipCap set, causing block production to stall
-	size := b.txPool.PendingSize(txpool.PendingFilter{})
-	return size > 0
+	return b.pendingTxs()
 }
 
 // signalCanBuild signals that a new block can be built.
@@ -85,6 +106,10 @@ func (b *blockBuilder) awaitSubmittedTxs() {
 	txSubmitChan := make(chan core.NewTxsEvent)
 	b.txPool.SubscribeTransactions(txSubmitChan, true)
 
+	// Bounded-wake backstop: guarantees a parked builder re-polls the mempool
+	// even if a tx-submit notification is ever delivered to no waiter.
+	b.startPendingTxPoll()
+
 	b.shutdownWg.Add(1)
 	go func() {
 		defer func() {
@@ -107,6 +132,40 @@ func (b *blockBuilder) awaitSubmittedTxs() {
 	}()
 }
 
+// startPendingTxPoll starts the bounded-wake backstop: a node-lifetime ticker
+// that periodically broadcasts pendingSignal so any builder parked in
+// waitForNeedToBuild re-evaluates needToBuild() (which polls the live mempool)
+// at least every builderPollInterval. This closes the subscribe-gap window
+// without a keep-warm hack: a tx that arrived while no builder was parked — so
+// the awaitSubmittedTxs broadcast reached no waiter — is picked up on the next
+// tick rather than sitting pending indefinitely. It is decomplected from the
+// tx-submit subscription so the two wake sources are independent (and the
+// backstop is testable without a live txpool).
+func (b *blockBuilder) startPendingTxPoll() {
+	b.shutdownWg.Add(1)
+	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Error("panic in pending-tx poll", "error", r)
+				panic(r)
+			}
+		}()
+		defer b.shutdownWg.Done()
+
+		ticker := time.NewTicker(builderPollInterval)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ticker.C:
+				b.signalCanBuild()
+			case <-b.shutdownChan:
+				return
+			}
+		}
+	}()
+}
+
 // waitForEvent waits until a block needs to be built, then paces the signal so
 // the next block is not built before [earliestBuild]. earliestBuild is
 // parent.Time + TargetBlockRate: building earlier makes the block gas cost rise

plugin/evm/block_builder_wake_test.go

@@ -0,0 +1,101 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package evm
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/luxfi/consensus/engine/chain/block"
+)
+
+// newWakeTestBuilder builds a blockBuilder wired with an injectable pending-tx
+// predicate and only the bounded-wake backstop running (no tx-submit
+// subscription). Starting only the backstop models a lost/missed submit
+// notification, so the periodic re-poll is the sole way a parked builder can
+// discover a pending tx — which is exactly the property under test.
+func newWakeTestBuilder(pending func() bool) (*blockBuilder, func()) {
+	shutdown := make(chan struct{})
+	b := &blockBuilder{
+		shutdownChan: shutdown,
+		shutdownWg:   &sync.WaitGroup{},
+		pendingTxs:   pending,
+	}
+	b.pendingSignal = sync.NewCond(&b.buildBlockLock)
+	b.startPendingTxPoll()
+	return b, func() { close(shutdown) }
+}
+
+// TestBuilderWakesOnPendingTxAfterIdle is the fail-on-old / pass-on-fix
+// regression for the idle-builder wake race. It reproduces the node bridge's
+// exact call — waitForEvent with a past earliestBuild (so pacing adds no delay)
+// — on an idle builder, then delivers a pending tx with NO pendingSignal
+// broadcast. On the pre-fix code the only wake source is the tx-submit
+// broadcast; a broadcast delivered to no parked waiter is lost, so a builder
+// that parks afterward never re-polls and the tx sits pending indefinitely
+// (this test times out). With the bounded-wake backstop the parked builder
+// re-polls the mempool within builderPollInterval and returns PendingTxs.
+func TestBuilderWakesOnPendingTxAfterIdle(t *testing.T) {
+	var pending atomic.Bool // starts false: mempool empty
+	b, stop := newWakeTestBuilder(pending.Load)
+	defer stop()
+
+	// earliestBuild in the past => no pacing delay, so the result is bounded by
+	// the wake alone, not by the target-block-rate pace.
+	earliestBuild := time.Now().Add(-time.Hour)
+
+	got := make(chan block.Message, 1)
+	go func() {
+		msg, err := b.waitForEvent(context.Background(), earliestBuild)
+		if err != nil {
+			t.Errorf("waitForEvent: %v", err)
+			return
+		}
+		got <- msg
+	}()
+
+	// While the mempool is empty the builder must stay parked: a demand-driven
+	// chain must not build empty blocks.
+	select {
+	case <-got:
+		t.Fatal("waitForEvent returned before any pending tx (idle build)")
+	case <-time.After(100 * time.Millisecond):
+	}
+
+	// A tx arrives, but no submit-signal broadcast is delivered for it — only
+	// the bounded-wake backstop can wake the parked builder.
+	pending.Store(true)
+
+	select {
+	case msg := <-got:
+		if msg.Type != block.PendingTxs {
+			t.Fatalf("got message type %v, want PendingTxs", msg.Type)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("builder did not wake on pending tx after idle within bound (lost wakeup)")
+	}
+}
+
+// TestBuilderPollBackstopStopsOnShutdown asserts the backstop goroutine exits on
+// shutdown (no leak) so the node-lifetime ticker is correctly torn down.
+func TestBuilderPollBackstopStopsOnShutdown(t *testing.T) {
+	b, stop := newWakeTestBuilder(func() bool { return false })
+
+	done := make(chan struct{})
+	go func() {
+		b.shutdownWg.Wait()
+		close(done)
+	}()
+
+	stop()
+
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("pending-tx poll goroutine did not exit on shutdown")
+	}
+}