bft/testutil/random_network/network.go

// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package random_network

import (
	"math/rand"
	"sync"
	"testing"
	"time"

	"github.com/luxfi/bft/common"
	"github.com/luxfi/bft/testutil"
	"github.com/luxfi/log"
)

type Network struct {
	*testutil.BasicInMemoryNetwork
	logger *testutil.TestLogger
	t      *testing.T

	lock       sync.Mutex
	nodes      []*Node
	numNodes   uint64
	randomness *rand.Rand
	config     *FuzzConfig

	// tx stats
	allIssuedTxs int
}

func NewNetwork(config *FuzzConfig, t *testing.T) *Network {
	logger := CreateNetworkLogger(t, config)
	logger.Info("Creating new network with random seed", log.Int64("seed", config.RandomSeed))
	r := rand.New(rand.NewSource(config.RandomSeed))

	numNodes := r.Intn(config.MaxNodes-config.MinNodes+1) + config.MinNodes
	nodeIds := make([]common.NodeID, numNodes)
	for i := range numNodes {
		nodeIds[i] = []byte{byte(i)}
	}

	nodes := make([]*Node, numNodes)

	logger.Info("Initiating network with nodes", log.Int("num nodes", numNodes))
	basicNetwork := testutil.NewBasicInMemoryNetwork(t, nodeIds)

	for i := range numNodes {
		node := NewNode(t, nodeIds[i], basicNetwork, config, randomNodeConfig{})
		basicNetwork.AddNode(node.BasicNode)
		nodes[i] = node
	}

	return &Network{
		BasicInMemoryNetwork: basicNetwork,
		nodes:                nodes,
		randomness:           r,
		logger:               logger,
		t:                    t,
		config:               config,
		numNodes:             uint64(numNodes),
	}
}

func (n *Network) StartInstances() {
	panic("Call Run() Instead")
}

func (n *Network) Run() {
	n.BasicInMemoryNetwork.StartInstances()
	defer n.BasicInMemoryNetwork.StopInstances()

	targetHeight := uint64(n.config.NumFinalizedBlocks)

	for {
		n.crashAndRecoverNodes()
		txs := n.issueTxs()

		maxHeight := n.getMaxHeight()
		minHeight := n.getMinHeight()
		n.logger.Info("Issued Transactions", log.Int("count", len(txs)), log.Uint64("min height", minHeight), log.Uint64("max height", maxHeight))
		n.waitForTxAcceptance(txs)

		maxHeight = n.getMaxHeight()
		minHeight = n.getMinHeight()
		n.logger.Info("All issued transactions accepted", log.Int("count", len(txs)), log.Uint64("min height", minHeight), log.Uint64("max height", maxHeight))
		// get the max height and ensure all nodes recover to that height
		n.recoverToHeight(n.getMaxHeight())

		if minHeight >= targetHeight {
			n.logger.Info("Reached target height", log.Uint64("targetHeight", targetHeight), log.Uint64("minHeight", minHeight))
			break
		}
	}

	// if we have gotten this far, the test has succeeded so we can clear the log directory
	clearLogDirectory(n.config.LogDirectory)
}

func (n *Network) getMinHeightNodeID() common.NodeID {
	minHeight := n.nodes[0].storage.NumBlocks()
	minHeightNodeID := n.nodes[0].E.ID

	for _, node := range n.nodes[1:] {
		height := node.storage.NumBlocks()

		if height < minHeight {
			minHeight = height
			minHeightNodeID = node.E.ID
		}
	}

	return minHeightNodeID
}

func (n *Network) recoverToHeight(height uint64) {
	for n.getMinHeight() < height {
		n.logger.Debug("Advancing network time", log.Uint64("num crashed nodes", n.numCrashedNodes()),
			log.Uint64("min height", n.getMinHeight()),
			log.Uint64("max height", n.getMaxHeight()),
			log.Stringer("Smallest node ID", n.getMinHeightNodeID()),
			log.Uint64("target height", height),
		)
		for i, node := range n.nodes {
			isCrashed := node.isCrashed.Load()
			if isCrashed {
				// randomly decide to recover based on NodeRecoverPercentage
				if n.randomness.Float64() < n.config.NodeRecoverProbability {
					n.logger.Debug("Recovering node", log.Stringer("nodeID", node.E.ID))
					n.startNode(i)
				}
			}
		}

		n.lock.Lock()
		n.BasicInMemoryNetwork.AdvanceTime(n.config.AdvanceTimeTickAmount)
		n.lock.Unlock()
	}

}

func (n *Network) issueTxs() []*TX {
	n.lock.Lock()
	defer n.lock.Unlock()

	numTxs := n.randomness.Intn(n.config.MaxTxsPerIssue-n.config.MinTxsPerIssue+1) + n.config.MinTxsPerIssue // randomize between min and max inclusive
	txs := make([]*TX, 0, numTxs)

	for range numTxs {
		tx := CreateNewTX()
		txs = append(txs, tx)
		n.allIssuedTxs++
	}

	// first add to all mempools
	for _, node := range n.nodes {
		node.mempool.AddPendingTXs(txs...)
	}

	// then notify all mempools that new txs are ready
	for _, node := range n.nodes {
		node.mempool.NotifyTxsReady()
	}

	return txs
}

func (n *Network) waitForTxAcceptance(txs []*TX) {
	for {
		allAccepted := true
		for _, node := range n.nodes {
			if node.isCrashed.Load() {
				continue
			}
			if accepted := node.areTxsAccepted(txs); !accepted {
				node.mempool.lock.Lock()
				n.logger.Debug("Not all txs accepted yet by node", log.Stringer("nodeID", node.E.ID), log.Int("unaccepted txs in mempool", len(node.mempool.unacceptedTxs)))
				node.mempool.lock.Unlock()
				allAccepted = false
			}
		}

		if allAccepted {
			return
		}

		n.lock.Lock()
		n.logger.Debug("Advancing network time to wait for tx acceptance", log.Uint64("num crashed nodes", n.numCrashedNodes()))
		n.BasicInMemoryNetwork.AdvanceTime(n.config.AdvanceTimeTickAmount)
		n.lock.Unlock()

		time.Sleep(20 * time.Millisecond)
	}
}

func (n *Network) numCrashedNodes() uint64 {
	numCrashed := 0
	for _, node := range n.nodes {
		if node.isCrashed.Load() {
			numCrashed++
		}
	}
	return uint64(numCrashed)
}

func (n *Network) crashAndRecoverNodes() {
	if n.config.NodeCrashProbability == 0 {
		return
	}

	f := (int(n.numNodes) - 1) / 3

	if f == 0 {
		n.logger.Info("Not enough nodes for crash testing", log.Uint64("numNodes", n.numNodes))
		return
	}

	crashedNodes := []common.NodeID{}
	recoveredNodes := []common.NodeID{}
	maxLeftToCrash := f - int(n.numCrashedNodes())
	// go through each node, randomly decide to crash based on NodeCrashPercentage
	for i, node := range n.nodes {
		isCrashed := node.isCrashed.Load()
		if isCrashed {
			// randomly decide to recover based on NodeRecoverPercentage
			if n.randomness.Float64() < n.config.NodeRecoverProbability {
				n.startNode(i)
				recoveredNodes = append(recoveredNodes, node.E.ID)
				maxLeftToCrash++
			}
			continue
		}

		// check if we can still crash more nodes
		if maxLeftToCrash <= 0 {
			continue
		}

		// randomly decide to crash the node
		if n.randomness.Float64() < n.config.NodeCrashProbability {
			maxLeftToCrash--
			n.crashNode(i)
			crashedNodes = append(crashedNodes, node.E.ID)
		}
	}

	if len(recoveredNodes)+len(crashedNodes) > 0 {
		n.logger.Info("Recovered and crashed nodes", log.Stringers("crashed", crashedNodes), log.Stringers("recovered", recoveredNodes), log.Uint64("num crashed", n.numCrashedNodes()))
	}
}

func (n *Network) getMinHeight() uint64 {
	minHeight := n.nodes[0].storage.NumBlocks()
	for _, node := range n.nodes[1:] {
		height := node.storage.NumBlocks()

		if height < minHeight {
			minHeight = height
		}
	}
	return minHeight
}

func (n *Network) getMaxHeight() uint64 {
	maxHeight := n.nodes[0].storage.NumBlocks()
	for _, node := range n.nodes[1:] {
		height := node.storage.NumBlocks()

		if height > maxHeight {
			maxHeight = height
		}
	}
	return maxHeight
}

func (n *Network) SetInfoLog() {
	n.lock.Lock()
	defer n.lock.Unlock()

	for _, node := range n.nodes {
		node.logger.SetLevel(log.InfoLevel)
	}
}

func (n *Network) PrintStatus() {
	n.lock.Lock()
	defer n.lock.Unlock()

	// prints the number of nodes
	n.logger.Info("Network Status", log.Int("num nodes", len(n.nodes)), log.Int64("Seed", n.config.RandomSeed))

	// prints the number of txs in each node's mempool
	for _, node := range n.nodes {
		n.logger.Info("Node Status", log.Stringer("nodeID", node.E.ID), log.Int("Short", int(node.E.ID[0])), log.Uint64("Round", node.E.Metadata().Round), log.Uint64("Height", node.storage.NumBlocks()))
		node.PrintMessageTypesSent()
	}

	// prints total issued txs and failed txs
	n.logger.Info("Transaction Stats", log.Int("total issued txs", n.allIssuedTxs))
}

func (n *Network) crashNode(idx int) {
	n.logger.Debug("Crashing node", log.Stringer("nodeID", n.nodes[idx].E.ID))
	n.nodes[idx].isCrashed.Store(true)
	n.nodes[idx].Stop()
}

func (n *Network) startNode(idx int) {
	instance := n.nodes[idx]
	nodeID := instance.E.ID
	mempool := instance.mempool
	clonedWal := instance.wal.Clone()
	clonedStorage := instance.storage.Clone()
	mempool.Clear()

	newNode := NewNode(n.t, nodeID, n.BasicInMemoryNetwork, n.config, randomNodeConfig{
		mempool: mempool,
		wal:     clonedWal,
		storage: clonedStorage,
		logger:  instance.logger,
	})
	n.nodes[idx] = newNode
	n.BasicInMemoryNetwork.ReplaceNode(newNode.BasicNode)

	newNode.Start()
}