all changes from v.13.2-x branch (#1141)

accounts/abi/abi.go

@@ -32,6 +32,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -372,24 +373,65 @@ func (abi *ABI) HasReceive() bool {
 // revertSelector is a special function selector for revert reason unpacking.
 var revertSelector = crypto.Keccak256([]byte("Error(string)"))[:4]
 
+// panicSelector is a special function selector for panic reason unpacking.
+var panicSelector = crypto.Keccak256([]byte("Panic(uint256)"))[:4]
+
+// panicReasons map is for readable panic codes
+// see this linkage for the deails
+// https://docs.soliditylang.org/en/v0.8.21/control-structures.html#panic-via-assert-and-error-via-require
+// the reason string list is copied from ether.js
+// https://github.com/ethers-io/ethers.js/blob/fa3a883ff7c88611ce766f58bdd4b8ac90814470/src.ts/abi/interface.ts#L207-L218
+var panicReasons = map[uint64]string{
+	0x00: "generic panic",
+	0x01: "assert(false)",
+	0x11: "arithmetic underflow or overflow",
+	0x12: "division or modulo by zero",
+	0x21: "enum overflow",
+	0x22: "invalid encoded storage byte array accessed",
+	0x31: "out-of-bounds array access; popping on an empty array",
+	0x32: "out-of-bounds access of an array or bytesN",
+	0x41: "out of memory",
+	0x51: "uninitialized function",
+}
+
 // UnpackRevert resolves the abi-encoded revert reason. According to the solidity
 // spec https://solidity.readthedocs.io/en/latest/control-structures.html#revert,
-// the provided revert reason is abi-encoded as if it were a call to a function
-// `Error(string)`. So it's a special tool for it.
+// the provided revert reason is abi-encoded as if it were a call to function
+// `Error(string)` or `Panic(uint256)`. So it's a special tool for it.
 func UnpackRevert(data []byte) (string, error) {
 	if len(data) < 4 {
 		return "", errors.New("invalid data for unpacking")
 	}
-	if !bytes.Equal(data[:4], revertSelector) {
+	switch {
+	case bytes.Equal(data[:4], revertSelector):
+		typ, err := NewType("string", "", nil)
+		if err != nil {
+			return "", err
+		}
+		unpacked, err := (Arguments{{Type: typ}}).Unpack(data[4:])
+		if err != nil {
+			return "", err
+		}
+		return unpacked[0].(string), nil
+	case bytes.Equal(data[:4], panicSelector):
+		typ, err := NewType("uint256", "", nil)
+		if err != nil {
+			return "", err
+		}
+		unpacked, err := (Arguments{{Type: typ}}).Unpack(data[4:])
+		if err != nil {
+			return "", err
+		}
+		pCode := unpacked[0].(*big.Int)
+		// uint64 safety check for future
+		// but the code is not bigger than MAX(uint64) now
+		if pCode.IsUint64() {
+			if reason, ok := panicReasons[pCode.Uint64()]; ok {
+				return reason, nil
+			}
+		}
+		return fmt.Sprintf("unknown panic code: %#x", pCode), nil
+	default:
 		return "", errors.New("invalid data for unpacking")
 	}
-	typ, err := NewType("string", "", nil)
-	if err != nil {
-		return "", err
-	}
-	unpacked, err := (Arguments{{Type: typ}}).Unpack(data[4:])
-	if err != nil {
-		return "", err
-	}
-	return unpacked[0].(string), nil
 }

accounts/abi/abi_test.go

@@ -1184,6 +1184,8 @@ func TestUnpackRevert(t *testing.T) {
 		{"", "", errors.New("invalid data for unpacking")},
 		{"08c379a1", "", errors.New("invalid data for unpacking")},
 		{"08c379a00000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000d72657665727420726561736f6e00000000000000000000000000000000000000", "revert reason", nil},
+		{"4e487b710000000000000000000000000000000000000000000000000000000000000000", "generic panic", nil},
+		{"4e487b7100000000000000000000000000000000000000000000000000000000000000ff", "unknown panic code: 0xff", nil},
 	}
 	for index, c := range cases {
 		t.Run(fmt.Sprintf("case %d", index), func(t *testing.T) {

accounts/abi/bind/backends/simulated.go

@@ -235,7 +235,6 @@ func (b *SimulatedBackend) CodeAt(ctx context.Context, contract common.Address,
 	if err != nil {
 		return nil, err
 	}
-
 	return stateDB.GetCode(contract), nil
 }
 
@@ -248,7 +247,6 @@ func (b *SimulatedBackend) BalanceAt(ctx context.Context, contract common.Addres
 	if err != nil {
 		return nil, err
 	}
-
 	return stateDB.GetBalance(contract), nil
 }
 
@@ -261,7 +259,6 @@ func (b *SimulatedBackend) NonceAt(ctx context.Context, contract common.Address,
 	if err != nil {
 		return 0, err
 	}
-
 	return stateDB.GetNonce(contract), nil
 }
 
@@ -274,7 +271,6 @@ func (b *SimulatedBackend) StorageAt(ctx context.Context, contract common.Addres
 	if err != nil {
 		return nil, err
 	}
-
 	val := stateDB.GetState(contract, key)
 	return val[:], nil
 }
@@ -739,8 +735,10 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
 	if err != nil {
 		return err
 	}
-	stateDB, _ := b.blockchain.State()
-
+	stateDB, err := b.blockchain.State()
+	if err != nil {
+		return err
+	}
 	b.acceptedBlock = blocks[0]
 	b.acceptedState, _ = state.New(b.acceptedBlock.Root(), stateDB.Database(), nil)
 	return nil
@@ -858,11 +856,12 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error {
 	blocks, _, _ := core.GenerateChain(b.config, block, dummy.NewFaker(), b.database, 1, 10, func(number int, block *core.BlockGen) {
 		block.OffsetTime(int64(adjustment.Seconds()))
 	})
-	stateDB, _ := b.blockchain.State()
-
+	stateDB, err := b.blockchain.State()
+	if err != nil {
+		return err
+	}
 	b.acceptedBlock = blocks[0]
 	b.acceptedState, _ = state.New(b.acceptedBlock.Root(), stateDB.Database(), nil)
-
 	return nil
 }
 

accounts/abi/method.go

@@ -137,11 +137,12 @@ func NewMethod(name string, rawName string, funType FunctionType, mutability str
 		state = state + " "
 	}
 	identity := fmt.Sprintf("function %v", rawName)
-	if funType == Fallback {
+	switch funType {
+	case Fallback:
 		identity = "fallback"
-	} else if funType == Receive {
+	case Receive:
 		identity = "receive"
-	} else if funType == Constructor {
+	case Constructor:
 		identity = "constructor"
 	}
 	str := fmt.Sprintf("%v(%v) %sreturns(%v)", identity, strings.Join(inputNames, ", "), state, strings.Join(outputNames, ", "))

accounts/abi/method_test.go

@@ -94,11 +94,12 @@ func TestMethodString(t *testing.T) {
 
 	for _, test := range table {
 		var got string
-		if test.method == "fallback" {
+		switch test.method {
+		case "fallback":
 			got = abi.Fallback.String()
-		} else if test.method == "receive" {
+		case "receive":
 			got = abi.Receive.String()
-		} else {
+		default:
 			got = abi.Methods[test.method].String()
 		}
 		if got != test.expectation {

accounts/abi/unpack.go

@@ -170,13 +170,14 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error)
 	// this value will become our slice or our array, depending on the type
 	var refSlice reflect.Value
 
-	if t.T == SliceTy {
+	switch t.T {
+	case SliceTy:
 		// declare our slice
 		refSlice = reflect.MakeSlice(t.GetType(), size, size)
-	} else if t.T == ArrayTy {
+	case ArrayTy:
 		// declare our array
 		refSlice = reflect.New(t.GetType()).Elem()
-	} else {
+	default:
 		return nil, errors.New("abi: invalid type in array/slice unpacking stage")
 	}
 

accounts/keystore/watch.go

@@ -30,6 +30,7 @@
 package keystore
 
 import (
+	"os"
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
@@ -87,7 +88,9 @@ func (w *watcher) loop() {
 	}
 	defer watcher.Close()
 	if err := watcher.Add(w.ac.keydir); err != nil {
-		logger.Warn("Failed to watch keystore folder", "err", err)
+		if !os.IsNotExist(err) {
+			logger.Warn("Failed to watch keystore folder", "err", err)
+		}
 		return
 	}
 

cmd/evm/internal/t8ntool/execution.go

@@ -76,25 +76,26 @@ type ommer struct {
 
 //go:generate go run github.com/fjl/gencodec -type stEnv -field-override stEnvMarshaling -out gen_stenv.go
 type stEnv struct {
-	Coinbase            common.Address                      `json:"currentCoinbase"   gencodec:"required"`
-	Difficulty          *big.Int                            `json:"currentDifficulty"`
-	Random              *big.Int                            `json:"currentRandom"`
-	ParentDifficulty    *big.Int                            `json:"parentDifficulty"`
-	ParentBaseFee       *big.Int                            `json:"parentBaseFee,omitempty"`
-	ParentGasUsed       uint64                              `json:"parentGasUsed,omitempty"`
-	ParentGasLimit      uint64                              `json:"parentGasLimit,omitempty"`
-	MinBaseFee          *big.Int                            `json:"minBaseFee,omitempty"`
-	GasLimit            uint64                              `json:"currentGasLimit"   gencodec:"required"`
-	Number              uint64                              `json:"currentNumber"     gencodec:"required"`
-	Timestamp           uint64                              `json:"currentTimestamp"  gencodec:"required"`
-	ParentTimestamp     uint64                              `json:"parentTimestamp,omitempty"`
-	BlockHashes         map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"`
-	Ommers              []ommer                             `json:"ommers,omitempty"`
-	BaseFee             *big.Int                            `json:"currentBaseFee,omitempty"`
-	ParentUncleHash     common.Hash                         `json:"parentUncleHash"`
-	ExcessBlobGas       *uint64                             `json:"excessBlobGas,omitempty"`
-	ParentExcessBlobGas *uint64                             `json:"parentExcessBlobGas,omitempty"`
-	ParentBlobGasUsed   *uint64                             `json:"parentBlobGasUsed,omitempty"`
+	Coinbase              common.Address                      `json:"currentCoinbase"   gencodec:"required"`
+	Difficulty            *big.Int                            `json:"currentDifficulty"`
+	Random                *big.Int                            `json:"currentRandom"`
+	ParentDifficulty      *big.Int                            `json:"parentDifficulty"`
+	ParentBaseFee         *big.Int                            `json:"parentBaseFee,omitempty"`
+	ParentGasUsed         uint64                              `json:"parentGasUsed,omitempty"`
+	ParentGasLimit        uint64                              `json:"parentGasLimit,omitempty"`
+	MinBaseFee            *big.Int                            `json:"minBaseFee,omitempty"`
+	GasLimit              uint64                              `json:"currentGasLimit"   gencodec:"required"`
+	Number                uint64                              `json:"currentNumber"     gencodec:"required"`
+	Timestamp             uint64                              `json:"currentTimestamp"  gencodec:"required"`
+	ParentTimestamp       uint64                              `json:"parentTimestamp,omitempty"`
+	BlockHashes           map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"`
+	Ommers                []ommer                             `json:"ommers,omitempty"`
+	BaseFee               *big.Int                            `json:"currentBaseFee,omitempty"`
+	ParentUncleHash       common.Hash                         `json:"parentUncleHash"`
+	ExcessBlobGas         *uint64                             `json:"excessBlobGas,omitempty"`
+	ParentExcessBlobGas   *uint64                             `json:"parentExcessBlobGas,omitempty"`
+	ParentBlobGasUsed     *uint64                             `json:"parentBlobGasUsed,omitempty"`
+	ParentBeaconBlockRoot *common.Hash                        `json:"parentBeaconBlockRoot"`
 }
 
 type stEnvMarshaling struct {
@@ -191,6 +192,10 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 	// 	chainConfig.DAOForkBlock.Cmp(new(big.Int).SetUint64(pre.Env.Number)) == 0 {
 	// 	misc.ApplyDAOHardFork(statedb)
 	// }
+	if beaconRoot := pre.Env.ParentBeaconBlockRoot; beaconRoot != nil {
+		evm := vm.NewEVM(vmContext, vm.TxContext{}, statedb, chainConfig, vmConfig)
+		core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb)
+	}
 	var blobGasUsed uint64
 	for i, tx := range txs {
 		if tx.Type() == types.BlobTxType && vmContext.ExcessBlobGas == nil {

cmd/evm/internal/t8ntool/gen_stenv.go

@@ -16,25 +16,26 @@ var _ = (*stEnvMarshaling)(nil)
 // MarshalJSON marshals as JSON.
 func (s stEnv) MarshalJSON() ([]byte, error) {
 	type stEnv struct {
-		Coinbase            common.UnprefixedAddress            `json:"currentCoinbase"   gencodec:"required"`
-		Difficulty          *math.HexOrDecimal256               `json:"currentDifficulty"`
-		Random              *math.HexOrDecimal256               `json:"currentRandom"`
-		ParentDifficulty    *math.HexOrDecimal256               `json:"parentDifficulty"`
-		ParentBaseFee       *math.HexOrDecimal256               `json:"parentBaseFee,omitempty"`
-		ParentGasUsed       math.HexOrDecimal64                 `json:"parentGasUsed,omitempty"`
-		ParentGasLimit      math.HexOrDecimal64                 `json:"parentGasLimit,omitempty"`
-		MinBaseFee          *math.HexOrDecimal256               `json:"minBaseFee,omitempty"`
-		GasLimit            math.HexOrDecimal64                 `json:"currentGasLimit"   gencodec:"required"`
-		Number              math.HexOrDecimal64                 `json:"currentNumber"     gencodec:"required"`
-		Timestamp           math.HexOrDecimal64                 `json:"currentTimestamp"  gencodec:"required"`
-		ParentTimestamp     math.HexOrDecimal64                 `json:"parentTimestamp,omitempty"`
-		BlockHashes         map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"`
-		Ommers              []ommer                             `json:"ommers,omitempty"`
-		BaseFee             *math.HexOrDecimal256               `json:"currentBaseFee,omitempty"`
-		ParentUncleHash     common.Hash                         `json:"parentUncleHash"`
-		ExcessBlobGas       *math.HexOrDecimal64                `json:"excessBlobGas,omitempty"`
-		ParentExcessBlobGas *math.HexOrDecimal64                `json:"parentExcessBlobGas,omitempty"`
-		ParentBlobGasUsed   *math.HexOrDecimal64                `json:"parentBlobGasUsed,omitempty"`
+		Coinbase              common.UnprefixedAddress            `json:"currentCoinbase"   gencodec:"required"`
+		Difficulty            *math.HexOrDecimal256               `json:"currentDifficulty"`
+		Random                *math.HexOrDecimal256               `json:"currentRandom"`
+		ParentDifficulty      *math.HexOrDecimal256               `json:"parentDifficulty"`
+		ParentBaseFee         *math.HexOrDecimal256               `json:"parentBaseFee,omitempty"`
+		ParentGasUsed         math.HexOrDecimal64                 `json:"parentGasUsed,omitempty"`
+		ParentGasLimit        math.HexOrDecimal64                 `json:"parentGasLimit,omitempty"`
+		MinBaseFee            *math.HexOrDecimal256               `json:"minBaseFee,omitempty"`
+		GasLimit              math.HexOrDecimal64                 `json:"currentGasLimit"   gencodec:"required"`
+		Number                math.HexOrDecimal64                 `json:"currentNumber"     gencodec:"required"`
+		Timestamp             math.HexOrDecimal64                 `json:"currentTimestamp"  gencodec:"required"`
+		ParentTimestamp       math.HexOrDecimal64                 `json:"parentTimestamp,omitempty"`
+		BlockHashes           map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"`
+		Ommers                []ommer                             `json:"ommers,omitempty"`
+		BaseFee               *math.HexOrDecimal256               `json:"currentBaseFee,omitempty"`
+		ParentUncleHash       common.Hash                         `json:"parentUncleHash"`
+		ExcessBlobGas         *math.HexOrDecimal64                `json:"excessBlobGas,omitempty"`
+		ParentExcessBlobGas   *math.HexOrDecimal64                `json:"parentExcessBlobGas,omitempty"`
+		ParentBlobGasUsed     *math.HexOrDecimal64                `json:"parentBlobGasUsed,omitempty"`
+		ParentBeaconBlockRoot *common.Hash                        `json:"parentBeaconBlockRoot"`
 	}
 	var enc stEnv
 	enc.Coinbase = common.UnprefixedAddress(s.Coinbase)
@@ -56,31 +57,33 @@ func (s stEnv) MarshalJSON() ([]byte, error) {
 	enc.ExcessBlobGas = (*math.HexOrDecimal64)(s.ExcessBlobGas)
 	enc.ParentExcessBlobGas = (*math.HexOrDecimal64)(s.ParentExcessBlobGas)
 	enc.ParentBlobGasUsed = (*math.HexOrDecimal64)(s.ParentBlobGasUsed)
+	enc.ParentBeaconBlockRoot = s.ParentBeaconBlockRoot
 	return json.Marshal(&enc)
 }
 
 // UnmarshalJSON unmarshals from JSON.
 func (s *stEnv) UnmarshalJSON(input []byte) error {
 	type stEnv struct {
-		Coinbase            *common.UnprefixedAddress           `json:"currentCoinbase"   gencodec:"required"`
-		Difficulty          *math.HexOrDecimal256               `json:"currentDifficulty"`
-		Random              *math.HexOrDecimal256               `json:"currentRandom"`
-		ParentDifficulty    *math.HexOrDecimal256               `json:"parentDifficulty"`
-		ParentBaseFee       *math.HexOrDecimal256               `json:"parentBaseFee,omitempty"`
-		ParentGasUsed       *math.HexOrDecimal64                `json:"parentGasUsed,omitempty"`
-		ParentGasLimit      *math.HexOrDecimal64                `json:"parentGasLimit,omitempty"`
-		MinBaseFee          *math.HexOrDecimal256               `json:"minBaseFee,omitempty"`
-		GasLimit            *math.HexOrDecimal64                `json:"currentGasLimit"   gencodec:"required"`
-		Number              *math.HexOrDecimal64                `json:"currentNumber"     gencodec:"required"`
-		Timestamp           *math.HexOrDecimal64                `json:"currentTimestamp"  gencodec:"required"`
-		ParentTimestamp     *math.HexOrDecimal64                `json:"parentTimestamp,omitempty"`
-		BlockHashes         map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"`
-		Ommers              []ommer                             `json:"ommers,omitempty"`
-		BaseFee             *math.HexOrDecimal256               `json:"currentBaseFee,omitempty"`
-		ParentUncleHash     *common.Hash                        `json:"parentUncleHash"`
-		ExcessBlobGas       *math.HexOrDecimal64                `json:"excessBlobGas,omitempty"`
-		ParentExcessBlobGas *math.HexOrDecimal64                `json:"parentExcessBlobGas,omitempty"`
-		ParentBlobGasUsed   *math.HexOrDecimal64                `json:"parentBlobGasUsed,omitempty"`
+		Coinbase              *common.UnprefixedAddress           `json:"currentCoinbase"   gencodec:"required"`
+		Difficulty            *math.HexOrDecimal256               `json:"currentDifficulty"`
+		Random                *math.HexOrDecimal256               `json:"currentRandom"`
+		ParentDifficulty      *math.HexOrDecimal256               `json:"parentDifficulty"`
+		ParentBaseFee         *math.HexOrDecimal256               `json:"parentBaseFee,omitempty"`
+		ParentGasUsed         *math.HexOrDecimal64                `json:"parentGasUsed,omitempty"`
+		ParentGasLimit        *math.HexOrDecimal64                `json:"parentGasLimit,omitempty"`
+		MinBaseFee            *math.HexOrDecimal256               `json:"minBaseFee,omitempty"`
+		GasLimit              *math.HexOrDecimal64                `json:"currentGasLimit"   gencodec:"required"`
+		Number                *math.HexOrDecimal64                `json:"currentNumber"     gencodec:"required"`
+		Timestamp             *math.HexOrDecimal64                `json:"currentTimestamp"  gencodec:"required"`
+		ParentTimestamp       *math.HexOrDecimal64                `json:"parentTimestamp,omitempty"`
+		BlockHashes           map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"`
+		Ommers                []ommer                             `json:"ommers,omitempty"`
+		BaseFee               *math.HexOrDecimal256               `json:"currentBaseFee,omitempty"`
+		ParentUncleHash       *common.Hash                        `json:"parentUncleHash"`
+		ExcessBlobGas         *math.HexOrDecimal64                `json:"excessBlobGas,omitempty"`
+		ParentExcessBlobGas   *math.HexOrDecimal64                `json:"parentExcessBlobGas,omitempty"`
+		ParentBlobGasUsed     *math.HexOrDecimal64                `json:"parentBlobGasUsed,omitempty"`
+		ParentBeaconBlockRoot *common.Hash                        `json:"parentBeaconBlockRoot"`
 	}
 	var dec stEnv
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -147,5 +150,8 @@ func (s *stEnv) UnmarshalJSON(input []byte) error {
 	if dec.ParentBlobGasUsed != nil {
 		s.ParentBlobGasUsed = (*uint64)(dec.ParentBlobGasUsed)
 	}
+	if dec.ParentBeaconBlockRoot != nil {
+		s.ParentBeaconBlockRoot = dec.ParentBeaconBlockRoot
+	}
 	return nil
 }

cmd/evm/internal/t8ntool/transition.go

@@ -202,93 +202,21 @@ func Transition(ctx *cli.Context) error {
 	// Set the chain id
 	chainConfig.ChainID = big.NewInt(ctx.Int64(ChainIDFlag.Name))
 
-	var txsWithKeys []*txWithKey
-	if txStr != stdinSelector {
-		inFile, err := os.Open(txStr)
-		if err != nil {
-			return NewError(ErrorIO, fmt.Errorf("failed reading txs file: %v", err))
-		}
-		defer inFile.Close()
-		decoder := json.NewDecoder(inFile)
-		if strings.HasSuffix(txStr, ".rlp") {
-			var body hexutil.Bytes
-			if err := decoder.Decode(&body); err != nil {
-				return err
-			}
-			var txs types.Transactions
-			if err := rlp.DecodeBytes(body, &txs); err != nil {
-				return err
-			}
-			for _, tx := range txs {
-				txsWithKeys = append(txsWithKeys, &txWithKey{
-					key: nil,
-					tx:  tx,
-				})
-			}
-		} else {
-			if err := decoder.Decode(&txsWithKeys); err != nil {
-				return NewError(ErrorJson, fmt.Errorf("failed unmarshaling txs-file: %v", err))
-			}
-		}
-	} else {
-		if len(inputData.TxRlp) > 0 {
-			// Decode the body of already signed transactions
-			body := common.FromHex(inputData.TxRlp)
-			var txs types.Transactions
-			if err := rlp.DecodeBytes(body, &txs); err != nil {
-				return err
-			}
-			for _, tx := range txs {
-				txsWithKeys = append(txsWithKeys, &txWithKey{
-					key: nil,
-					tx:  tx,
-				})
-			}
-		} else {
-			// JSON encoded transactions
-			txsWithKeys = inputData.Txs
-		}
+	if txs, err = loadTransactions(txStr, inputData, prestate.Env, chainConfig); err != nil {
+		return err
 	}
-	// We may have to sign the transactions.
-	signer := types.MakeSigner(chainConfig, big.NewInt(int64(prestate.Env.Number)), prestate.Env.Timestamp)
-
-	if txs, err = signUnsignedTransactions(txsWithKeys, signer); err != nil {
-		return NewError(ErrorJson, fmt.Errorf("failed signing transactions: %v", err))
-	}
-	// Sanity check, to not `panic` in state_transition
-	// NOTE: IsLondon replaced with IsSubnetEVM here
-	if chainConfig.IsSubnetEVM(prestate.Env.Timestamp) {
-		if prestate.Env.BaseFee != nil {
-			// Already set, base fee has precedent over parent base fee.
-		} else if prestate.Env.ParentBaseFee != nil && prestate.Env.Number != 0 {
-			parent := &types.Header{
-				Number:   new(big.Int).SetUint64(prestate.Env.Number - 1),
-				Time:     prestate.Env.ParentTimestamp,
-				BaseFee:  prestate.Env.ParentBaseFee,
-				GasUsed:  prestate.Env.ParentGasUsed,
-				GasLimit: prestate.Env.ParentGasLimit,
-				Extra:    make([]byte, params.DynamicFeeExtraDataSize), // TODO: consider passing extra through env
-			}
-			feeConfig := params.DefaultFeeConfig
-			if prestate.Env.MinBaseFee != nil {
-				// Override the default min base fee if it's set in the env
-				feeConfig.MinBaseFee = prestate.Env.MinBaseFee
-			}
-			_, prestate.Env.BaseFee, err = dummy.CalcBaseFee(chainConfig, feeConfig, parent, prestate.Env.Timestamp)
-			if err != nil {
-				return NewError(ErrorConfig, fmt.Errorf("failed calculating base fee: %v", err))
-			}
-		} else {
-			return NewError(ErrorConfig, errors.New("EIP-1559 config but missing 'currentBaseFee' in env section"))
-		}
+	if err := applyLondonChecks(&prestate.Env, chainConfig); err != nil {
+		return err
 	}
 	// NOTE: Removed isMerged logic here.
-	// isMerged := chainConfig.TerminalTotalDifficulty != nil && chainConfig.TerminalTotalDifficulty.BitLen() == 0
 	if prestate.Env.Random != nil {
 		// NOTE: subnet-evm continues to return the difficulty value for the RANDOM opcode,
 		// so for testing if Random is set in the environment, we copy it to difficulty instead.
 		prestate.Env.Difficulty = prestate.Env.Random
 	}
+	if err := applyCancunChecks(&prestate.Env, chainConfig); err != nil {
+		return err
+	}
 	// Run the test and aggregate the result
 	s, result, err := prestate.Apply(vmConfig, chainConfig, txs, ctx.Int64(RewardFlag.Name), getTracer)
 	if err != nil {
@@ -355,33 +283,118 @@ func (t *txWithKey) UnmarshalJSON(input []byte) error {
 // and secondly to read them with the standard tx json format
 func signUnsignedTransactions(txs []*txWithKey, signer types.Signer) (types.Transactions, error) {
 	var signedTxs []*types.Transaction
-	for i, txWithKey := range txs {
-		tx := txWithKey.tx
-		key := txWithKey.key
-		v, r, s := tx.RawSignatureValues()
-		if key != nil && v.BitLen()+r.BitLen()+s.BitLen() == 0 {
-			// This transaction needs to be signed
-			var (
-				signed *types.Transaction
-				err    error
-			)
-			if txWithKey.protected {
-				signed, err = types.SignTx(tx, signer, key)
-			} else {
-				signed, err = types.SignTx(tx, types.FrontierSigner{}, key)
-			}
-			if err != nil {
-				return nil, NewError(ErrorJson, fmt.Errorf("tx %d: failed to sign tx: %v", i, err))
-			}
-			signedTxs = append(signedTxs, signed)
-		} else {
+	for i, tx := range txs {
+		var (
+			v, r, s = tx.tx.RawSignatureValues()
+			signed  *types.Transaction
+			err     error
+		)
+		if tx.key == nil || v.BitLen()+r.BitLen()+s.BitLen() != 0 {
 			// Already signed
-			signedTxs = append(signedTxs, tx)
+			signedTxs = append(signedTxs, tx.tx)
+			continue
 		}
+		// This transaction needs to be signed
+		if tx.protected {
+			signed, err = types.SignTx(tx.tx, signer, tx.key)
+		} else {
+			signed, err = types.SignTx(tx.tx, types.FrontierSigner{}, tx.key)
+		}
+		if err != nil {
+			return nil, NewError(ErrorJson, fmt.Errorf("tx %d: failed to sign tx: %v", i, err))
+		}
+		signedTxs = append(signedTxs, signed)
 	}
 	return signedTxs, nil
 }
 
+func loadTransactions(txStr string, inputData *input, env stEnv, chainConfig *params.ChainConfig) (types.Transactions, error) {
+	var txsWithKeys []*txWithKey
+	var signed types.Transactions
+	if txStr != stdinSelector {
+		data, err := os.ReadFile(txStr)
+		if err != nil {
+			return nil, NewError(ErrorIO, fmt.Errorf("failed reading txs file: %v", err))
+		}
+		if strings.HasSuffix(txStr, ".rlp") { // A file containing an rlp list
+			var body hexutil.Bytes
+			if err := json.Unmarshal(data, &body); err != nil {
+				return nil, err
+			}
+			// Already signed transactions
+			if err := rlp.DecodeBytes(body, &signed); err != nil {
+				return nil, err
+			}
+			return signed, nil
+		}
+		if err := json.Unmarshal(data, &txsWithKeys); err != nil {
+			return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshaling txs-file: %v", err))
+		}
+	} else {
+		if len(inputData.TxRlp) > 0 {
+			// Decode the body of already signed transactions
+			body := common.FromHex(inputData.TxRlp)
+			// Already signed transactions
+			if err := rlp.DecodeBytes(body, &signed); err != nil {
+				return nil, err
+			}
+			return signed, nil
+		}
+		// JSON encoded transactions
+		txsWithKeys = inputData.Txs
+	}
+	// We may have to sign the transactions.
+	signer := types.MakeSigner(chainConfig, big.NewInt(int64(env.Number)), env.Timestamp)
+	return signUnsignedTransactions(txsWithKeys, signer)
+}
+
+func applyLondonChecks(env *stEnv, chainConfig *params.ChainConfig) error {
+	// NOTE: IsLondon replaced with IsSubnetEVM here
+	if !chainConfig.IsSubnetEVM(env.Timestamp) {
+		return nil
+	}
+	// Sanity check, to not `panic` in state_transition
+	if env.BaseFee != nil {
+		// Already set, base fee has precedent over parent base fee.
+		return nil
+	}
+	if env.ParentBaseFee == nil || env.Number == 0 {
+		return NewError(ErrorConfig, errors.New("EIP-1559 config but missing 'currentBaseFee' in env section"))
+	}
+	parent := &types.Header{
+		Number:   new(big.Int).SetUint64(env.Number - 1),
+		Time:     env.ParentTimestamp,
+		BaseFee:  env.ParentBaseFee,
+		GasUsed:  env.ParentGasUsed,
+		GasLimit: env.ParentGasLimit,
+		Extra:    make([]byte, params.DynamicFeeExtraDataSize), // TODO: consider passing extra through env
+	}
+	feeConfig := params.DefaultFeeConfig
+	if env.MinBaseFee != nil {
+		// Override the default min base fee if it's set in the env
+		feeConfig.MinBaseFee = env.MinBaseFee
+	}
+	var err error
+	_, env.BaseFee, err = dummy.CalcBaseFee(chainConfig, feeConfig, parent, env.Timestamp)
+	if err != nil {
+		return NewError(ErrorConfig, fmt.Errorf("failed calculating base fee: %v", err))
+	}
+	return nil
+}
+
+func applyCancunChecks(env *stEnv, chainConfig *params.ChainConfig) error {
+	if !chainConfig.IsCancun(big.NewInt(int64(env.Number)), env.Timestamp) {
+		env.ParentBeaconBlockRoot = nil // un-set it if it has been set too early
+		return nil
+	}
+	// Post-cancun
+	// We require EIP-4788 beacon root to be set in the env
+	if env.ParentBeaconBlockRoot == nil {
+		return NewError(ErrorConfig, errors.New("post-cancun env requires parentBeaconBlockRoot to be set"))
+	}
+	return nil
+}
+
 type Alloc map[common.Address]core.GenesisAccount
 
 func (g Alloc) OnRoot(common.Hash) {}

cmd/evm/main.go

@@ -33,107 +33,116 @@ import (
 	"os"
 
 	"github.com/ava-labs/subnet-evm/cmd/evm/internal/t8ntool"
+	"github.com/ava-labs/subnet-evm/internal/debug"
 	"github.com/ava-labs/subnet-evm/internal/flags"
 	"github.com/urfave/cli/v2"
 )
 
 var (
 	DebugFlag = &cli.BoolFlag{
-		Name:  "debug",
-		Usage: "output full trace logs",
-	}
-	MemProfileFlag = &cli.StringFlag{
-		Name:  "memprofile",
-		Usage: "creates a memory profile at the given path",
-	}
-	CPUProfileFlag = &cli.StringFlag{
-		Name:  "cpuprofile",
-		Usage: "creates a CPU profile at the given path",
+		Name:     "debug",
+		Usage:    "output full trace logs",
+		Category: flags.VMCategory,
 	}
 	StatDumpFlag = &cli.BoolFlag{
-		Name:  "statdump",
-		Usage: "displays stack and heap memory information",
+		Name:     "statdump",
+		Usage:    "displays stack and heap memory information",
+		Category: flags.VMCategory,
 	}
 	CodeFlag = &cli.StringFlag{
-		Name:  "code",
-		Usage: "EVM code",
+		Name:     "code",
+		Usage:    "EVM code",
+		Category: flags.VMCategory,
 	}
 	CodeFileFlag = &cli.StringFlag{
-		Name:  "codefile",
-		Usage: "File containing EVM code. If '-' is specified, code is read from stdin ",
+		Name:     "codefile",
+		Usage:    "File containing EVM code. If '-' is specified, code is read from stdin ",
+		Category: flags.VMCategory,
 	}
 	GasFlag = &cli.Uint64Flag{
-		Name:  "gas",
-		Usage: "gas limit for the evm",
-		Value: 10000000000,
+		Name:     "gas",
+		Usage:    "gas limit for the evm",
+		Value:    10000000000,
+		Category: flags.VMCategory,
 	}
 	PriceFlag = &flags.BigFlag{
-		Name:  "price",
-		Usage: "price set for the evm",
-		Value: new(big.Int),
+		Name:     "price",
+		Usage:    "price set for the evm",
+		Value:    new(big.Int),
+		Category: flags.VMCategory,
 	}
 	ValueFlag = &flags.BigFlag{
-		Name:  "value",
-		Usage: "value set for the evm",
-		Value: new(big.Int),
+		Name:     "value",
+		Usage:    "value set for the evm",
+		Value:    new(big.Int),
+		Category: flags.VMCategory,
 	}
 	DumpFlag = &cli.BoolFlag{
-		Name:  "dump",
-		Usage: "dumps the state after the run",
+		Name:     "dump",
+		Usage:    "dumps the state after the run",
+		Category: flags.VMCategory,
 	}
 	InputFlag = &cli.StringFlag{
-		Name:  "input",
-		Usage: "input for the EVM",
+		Name:     "input",
+		Usage:    "input for the EVM",
+		Category: flags.VMCategory,
 	}
 	InputFileFlag = &cli.StringFlag{
-		Name:  "inputfile",
-		Usage: "file containing input for the EVM",
-	}
-	VerbosityFlag = &cli.IntFlag{
-		Name:  "verbosity",
-		Usage: "sets the verbosity level",
+		Name:     "inputfile",
+		Usage:    "file containing input for the EVM",
+		Category: flags.VMCategory,
 	}
 	BenchFlag = &cli.BoolFlag{
-		Name:  "bench",
-		Usage: "benchmark the execution",
+		Name:     "bench",
+		Usage:    "benchmark the execution",
+		Category: flags.VMCategory,
 	}
 	CreateFlag = &cli.BoolFlag{
-		Name:  "create",
-		Usage: "indicates the action should be create rather than call",
+		Name:     "create",
+		Usage:    "indicates the action should be create rather than call",
+		Category: flags.VMCategory,
 	}
 	GenesisFlag = &cli.StringFlag{
-		Name:  "prestate",
-		Usage: "JSON file with prestate (genesis) config",
+		Name:     "prestate",
+		Usage:    "JSON file with prestate (genesis) config",
+		Category: flags.VMCategory,
 	}
 	MachineFlag = &cli.BoolFlag{
-		Name:  "json",
-		Usage: "output trace logs in machine readable format (json)",
+		Name:     "json",
+		Usage:    "output trace logs in machine readable format (json)",
+		Category: flags.VMCategory,
 	}
 	SenderFlag = &cli.StringFlag{
-		Name:  "sender",
-		Usage: "The transaction origin",
+		Name:     "sender",
+		Usage:    "The transaction origin",
+		Category: flags.VMCategory,
 	}
 	ReceiverFlag = &cli.StringFlag{
-		Name:  "receiver",
-		Usage: "The transaction receiver (execution context)",
+		Name:     "receiver",
+		Usage:    "The transaction receiver (execution context)",
+		Category: flags.VMCategory,
 	}
 	DisableMemoryFlag = &cli.BoolFlag{
-		Name:  "nomemory",
-		Value: true,
-		Usage: "disable memory output",
+		Name:     "nomemory",
+		Value:    true,
+		Usage:    "disable memory output",
+		Category: flags.VMCategory,
 	}
 	DisableStackFlag = &cli.BoolFlag{
-		Name:  "nostack",
-		Usage: "disable stack output",
+		Name:     "nostack",
+		Usage:    "disable stack output",
+		Category: flags.VMCategory,
 	}
 	DisableStorageFlag = &cli.BoolFlag{
-		Name:  "nostorage",
-		Usage: "disable storage output",
+		Name:     "nostorage",
+		Usage:    "disable storage output",
+		Category: flags.VMCategory,
 	}
 	DisableReturnDataFlag = &cli.BoolFlag{
-		Name:  "noreturndata",
-		Value: true,
-		Usage: "enable return data output",
+		Name:     "noreturndata",
+		Value:    true,
+		Usage:    "enable return data output",
+		Category: flags.VMCategory,
 	}
 )
 
@@ -192,34 +201,38 @@ var blockBuilderCommand = &cli.Command{
 	},
 }
 
+// vmFlags contains flags related to running the EVM.
+var vmFlags = []cli.Flag{
+	CodeFlag,
+	CodeFileFlag,
+	CreateFlag,
+	GasFlag,
+	PriceFlag,
+	ValueFlag,
+	InputFlag,
+	InputFileFlag,
+	GenesisFlag,
+	SenderFlag,
+	ReceiverFlag,
+}
+
+// traceFlags contains flags that configure tracing output.
+var traceFlags = []cli.Flag{
+	BenchFlag,
+	DebugFlag,
+	DumpFlag,
+	MachineFlag,
+	StatDumpFlag,
+	DisableMemoryFlag,
+	DisableStackFlag,
+	DisableStorageFlag,
+	DisableReturnDataFlag,
+}
+
 var app = flags.NewApp("the evm command line interface")
 
 func init() {
-	app.Flags = []cli.Flag{
-		BenchFlag,
-		CreateFlag,
-		DebugFlag,
-		VerbosityFlag,
-		CodeFlag,
-		CodeFileFlag,
-		GasFlag,
-		PriceFlag,
-		ValueFlag,
-		DumpFlag,
-		InputFlag,
-		InputFileFlag,
-		MemProfileFlag,
-		CPUProfileFlag,
-		StatDumpFlag,
-		GenesisFlag,
-		MachineFlag,
-		SenderFlag,
-		ReceiverFlag,
-		DisableMemoryFlag,
-		DisableStackFlag,
-		DisableStorageFlag,
-		DisableReturnDataFlag,
-	}
+	app.Flags = flags.Merge(vmFlags, traceFlags, debug.Flags)
 	app.Commands = []*cli.Command{
 		compileCommand,
 		disasmCommand,
@@ -229,6 +242,14 @@ func init() {
 		transactionCommand,
 		blockBuilderCommand,
 	}
+	app.Before = func(ctx *cli.Context) error {
+		flags.MigrateGlobalFlags(ctx)
+		return debug.Setup(ctx)
+	}
+	app.After = func(ctx *cli.Context) error {
+		debug.Exit()
+		return nil
+	}
 }
 
 func main() {

cmd/evm/runner.go

@@ -34,7 +34,6 @@ import (
 	"math/big"
 	"os"
 	goruntime "runtime"
-	"runtime/pprof"
 	"testing"
 	"time"
 
@@ -43,15 +42,14 @@ import (
 	"github.com/ava-labs/subnet-evm/core"
 	"github.com/ava-labs/subnet-evm/core/rawdb"
 	"github.com/ava-labs/subnet-evm/core/state"
-	"github.com/ava-labs/subnet-evm/core/types"
 	"github.com/ava-labs/subnet-evm/core/vm"
 	"github.com/ava-labs/subnet-evm/core/vm/runtime"
 	"github.com/ava-labs/subnet-evm/eth/tracers/logger"
 	"github.com/ava-labs/subnet-evm/internal/flags"
 	"github.com/ava-labs/subnet-evm/params"
 	"github.com/ava-labs/subnet-evm/trie"
+	"github.com/ava-labs/subnet-evm/trie/triedb/hashdb"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/urfave/cli/v2"
 )
 
@@ -61,6 +59,7 @@ var runCommand = &cli.Command{
 	Usage:       "run arbitrary evm binary",
 	ArgsUsage:   "<code>",
 	Description: `The run command runs arbitrary EVM code.`,
+	Flags:       flags.Merge(vmFlags, traceFlags),
 }
 
 // readGenesis will read the given JSON format genesis file and return
@@ -118,9 +117,6 @@ func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) (output []by
 }
 
 func runCmd(ctx *cli.Context) error {
-	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
-	glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name)))
-	log.Root().SetHandler(glogger)
 	logconfig := &logger.Config{
 		EnableMemory:     !ctx.Bool(DisableMemoryFlag.Name),
 		DisableStack:     ctx.Bool(DisableStackFlag.Name),
@@ -130,15 +126,14 @@ func runCmd(ctx *cli.Context) error {
 	}
 
 	var (
-		tracer        vm.EVMLogger
-		debugLogger   *logger.StructLogger
-		statedb       *state.StateDB
-		chainConfig   *params.ChainConfig
-		sender        = common.BytesToAddress([]byte("sender"))
-		receiver      = common.BytesToAddress([]byte("receiver"))
-		genesisConfig *core.Genesis
-		preimages     = ctx.Bool(DumpFlag.Name)
-		blobHashes    []common.Hash // TODO (MariusVanDerWijden) implement blob hashes in state tests
+		tracer      vm.EVMLogger
+		debugLogger *logger.StructLogger
+		statedb     *state.StateDB
+		chainConfig *params.ChainConfig
+		sender      = common.BytesToAddress([]byte("sender"))
+		receiver    = common.BytesToAddress([]byte("receiver"))
+		preimages   = ctx.Bool(DumpFlag.Name)
+		blobHashes  []common.Hash // TODO (MariusVanDerWijden) implement blob hashes in state tests
 	)
 	if ctx.Bool(MachineFlag.Name) {
 		tracer = logger.NewJSONLogger(logconfig, os.Stdout)
@@ -148,19 +143,30 @@ func runCmd(ctx *cli.Context) error {
 	} else {
 		debugLogger = logger.NewStructLogger(logconfig)
 	}
+
+	initialGas := ctx.Uint64(GasFlag.Name)
+	genesisConfig := new(core.Genesis)
+	genesisConfig.GasLimit = initialGas
 	if ctx.String(GenesisFlag.Name) != "" {
-		gen := readGenesis(ctx.String(GenesisFlag.Name))
-		genesisConfig = gen
-		db := rawdb.NewMemoryDatabase()
-		genesis := gen.MustCommit(db)
-		sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: preimages})
-		statedb, _ = state.New(genesis.Root(), sdb, nil)
-		chainConfig = gen.Config
+		genesisConfig = readGenesis(ctx.String(GenesisFlag.Name))
+		if genesisConfig.GasLimit != 0 {
+			initialGas = genesisConfig.GasLimit
+		}
 	} else {
-		sdb := state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: preimages})
-		statedb, _ = state.New(types.EmptyRootHash, sdb, nil)
-		genesisConfig = new(core.Genesis)
+		genesisConfig.Config = params.TestSubnetEVMConfig
 	}
+
+	db := rawdb.NewMemoryDatabase()
+	triedb := trie.NewDatabase(db, &trie.Config{
+		Preimages: preimages,
+		HashDB:    hashdb.Defaults,
+	})
+	defer triedb.Close()
+	genesis := genesisConfig.MustCommit(db, triedb)
+	sdb := state.NewDatabaseWithNodeDB(db, triedb)
+	statedb, _ = state.New(genesis.Root(), sdb, nil)
+	chainConfig = genesisConfig.Config
+
 	if ctx.String(SenderFlag.Name) != "" {
 		sender = common.HexToAddress(ctx.String(SenderFlag.Name))
 	}
@@ -214,10 +220,6 @@ func runCmd(ctx *cli.Context) error {
 		}
 		code = common.Hex2Bytes(bin)
 	}
-	initialGas := ctx.Uint64(GasFlag.Name)
-	if genesisConfig.GasLimit != 0 {
-		initialGas = genesisConfig.GasLimit
-	}
 	runtimeConfig := runtime.Config{
 		Origin:      sender,
 		State:       statedb,
@@ -234,19 +236,6 @@ func runCmd(ctx *cli.Context) error {
 		},
 	}
 
-	if cpuProfilePath := ctx.String(CPUProfileFlag.Name); cpuProfilePath != "" {
-		f, err := os.Create(cpuProfilePath)
-		if err != nil {
-			fmt.Println("could not create CPU profile: ", err)
-			os.Exit(1)
-		}
-		if err := pprof.StartCPUProfile(f); err != nil {
-			fmt.Println("could not start CPU profile: ", err)
-			os.Exit(1)
-		}
-		defer pprof.StopCPUProfile()
-	}
-
 	if chainConfig != nil {
 		runtimeConfig.ChainConfig = chainConfig
 	} else {
@@ -294,19 +283,6 @@ func runCmd(ctx *cli.Context) error {
 		fmt.Println(string(statedb.Dump(nil)))
 	}
 
-	if memProfilePath := ctx.String(MemProfileFlag.Name); memProfilePath != "" {
-		f, err := os.Create(memProfilePath)
-		if err != nil {
-			fmt.Println("could not create memory profile: ", err)
-			os.Exit(1)
-		}
-		if err := pprof.WriteHeapProfile(f); err != nil {
-			fmt.Println("could not write memory profile: ", err)
-			os.Exit(1)
-		}
-		f.Close()
-	}
-
 	if ctx.Bool(DebugFlag.Name) {
 		if debugLogger != nil {
 			fmt.Fprintln(os.Stderr, "#### TRACE ####")

cmd/evm/staterunner.go

@@ -32,12 +32,13 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/ava-labs/subnet-evm/core/rawdb"
 	"github.com/ava-labs/subnet-evm/core/state"
+	"github.com/ava-labs/subnet-evm/core/state/snapshot"
 	"github.com/ava-labs/subnet-evm/core/vm"
 	"github.com/ava-labs/subnet-evm/eth/tracers/logger"
 	"github.com/ava-labs/subnet-evm/tests"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/urfave/cli/v2"
 )
 
@@ -60,11 +61,6 @@ type StatetestResult struct {
 }
 
 func stateTestCmd(ctx *cli.Context) error {
-	// Configure the go-ethereum logger
-	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
-	glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name)))
-	log.Root().SetHandler(glogger)
-
 	// Configure the EVM logger
 	config := &logger.Config{
 		EnableMemory:     !ctx.Bool(DisableMemoryFlag.Name),
@@ -114,25 +110,23 @@ func runStateTest(fname string, cfg vm.Config, jsonOut, dump bool) error {
 		for _, st := range test.Subtests() {
 			// Run the test and aggregate the result
 			result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
-			_, s, err := test.Run(st, cfg, false)
-			// print state root for evmlab tracing
-			if s != nil {
-				root := s.IntermediateRoot(false)
-				result.Root = &root
-				if jsonOut {
-					fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root)
+			test.Run(st, cfg, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
+				if state != nil {
+					root := state.IntermediateRoot(false)
+					result.Root = &root
+					if jsonOut {
+						fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root)
+					}
 				}
-			}
-			if err != nil {
-				// Test failed, mark as so and dump any state to aid debugging
-				result.Pass, result.Error = false, err.Error()
-				if dump && s != nil {
-					s, _ = state.New(*result.Root, s.Database(), nil)
-					dump := s.RawDump(nil)
-					result.State = &dump
+				if err != nil {
+					// Test failed, mark as so and dump any state to aid debugging
+					result.Pass, result.Error = false, err.Error()
+					if dump {
+						dump := state.RawDump(nil)
+						result.State = &dump
+					}
 				}
-			}
-
+			})
 			results = append(results, *result)
 		}
 	}

cmd/evm/t8n_test.go

@@ -284,6 +284,14 @@ func TestT8n(t *testing.T) {
 			output: t8nOutput{alloc: true, result: true},
 			expOut: "exp.json",
 		},
+		{ // More cancun tests
+			base: "./testdata/29",
+			input: t8nInput{
+				"alloc.json", "txs.json", "env.json", "Cancun", "",
+			},
+			output: t8nOutput{alloc: true, result: true},
+			expOut: "exp.json",
+		},
 	} {
 		args := []string{"t8n"}
 		args = append(args, tc.output.get()...)

cmd/evm/testdata/28/env.json

@@ -1,4 +1,5 @@
 {
+    "minBaseFee" : "0x9",
     "currentCoinbase" : "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba",
     "currentNumber" : "0x01",
     "currentTimestamp" : "0x079e",
@@ -19,5 +20,5 @@
     "blockHashes" : {
         "0" : "0x3a9b485972e7353edd9152712492f0c58d89ef80623686b6bf947a4a6dce6cb6"
     },
-    "minBaseFee" : "0x9"
+    "parentBeaconBlockRoot": "0x0000beac00beac00beac00beac00beac00beac00beac00beac00beac00beac00"
 }

cmd/evm/testdata/29/alloc.json

@@ -0,0 +1,16 @@
+{
+  "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" : {
+    "balance" : "0x016345785d8a0000",
+    "code" : "0x",
+    "nonce" : "0x00",
+    "storage" : {
+    }
+  },
+  "0xbEac00dDB15f3B6d645C48263dC93862413A222D" : {
+    "balance" : "0x1",
+    "code" : "0x3373fffffffffffffffffffffffffffffffffffffffe14604457602036146024575f5ffd5b620180005f350680545f35146037575f5ffd5b6201800001545f5260205ff35b6201800042064281555f359062018000015500",
+    "nonce" : "0x00",
+    "storage" : {
+    }
+  }
+}

cmd/evm/testdata/29/env.json

@@ -0,0 +1,21 @@
+{
+    "minBaseFee" : "0x9",
+    "currentCoinbase" : "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba",
+    "currentNumber" : "0x01",
+    "currentTimestamp" : "0x079e",
+    "currentGasLimit" : "0x7fffffffffffffff",
+    "previousHash" : "0x3a9b485972e7353edd9152712492f0c58d89ef80623686b6bf947a4a6dce6cb6",
+    "currentBlobGasUsed" : "0x00",
+    "parentTimestamp" : "0x03b6",
+    "parentDifficulty" : "0x00",
+    "parentUncleHash" : "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+    "currentRandom" : "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+    "withdrawals" : [
+    ],
+    "parentBaseFee" : "0x0a",
+    "parentGasUsed" : "0x00",
+    "parentGasLimit" : "0x7fffffffffffffff",
+    "parentExcessBlobGas" : "0x00",
+    "parentBlobGasUsed" : "0x00",
+    "parentBeaconBlockRoot": "0x0000beac00beac00beac00beac00beac00beac00beac00beac00beac00beac00"
+}

cmd/evm/testdata/29/exp.json

@@ -0,0 +1,47 @@
+{
+  "alloc": {
+    "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": {
+      "balance": "0x2e248"
+    },
+    "0xbeac00ddb15f3b6d645c48263dc93862413a222d": {
+      "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604457602036146024575f5ffd5b620180005f350680545f35146037575f5ffd5b6201800001545f5260205ff35b6201800042064281555f359062018000015500",
+      "storage": {
+        "0x000000000000000000000000000000000000000000000000000000000000079e": "0x000000000000000000000000000000000000000000000000000000000000079e",
+        "0x000000000000000000000000000000000000000000000000000000000001879e": "0x0000beac00beac00beac00beac00beac00beac00beac00beac00beac00beac00"
+      },
+      "balance": "0x1"
+    },
+    "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+      "balance": "0x16345785d871db8",
+      "nonce": "0x1"
+    }
+  },
+  "result": {
+    "stateRoot": "0x79f47a970555f9f8aab12135f754201a46f48affaba011593339523127496c73",
+    "txRoot": "0x248074fabe112f7d93917f292b64932394f835bb98da91f21501574d58ec92ab",
+    "receiptsRoot": "0xf78dfb743fbd92ade140711c8bbc542b5e307f0ab7984eff35d751969fe57efa",
+    "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+    "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+    "receipts": [
+      {
+        "type": "0x2",
+        "root": "0x",
+        "status": "0x1",
+        "cumulativeGasUsed": "0x5208",
+        "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+        "logs": null,
+        "transactionHash": "0x84f70aba406a55628a0620f26d260f90aeb6ccc55fed6ec2ac13dd4f727032ed",
+        "contractAddress": "0x0000000000000000000000000000000000000000",
+        "gasUsed": "0x5208",
+        "effectiveGasPrice": null,
+        "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+        "transactionIndex": "0x0"
+      }
+    ],
+    "currentDifficulty": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+    "gasUsed": "0x5208",
+    "currentBaseFee": "0x9",
+    "currentExcessBlobGas": "0x0",
+    "currentBlobGasUsed": "0x0"
+  }
+}

cmd/evm/testdata/29/readme.md

@@ -0,0 +1,29 @@
+## EIP 4788
+
+This test contains testcases for EIP-4788. The 4788-contract is 
+located at address `0xbeac00ddb15f3b6d645c48263dc93862413a222d`, and this test executes a simple transaction. It also
+implicitly invokes the system tx, which sets calls the contract and sets the 
+storage values
+```
+$ dir=./testdata/29/ && go run . t8n --state.fork=Cancun  --input.alloc=$dir/alloc.json --input.txs=$dir/txs.json --input.env=$dir/env.json --output.alloc=stdout
+INFO [08-15|20:07:56.335] Trie dumping started                     root=ecde45..2af8a7
+INFO [08-15|20:07:56.335] Trie dumping complete                    accounts=2 elapsed="225.848µs"
+INFO [08-15|20:07:56.335] Wrote file                               file=result.json
+{
+  "alloc": {
+    "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+      "balance": "0x16345785d871db8",
+      "nonce": "0x1"
+    },
+    "0xbeac00541d49391ed88abf392bfc1f4dea8c4143": {
+      "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604457602036146024575f5ffd5b620180005f350680545f35146037575f5ffd5b6201800001545f5260205ff35b6201800042064281555f359062018000015500",
+      "storage": {
+        "0x000000000000000000000000000000000000000000000000000000000000079e": "0x000000000000000000000000000000000000000000000000000000000000079e",
+        "0x000000000000000000000000000000000000000000000000000000000001879e": "0x0000beac00beac00beac00beac00beac00beac00beac00beac00beac00beac00"
+      },
+      "balance": "0x
+    }
+  }
+}
+
+```

cmd/evm/testdata/29/txs.json

@@ -0,0 +1,19 @@
+[
+  {
+    "input" : "0x",
+    "gas" : "0x10000000",
+    "nonce" : "0x0",
+    "to" : "0x1111111111111111111111111111111111111111",
+    "value" : "0x0",
+    "secretKey" : "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8",
+    "chainId" : "0x1",
+    "type" : "0x2",
+    "v": "0x0",
+    "r": "0x0",
+    "s": "0x0",
+    "maxFeePerGas" : "0xfa0",
+    "maxPriorityFeePerGas" : "0x0",
+    "accessList" : [
+    ]
+  }
+]

consensus/misc/eip4844/eip4844.go

@@ -52,8 +52,8 @@ func VerifyEIP4844Header(parent, header *types.Header) error {
 		return errors.New("header is missing blobGasUsed")
 	}
 	// Verify that the blob gas used remains within reasonable limits.
-	if *header.BlobGasUsed > params.BlobTxMaxBlobGasPerBlock {
-		return fmt.Errorf("blob gas used %d exceeds maximum allowance %d", *header.BlobGasUsed, params.BlobTxMaxBlobGasPerBlock)
+	if *header.BlobGasUsed > params.MaxBlobGasPerBlock {
+		return fmt.Errorf("blob gas used %d exceeds maximum allowance %d", *header.BlobGasUsed, params.MaxBlobGasPerBlock)
 	}
 	if *header.BlobGasUsed%params.BlobTxBlobGasPerBlob != 0 {
 		return fmt.Errorf("blob gas used %d not a multiple of blob gas per blob %d", header.BlobGasUsed, params.BlobTxBlobGasPerBlob)

consensus/misc/eip4844/eip4844_test.go

@@ -45,14 +45,14 @@ func TestCalcExcessBlobGas(t *testing.T) {
 		// The excess blob gas should decrease by however much the target was
 		// under-shot, capped at zero.
 		{params.BlobTxTargetBlobGasPerBlock, params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob, params.BlobTxTargetBlobGasPerBlock},
-		{params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 1, params.BlobTxBlobGasPerBlob},
-		{params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 2, 0},
+		{params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 1, params.BlobTxTargetBlobGasPerBlock - params.BlobTxBlobGasPerBlob},
+		{params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 2, params.BlobTxTargetBlobGasPerBlock - (2 * params.BlobTxBlobGasPerBlob)},
 		{params.BlobTxBlobGasPerBlob - 1, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 1, 0},
 	}
-	for _, tt := range tests {
+	for i, tt := range tests {
 		result := CalcExcessBlobGas(tt.excess, tt.blobs*params.BlobTxBlobGasPerBlob)
 		if result != tt.want {
-			t.Errorf("excess blob gas mismatch: have %v, want %v", result, tt.want)
+			t.Errorf("test %d: excess blob gas mismatch: have %v, want %v", i, result, tt.want)
 		}
 	}
 }
@@ -63,9 +63,9 @@ func TestCalcBlobFee(t *testing.T) {
 		blobfee       int64
 	}{
 		{0, 1},
-		{1542706, 1},
-		{1542707, 2},
-		{10 * 1024 * 1024, 111},
+		{2314057, 1},
+		{2314058, 2},
+		{10 * 1024 * 1024, 23},
 	}
 	for i, tt := range tests {
 		have := CalcBlobFee(tt.excessBlobGas)

core/block_validator.go

@@ -78,14 +78,23 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash {
 		return fmt.Errorf("transaction root hash mismatch (header value %x, calculated %x)", header.TxHash, hash)
 	}
+
 	// Blob transactions may be present after the Cancun fork.
 	var blobs int
-	for _, tx := range block.Transactions() {
+	for i, tx := range block.Transactions() {
 		// Count the number of blobs to validate against the header's blobGasUsed
 		blobs += len(tx.BlobHashes())
+
+		// If the tx is a blob tx, it must NOT have a sidecar attached to be valid in a block.
+		if tx.BlobTxSidecar() != nil {
+			return fmt.Errorf("unexpected blob sidecar in transaction at index %d", i)
+		}
+
 		// The individual checks for blob validity (version-check + not empty)
-		// happens in the state_transition check.
+		// happens in StateTransition.
 	}
+
+	// Check blob gas usage.
 	if header.BlobGasUsed != nil {
 		if want := *header.BlobGasUsed / params.BlobTxBlobGasPerBlob; uint64(blobs) != want { // div because the header is surely good vs the body might be bloated
 			return fmt.Errorf("blob gas used mismatch (header %v, calculated %v)", *header.BlobGasUsed, blobs*params.BlobTxBlobGasPerBlob)
@@ -95,6 +104,8 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 			return errors.New("data blobs present in block body")
 		}
 	}
+
+	// Ancestor block must be known.
 	if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
 		if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
 			return consensus.ErrUnknownAncestor

core/blockchain.go

@@ -51,6 +51,8 @@ import (
 	"github.com/ava-labs/subnet-evm/metrics"
 	"github.com/ava-labs/subnet-evm/params"
 	"github.com/ava-labs/subnet-evm/trie"
+	"github.com/ava-labs/subnet-evm/trie/triedb/hashdb"
+	"github.com/ava-labs/subnet-evm/trie/triedb/pathdb"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/lru"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -164,7 +166,7 @@ type cacheableCoinbaseConfig struct {
 }
 
 // CacheConfig contains the configuration values for the trie database
-// that's resident in a blockchain.
+// and state snapshot these are resident in a blockchain.
 type CacheConfig struct {
 	TrieCleanLimit                  int     // Memory allowance (MB) to use for caching trie nodes in memory
 	TrieDirtyLimit                  int     // Memory limit (MB) at which to block on insert and force a flush of dirty trie nodes to disk
@@ -183,11 +185,34 @@ type CacheConfig struct {
 	AcceptedCacheSize               int     // Depth of accepted headers cache and accepted logs cache at the accepted tip
 	TxLookupLimit                   uint64  // Number of recent blocks for which to maintain transaction lookup indices
 	SkipTxIndexing                  bool    // Whether to skip transaction indexing
+	StateHistory                    uint64  // Number of blocks from head whose state histories are reserved.
+	StateScheme                     string  // Scheme used to store ethereum states and merkle tree nodes on top
 
 	SnapshotNoBuild bool // Whether the background generation is allowed
 	SnapshotWait    bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it
 }
 
+// triedbConfig derives the configures for trie database.
+func (c *CacheConfig) triedbConfig() *trie.Config {
+	config := &trie.Config{Preimages: c.Preimages}
+	if c.StateScheme == rawdb.HashScheme {
+		config.HashDB = &hashdb.Config{
+			CleanCacheSize: c.TrieCleanLimit * 1024 * 1024,
+			StatsPrefix:    trieCleanCacheStatsNamespace,
+		}
+	}
+	if c.StateScheme == rawdb.PathScheme {
+		config.PathDB = &pathdb.Config{
+			StateHistory:   c.StateHistory,
+			CleanCacheSize: c.TrieCleanLimit * 1024 * 1024,
+			DirtyCacheSize: c.TrieDirtyLimit * 1024 * 1024,
+		}
+	}
+	return config
+}
+
+// DefaultCacheConfig are the default caching values if none are specified by the
+// user (also used during testing).
 var DefaultCacheConfig = &CacheConfig{
 	TrieCleanLimit:            256,
 	TrieDirtyLimit:            256,
@@ -198,6 +223,15 @@ var DefaultCacheConfig = &CacheConfig{
 	AcceptorQueueLimit:        64, // Provides 2 minutes of buffer (2s block target) for a commit delay
 	SnapshotLimit:             256,
 	AcceptedCacheSize:         32,
+	StateScheme:               rawdb.HashScheme,
+}
+
+// DefaultCacheConfigWithScheme returns a deep copied default cache config with
+// a provided trie node scheme.
+func DefaultCacheConfigWithScheme(scheme string) *CacheConfig {
+	config := *DefaultCacheConfig
+	config.StateScheme = scheme
+	return &config
 }
 
 // BlockChain represents the canonical chain given a database with a genesis
@@ -314,11 +348,8 @@ func NewBlockChain(
 		return nil, errCacheConfigNotSpecified
 	}
 	// Open trie database with provided config
-	triedb := trie.NewDatabaseWithConfig(db, &trie.Config{
-		Cache:       cacheConfig.TrieCleanLimit,
-		Preimages:   cacheConfig.Preimages,
-		StatsPrefix: trieCleanCacheStatsNamespace,
-	})
+	triedb := trie.NewDatabase(db, cacheConfig.triedbConfig())
+
 	// Setup the genesis block, commit the provided genesis specification
 	// to database if the genesis block is not present yet, or load the
 	// stored one from database.
@@ -433,6 +464,12 @@ func (bc *BlockChain) unindexBlocks(tail uint64, head uint64, done chan struct{}
 		close(done)
 	}()
 
+	// If head is 0, it means the chain is just initialized and no blocks are inserted,
+	// so don't need to indexing anything.
+	if head == 0 {
+		return
+	}
+
 	if head-txLookupLimit+1 >= tail {
 		// Unindex a part of stale indices and forward index tail to HEAD-limit
 		rawdb.UnindexTransactions(bc.db, tail, head-txLookupLimit+1, bc.quit)
@@ -464,6 +501,17 @@ func (bc *BlockChain) dispatchTxUnindexer() {
 		return
 	}
 	defer sub.Unsubscribe()
+	log.Info("Initialized transaction unindexer", "limit", txLookupLimit)
+
+	// TODO: Uncomment this code when the tx-unindexer fix is ready.
+	// Launch the initial processing if chain is not empty. This step is
+	// useful in these scenarios that chain has no progress and indexer
+	// is never triggered.
+	// if head := bc.lastAccepted; head != nil && head.NumberU64() > txLookupLimit {
+	// 	done = make(chan struct{})
+	// 	tail := rawdb.ReadTxIndexTail(bc.db)
+	// 	go bc.unindexBlocks(*tail, head.NumberU64(), done)
+	// }
 
 	for {
 		select {
@@ -968,15 +1016,21 @@ func (bc *BlockChain) stopWithoutSaving() {
 func (bc *BlockChain) Stop() {
 	bc.stopWithoutSaving()
 
+	if bc.triedb.Scheme() == rawdb.PathScheme {
+		// Ensure that the in-memory trie nodes are journaled to disk properly.
+		if err := bc.triedb.Journal(bc.CurrentBlock().Root); err != nil {
+			log.Info("Failed to journal in-memory trie nodes", "err", err)
+		}
+	}
 	log.Info("Shutting down state manager")
 	start := time.Now()
 	if err := bc.stateManager.Shutdown(); err != nil {
 		log.Error("Failed to Shutdown state manager", "err", err)
 	}
 	log.Info("State manager shut down", "t", time.Since(start))
-	// Flush the collected preimages to disk
-	if err := bc.stateCache.TrieDB().Close(); err != nil {
-		log.Error("Failed to close trie db", "err", err)
+	// Close the trie database, release all the held resources as the last step.
+	if err := bc.triedb.Close(); err != nil {
+		log.Error("Failed to close trie database", "err", err)
 	}
 	log.Info("Blockchain stopped")
 }
@@ -1224,6 +1278,11 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 	if err != nil {
 		return err
 	}
+	// If node is running in path mode, skip explicit gc operation
+	// which is unnecessary in this mode.
+	if bc.triedb.Scheme() == rawdb.PathScheme {
+		return nil
+	}
 
 	// Note: if InsertTrie must be the last step in verification that can return an error.
 	// This allows [stateManager] to assume that if it inserts a trie without returning an
@@ -1927,7 +1986,7 @@ func (bc *BlockChain) reprocessState(current *types.Block, reexec uint64) error
 		}
 	}
 
-	nodes, imgs := triedb.Size()
+	_, nodes, imgs := triedb.Size()
 	log.Info("Historical state regenerated", "block", current.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs)
 	if previousRoot != (common.Hash{}) {
 		return triedb.Commit(previousRoot, true)
@@ -2028,7 +2087,7 @@ func (bc *BlockChain) populateMissingTries() error {
 		return fmt.Errorf("failed to write offline pruning success marker: %w", err)
 	}
 
-	nodes, imgs := triedb.Size()
+	_, nodes, imgs := triedb.Size()
 	log.Info("All missing tries populated", "startHeight", startHeight, "lastAcceptedHeight", lastAccepted, "missing", missing, "elapsed", time.Since(startTime), "nodes", nodes, "preimages", imgs)
 	return nil
 }

core/blockchain_reader.go

@@ -236,12 +236,6 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool {
 	return bc.HasState(block.Root())
 }
 
-// TrieNode retrieves a blob of data associated with a trie node
-// either from ephemeral in-memory cache, or from persistent storage.
-func (bc *BlockChain) TrieNode(hash common.Hash) ([]byte, error) {
-	return bc.stateCache.TrieDB().Node(hash)
-}
-
 // State returns a new mutable state based on the current HEAD block.
 func (bc *BlockChain) State() (*state.StateDB, error) {
 	return bc.StateAt(bc.CurrentBlock().Root)

core/blockchain_repair_test.go

@@ -39,7 +39,10 @@ import (
 	"github.com/ava-labs/subnet-evm/core/types"
 	"github.com/ava-labs/subnet-evm/core/vm"
 	"github.com/ava-labs/subnet-evm/params"
+	"github.com/ava-labs/subnet-evm/trie"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/stretchr/testify/require"
 )
 
 // rewindTest is a test case for chain rollback upon user request.
@@ -501,6 +504,12 @@ func testLongReorgedDeepRepair(t *testing.T, snapshots bool) {
 }
 
 func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
+	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
+		testRepairWithScheme(t, tt, snapshots, scheme)
+	}
+}
+
+func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) {
 	// It's hard to follow the test case, visualize the input
 	//log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
 	// fmt.Println(tt.dump(true))
@@ -510,6 +519,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
 
 	db, err := rawdb.Open(rawdb.OpenOptions{
 		Directory: datadir,
+		Ephemeral: true,
 	})
 	if err != nil {
 		t.Fatalf("Failed to create persistent database: %v", err)
@@ -517,22 +527,31 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
 	defer db.Close() // Might double close, should be fine
 
 	// Initialize a fresh chain
+	chainConfig := *params.TestChainConfig
+	chainConfig.FeeConfig.MinBaseFee = big.NewInt(1)
 	var (
-		gspec = &Genesis{
-			BaseFee: big.NewInt(params.TestInitialBaseFee),
-			Config:  params.TestChainConfig,
+		require = require.New(t)
+		key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+		addr1   = crypto.PubkeyToAddress(key1.PublicKey)
+		gspec   = &Genesis{
+			BaseFee: chainConfig.FeeConfig.MinBaseFee,
+			Config:  &chainConfig,
+			Alloc:   GenesisAlloc{addr1: {Balance: big.NewInt(params.Ether)}},
 		}
+		signer = types.LatestSigner(gspec.Config)
 		engine = dummy.NewFullFaker()
 		config = &CacheConfig{
 			TrieCleanLimit:            256,
 			TrieDirtyLimit:            256,
 			TriePrefetcherParallelism: 4,
 			SnapshotLimit:             0, // Disable snapshot by default
+			StateScheme:               scheme,
 		}
 	)
 	defer engine.Close()
 	if snapshots {
 		config.SnapshotLimit = 256
+		config.SnapshotWait = true
 	}
 	chain, err := NewBlockChain(db, config, gspec, engine, vm.Config{}, common.Hash{}, false)
 	if err != nil {
@@ -544,17 +563,29 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
 	// If sidechain blocks are needed, make a light chain and import it
 	var sideblocks types.Blocks
 	if tt.sidechainBlocks > 0 {
-		sideblocks, _, _ = GenerateChain(gspec.Config, gspec.ToBlock(), engine, rawdb.NewMemoryDatabase(), tt.sidechainBlocks, 10, func(i int, b *BlockGen) {
+		genDb := rawdb.NewMemoryDatabase()
+		gspec.MustCommit(genDb, trie.NewDatabase(genDb, nil))
+		sideblocks, _, err = GenerateChain(gspec.Config, gspec.ToBlock(), engine, genDb, tt.sidechainBlocks, 10, func(i int, b *BlockGen) {
 			b.SetCoinbase(common.Address{0x01})
+			tx, err := types.SignTx(types.NewTransaction(b.TxNonce(addr1), common.Address{0x01}, big.NewInt(10000), params.TxGas, common.Big1, nil), signer, key1)
+			require.NoError(err)
+			b.AddTx(tx)
 		})
+		require.NoError(err)
 		if _, err := chain.InsertChain(sideblocks); err != nil {
 			t.Fatalf("Failed to import side chain: %v", err)
 		}
 	}
-	canonblocks, _, _ := GenerateChain(gspec.Config, gspec.ToBlock(), engine, rawdb.NewMemoryDatabase(), tt.canonicalBlocks, 10, func(i int, b *BlockGen) {
+	genDb := rawdb.NewMemoryDatabase()
+	gspec.MustCommit(genDb, trie.NewDatabase(genDb, nil))
+	canonblocks, _, err := GenerateChain(gspec.Config, gspec.ToBlock(), engine, genDb, tt.canonicalBlocks, 10, func(i int, b *BlockGen) {
 		b.SetCoinbase(common.Address{0x02})
 		b.SetDifficulty(big.NewInt(1000000))
+		tx, err := types.SignTx(types.NewTransaction(b.TxNonce(addr1), common.Address{0x02}, big.NewInt(10000), params.TxGas, common.Big1, nil), signer, key1)
+		require.NoError(err)
+		b.AddTx(tx)
 	})
+	require.NoError(err)
 	if _, err := chain.InsertChain(canonblocks[:tt.commitBlock]); err != nil {
 		t.Fatalf("Failed to import canonical chain start: %v", err)
 	}
@@ -574,12 +605,14 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
 	}
 
 	// Pull the plug on the database, simulating a hard crash
+	chain.triedb.Close()
 	db.Close()
 	chain.stopWithoutSaving()
 
 	// Start a new blockchain back up and see where the repair leads us
 	db, err = rawdb.Open(rawdb.OpenOptions{
 		Directory: datadir,
+		Ephemeral: true,
 	})
 	if err != nil {
 		t.Fatalf("Failed to reopen persistent database: %v", err)

core/blockchain_snapshot_test.go

@@ -34,6 +34,7 @@ import (
 	"fmt"
 	"math/big"
 	"os"
+	"path"
 	"strings"
 	"testing"
 
@@ -49,6 +50,7 @@ import (
 
 // snapshotTestBasic wraps the common testing fields in the snapshot tests.
 type snapshotTestBasic struct {
+	scheme        string // Disk scheme used for storing trie nodes
 	chainBlocks   int    // Number of blocks to generate for the canonical chain
 	snapshotBlock uint64 // Block number of the relevant snapshot disk layer
 
@@ -58,6 +60,7 @@ type snapshotTestBasic struct {
 
 	// share fields, set in runtime
 	datadir string
+	ancient string
 	db      ethdb.Database
 	genDb   ethdb.Database
 	engine  consensus.Engine
@@ -69,9 +72,11 @@ type snapshotTestBasic struct {
 func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Block) {
 	// Create a temporary persistent database
 	datadir := t.TempDir()
+	ancient := path.Join(datadir, "ancient")
 
 	db, err := rawdb.Open(rawdb.OpenOptions{
 		Directory: datadir,
+		Ephemeral: true,
 	})
 	if err != nil {
 		t.Fatalf("Failed to create persistent database: %v", err)
@@ -83,13 +88,8 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
 			Config:  params.TestChainConfig,
 		}
 		engine = dummy.NewFullFaker()
-
-		// Snapshot is enabled, the first snapshot is created from the Genesis.
-		// The snapshot memory allowance is 256MB, it means no snapshot flush
-		// will happen during the block insertion.
-		cacheConfig = DefaultCacheConfig
 	)
-	chain, err := NewBlockChain(db, cacheConfig, gspec, engine, vm.Config{}, common.Hash{}, false)
+	chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(basic.scheme), gspec, engine, vm.Config{}, common.Hash{}, false)
 	if err != nil {
 		t.Fatalf("Failed to create chain: %v", err)
 	}
@@ -130,6 +130,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
 
 	// Set runtime fields
 	basic.datadir = datadir
+	basic.ancient = ancient
 	basic.db = db
 	basic.genDb = genDb
 	basic.engine = engine
@@ -211,6 +212,7 @@ func (basic *snapshotTestBasic) teardown() {
 	basic.db.Close()
 	basic.genDb.Close()
 	os.RemoveAll(basic.datadir)
+	os.RemoveAll(basic.ancient)
 }
 
 // snapshotTest is a test case type for normal snapshot recovery.
@@ -227,7 +229,7 @@ func (snaptest *snapshotTest) test(t *testing.T) {
 
 	// Restart the chain normally
 	chain.Stop()
-	newchain, err := NewBlockChain(snaptest.db, DefaultCacheConfig, snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false)
+	newchain, err := NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}
@@ -236,7 +238,7 @@ func (snaptest *snapshotTest) test(t *testing.T) {
 	snaptest.verify(t, newchain, blocks)
 }
 
-// crashSnapshotTest is a test case type for innormal snapshot recovery.
+// crashSnapshotTest is a test case type for irregular snapshot recovery.
 // It can be used for testing that restart Geth after the crash.
 type crashSnapshotTest struct {
 	snapshotTestBasic
@@ -252,10 +254,12 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
 	db := chain.db
 	db.Close()
 	chain.stopWithoutSaving()
+	chain.triedb.Close()
 
 	// Start a new blockchain back up and see where the repair leads us
 	newdb, err := rawdb.Open(rawdb.OpenOptions{
 		Directory: snaptest.datadir,
+		Ephemeral: true,
 	})
 	if err != nil {
 		t.Fatalf("Failed to reopen persistent database: %v", err)
@@ -266,13 +270,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
 	// the crash, we do restart twice here: one after the crash and one
 	// after the normal stop. It's used to ensure the broken snapshot
 	// can be detected all the time.
-	newchain, err := NewBlockChain(newdb, DefaultCacheConfig, snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false)
+	newchain, err := NewBlockChain(newdb, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}
 	newchain.Stop()
 
-	newchain, err = NewBlockChain(newdb, DefaultCacheConfig, snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false)
+	newchain, err = NewBlockChain(newdb, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}
@@ -299,7 +303,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) {
 
 	// Insert blocks without enabling snapshot if gapping is required.
 	chain.Stop()
-	gappedBlocks, _, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.gapped, 10, func(i int, b *BlockGen) {})
+	gappedBlocks, _, _ := GenerateChain(snaptest.gspec.Config, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.gapped, 10, func(i int, b *BlockGen) {})
 
 	// Insert a few more blocks without enabling snapshot
 	var cacheConfig = &CacheConfig{
@@ -308,6 +312,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) {
 		SnapshotLimit:  0,
 		Pruning:        true,
 		CommitInterval: 4096,
+		StateScheme:    snaptest.scheme,
 	}
 	newchain, err := NewBlockChain(snaptest.db, cacheConfig, snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false)
 	if err != nil {
@@ -317,7 +322,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) {
 	newchain.Stop()
 
 	// Restart the chain with enabling the snapshot
-	newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfig, snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false)
+	newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}
@@ -352,22 +357,25 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) {
 		SnapshotLimit:  0,
 		Pruning:        true,
 		CommitInterval: 4096,
+		StateScheme:    snaptest.scheme,
 	}
 	newchain, err := NewBlockChain(snaptest.db, config, snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}
-	newBlocks, _, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.newBlocks, 10, func(i int, b *BlockGen) {})
+	newBlocks, _, _ := GenerateChain(snaptest.gspec.Config, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.newBlocks, 10, func(i int, b *BlockGen) {})
 	newchain.InsertChain(newBlocks)
 	newchain.Stop()
 
-	// Restart the chain, the wiper should starts working
+	// Restart the chain, the wiper should start working
 	config = &CacheConfig{
 		TrieCleanLimit: 256,
 		TrieDirtyLimit: 256,
 		SnapshotLimit:  256,
 		Pruning:        true,
 		CommitInterval: 4096,
+		SnapshotWait:   false, // Don't wait rebuild
+		StateScheme:    snaptest.scheme,
 	}
 	tmp, err := NewBlockChain(snaptest.db, config, snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false)
 	if err != nil {
@@ -375,14 +383,15 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) {
 	}
 
 	// Simulate the blockchain crash.
+	tmp.triedb.Close()
 	tmp.stopWithoutSaving()
 
-	newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfig, snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false)
+	newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, snaptest.engine, vm.Config{}, snaptest.lastAcceptedHash, false)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}
-	defer newchain.Stop()
 	snaptest.verify(t, newchain, blocks)
+	newchain.Stop()
 }
 
 // Tests a Geth restart with valid snapshot. Before the shutdown, all snapshot
@@ -402,17 +411,20 @@ func TestRestartWithNewSnapshot(t *testing.T) {
 	// Expected head header    : C8
 	// Expected head block     : C4
 	// Expected snapshot disk  : C4
-	test := &snapshotTest{
-		snapshotTestBasic{
-			chainBlocks:        8,
-			snapshotBlock:      4,
-			expCanonicalBlocks: 8,
-			expHeadBlock:       4,
-			expSnapshotBottom:  4, // Initial disk layer built from genesis
-		},
+	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
+		test := &snapshotTest{
+			snapshotTestBasic{
+				scheme:             scheme,
+				chainBlocks:        8,
+				snapshotBlock:      4,
+				expCanonicalBlocks: 8,
+				expHeadBlock:       4,
+				expSnapshotBottom:  4, // Initial disk layer built from genesis
+			},
+		}
+		test.test(t)
+		test.teardown()
 	}
-	test.test(t)
-	test.teardown()
 }
 
 // Tests a Geth was crashed and restarts with a broken snapshot. In this case the
@@ -435,17 +447,20 @@ func TestNoCommitCrashWithNewSnapshot(t *testing.T) {
 	//
 	// Expected head block     : C4
 	// Expected snapshot disk  : C4
-	test := &crashSnapshotTest{
-		snapshotTestBasic{
-			chainBlocks:        8,
-			snapshotBlock:      4,
-			expCanonicalBlocks: 8,
-			expHeadBlock:       4,
-			expSnapshotBottom:  4, // Last committed disk layer, wait recovery
-		},
+	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
+		test := &crashSnapshotTest{
+			snapshotTestBasic{
+				scheme:             scheme,
+				chainBlocks:        8,
+				snapshotBlock:      4,
+				expCanonicalBlocks: 8,
+				expHeadBlock:       4,
+				expSnapshotBottom:  4, // Last committed disk layer, wait recovery
+			},
+		}
+		test.test(t)
+		test.teardown()
 	}
-	test.test(t)
-	test.teardown()
 }
 
 // Tests a Geth was crashed and restarts with a broken snapshot. In this case the
@@ -468,17 +483,20 @@ func TestLowCommitCrashWithNewSnapshot(t *testing.T) {
 	//
 	// Expected head block     : C4
 	// Expected snapshot disk  : C4
-	test := &crashSnapshotTest{
-		snapshotTestBasic{
-			chainBlocks:        8,
-			snapshotBlock:      4,
-			expCanonicalBlocks: 8,
-			expHeadBlock:       4,
-			expSnapshotBottom:  4, // Last committed disk layer, wait recovery
-		},
+	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
+		test := &crashSnapshotTest{
+			snapshotTestBasic{
+				scheme:             scheme,
+				chainBlocks:        8,
+				snapshotBlock:      4,
+				expCanonicalBlocks: 8,
+				expHeadBlock:       4,
+				expSnapshotBottom:  4, // Last committed disk layer, wait recovery
+			},
+		}
+		test.test(t)
+		test.teardown()
 	}
-	test.test(t)
-	test.teardown()
 }
 
 // Tests a Geth was crashed and restarts with a broken snapshot. In this case
@@ -501,17 +519,20 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) {
 	//
 	// Expected head block     : C4
 	// Expected snapshot disk  : C4
-	test := &crashSnapshotTest{
-		snapshotTestBasic{
-			chainBlocks:        8,
-			snapshotBlock:      4,
-			expCanonicalBlocks: 8,
-			expHeadBlock:       4,
-			expSnapshotBottom:  4, // Last committed disk layer, wait recovery
-		},
+	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
+		test := &crashSnapshotTest{
+			snapshotTestBasic{
+				scheme:             scheme,
+				chainBlocks:        8,
+				snapshotBlock:      4,
+				expCanonicalBlocks: 8,
+				expHeadBlock:       4,
+				expSnapshotBottom:  4, // Last committed disk layer, wait recovery
+			},
+		}
+		test.test(t)
+		test.teardown()
 	}
-	test.test(t)
-	test.teardown()
 }
 
 // Tests a Geth was running with snapshot enabled. Then restarts without
@@ -530,18 +551,21 @@ func TestGappedNewSnapshot(t *testing.T) {
 	//
 	// Expected head block     : G
 	// Expected snapshot disk  : G
-	test := &gappedSnapshotTest{
-		snapshotTestBasic: snapshotTestBasic{
-			chainBlocks:        8,
-			snapshotBlock:      0,
-			expCanonicalBlocks: 10,
-			expHeadBlock:       0,
-			expSnapshotBottom:  0, // Rebuilt snapshot from the latest HEAD
-		},
-		gapped: 2,
+	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
+		test := &gappedSnapshotTest{
+			snapshotTestBasic: snapshotTestBasic{
+				scheme:             scheme,
+				chainBlocks:        8,
+				snapshotBlock:      0,
+				expCanonicalBlocks: 10,
+				expHeadBlock:       0,
+				expSnapshotBottom:  0, // Rebuilt snapshot from the latest HEAD
+			},
+			gapped: 2,
+		}
+		test.test(t)
+		test.teardown()
 	}
-	test.test(t)
-	test.teardown()
 }
 
 // Tests the Geth was running with a complete snapshot and then imports a few
@@ -560,16 +584,19 @@ func TestRecoverSnapshotFromWipingCrash(t *testing.T) {
 	//
 	// Expected head block     : C4
 	// Expected snapshot disk  : C4
-	test := &wipeCrashSnapshotTest{
-		snapshotTestBasic: snapshotTestBasic{
-			chainBlocks:        8,
-			snapshotBlock:      4,
-			expCanonicalBlocks: 10,
-			expHeadBlock:       4,
-			expSnapshotBottom:  4,
-		},
-		newBlocks: 2,
+	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
+		test := &wipeCrashSnapshotTest{
+			snapshotTestBasic: snapshotTestBasic{
+				scheme:             scheme,
+				chainBlocks:        8,
+				snapshotBlock:      4,
+				expCanonicalBlocks: 10,
+				expHeadBlock:       4,
+				expSnapshotBottom:  4,
+			},
+			newBlocks: 2,
+		}
+		test.test(t)
+		test.teardown()
 	}
-	test.test(t)
-	test.teardown()
 }

core/blockchain_test.go

@@ -541,7 +541,6 @@ func TestUngracefulAsyncShutdown(t *testing.T) {
 // TODO: simplify the unindexer logic and this test.
 func TestTransactionIndices(t *testing.T) {
 	// Configure and generate a sample block chain
-	require := require.New(t)
 	var (
 		key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 		key2, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
@@ -556,19 +555,20 @@ func TestTransactionIndices(t *testing.T) {
 	)
 	genDb, blocks, _, err := GenerateChainWithGenesis(gspec, dummy.NewFaker(), 128, 10, func(i int, block *BlockGen) {
 		tx, err := types.SignTx(types.NewTransaction(block.TxNonce(addr1), addr2, big.NewInt(10000), params.TxGas, nil, nil), signer, key1)
-		require.NoError(err)
+		require.NoError(t, err)
 		block.AddTx(tx)
 	})
-	require.NoError(err)
+	require.NoError(t, err)
 
 	blocks2, _, err := GenerateChain(gspec.Config, blocks[len(blocks)-1], dummy.NewFaker(), genDb, 10, 10, func(i int, block *BlockGen) {
 		tx, err := types.SignTx(types.NewTransaction(block.TxNonce(addr1), addr2, big.NewInt(10000), params.TxGas, nil, nil), signer, key1)
-		require.NoError(err)
+		require.NoError(t, err)
 		block.AddTx(tx)
 	})
-	require.NoError(err)
+	require.NoError(t, err)
 
-	check := func(tail *uint64, chain *BlockChain) {
+	check := func(t *testing.T, tail *uint64, chain *BlockChain) {
+		require := require.New(t)
 		stored := rawdb.ReadTxIndexTail(chain.db)
 		var tailValue uint64
 		if tail == nil {
@@ -617,19 +617,19 @@ func TestTransactionIndices(t *testing.T) {
 	// Init block chain and check all needed indices has been indexed.
 	chainDB := rawdb.NewMemoryDatabase()
 	chain, err := createBlockChain(chainDB, conf, gspec, common.Hash{})
-	require.NoError(err)
+	require.NoError(t, err)
 
 	_, err = chain.InsertChain(blocks)
-	require.NoError(err)
+	require.NoError(t, err)
 
 	for _, block := range blocks {
 		err := chain.Accept(block)
-		require.NoError(err)
+		require.NoError(t, err)
 	}
 	chain.DrainAcceptorQueue()
 
 	chain.Stop()
-	check(nil, chain) // check all indices has been indexed
+	check(t, nil, chain) // check all indices has been indexed
 
 	lastAcceptedHash := chain.CurrentHeader().Hash()
 
@@ -647,14 +647,14 @@ func TestTransactionIndices(t *testing.T) {
 			conf.TxLookupLimit = l
 
 			chain, err := createBlockChain(chainDB, conf, gspec, lastAcceptedHash)
-			require.NoError(err)
+			require.NoError(t, err)
 
 			newBlks := blocks2[i : i+1]
 			_, err = chain.InsertChain(newBlks) // Feed chain a higher block to trigger indices updater.
-			require.NoError(err)
+			require.NoError(t, err)
 
 			err = chain.Accept(newBlks[0]) // Accept the block to trigger indices updater.
-			require.NoError(err)
+			require.NoError(t, err)
 
 			chain.DrainAcceptorQueue()
 			time.Sleep(50 * time.Millisecond) // Wait for indices initialisation
@@ -673,7 +673,7 @@ func TestTransactionIndices(t *testing.T) {
 				tail = &tl
 			}
 
-			check(tail, chain)
+			check(t, tail, chain)
 
 			lastAcceptedHash = chain.CurrentHeader().Hash()
 		})
@@ -801,6 +801,11 @@ func TestTransactionSkipIndexing(t *testing.T) {
 // TestCanonicalHashMarker tests all the canonical hash markers are updated/deleted
 // correctly in case reorg is called.
 func TestCanonicalHashMarker(t *testing.T) {
+	testCanonicalHashMarker(t, rawdb.HashScheme)
+	testCanonicalHashMarker(t, rawdb.PathScheme)
+}
+
+func testCanonicalHashMarker(t *testing.T, scheme string) {
 	var cases = []struct {
 		forkA int
 		forkB int
@@ -854,8 +859,7 @@ func TestCanonicalHashMarker(t *testing.T) {
 		}
 
 		// Initialize test chain
-		diskdb := rawdb.NewMemoryDatabase()
-		chain, err := NewBlockChain(diskdb, DefaultCacheConfig, gspec, engine, vm.Config{}, common.Hash{}, false)
+		chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, engine, vm.Config{}, common.Hash{}, false)
 		if err != nil {
 			t.Fatalf("failed to create tester chain: %v", err)
 		}
@@ -1049,6 +1053,120 @@ func testCreateThenDelete(t *testing.T, config *params.ChainConfig) {
 	}
 }
 
+func TestDeleteThenCreate(t *testing.T) {
+	var (
+		engine      = dummy.NewFaker()
+		key, _      = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+		address     = crypto.PubkeyToAddress(key.PublicKey)
+		factoryAddr = crypto.CreateAddress(address, 0)
+		funds       = big.NewInt(params.Ether) // Note: additional funds are provided here compared to go-ethereum so test completes.
+	)
+	/*
+		contract Factory {
+		  function deploy(bytes memory code) public {
+			address addr;
+			assembly {
+			  addr := create2(0, add(code, 0x20), mload(code), 0)
+			  if iszero(extcodesize(addr)) {
+				revert(0, 0)
+			  }
+			}
+		  }
+		}
+	*/
+	factoryBIN := common.Hex2Bytes("608060405234801561001057600080fd5b50610241806100206000396000f3fe608060405234801561001057600080fd5b506004361061002a5760003560e01c80627743601461002f575b600080fd5b610049600480360381019061004491906100d8565b61004b565b005b6000808251602084016000f59050803b61006457600080fd5b5050565b600061007b61007684610146565b610121565b905082815260208101848484011115610097576100966101eb565b5b6100a2848285610177565b509392505050565b600082601f8301126100bf576100be6101e6565b5b81356100cf848260208601610068565b91505092915050565b6000602082840312156100ee576100ed6101f5565b5b600082013567ffffffffffffffff81111561010c5761010b6101f0565b5b610118848285016100aa565b91505092915050565b600061012b61013c565b90506101378282610186565b919050565b6000604051905090565b600067ffffffffffffffff821115610161576101606101b7565b5b61016a826101fa565b9050602081019050919050565b82818337600083830152505050565b61018f826101fa565b810181811067ffffffffffffffff821117156101ae576101ad6101b7565b5b80604052505050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b600080fd5b600080fd5b600080fd5b600080fd5b6000601f19601f830116905091905056fea2646970667358221220ea8b35ed310d03b6b3deef166941140b4d9e90ea2c92f6b41eb441daf49a59c364736f6c63430008070033")
+
+	/*
+		contract C {
+			uint256 value;
+			constructor() {
+				value = 100;
+			}
+			function destruct() public payable {
+				selfdestruct(payable(msg.sender));
+			}
+			receive() payable external {}
+		}
+	*/
+	contractABI := common.Hex2Bytes("6080604052348015600f57600080fd5b5060646000819055506081806100266000396000f3fe608060405260043610601f5760003560e01c80632b68b9c614602a576025565b36602557005b600080fd5b60306032565b005b3373ffffffffffffffffffffffffffffffffffffffff16fffea2646970667358221220ab749f5ed1fcb87bda03a74d476af3f074bba24d57cb5a355e8162062ad9a4e664736f6c63430008070033")
+	contractAddr := crypto.CreateAddress2(factoryAddr, [32]byte{}, crypto.Keccak256(contractABI))
+
+	gspec := &Genesis{
+		Config: params.TestChainConfig,
+		Alloc: GenesisAlloc{
+			address: {Balance: funds},
+		},
+	}
+	nonce := uint64(0)
+	signer := types.HomesteadSigner{}
+	_, blocks, _, err := GenerateChainWithGenesis(gspec, engine, 2, 10, func(i int, b *BlockGen) {
+		fee := big.NewInt(1)
+		if b.header.BaseFee != nil {
+			fee = b.header.BaseFee
+		}
+		b.SetCoinbase(common.Address{1})
+
+		// Block 1
+		if i == 0 {
+			tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{
+				Nonce:    nonce,
+				GasPrice: new(big.Int).Set(fee),
+				Gas:      500000,
+				Data:     factoryBIN,
+			})
+			nonce++
+			b.AddTx(tx)
+
+			data := common.Hex2Bytes("00774360000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000a76080604052348015600f57600080fd5b5060646000819055506081806100266000396000f3fe608060405260043610601f5760003560e01c80632b68b9c614602a576025565b36602557005b600080fd5b60306032565b005b3373ffffffffffffffffffffffffffffffffffffffff16fffea2646970667358221220ab749f5ed1fcb87bda03a74d476af3f074bba24d57cb5a355e8162062ad9a4e664736f6c6343000807003300000000000000000000000000000000000000000000000000")
+			tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{
+				Nonce:    nonce,
+				GasPrice: new(big.Int).Set(fee),
+				Gas:      500000,
+				To:       &factoryAddr,
+				Data:     data,
+			})
+			b.AddTx(tx)
+			nonce++
+		} else {
+			// Block 2
+			tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{
+				Nonce:    nonce,
+				GasPrice: new(big.Int).Set(fee),
+				Gas:      500000,
+				To:       &contractAddr,
+				Data:     common.Hex2Bytes("2b68b9c6"), // destruct
+			})
+			nonce++
+			b.AddTx(tx)
+
+			data := common.Hex2Bytes("00774360000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000a76080604052348015600f57600080fd5b5060646000819055506081806100266000396000f3fe608060405260043610601f5760003560e01c80632b68b9c614602a576025565b36602557005b600080fd5b60306032565b005b3373ffffffffffffffffffffffffffffffffffffffff16fffea2646970667358221220ab749f5ed1fcb87bda03a74d476af3f074bba24d57cb5a355e8162062ad9a4e664736f6c6343000807003300000000000000000000000000000000000000000000000000")
+			tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{
+				Nonce:    nonce,
+				GasPrice: new(big.Int).Set(fee),
+				Gas:      500000,
+				To:       &factoryAddr, // re-creation
+				Data:     data,
+			})
+			b.AddTx(tx)
+			nonce++
+		}
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Import the canonical chain
+	chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfig, gspec, engine, vm.Config{}, common.Hash{}, false)
+	if err != nil {
+		t.Fatalf("failed to create tester chain: %v", err)
+	}
+	defer chain.Stop()
+	for _, block := range blocks {
+		if _, err := chain.InsertChain([]*types.Block{block}); err != nil {
+			t.Fatalf("block %d: failed to insert into chain: %v", block.NumberU64(), err)
+		}
+	}
+}
+
 // TestTransientStorageReset ensures the transient storage is wiped correctly
 // between transactions.
 func TestTransientStorageReset(t *testing.T) {

core/chain_makers.go

@@ -33,6 +33,7 @@ import (
 	"github.com/ava-labs/subnet-evm/commontype"
 	"github.com/ava-labs/subnet-evm/consensus"
 	"github.com/ava-labs/subnet-evm/consensus/dummy"
+	"github.com/ava-labs/subnet-evm/consensus/misc/eip4844"
 	"github.com/ava-labs/subnet-evm/constants"
 	"github.com/ava-labs/subnet-evm/core/rawdb"
 	"github.com/ava-labs/subnet-evm/core/state"
@@ -98,6 +99,11 @@ func (b *BlockGen) SetDifficulty(diff *big.Int) {
 	b.header.Difficulty = diff
 }
 
+// SetBlobGas sets the data gas used by the blob in the generated block.
+func (b *BlockGen) SetBlobGas(blobGasUsed uint64) {
+	b.header.BlobGasUsed = &blobGasUsed
+}
+
 // addTx adds a transaction to the generated block. If no coinbase has
 // been set, the block's coinbase is set to the zero address.
 //
@@ -250,7 +256,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 	}
 	blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
 	chainreader := &fakeChainReader{config: config}
-	genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts, error) {
+	genblock := func(i int, parent *types.Block, triedb *trie.Database, statedb *state.StateDB) (*types.Block, types.Receipts, error) {
 		b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine}
 		b.header = makeHeader(chainreader, config, parent, gap, statedb, b.engine)
 
@@ -275,7 +281,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 			if err != nil {
 				panic(fmt.Sprintf("state write error: %v", err))
 			}
-			if err := statedb.Database().TrieDB().Commit(root, false); err != nil {
+			if err = triedb.Commit(root, false); err != nil {
 				panic(fmt.Sprintf("trie write error: %v", err))
 			}
 			if b.onBlockGenerated != nil {
@@ -285,12 +291,16 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		}
 		return nil, nil, nil
 	}
+	// Forcibly use hash-based state scheme for retaining all nodes in disk.
+	triedb := trie.NewDatabase(db, trie.HashDefaults)
+	defer triedb.Close()
+
 	for i := 0; i < n; i++ {
-		statedb, err := state.New(parent.Root(), state.NewDatabase(db), nil)
+		statedb, err := state.New(parent.Root(), state.NewDatabaseWithNodeDB(db, triedb), nil)
 		if err != nil {
 			return nil, nil, err
 		}
-		block, receipt, err := genblock(i, parent, statedb)
+		block, receipt, err := genblock(i, parent, triedb, statedb)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -306,7 +316,9 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 // then generate chain on top.
 func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gap uint64, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts, error) {
 	db := rawdb.NewMemoryDatabase()
-	_, err := genesis.Commit(db, trie.NewDatabase(db))
+	triedb := trie.NewDatabase(db, trie.HashDefaults)
+	defer triedb.Close()
+	_, err := genesis.Commit(db, triedb)
 	if err != nil {
 		return nil, nil, nil, err
 	}
@@ -349,6 +361,20 @@ func makeHeader(chain consensus.ChainReader, config *params.ChainConfig, parent
 	} else {
 		header.GasLimit = CalcGasLimit(parent.GasUsed(), parent.GasLimit(), parent.GasLimit(), parent.GasLimit())
 	}
+	if chain.Config().IsCancun(header.Number, header.Time) {
+		var (
+			parentExcessBlobGas uint64
+			parentBlobGasUsed   uint64
+		)
+		if parent.ExcessBlobGas() != nil {
+			parentExcessBlobGas = *parent.ExcessBlobGas()
+			parentBlobGasUsed = *parent.BlobGasUsed()
+		}
+		excessBlobGas := eip4844.CalcExcessBlobGas(parentExcessBlobGas, parentBlobGasUsed)
+		header.ExcessBlobGas = &excessBlobGas
+		header.BlobGasUsed = new(uint64)
+		header.ParentBeaconRoot = new(common.Hash)
+	}
 	return header
 }
 

core/chain_makers_test.go

@@ -35,6 +35,7 @@ import (
 	"github.com/ava-labs/subnet-evm/core/types"
 	"github.com/ava-labs/subnet-evm/core/vm"
 	"github.com/ava-labs/subnet-evm/params"
+	"github.com/ava-labs/subnet-evm/trie"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 )
@@ -48,6 +49,7 @@ func ExampleGenerateChain() {
 		addr2   = crypto.PubkeyToAddress(key2.PublicKey)
 		addr3   = crypto.PubkeyToAddress(key3.PublicKey)
 		db      = rawdb.NewMemoryDatabase()
+		genDb   = rawdb.NewMemoryDatabase()
 	)
 
 	// Ensure that key1 has some funds in the genesis block.
@@ -55,12 +57,13 @@ func ExampleGenerateChain() {
 		Config: &params.ChainConfig{HomesteadBlock: new(big.Int)},
 		Alloc:  GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}},
 	}
+	genesis := gspec.MustCommit(genDb, trie.NewDatabase(genDb, trie.HashDefaults))
 
 	// This call generates a chain of 3 blocks. The function runs for
 	// each block and adds different features to gen based on the
 	// block index.
 	signer := types.HomesteadSigner{}
-	_, chain, _, err := GenerateChainWithGenesis(gspec, dummy.NewCoinbaseFaker(), 3, 10, func(i int, gen *BlockGen) {
+	chain, _, err := GenerateChain(gspec.Config, genesis, dummy.NewCoinbaseFaker(), genDb, 3, 10, func(i int, gen *BlockGen) {
 		switch i {
 		case 0:
 			// In block 1, addr1 sends addr2 some ether.
@@ -81,7 +84,7 @@ func ExampleGenerateChain() {
 	}
 
 	// Import the chain. This runs all block validation rules.
-	blockchain, _ := NewBlockChain(db, DefaultCacheConfig, gspec, dummy.NewCoinbaseFaker(), vm.Config{}, common.Hash{}, false)
+	blockchain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.HashScheme), gspec, dummy.NewCoinbaseFaker(), vm.Config{}, common.Hash{}, false)
 	defer blockchain.Stop()
 
 	if i, err := blockchain.InsertChain(chain); err != nil {

core/genesis.go

@@ -197,9 +197,12 @@ func SetupGenesisBlock(
 		}
 		return genesis.Config, block.Hash(), nil
 	}
-	// We have the genesis block in database but the corresponding state is missing.
+	// The genesis block is present(perhaps in ancient database) while the
+	// state database is not initialized yet. It can happen that the node
+	// is initialized with an external ancient store. Commit genesis state
+	// in this case.
 	header := rawdb.ReadHeader(db, stored, 0)
-	if header.Root != types.EmptyRootHash && !rawdb.HasLegacyTrieNode(db, header.Root) {
+	if header.Root != types.EmptyRootHash && !triedb.Initialized(header.Root) {
 		// Ensure the stored genesis matches with the given one.
 		hash := genesis.ToBlock().Hash()
 		if hash != stored {
@@ -261,7 +264,7 @@ func SetupGenesisBlock(
 // to the given database (or discards it if nil).
 func (g *Genesis) ToBlock() *types.Block {
 	db := rawdb.NewMemoryDatabase()
-	return g.toBlock(db, trie.NewDatabase(db))
+	return g.toBlock(db, trie.NewDatabase(db, nil))
 }
 
 // TODO: migrate this function to "flush" for more similarity with upstream.
@@ -339,6 +342,11 @@ func (g *Genesis) toBlock(db ethdb.Database, triedb *trie.Database) *types.Block
 			}
 		}
 		if conf.IsCancun(num, g.Timestamp) {
+			// EIP-4788: The parentBeaconBlockRoot of the genesis block is always
+			// the zero hash. This is because the genesis block does not have a parent
+			// by definition.
+			head.ParentBeaconRoot = new(common.Hash)
+			// EIP-4844 fields
 			head.ExcessBlobGas = g.ExcessBlobGas
 			head.BlobGasUsed = g.BlobGasUsed
 			if head.ExcessBlobGas == nil {
@@ -389,10 +397,8 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database) (*types.Block
 
 // MustCommit writes the genesis block and state to db, panicking on error.
 // The block is committed as the canonical head block.
-// Note the state changes will be committed in hash-based scheme, use Commit
-// if path-scheme is preferred.
-func (g *Genesis) MustCommit(db ethdb.Database) *types.Block {
-	block, err := g.Commit(db, trie.NewDatabase(db))
+func (g *Genesis) MustCommit(db ethdb.Database, triedb *trie.Database) *types.Block {
+	block, err := g.Commit(db, triedb)
 	if err != nil {
 		panic(err)
 	}
@@ -423,7 +429,7 @@ func GenesisBlockForTesting(db ethdb.Database, addr common.Address, balance *big
 		Alloc:   GenesisAlloc{addr: {Balance: balance}},
 		BaseFee: big.NewInt(params.TestMaxBaseFee),
 	}
-	return g.MustCommit(db)
+	return g.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults))
 }
 
 // ReadBlockByHash reads the block with the given hash from the database.

core/genesis_test.go

@@ -41,6 +41,7 @@ import (
 	"github.com/ava-labs/subnet-evm/precompile/allowlist"
 	"github.com/ava-labs/subnet-evm/precompile/contracts/deployerallowlist"
 	"github.com/ava-labs/subnet-evm/trie"
+	"github.com/ava-labs/subnet-evm/trie/triedb/pathdb"
 	"github.com/ava-labs/subnet-evm/utils"
 	"github.com/davecgh/go-spew/spew"
 	"github.com/ethereum/go-ethereum/common"
@@ -62,6 +63,11 @@ func TestGenesisBlockForTesting(t *testing.T) {
 }
 
 func TestSetupGenesis(t *testing.T) {
+	testSetupGenesis(t, rawdb.HashScheme)
+	testSetupGenesis(t, rawdb.PathScheme)
+}
+
+func testSetupGenesis(t *testing.T, scheme string) {
 	preSubnetConfig := *params.TestPreSubnetEVMConfig
 	preSubnetConfig.SubnetEVMTimestamp = utils.NewUint64(100)
 	var (
@@ -79,6 +85,7 @@ func TestSetupGenesis(t *testing.T) {
 	rollbackpreSubnetConfig := preSubnetConfig
 	rollbackpreSubnetConfig.SubnetEVMTimestamp = utils.NewUint64(90)
 	oldcustomg.Config = &rollbackpreSubnetConfig
+
 	tests := []struct {
 		name       string
 		fn         func(ethdb.Database) (*params.ChainConfig, common.Hash, error)
@@ -89,7 +96,7 @@ func TestSetupGenesis(t *testing.T) {
 		{
 			name: "genesis without ChainConfig",
 			fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
-				return setupGenesisBlock(db, trie.NewDatabase(db), new(Genesis), common.Hash{})
+				return setupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), new(Genesis), common.Hash{})
 			},
 			wantErr:    errGenesisNoConfig,
 			wantConfig: nil,
@@ -97,7 +104,7 @@ func TestSetupGenesis(t *testing.T) {
 		{
 			name: "no block in DB, genesis == nil",
 			fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
-				return setupGenesisBlock(db, trie.NewDatabase(db), nil, common.Hash{})
+				return setupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), nil, common.Hash{})
 			},
 			wantErr:    ErrNoGenesis,
 			wantConfig: nil,
@@ -105,8 +112,9 @@ func TestSetupGenesis(t *testing.T) {
 		{
 			name: "custom block in DB, genesis == nil",
 			fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
-				customg.MustCommit(db)
-				return setupGenesisBlock(db, trie.NewDatabase(db), nil, common.Hash{})
+				tdb := trie.NewDatabase(db, newDbConfig(scheme))
+				customg.Commit(db, tdb)
+				return setupGenesisBlock(db, tdb, nil, common.Hash{})
 			},
 			wantErr:    ErrNoGenesis,
 			wantConfig: nil,
@@ -114,8 +122,9 @@ func TestSetupGenesis(t *testing.T) {
 		{
 			name: "compatible config in DB",
 			fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
-				oldcustomg.MustCommit(db)
-				return setupGenesisBlock(db, trie.NewDatabase(db), &customg, customghash)
+				tdb := trie.NewDatabase(db, newDbConfig(scheme))
+				oldcustomg.Commit(db, tdb)
+				return setupGenesisBlock(db, tdb, &customg, customghash)
 			},
 			wantHash:   customghash,
 			wantConfig: customg.Config,
@@ -125,12 +134,19 @@ func TestSetupGenesis(t *testing.T) {
 			fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
 				// Commit the 'old' genesis block with SubnetEVM transition at 90.
 				// Advance to block #4, past the SubnetEVM transition block of customg.
-				genesis := oldcustomg.MustCommit(db)
+				tdb := trie.NewDatabase(db, newDbConfig(scheme))
+				genesis, err := oldcustomg.Commit(db, tdb)
+				if err != nil {
+					t.Fatal(err)
+				}
 
-				bc, _ := NewBlockChain(db, DefaultCacheConfig, &oldcustomg, dummy.NewFullFaker(), vm.Config{}, genesis.Hash(), false)
+				bc, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), &oldcustomg, dummy.NewFullFaker(), vm.Config{}, genesis.Hash(), false)
 				defer bc.Stop()
 
-				blocks, _, _ := GenerateChain(oldcustomg.Config, genesis, dummy.NewFullFaker(), db, 4, 25, nil)
+				_, blocks, _, err := GenerateChainWithGenesis(&oldcustomg, dummy.NewFullFaker(), 4, 25, nil)
+				if err != nil {
+					t.Fatal(err)
+				}
 				bc.InsertChain(blocks)
 
 				for _, block := range blocks {
@@ -140,7 +156,7 @@ func TestSetupGenesis(t *testing.T) {
 				}
 
 				// This should return a compatibility error.
-				return setupGenesisBlock(db, trie.NewDatabase(db), &customg, bc.lastAccepted.Hash())
+				return setupGenesisBlock(db, tdb, &customg, bc.lastAccepted.Hash())
 			},
 			wantHash:   customghash,
 			wantConfig: customg.Config,
@@ -218,7 +234,7 @@ func TestStatefulPrecompilesConfigure(t *testing.T) {
 			genesisBlock := genesis.ToBlock()
 			genesisRoot := genesisBlock.Root()
 
-			_, _, err := setupGenesisBlock(db, trie.NewDatabase(db), genesis, genesisBlock.Hash())
+			_, _, err := setupGenesisBlock(db, trie.NewDatabase(db, trie.HashDefaults), genesis, genesisBlock.Hash())
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -280,7 +296,7 @@ func TestPrecompileActivationAfterHeaderBlock(t *testing.T) {
 	require.Less(bc.lastAccepted.Time(), *contractDeployerConfig.Timestamp())
 
 	// This should not return any error since the last accepted block is before the activation block.
-	config, _, err := setupGenesisBlock(db, trie.NewDatabase(db), &customg, bc.lastAccepted.Hash())
+	config, _, err := setupGenesisBlock(db, trie.NewDatabase(db, nil), &customg, bc.lastAccepted.Hash())
 	require.NoError(err)
 	if !reflect.DeepEqual(config, customg.Config) {
 		t.Errorf("returned %v\nwant     %v", config, customg.Config)
@@ -299,8 +315,8 @@ func TestGenesisWriteUpgradesRegression(t *testing.T) {
 	}
 
 	db := rawdb.NewMemoryDatabase()
-	genesisBlock := genesis.ToBlock()
-	trieDB := trie.NewDatabase(db)
+	trieDB := trie.NewDatabase(db, trie.HashDefaults)
+	genesisBlock := genesis.MustCommit(db, trieDB)
 
 	_, _, err := SetupGenesisBlock(db, trieDB, genesis, genesisBlock.Hash(), false)
 	require.NoError(err)
@@ -328,3 +344,10 @@ func TestGenesisWriteUpgradesRegression(t *testing.T) {
 	_, _, err = SetupGenesisBlock(db, trieDB, genesis, lastAcceptedBlock.Hash(), false)
 	require.NoError(err)
 }
+
+func newDbConfig(scheme string) *trie.Config {
+	if scheme == rawdb.HashScheme {
+		return trie.HashDefaults
+	}
+	return &trie.Config{PathDB: pathdb.Defaults}
+}

core/mkalloc.go

@@ -43,23 +43,50 @@ import (
 	"strconv"
 
 	"github.com/ava-labs/subnet-evm/core"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/rlp"
 	"golang.org/x/exp/slices"
 )
 
-type allocItem struct{ Addr, Balance *big.Int }
+type allocItem struct {
+	Addr    *big.Int
+	Balance *big.Int
+	Misc    *allocItemMisc `rlp:"optional"`
+}
+
+type allocItemMisc struct {
+	Nonce uint64
+	Code  []byte
+	Slots []allocItemStorageItem
+}
+
+type allocItemStorageItem struct {
+	Key common.Hash
+	Val common.Hash
+}
 
 func makelist(g *core.Genesis) []allocItem {
 	items := make([]allocItem, 0, len(g.Alloc))
 	for addr, account := range g.Alloc {
+		var misc *allocItemMisc
 		if len(account.Storage) > 0 || len(account.Code) > 0 || account.Nonce != 0 {
-			panic(fmt.Sprintf("can't encode account %x", addr))
+			misc = &allocItemMisc{
+				Nonce: account.Nonce,
+				Code:  account.Code,
+				Slots: make([]allocItemStorageItem, 0, len(account.Storage)),
+			}
+			for key, val := range account.Storage {
+				misc.Slots = append(misc.Slots, allocItemStorageItem{key, val})
+			}
+			slices.SortFunc(misc.Slots, func(a, b allocItemStorageItem) int {
+				return a.Key.Cmp(b.Key)
+			})
 		}
 		bigAddr := new(big.Int).SetBytes(addr.Bytes())
-		items = append(items, allocItem{bigAddr, account.Balance})
+		items = append(items, allocItem{bigAddr, account.Balance, misc})
 	}
-	slices.SortFunc(items, func(a, b allocItem) bool {
-		return a.Addr.Cmp(b.Addr) < 0
+	slices.SortFunc(items, func(a, b allocItem) int {
+		return a.Addr.Cmp(b.Addr)
 	})
 	return items
 }

core/rawdb/accessors_chain.go

@@ -227,7 +227,7 @@ func ReadHeader(db ethdb.Reader, hash common.Hash, number uint64) *types.Header
 		return nil
 	}
 	header := new(types.Header)
-	if err := rlp.Decode(bytes.NewReader(data), header); err != nil {
+	if err := rlp.DecodeBytes(data, header); err != nil {
 		log.Error("Invalid block header RLP", "hash", hash, "err", err)
 		return nil
 	}
@@ -313,7 +313,7 @@ func ReadBody(db ethdb.Reader, hash common.Hash, number uint64) *types.Body {
 		return nil
 	}
 	body := new(types.Body)
-	if err := rlp.Decode(bytes.NewReader(data), body); err != nil {
+	if err := rlp.DecodeBytes(data, body); err != nil {
 		log.Error("Invalid block body RLP", "hash", hash, "err", err)
 		return nil
 	}

core/rawdb/accessors_metadata.go

@@ -127,10 +127,10 @@ const crashesToKeep = 10
 func PushUncleanShutdownMarker(db ethdb.KeyValueStore) ([]uint64, uint64, error) {
 	var uncleanShutdowns crashList
 	// Read old data
-	if data, err := db.Get(uncleanShutdownKey); err != nil {
-		log.Warn("Error reading unclean shutdown markers", "error", err)
-	} else if err := rlp.DecodeBytes(data, &uncleanShutdowns); err != nil {
-		return nil, 0, err
+	if data, err := db.Get(uncleanShutdownKey); err == nil {
+		if err := rlp.DecodeBytes(data, &uncleanShutdowns); err != nil {
+			return nil, 0, err
+		}
 	}
 	var discarded = uncleanShutdowns.Discarded
 	var previous = make([]uint64, len(uncleanShutdowns.Recent))

core/rawdb/accessors_trie.go

@@ -46,7 +46,7 @@ import (
 //
 // Now this scheme is still kept for backward compatibility, and it will be used
 // for archive node and some other tries(e.g. light trie).
-const HashScheme = "hashScheme"
+const HashScheme = "hash"
 
 // PathScheme is the new path-based state scheme with which trie nodes are stored
 // in the disk with node path as the database key. This scheme will only store one
@@ -54,7 +54,7 @@ const HashScheme = "hashScheme"
 // is native. At the same time, this scheme will put adjacent trie nodes in the same
 // area of the disk with good data locality property. But this scheme needs to rely
 // on extra state diffs to survive deep reorg.
-const PathScheme = "pathScheme"
+const PathScheme = "path"
 
 // hasher is used to compute the sha256 hash of the provided data.
 type hasher struct{ sha crypto.KeccakState }
@@ -99,6 +99,16 @@ func HasAccountTrieNode(db ethdb.KeyValueReader, path []byte, hash common.Hash)
 	return h.hash(data) == hash
 }
 
+// ExistsAccountTrieNode checks the presence of the account trie node with the
+// specified node path, regardless of the node hash.
+func ExistsAccountTrieNode(db ethdb.KeyValueReader, path []byte) bool {
+	has, err := db.Has(accountTrieNodeKey(path))
+	if err != nil {
+		return false
+	}
+	return has
+}
+
 // WriteAccountTrieNode writes the provided account trie node into database.
 func WriteAccountTrieNode(db ethdb.KeyValueWriter, path []byte, node []byte) {
 	if err := db.Put(accountTrieNodeKey(path), node); err != nil {
@@ -137,6 +147,16 @@ func HasStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path [
 	return h.hash(data) == hash
 }
 
+// ExistsStorageTrieNode checks the presence of the storage trie node with the
+// specified account hash and node path, regardless of the node hash.
+func ExistsStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte) bool {
+	has, err := db.Has(storageTrieNodeKey(accountHash, path))
+	if err != nil {
+		return false
+	}
+	return has
+}
+
 // WriteStorageTrieNode writes the provided storage trie node into database.
 func WriteStorageTrieNode(db ethdb.KeyValueWriter, accountHash common.Hash, path []byte, node []byte) {
 	if err := db.Put(storageTrieNodeKey(accountHash, path), node); err != nil {
@@ -273,3 +293,25 @@ func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, has
 		panic(fmt.Sprintf("Unknown scheme %v", scheme))
 	}
 }
+
+// ReadStateScheme reads the state scheme of persistent state, or none
+// if the state is not present in database.
+func ReadStateScheme(db ethdb.Reader) string {
+	// Check if state in path-based scheme is present
+	blob, _ := ReadAccountTrieNode(db, nil)
+	if len(blob) != 0 {
+		return PathScheme
+	}
+	// In a hash-based scheme, the genesis state is consistently stored
+	// on the disk. To assess the scheme of the persistent state, it
+	// suffices to inspect the scheme of the genesis state.
+	header := ReadHeader(db, ReadCanonicalHash(db, 0), 0)
+	if header == nil {
+		return "" // empty datadir
+	}
+	blob = ReadLegacyTrieNode(db, header.Root)
+	if len(blob) == 0 {
+		return "" // no state in disk
+	}
+	return HashScheme
+}

core/rawdb/database.go

@@ -159,10 +159,10 @@ const (
 	dbLeveldb = "leveldb"
 )
 
-// hasPreexistingDb checks the given data directory whether a database is already
+// PreexistingDatabase checks the given data directory whether a database is already
 // instantiated at that location, and if so, returns the type of database (or the
 // empty string).
-func hasPreexistingDb(path string) string {
+func PreexistingDatabase(path string) string {
 	if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
 		return "" // No pre-existing db
 	}
@@ -184,6 +184,9 @@ type OpenOptions struct {
 	Cache     int    // the capacity(in megabytes) of the data caching
 	Handles   int    // number of files to be open simultaneously
 	ReadOnly  bool
+	// Ephemeral means that filesystem sync operations should be avoided: data integrity in the face of
+	// a crash is not important. This option should typically be used in tests.
+	Ephemeral bool
 }
 
 // openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble.
@@ -199,14 +202,14 @@ func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) {
 	}
 	// Retrieve any pre-existing database's type and use that or the requested one
 	// as long as there's no conflict between the two types
-	existingDb := hasPreexistingDb(o.Directory)
+	existingDb := PreexistingDatabase(o.Directory)
 	if len(existingDb) != 0 && len(o.Type) != 0 && o.Type != existingDb {
 		return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb)
 	}
 	if o.Type == dbPebble || existingDb == dbPebble {
 		if PebbleEnabled {
 			log.Info("Using pebble as the backing database")
-			return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
+			return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral)
 		} else {
 			return nil, errors.New("db.engine 'pebble' not supported on this platform")
 		}
@@ -219,7 +222,7 @@ func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) {
 	// on supported platforms and LevelDB on anything else.
 	if PebbleEnabled {
 		log.Info("Defaulting to pebble as the backing database")
-		return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
+		return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral)
 	} else {
 		log.Info("Defaulting to leveldb as the backing database")
 		return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
@@ -286,7 +289,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		receipts        stat
 		numHashPairings stat
 		hashNumPairings stat
-		tries           stat
+		legacyTries     stat
+		stateLookups    stat
+		accountTries    stat
+		storageTries    stat
 		codes           stat
 		txLookups       stat
 		accountSnaps    stat
@@ -330,8 +336,14 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 			numHashPairings.Add(size)
 		case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):
 			hashNumPairings.Add(size)
-		case len(key) == common.HashLength:
-			tries.Add(size)
+		case IsLegacyTrieNode(key, it.Value()):
+			legacyTries.Add(size)
+		case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength:
+			stateLookups.Add(size)
+		case IsAccountTrieNode(key):
+			accountTries.Add(size)
+		case IsStorageTrieNode(key):
+			storageTries.Add(size)
 		case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength:
 			codes.Add(size)
 		case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
@@ -364,6 +376,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 				databaseVersionKey, headHeaderKey, headBlockKey,
 				snapshotRootKey, snapshotBlockHashKey, snapshotGeneratorKey,
 				uncleanShutdownKey, syncRootKey, txIndexTailKey,
+				persistentStateIDKey, trieJournalKey,
 			} {
 				if bytes.Equal(key, meta) {
 					metadata.Add(size)
@@ -391,7 +404,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		{"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()},
 		{"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()},
 		{"Key-Value store", "Contract codes", codes.Size(), codes.Count()},
-		{"Key-Value store", "Trie nodes", tries.Size(), tries.Count()},
+		{"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()},
+		{"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()},
+		{"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()},
+		{"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()},
 		{"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()},
 		{"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()},
 		{"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()},

core/rawdb/databases_64bit.go

@@ -28,8 +28,8 @@ const PebbleEnabled = true
 
 // NewPebbleDBDatabase creates a persistent key-value database without a freezer
 // moving immutable chain segments into cold storage.
-func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
-	db, err := pebble.New(file, cache, handles, namespace, readonly)
+func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly, ephemeral bool) (ethdb.Database, error) {
+	db, err := pebble.New(file, cache, handles, namespace, readonly, ephemeral)
 	if err != nil {
 		return nil, err
 	}

core/rawdb/databases_non64bit.go

@@ -39,6 +39,6 @@ const PebbleEnabled = false
 
 // NewPebbleDBDatabase creates a persistent key-value database without a freezer
 // moving immutable chain segments into cold storage.
-func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
+func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly, ephemeral bool) (ethdb.Database, error) {
 	return nil, errors.New("pebble is not supported on this platform")
 }

core/rawdb/schema.go

@@ -184,7 +184,11 @@ func accountSnapshotKey(hash common.Hash) []byte {
 
 // storageSnapshotKey = SnapshotStoragePrefix + account hash + storage hash
 func storageSnapshotKey(accountHash, storageHash common.Hash) []byte {
-	return append(append(SnapshotStoragePrefix, accountHash.Bytes()...), storageHash.Bytes()...)
+	buf := make([]byte, len(SnapshotStoragePrefix)+common.HashLength+common.HashLength)
+	n := copy(buf, SnapshotStoragePrefix)
+	n += copy(buf[n:], accountHash.Bytes())
+	copy(buf[n:], storageHash.Bytes())
+	return buf
 }
 
 // storageSnapshotsKey = SnapshotStoragePrefix + account hash + storage hash
@@ -243,7 +247,11 @@ func accountTrieNodeKey(path []byte) []byte {
 
 // storageTrieNodeKey = trieNodeStoragePrefix + accountHash + nodePath.
 func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte {
-	return append(append(trieNodeStoragePrefix, accountHash.Bytes()...), path...)
+	buf := make([]byte, len(trieNodeStoragePrefix)+common.HashLength+len(path))
+	n := copy(buf, trieNodeStoragePrefix)
+	n += copy(buf[n:], accountHash.Bytes())
+	copy(buf[n:], path)
+	return buf
 }
 
 // IsLegacyTrieNode reports whether a provided database entry is a legacy trie
@@ -257,9 +265,10 @@ func IsLegacyTrieNode(key []byte, val []byte) bool {
 	return bytes.Equal(key, crypto.Keccak256(val))
 }
 
-// IsAccountTrieNode reports whether a provided database entry is an account
-// trie node in path-based state scheme.
-func IsAccountTrieNode(key []byte) (bool, []byte) {
+// ResolveAccountTrieNodeKey reports whether a provided database entry is an
+// account trie node in path-based state scheme, and returns the resolved
+// node path if so.
+func ResolveAccountTrieNodeKey(key []byte) (bool, []byte) {
 	if !bytes.HasPrefix(key, trieNodeAccountPrefix) {
 		return false, nil
 	}
@@ -272,9 +281,17 @@ func IsAccountTrieNode(key []byte) (bool, []byte) {
 	return true, key[len(trieNodeAccountPrefix):]
 }
 
-// IsStorageTrieNode reports whether a provided database entry is a storage
+// IsAccountTrieNode reports whether a provided database entry is an account
 // trie node in path-based state scheme.
-func IsStorageTrieNode(key []byte) (bool, common.Hash, []byte) {
+func IsAccountTrieNode(key []byte) bool {
+	ok, _ := ResolveAccountTrieNodeKey(key)
+	return ok
+}
+
+// ResolveStorageTrieNode reports whether a provided database entry is a storage
+// trie node in path-based state scheme, and returns the resolved account hash
+// and node path if so.
+func ResolveStorageTrieNode(key []byte) (bool, common.Hash, []byte) {
 	if !bytes.HasPrefix(key, trieNodeStoragePrefix) {
 		return false, common.Hash{}, nil
 	}
@@ -290,3 +307,10 @@ func IsStorageTrieNode(key []byte) (bool, common.Hash, []byte) {
 	accountHash := common.BytesToHash(key[len(trieNodeStoragePrefix) : len(trieNodeStoragePrefix)+common.HashLength])
 	return true, accountHash, key[len(trieNodeStoragePrefix)+common.HashLength:]
 }
+
+// IsStorageTrieNode reports whether a provided database entry is a storage
+// trie node in path-based state scheme.
+func IsStorageTrieNode(key []byte) bool {
+	ok, _, _ := ResolveStorageTrieNode(key)
+	return ok
+}

core/state/database.go

@@ -68,7 +68,7 @@ type Database interface {
 	// DiskDB returns the underlying key-value disk database.
 	DiskDB() ethdb.KeyValueStore
 
-	// TrieDB retrieves the low level trie database used for data storage.
+	// TrieDB returns the underlying trie database for managing trie nodes.
 	TrieDB() *trie.Database
 }
 
@@ -157,7 +157,7 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database {
 		disk:          db,
 		codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize),
 		codeCache:     lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize),
-		triedb:        trie.NewDatabaseWithConfig(db, config),
+		triedb:        trie.NewDatabase(db, config),
 	}
 }
 

core/state/iterator.go

@@ -122,7 +122,7 @@ func (it *nodeIterator) step() error {
 	}
 	// Otherwise we've reached an account node, initiate data iteration
 	var account types.StateAccount
-	if err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob()), &account); err != nil {
+	if err := rlp.DecodeBytes(it.stateIt.LeafBlob(), &account); err != nil {
 		return err
 	}
 	// Lookup the preimage of account hash

core/state/iterator_test.go

@@ -36,9 +36,14 @@ import (
 
 // Tests that the node iterator indeed walks over the entire database contents.
 func TestNodeIteratorCoverage(t *testing.T) {
+	testNodeIteratorCoverage(t, rawdb.HashScheme)
+	testNodeIteratorCoverage(t, rawdb.PathScheme)
+}
+
+func testNodeIteratorCoverage(t *testing.T, scheme string) {
 	// Create some arbitrary test state to iterate
-	db, sdb, root, _ := makeTestState()
-	sdb.TrieDB().Commit(root, false)
+	db, sdb, ndb, root, _ := makeTestState(scheme)
+	ndb.Commit(root, false)
 
 	state, err := New(root, sdb, nil)
 	if err != nil {
@@ -58,7 +63,7 @@ func TestNodeIteratorCoverage(t *testing.T) {
 	)
 	it := db.NewIterator(nil, nil)
 	for it.Next() {
-		ok, hash := isTrieNode(sdb.TrieDB().Scheme(), it.Key(), it.Value())
+		ok, hash := isTrieNode(scheme, it.Key(), it.Value())
 		if !ok {
 			continue
 		}
@@ -100,11 +105,11 @@ func isTrieNode(scheme string, key, val []byte) (bool, common.Hash) {
 			return true, common.BytesToHash(key)
 		}
 	} else {
-		ok, _ := rawdb.IsAccountTrieNode(key)
+		ok := rawdb.IsAccountTrieNode(key)
 		if ok {
 			return true, crypto.Keccak256Hash(val)
 		}
-		ok, _, _ = rawdb.IsStorageTrieNode(key)
+		ok = rawdb.IsStorageTrieNode(key)
 		if ok {
 			return true, crypto.Keccak256Hash(val)
 		}

core/state/pruner/pruner.go

@@ -95,16 +95,20 @@ func NewPruner(db ethdb.Database, config Config) (*Pruner, error) {
 	if headBlock == nil {
 		return nil, errors.New("failed to load head block")
 	}
+	// Offline pruning is only supported in legacy hash based scheme.
+	triedb := trie.NewDatabase(db, trie.HashDefaults)
+
 	// Note: we refuse to start a pruning session unless the snapshot disk layer exists, which should prevent
 	// us from ever needing to enter RecoverPruning in an invalid pruning session (a session where we do not have
 	// the protected trie in the triedb and in the snapshot disk layer).
+
 	snapconfig := snapshot.Config{
 		CacheSize:  256,
 		AsyncBuild: false,
 		NoBuild:    true,
 		SkipVerify: true,
 	}
-	snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Hash(), headBlock.Root())
+	snaptree, err := snapshot.New(snapconfig, db, triedb, headBlock.Hash(), headBlock.Root())
 	if err != nil {
 		return nil, fmt.Errorf("failed to create snapshot for pruning, must restart without offline pruning disabled to recover: %w", err) // The relevant snapshot(s) might not exist
 	}
@@ -342,7 +346,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
 	if genesis == nil {
 		return errors.New("missing genesis block")
 	}
-	t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db))
+	t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db, trie.HashDefaults))
 	if err != nil {
 		return err
 	}
@@ -366,7 +370,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
 			}
 			if acc.Root != types.EmptyRootHash {
 				id := trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root)
-				storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db))
+				storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db, trie.HashDefaults))
 				if err != nil {
 					return err
 				}

core/state/snapshot/generate_test.go

@@ -36,6 +36,8 @@ import (
 	"github.com/ava-labs/subnet-evm/core/rawdb"
 	"github.com/ava-labs/subnet-evm/core/types"
 	"github.com/ava-labs/subnet-evm/trie"
+	"github.com/ava-labs/subnet-evm/trie/triedb/hashdb"
+	"github.com/ava-labs/subnet-evm/trie/triedb/pathdb"
 	"github.com/ava-labs/subnet-evm/trie/trienode"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -57,9 +59,15 @@ func hashData(input []byte) common.Hash {
 
 // Tests that snapshot generation from an empty database.
 func TestGeneration(t *testing.T) {
+	testGeneration(t, rawdb.HashScheme)
+	testGeneration(t, rawdb.PathScheme)
+}
+
+func testGeneration(t *testing.T, scheme string) {
 	// We can't use statedb to make a test trie (circular dependency), so make
 	// a fake one manually. We're going with a small account trie of 3 accounts,
-	var helper = newHelper()
+	// two of which also has the same 3-slot storage trie attached.
+	var helper = newHelper(scheme)
 	stRoot := helper.makeStorageTrie(common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false)
 
 	helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
@@ -91,10 +99,15 @@ func TestGeneration(t *testing.T) {
 
 // Tests that snapshot generation with existent flat state.
 func TestGenerateExistentState(t *testing.T) {
+	testGenerateExistentState(t, rawdb.HashScheme)
+	testGenerateExistentState(t, rawdb.PathScheme)
+}
+
+func testGenerateExistentState(t *testing.T, scheme string) {
 	// We can't use statedb to make a test trie (circular dependency), so make
 	// a fake one manually. We're going with a small account trie of 3 accounts,
 	// two of which also has the same 3-slot storage trie attached.
-	var helper = newHelper()
+	var helper = newHelper(scheme)
 
 	stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 	helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
@@ -158,9 +171,15 @@ type testHelper struct {
 	nodes   *trienode.MergedNodeSet
 }
 
-func newHelper() *testHelper {
+func newHelper(scheme string) *testHelper {
 	diskdb := rawdb.NewMemoryDatabase()
-	triedb := trie.NewDatabase(diskdb)
+	config := &trie.Config{}
+	if scheme == rawdb.PathScheme {
+		config.PathDB = &pathdb.Config{} // disable caching
+	} else {
+		config.HashDB = &hashdb.Config{} // disable caching
+	}
+	triedb := trie.NewDatabase(diskdb, config)
 	accTrie, _ := trie.NewStateTrie(trie.StateTrieID(types.EmptyRootHash), triedb)
 	return &testHelper{
 		diskdb:  diskdb,
@@ -243,7 +262,12 @@ func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) {
 //   - extra slots in the middle
 //   - extra slots in the end
 func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
-	helper := newHelper()
+	testGenerateExistentStateWithWrongStorage(t, rawdb.HashScheme)
+	testGenerateExistentStateWithWrongStorage(t, rawdb.PathScheme)
+}
+
+func testGenerateExistentStateWithWrongStorage(t *testing.T, scheme string) {
+	helper := newHelper(scheme)
 
 	// Account one, empty root but non-empty database
 	helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()})
@@ -335,7 +359,12 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
 // - wrong accounts
 // - extra accounts
 func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
-	helper := newHelper()
+	testGenerateExistentStateWithWrongAccounts(t, rawdb.HashScheme)
+	testGenerateExistentStateWithWrongAccounts(t, rawdb.PathScheme)
+}
+
+func testGenerateExistentStateWithWrongAccounts(t *testing.T, scheme string) {
+	helper := newHelper(scheme)
 
 	helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 	helper.makeStorageTrie(hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
@@ -390,10 +419,15 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
 // Tests that snapshot generation errors out correctly in case of a missing trie
 // node in the account trie.
 func TestGenerateCorruptAccountTrie(t *testing.T) {
+	testGenerateCorruptAccountTrie(t, rawdb.HashScheme)
+	testGenerateCorruptAccountTrie(t, rawdb.PathScheme)
+}
+
+func testGenerateCorruptAccountTrie(t *testing.T, scheme string) {
 	// We can't use statedb to make a test trie (circular dependency), so make
 	// a fake one manually. We're going with a small account trie of 3 accounts,
 	// without any storage slots to keep the test smaller.
-	helper := newHelper()
+	helper := newHelper(scheme)
 
 	helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074
 	helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
@@ -401,9 +435,11 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
 
 	root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978
 
-	// Delete an account trie leaf and ensure the generator chokes
-	helper.triedb.Commit(root, false)
-	helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes())
+	// Delete an account trie node and ensure the generator chokes
+	targetPath := []byte{0xc}
+	targetHash := common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7")
+
+	rawdb.DeleteTrieNode(helper.diskdb, common.Hash{}, targetPath, targetHash, scheme)
 
 	snap := generateSnapshot(helper.diskdb, helper.triedb, 16, testBlockHash, root, nil)
 	select {
@@ -424,11 +460,19 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
 // trie node for a storage trie. It's similar to internal corruption but it is
 // handled differently inside the generator.
 func TestGenerateMissingStorageTrie(t *testing.T) {
+	testGenerateMissingStorageTrie(t, rawdb.HashScheme)
+	testGenerateMissingStorageTrie(t, rawdb.PathScheme)
+}
+
+func testGenerateMissingStorageTrie(t *testing.T, scheme string) {
 	// We can't use statedb to make a test trie (circular dependency), so make
 	// a fake one manually. We're going with a small account trie of 3 accounts,
 	// two of which also has the same 3-slot storage trie attached.
-	helper := newHelper()
-
+	var (
+		acc1   = hashData([]byte("acc-1"))
+		acc3   = hashData([]byte("acc-3"))
+		helper = newHelper(scheme)
+	)
 	stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)   // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
 	helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})              // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
 	helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
@@ -437,8 +481,9 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
 
 	root := helper.Commit()
 
-	// Delete a storage trie root and ensure the generator chokes
-	helper.diskdb.Delete(stRoot.Bytes())
+	// Delete storage trie root of account one and three.
+	rawdb.DeleteTrieNode(helper.diskdb, acc1, nil, stRoot, scheme)
+	rawdb.DeleteTrieNode(helper.diskdb, acc3, nil, stRoot, scheme)
 
 	snap := generateSnapshot(helper.diskdb, helper.triedb, 16, testBlockHash, root, nil)
 	select {
@@ -458,10 +503,15 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
 // Tests that snapshot generation errors out correctly in case of a missing trie
 // node in a storage trie.
 func TestGenerateCorruptStorageTrie(t *testing.T) {
+	testGenerateCorruptStorageTrie(t, rawdb.HashScheme)
+	testGenerateCorruptStorageTrie(t, rawdb.PathScheme)
+}
+
+func testGenerateCorruptStorageTrie(t *testing.T, scheme string) {
 	// We can't use statedb to make a test trie (circular dependency), so make
 	// a fake one manually. We're going with a small account trie of 3 accounts,
 	// two of which also has the same 3-slot storage trie attached.
-	helper := newHelper()
+	helper := newHelper(scheme)
 
 	stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)   // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
 	helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})              // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
@@ -471,8 +521,11 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
 
 	root := helper.Commit()
 
-	// Delete a storage trie leaf and ensure the generator chokes
-	helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes())
+	// Delete a node in the storage trie.
+	targetPath := []byte{0x4}
+	targetHash := common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371")
+	rawdb.DeleteTrieNode(helper.diskdb, hashData([]byte("acc-1")), targetPath, targetHash, scheme)
+	rawdb.DeleteTrieNode(helper.diskdb, hashData([]byte("acc-3")), targetPath, targetHash, scheme)
 
 	snap := generateSnapshot(helper.diskdb, helper.triedb, 16, testBlockHash, root, nil)
 	select {
@@ -491,7 +544,12 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
 
 // Tests that snapshot generation when an extra account with storage exists in the snap state.
 func TestGenerateWithExtraAccounts(t *testing.T) {
-	helper := newHelper()
+	testGenerateWithExtraAccounts(t, rawdb.HashScheme)
+	testGenerateWithExtraAccounts(t, rawdb.PathScheme)
+}
+
+func testGenerateWithExtraAccounts(t *testing.T, scheme string) {
+	helper := newHelper(scheme)
 	{
 		// Account one in the trie
 		stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")),
@@ -559,10 +617,15 @@ func enableLogging() {
 
 // Tests that snapshot generation when an extra account with storage exists in the snap state.
 func TestGenerateWithManyExtraAccounts(t *testing.T) {
+	testGenerateWithManyExtraAccounts(t, rawdb.HashScheme)
+	testGenerateWithManyExtraAccounts(t, rawdb.PathScheme)
+}
+
+func testGenerateWithManyExtraAccounts(t *testing.T, scheme string) {
 	if false {
 		enableLogging()
 	}
-	helper := newHelper()
+	helper := newHelper(scheme)
 	{
 		// Account one in the trie
 		stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")),
@@ -615,10 +678,15 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) {
 // So in trie, we iterate 2 entries 0x03, 0x07. We create the 0x07 in the database and abort the procedure, because the trie is exhausted.
 // But in the database, we still have the stale storage slots 0x04, 0x05. They are not iterated yet, but the procedure is finished.
 func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
+	testGenerateWithExtraBeforeAndAfter(t, rawdb.HashScheme)
+	testGenerateWithExtraBeforeAndAfter(t, rawdb.PathScheme)
+}
+
+func testGenerateWithExtraBeforeAndAfter(t *testing.T, scheme string) {
 	if false {
 		enableLogging()
 	}
-	helper := newHelper()
+	helper := newHelper(scheme)
 	{
 		acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}
 		val, _ := rlp.EncodeToBytes(acc)
@@ -651,10 +719,15 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
 // TestGenerateWithMalformedSnapdata tests what happes if we have some junk
 // in the snapshot database, which cannot be parsed back to an account
 func TestGenerateWithMalformedSnapdata(t *testing.T) {
+	testGenerateWithMalformedSnapdata(t, rawdb.HashScheme)
+	testGenerateWithMalformedSnapdata(t, rawdb.PathScheme)
+}
+
+func testGenerateWithMalformedSnapdata(t *testing.T, scheme string) {
 	if false {
 		enableLogging()
 	}
-	helper := newHelper()
+	helper := newHelper(scheme)
 	{
 		acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}
 		val, _ := rlp.EncodeToBytes(acc)
@@ -687,8 +760,13 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) {
 }
 
 func TestGenerateFromEmptySnap(t *testing.T) {
+	testGenerateFromEmptySnap(t, rawdb.HashScheme)
+	testGenerateFromEmptySnap(t, rawdb.PathScheme)
+}
+
+func testGenerateFromEmptySnap(t *testing.T, scheme string) {
 	//enableLogging()
-	helper := newHelper()
+	helper := newHelper(scheme)
 	// Add 1K accounts to the trie
 	for i := 0; i < 400; i++ {
 		stRoot := helper.makeStorageTrie(hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
@@ -720,7 +798,12 @@ func TestGenerateFromEmptySnap(t *testing.T) {
 // This hits a case where the snap verification passes, but there are more elements in the trie
 // which we must also add.
 func TestGenerateWithIncompleteStorage(t *testing.T) {
-	helper := newHelper()
+	testGenerateWithIncompleteStorage(t, rawdb.HashScheme)
+	testGenerateWithIncompleteStorage(t, rawdb.PathScheme)
+}
+
+func testGenerateWithIncompleteStorage(t *testing.T, scheme string) {
+	helper := newHelper(scheme)
 	stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"}
 	stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"}
 	// We add 8 accounts, each one is missing exactly one of the storage slots. This means
@@ -818,7 +901,12 @@ func populateDangling(disk ethdb.KeyValueStore) {
 //
 // This test will populate some dangling storages to see if they can be cleaned up.
 func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
-	var helper = newHelper()
+	testGenerateCompleteSnapshotWithDanglingStorage(t, rawdb.HashScheme)
+	testGenerateCompleteSnapshotWithDanglingStorage(t, rawdb.PathScheme)
+}
+
+func testGenerateCompleteSnapshotWithDanglingStorage(t *testing.T, scheme string) {
+	var helper = newHelper(scheme)
 
 	stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 	helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
@@ -853,7 +941,12 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
 //
 // This test will populate some dangling storages to see if they can be cleaned up.
 func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) {
-	var helper = newHelper()
+	testGenerateBrokenSnapshotWithDanglingStorage(t, rawdb.HashScheme)
+	testGenerateBrokenSnapshotWithDanglingStorage(t, rawdb.PathScheme)
+}
+
+func testGenerateBrokenSnapshotWithDanglingStorage(t *testing.T, scheme string) {
+	var helper = newHelper(scheme)
 
 	stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 	helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})

core/state/snapshot/snapshot.go

@@ -38,7 +38,6 @@ import (
 	"github.com/ava-labs/subnet-evm/core/types"
 	"github.com/ava-labs/subnet-evm/metrics"
 	"github.com/ava-labs/subnet-evm/trie"
-	"github.com/ava-labs/subnet-evm/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
@@ -921,51 +920,20 @@ func (t *Tree) DiskRoot() common.Hash {
 	return t.diskRoot()
 }
 
-func (t *Tree) DiskAccountIterator(seek common.Hash) AccountIterator {
-	t.lock.Lock()
-	defer t.lock.Unlock()
+// Size returns the memory usage of the diff layers above the disk layer and the
+// dirty nodes buffered in the disk layer. Currently, the implementation uses a
+// special diff layer (the first) as an aggregator simulating a dirty buffer, so
+// the second return will always be 0. However, this will be made consistent with
+// the pathdb, which will require a second return.
+func (t *Tree) Size() (diffs common.StorageSize, buf common.StorageSize) {
+	t.lock.RLock()
+	defer t.lock.RUnlock()
 
-	return t.disklayer().AccountIterator(seek)
-}
-
-func (t *Tree) DiskStorageIterator(account common.Hash, seek common.Hash) StorageIterator {
-	t.lock.Lock()
-	defer t.lock.Unlock()
-
-	it, _ := t.disklayer().StorageIterator(account, seek)
-	return it
-}
-
-// NewDiskLayer creates a diskLayer for direct access to the contents of the on-disk
-// snapshot. Does not perform any validation.
-func NewDiskLayer(diskdb ethdb.KeyValueStore) Snapshot {
-	return &diskLayer{
-		diskdb:  diskdb,
-		created: time.Now(),
-
-		// state sync uses iterators to access data, so this cache is not used.
-		// initializing it out of caution.
-		cache: utils.NewMeteredCache(32*1024, "", 0),
-	}
-}
-
-// NewTestTree creates a *Tree with a pre-populated diskLayer
-func NewTestTree(diskdb ethdb.KeyValueStore, blockHash, root common.Hash) *Tree {
-	base := &diskLayer{
-		diskdb:    diskdb,
-		root:      root,
-		blockHash: blockHash,
-		cache:     utils.NewMeteredCache(128*256, "", 0),
-		created:   time.Now(),
-	}
-	return &Tree{
-		blockLayers: map[common.Hash]snapshot{
-			blockHash: base,
-		},
-		stateLayers: map[common.Hash]map[common.Hash]snapshot{
-			root: {
-				blockHash: base,
-			},
-		},
+	var size common.StorageSize
+	for _, layer := range t.blockLayers {
+		if layer, ok := layer.(*diffLayer); ok {
+			size += common.StorageSize(layer.memory)
+		}
 	}
+	return size, 0
 }

core/state/snapshot/snapshot_ext.go

@@ -0,0 +1,58 @@
+package snapshot
+
+import (
+	"time"
+
+	"github.com/ava-labs/subnet-evm/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+func (t *Tree) DiskAccountIterator(seek common.Hash) AccountIterator {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	return t.disklayer().AccountIterator(seek)
+}
+
+func (t *Tree) DiskStorageIterator(account common.Hash, seek common.Hash) StorageIterator {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+
+	it, _ := t.disklayer().StorageIterator(account, seek)
+	return it
+}
+
+// NewDiskLayer creates a diskLayer for direct access to the contents of the on-disk
+// snapshot. Does not perform any validation.
+func NewDiskLayer(diskdb ethdb.KeyValueStore) Snapshot {
+	return &diskLayer{
+		diskdb:  diskdb,
+		created: time.Now(),
+
+		// state sync uses iterators to access data, so this cache is not used.
+		// initializing it out of caution.
+		cache: utils.NewMeteredCache(32*1024, "", 0),
+	}
+}
+
+// NewTestTree creates a *Tree with a pre-populated diskLayer
+func NewTestTree(diskdb ethdb.KeyValueStore, blockHash, root common.Hash) *Tree {
+	base := &diskLayer{
+		diskdb:    diskdb,
+		root:      root,
+		blockHash: blockHash,
+		cache:     utils.NewMeteredCache(128*256, "", 0),
+		created:   time.Now(),
+	}
+	return &Tree{
+		blockLayers: map[common.Hash]snapshot{
+			blockHash: base,
+		},
+		stateLayers: map[common.Hash]map[common.Hash]snapshot{
+			root: {
+				blockHash: base,
+			},
+		},
+	}
+}

core/state/state_object.go

@@ -274,12 +274,17 @@ func (s *stateObject) finalise(prefetch bool) {
 	}
 }
 
-// updateTrie writes cached storage modifications into the object's storage trie.
-// It will return nil if the trie has not been loaded and no changes have been
-// made. An error will be returned if the trie can't be loaded/updated correctly.
+// updateTrie is responsible for persisting cached storage changes into the
+// object's storage trie. In case the storage trie is not yet loaded, this
+// function will load the trie automatically. If any issues arise during the
+// loading or updating of the trie, an error will be returned. Furthermore,
+// this function will return the mutated storage trie, or nil if there is no
+// storage change at all.
 func (s *stateObject) updateTrie() (Trie, error) {
 	// Make sure all dirty slots are finalized into the pending storage area
-	s.finalise(false) // Don't prefetch anymore, pull directly if need be
+	s.finalise(false)
+
+	// Short circuit if nothing changed, don't bother with hashing anything
 	if len(s.pendingStorage) == 0 {
 		return s.trie, nil
 	}
@@ -291,14 +296,13 @@ func (s *stateObject) updateTrie() (Trie, error) {
 	var (
 		storage map[common.Hash][]byte
 		origin  map[common.Hash][]byte
-		hasher  = s.db.hasher
 	)
 	tr, err := s.getTrie()
 	if err != nil {
 		s.db.setError(err)
 		return nil, err
 	}
-	// Insert all the pending updates into the trie
+	// Insert all the pending storage updates into the trie
 	usedStorage := make([][]byte, 0, len(s.pendingStorage))
 	for key, value := range s.pendingStorage {
 		// Skip noop changes, persist actual changes
@@ -308,8 +312,7 @@ func (s *stateObject) updateTrie() (Trie, error) {
 		prev := s.originStorage[key]
 		s.originStorage[key] = value
 
-		// rlp-encoded value to be used by the snapshot
-		var snapshotVal []byte
+		var encoded []byte // rlp-encoded value to be used by the snapshot
 		if (value == common.Hash{}) {
 			if err := tr.DeleteStorage(s.address, key[:]); err != nil {
 				s.db.setError(err)
@@ -317,10 +320,10 @@ func (s *stateObject) updateTrie() (Trie, error) {
 			}
 			s.db.StorageDeleted += 1
 		} else {
-			trimmedVal := common.TrimLeftZeroes(value[:])
 			// Encoding []byte cannot fail, ok to ignore the error.
-			snapshotVal, _ = rlp.EncodeToBytes(trimmedVal)
-			if err := tr.UpdateStorage(s.address, key[:], trimmedVal); err != nil {
+			trimmed := common.TrimLeftZeroes(value[:])
+			encoded, _ = rlp.EncodeToBytes(trimmed)
+			if err := tr.UpdateStorage(s.address, key[:], trimmed); err != nil {
 				s.db.setError(err)
 				return nil, err
 			}
@@ -333,8 +336,8 @@ func (s *stateObject) updateTrie() (Trie, error) {
 				s.db.storages[s.addrHash] = storage
 			}
 		}
-		khash := crypto.HashData(hasher, key[:])
-		storage[khash] = snapshotVal // snapshotVal will be nil if it's deleted
+		khash := crypto.HashData(s.db.hasher, key[:])
+		storage[khash] = encoded // encoded will be nil if it's deleted
 
 		// Cache the original value of mutated storage slots
 		if origin == nil {
@@ -359,21 +362,17 @@ func (s *stateObject) updateTrie() (Trie, error) {
 	if s.db.prefetcher != nil {
 		s.db.prefetcher.used(s.addrHash, s.data.Root, usedStorage)
 	}
-	if len(s.pendingStorage) > 0 {
-		s.pendingStorage = make(Storage)
-	}
+	s.pendingStorage = make(Storage) // reset pending map
 	return tr, nil
 }
 
-// UpdateRoot sets the trie root to the current root hash of. An error
-// will be returned if trie root hash is not computed correctly.
+// updateRoot flushes all cached storage mutations to trie, recalculating the
+// new storage trie root.
 func (s *stateObject) updateRoot() {
+	// Flush cached storage mutations into trie, short circuit if any error
+	// is occurred or there is not change in the trie.
 	tr, err := s.updateTrie()
-	if err != nil {
-		return
-	}
-	// If nothing changed, don't bother with hashing anything
-	if tr == nil {
+	if err != nil || tr == nil {
 		return
 	}
 	// Track the amount of time wasted on hashing the storage trie
@@ -383,14 +382,12 @@ func (s *stateObject) updateRoot() {
 	s.data.Root = tr.Hash()
 }
 
-// commit returns the changes made in storage trie and updates the account data.
+// commit obtains a set of dirty storage trie nodes and updates the account data.
+// The returned set can be nil if nothing to commit. This function assumes all
+// storage mutations have already been flushed into trie by updateRoot.
 func (s *stateObject) commit() (*trienode.NodeSet, error) {
-	tr, err := s.updateTrie()
-	if err != nil {
-		return nil, err
-	}
-	// If nothing changed, don't bother with committing anything
-	if tr == nil {
+	// Short circuit if trie is not even loaded, don't bother with committing anything
+	if s.trie == nil {
 		s.origin = s.data.Copy()
 		return nil, nil
 	}
@@ -398,7 +395,10 @@ func (s *stateObject) commit() (*trienode.NodeSet, error) {
 	if metrics.EnabledExpensive {
 		defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now())
 	}
-	root, nodes, err := tr.Commit(false)
+	// The trie is currently in an open state and could potentially contain
+	// cached mutations. Call commit to acquire a set of nodes that have been
+	// modified, the set can be nil if nothing to commit.
+	root, nodes, err := s.trie.Commit(false)
 	if err != nil {
 		return nil, err
 	}
@@ -546,3 +546,7 @@ func (s *stateObject) Balance() *big.Int {
 func (s *stateObject) Nonce() uint64 {
 	return s.data.Nonce
 }
+
+func (s *stateObject) Root() common.Hash {
+	return s.data.Root
+}

core/state/statedb.go

@@ -28,7 +28,6 @@
 package state
 
 import (
-	"errors"
 	"fmt"
 	"math/big"
 	"sort"
@@ -46,7 +45,12 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/rlp"
+)
+
+const (
+	// storageDeleteLimit denotes the highest permissible memory allocation
+	// employed for contract storage deletion.
+	storageDeleteLimit = 512 * 1024 * 1024
 )
 
 type revision struct {
@@ -54,17 +58,6 @@ type revision struct {
 	journalIndex int
 }
 
-type proofList [][]byte
-
-func (n *proofList) Put(key []byte, value []byte) error {
-	*n = append(*n, value)
-	return nil
-}
-
-func (n *proofList) Delete(key []byte) error {
-	panic("not supported")
-}
-
 // StateDB structs within the ethereum protocol are used to store anything
 // within the merkle trie. StateDBs take care of caching and storing
 // nested states. It's the general query interface to retrieve:
@@ -155,6 +148,9 @@ type StateDB struct {
 	StorageUpdated int
 	AccountDeleted int
 	StorageDeleted int
+
+	// Testing hooks
+	onCommit func(states *triestate.Set) // Hook invoked when commit is performed
 }
 
 // New creates a new state from a given trie.
@@ -345,6 +341,7 @@ func (s *StateDB) GetBalance(addr common.Address) *big.Int {
 	return new(big.Int).Set(common.Big0)
 }
 
+// GetNonce retrieves the nonce from the given address or 0 if object not found
 func (s *StateDB) GetNonce(addr common.Address) uint64 {
 	stateObject := s.getStateObject(addr)
 	if stateObject != nil {
@@ -354,6 +351,16 @@ func (s *StateDB) GetNonce(addr common.Address) uint64 {
 	return 0
 }
 
+// GetStorageRoot retrieves the storage root from the given address or empty
+// if object not found.
+func (s *StateDB) GetStorageRoot(addr common.Address) common.Hash {
+	stateObject := s.getStateObject(addr)
+	if stateObject != nil {
+		return stateObject.Root()
+	}
+	return common.Hash{}
+}
+
 // TxIndex returns the current transaction index set by Prepare.
 func (s *StateDB) TxIndex() int {
 	return s.txIndex
@@ -392,35 +399,6 @@ func (s *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash {
 	return common.Hash{}
 }
 
-// GetProof returns the Merkle proof for a given account.
-func (s *StateDB) GetProof(addr common.Address) ([][]byte, error) {
-	return s.GetProofByHash(crypto.Keccak256Hash(addr.Bytes()))
-}
-
-// GetProofByHash returns the Merkle proof for a given account.
-func (s *StateDB) GetProofByHash(addrHash common.Hash) ([][]byte, error) {
-	var proof proofList
-	err := s.trie.Prove(addrHash[:], &proof)
-	return proof, err
-}
-
-// GetStorageProof returns the Merkle proof for given storage slot.
-func (s *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, error) {
-	trie, err := s.StorageTrie(a)
-	if err != nil {
-		return nil, err
-	}
-	if trie == nil {
-		return nil, errors.New("storage trie for requested address does not exist")
-	}
-	var proof proofList
-	err = trie.Prove(crypto.Keccak256(key.Bytes()), &proof)
-	if err != nil {
-		return nil, err
-	}
-	return proof, nil
-}
-
 // GetCommittedState retrieves a value from the given account's committed storage trie.
 func (s *StateDB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash {
 	stateObject := s.getStateObject(addr)
@@ -435,21 +413,6 @@ func (s *StateDB) Database() Database {
 	return s.db
 }
 
-// StorageTrie returns the storage trie of an account. The return value is a copy
-// and is nil for non-existent accounts. An error will be returned if storage trie
-// is existent but can't be loaded correctly.
-func (s *StateDB) StorageTrie(addr common.Address) (Trie, error) {
-	stateObject := s.getStateObject(addr)
-	if stateObject == nil {
-		return nil, nil
-	}
-	cpy := stateObject.deepCopy(s)
-	if _, err := cpy.updateTrie(); err != nil {
-		return nil, err
-	}
-	return cpy.getTrie()
-}
-
 func (s *StateDB) HasSelfDestructed(addr common.Address) bool {
 	stateObject := s.getStateObject(addr)
 	if stateObject != nil {
@@ -775,43 +738,6 @@ func (s *StateDB) CreateAccount(addr common.Address) {
 	}
 }
 
-func (s *StateDB) ForEachStorage(addr common.Address, cb func(key, value common.Hash) bool) error {
-	so := s.getStateObject(addr)
-	if so == nil {
-		return nil
-	}
-	tr, err := so.getTrie()
-	if err != nil {
-		return err
-	}
-	trieIt, err := tr.NodeIterator(nil)
-	if err != nil {
-		return err
-	}
-	it := trie.NewIterator(trieIt)
-
-	for it.Next() {
-		key := common.BytesToHash(s.trie.GetKey(it.Key))
-		if value, dirty := so.dirtyStorage[key]; dirty {
-			if !cb(key, value) {
-				return nil
-			}
-			continue
-		}
-
-		if len(it.Value) > 0 {
-			_, content, _, err := rlp.Split(it.Value)
-			if err != nil {
-				return err
-			}
-			if !cb(key, common.BytesToHash(content)) {
-				return nil
-			}
-		}
-	}
-	return nil
-}
-
 // copyPredicateStorageSlots creates a deep copy of the provided predicateStorageSlots map.
 func copyPredicateStorageSlots(predicateStorageSlots map[common.Address][][]byte) map[common.Address][][]byte {
 	res := make(map[common.Address][][]byte, len(predicateStorageSlots))
@@ -1088,59 +1014,125 @@ func (s *StateDB) clearJournalAndRefund() {
 	s.validRevisions = s.validRevisions[:0] // Snapshots can be created without journal entries
 }
 
-// deleteStorage iterates the storage trie belongs to the account and mark all
-// slots inside as deleted.
-func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, map[common.Hash][]byte, *trienode.NodeSet, error) {
-	start := time.Now()
+// fastDeleteStorage is the function that efficiently deletes the storage trie
+// of a specific account. It leverages the associated state snapshot for fast
+// storage iteration and constructs trie node deletion markers by creating
+// stack trie with iterated slots.
+func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (bool, common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) {
+	iter, _ := s.snap.StorageIterator(addrHash, common.Hash{})
+	defer iter.Release()
+
+	var (
+		size  common.StorageSize
+		nodes = trienode.NewNodeSet(addrHash)
+		slots = make(map[common.Hash][]byte)
+	)
+	stack := trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) {
+		nodes.AddNode(path, trienode.NewDeleted())
+		size += common.StorageSize(len(path))
+	})
+	for iter.Next() {
+		if size > storageDeleteLimit {
+			return true, size, nil, nil, nil
+		}
+		slot := common.CopyBytes(iter.Slot())
+		if err := iter.Error(); err != nil { // error might occur after Slot function
+			return false, 0, nil, nil, err
+		}
+		size += common.StorageSize(common.HashLength + len(slot))
+		slots[iter.Hash()] = slot
+
+		if err := stack.Update(iter.Hash().Bytes(), slot); err != nil {
+			return false, 0, nil, nil, err
+		}
+	}
+	if err := iter.Error(); err != nil { // error might occur during iteration
+		return false, 0, nil, nil, err
+	}
+	if stack.Hash() != root {
+		return false, 0, nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash())
+	}
+	return false, size, slots, nodes, nil
+}
+
+// slowDeleteStorage serves as a less-efficient alternative to "fastDeleteStorage,"
+// employed when the associated state snapshot is not available. It iterates the
+// storage slots along with all internal trie nodes via trie directly.
+func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) {
 	tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root)
 	if err != nil {
-		return false, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err)
+		return false, 0, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err)
 	}
 	it, err := tr.NodeIterator(nil)
 	if err != nil {
-		return false, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err)
+		return false, 0, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err)
 	}
 	var (
-		set       = trienode.NewNodeSet(addrHash)
-		slots     = make(map[common.Hash][]byte)
-		stateSize common.StorageSize
-		nodeSize  common.StorageSize
+		size  common.StorageSize
+		nodes = trienode.NewNodeSet(addrHash)
+		slots = make(map[common.Hash][]byte)
 	)
 	for it.Next(true) {
-		// arbitrary stateSize limit, make it configurable
-		if stateSize+nodeSize > 512*1024*1024 {
-			log.Info("Skip large storage deletion", "address", addr.Hex(), "states", stateSize, "nodes", nodeSize)
-			if metrics.EnabledExpensive {
-				slotDeletionSkip.Inc(1)
-			}
-			return true, nil, nil, nil
+		if size > storageDeleteLimit {
+			return true, size, nil, nil, nil
 		}
 		if it.Leaf() {
 			slots[common.BytesToHash(it.LeafKey())] = common.CopyBytes(it.LeafBlob())
-			stateSize += common.StorageSize(common.HashLength + len(it.LeafBlob()))
+			size += common.StorageSize(common.HashLength + len(it.LeafBlob()))
 			continue
 		}
 		if it.Hash() == (common.Hash{}) {
 			continue
 		}
-		nodeSize += common.StorageSize(len(it.Path()))
-		set.AddNode(it.Path(), trienode.NewDeleted())
+		size += common.StorageSize(len(it.Path()))
+		nodes.AddNode(it.Path(), trienode.NewDeleted())
 	}
 	if err := it.Error(); err != nil {
+		return false, 0, nil, nil, err
+	}
+	return false, size, slots, nodes, nil
+}
+
+// deleteStorage is designed to delete the storage trie of a designated account.
+// It could potentially be terminated if the storage size is excessively large,
+// potentially leading to an out-of-memory panic. The function will make an attempt
+// to utilize an efficient strategy if the associated state snapshot is reachable;
+// otherwise, it will resort to a less-efficient approach.
+func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, map[common.Hash][]byte, *trienode.NodeSet, error) {
+	var (
+		start   = time.Now()
+		err     error
+		aborted bool
+		size    common.StorageSize
+		slots   map[common.Hash][]byte
+		nodes   *trienode.NodeSet
+	)
+	// The fast approach can be failed if the snapshot is not fully
+	// generated, or it's internally corrupted. Fallback to the slow
+	// one just in case.
+	if s.snap != nil {
+		aborted, size, slots, nodes, err = s.fastDeleteStorage(addrHash, root)
+	}
+	if s.snap == nil || err != nil {
+		aborted, size, slots, nodes, err = s.slowDeleteStorage(addr, addrHash, root)
+	}
+	if err != nil {
 		return false, nil, nil, err
 	}
 	if metrics.EnabledExpensive {
-		if int64(len(slots)) > slotDeletionMaxCount.Value() {
-			slotDeletionMaxCount.Update(int64(len(slots)))
-		}
-		if int64(stateSize+nodeSize) > slotDeletionMaxSize.Value() {
-			slotDeletionMaxSize.Update(int64(stateSize + nodeSize))
+		if aborted {
+			slotDeletionSkip.Inc(1)
 		}
+		n := int64(len(slots))
+
+		slotDeletionMaxCount.UpdateIfGt(int64(len(slots)))
+		slotDeletionMaxSize.UpdateIfGt(int64(size))
+
 		slotDeletionTimer.UpdateSince(start)
-		slotDeletionCount.Mark(int64(len(slots)))
-		slotDeletionSize.Mark(int64(stateSize + nodeSize))
+		slotDeletionCount.Mark(n)
+		slotDeletionSize.Mark(int64(size))
 	}
-	return false, slots, set, nil
+	return aborted, slots, nodes, nil
 }
 
 // handleDestruction processes all destruction markers and deletes the account
@@ -1168,7 +1160,13 @@ func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root
 // In case (d), **original** account along with its storages should be deleted,
 // with their values be tracked as original value.
 func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.Address]struct{}, error) {
+	// Short circuit if geth is running with hash mode. This procedure can consume
+	// considerable time and storage deletion isn't supported in hash mode, thus
+	// preemptively avoiding unnecessary expenses.
 	incomplete := make(map[common.Address]struct{})
+	if s.db.TrieDB().Scheme() == rawdb.HashScheme {
+		return incomplete, nil
+	}
 	for addr, prev := range s.stateObjectsDestruct {
 		// The original account was non-existing, and it's marked as destructed
 		// in the scope of block. It can be case (a) or (b).
@@ -1345,12 +1343,13 @@ func (s *StateDB) commit(block uint64, deleteEmptyObjects bool, snaps *snapshot.
 	}
 	if root != origin {
 		start := time.Now()
+		set := triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete)
 		if referenceRoot {
-			if err := s.db.TrieDB().UpdateAndReferenceRoot(root, origin, block, nodes, triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete)); err != nil {
+			if err := s.db.TrieDB().UpdateAndReferenceRoot(root, origin, block, nodes, set); err != nil {
 				return common.Hash{}, err
 			}
 		} else {
-			if err := s.db.TrieDB().Update(root, origin, block, nodes, triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete)); err != nil {
+			if err := s.db.TrieDB().Update(root, origin, block, nodes, set); err != nil {
 				return common.Hash{}, err
 			}
 		}
@@ -1358,6 +1357,9 @@ func (s *StateDB) commit(block uint64, deleteEmptyObjects bool, snaps *snapshot.
 		if metrics.EnabledExpensive {
 			s.TrieDBCommits += time.Since(start)
 		}
+		if s.onCommit != nil {
+			s.onCommit(set)
+		}
 	}
 	// Clear all internal flags at the end of commit operation.
 	s.accounts = make(map[common.Hash][]byte)

core/state/statedb_fuzz_test.go

@@ -40,8 +40,10 @@ import (
 	"testing/quick"
 
 	"github.com/ava-labs/subnet-evm/core/rawdb"
+	"github.com/ava-labs/subnet-evm/core/state/snapshot"
 	"github.com/ava-labs/subnet-evm/core/types"
 	"github.com/ava-labs/subnet-evm/trie"
+	"github.com/ava-labs/subnet-evm/trie/triedb/pathdb"
 	"github.com/ava-labs/subnet-evm/trie/triestate"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -189,19 +191,32 @@ func (test *stateTest) run() bool {
 			storageList = append(storageList, copy2DSet(states.Storages))
 		}
 		disk      = rawdb.NewMemoryDatabase()
-		tdb       = trie.NewDatabaseWithConfig(disk, &trie.Config{OnCommit: onCommit})
+		tdb       = trie.NewDatabase(disk, &trie.Config{PathDB: pathdb.Defaults})
 		sdb       = NewDatabaseWithNodeDB(disk, tdb)
 		byzantium = rand.Intn(2) == 0
 	)
+	defer disk.Close()
+	defer tdb.Close()
+
+	var snaps *snapshot.Tree
+	if rand.Intn(3) == 0 {
+		snaps, _ = snapshot.New(snapshot.Config{
+			CacheSize:  1,
+			NoBuild:    false,
+			AsyncBuild: false,
+		}, disk, tdb, common.Hash{}, types.EmptyRootHash)
+	}
 	for i, actions := range test.actions {
 		root := types.EmptyRootHash
 		if i != 0 {
 			root = roots[len(roots)-1]
 		}
-		state, err := New(root, sdb, nil)
+		state, err := New(root, sdb, snaps)
 		if err != nil {
 			panic(err)
 		}
+		state.onCommit = onCommit
+
 		for i, action := range actions {
 			if i%test.chunk == 0 && i != 0 {
 				if byzantium {

core/state/statedb_test.go

@@ -44,16 +44,24 @@ import (
 	"github.com/ava-labs/subnet-evm/core/state/snapshot"
 	"github.com/ava-labs/subnet-evm/core/types"
 	"github.com/ava-labs/subnet-evm/trie"
+	"github.com/ava-labs/subnet-evm/trie/triedb/hashdb"
+	"github.com/ava-labs/subnet-evm/trie/triedb/pathdb"
+	"github.com/ava-labs/subnet-evm/trie/trienode"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/holiman/uint256"
 )
 
 // Tests that updating a state trie does not leak any database writes prior to
 // actually committing the state.
 func TestUpdateLeaks(t *testing.T) {
 	// Create an empty state database
-	db := rawdb.NewMemoryDatabase()
-	state, _ := New(types.EmptyRootHash, NewDatabase(db), nil)
+	var (
+		db  = rawdb.NewMemoryDatabase()
+		tdb = trie.NewDatabase(db, nil)
+	)
+	state, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(db, tdb), nil)
 
 	// Update it with some accounts
 	for i := byte(0); i < 255; i++ {
@@ -69,7 +77,7 @@ func TestUpdateLeaks(t *testing.T) {
 	}
 
 	root := state.IntermediateRoot(false)
-	if err := state.Database().TrieDB().Commit(root, false); err != nil {
+	if err := tdb.Commit(root, false); err != nil {
 		t.Errorf("can not commit trie %v to persistent database", root.Hex())
 	}
 
@@ -87,8 +95,10 @@ func TestIntermediateLeaks(t *testing.T) {
 	// Create two state databases, one transitioning to the final state, the other final from the beginning
 	transDb := rawdb.NewMemoryDatabase()
 	finalDb := rawdb.NewMemoryDatabase()
-	transState, _ := New(types.EmptyRootHash, NewDatabase(transDb), nil)
-	finalState, _ := New(types.EmptyRootHash, NewDatabase(finalDb), nil)
+	transNdb := trie.NewDatabase(transDb, nil)
+	finalNdb := trie.NewDatabase(finalDb, nil)
+	transState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(transDb, transNdb), nil)
+	finalState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(finalDb, finalNdb), nil)
 
 	modify := func(state *StateDB, addr common.Address, i, tweak byte) {
 		state.SetBalance(addr, big.NewInt(int64(11*i)+int64(tweak)))
@@ -120,7 +130,7 @@ func TestIntermediateLeaks(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to commit transition state: %v", err)
 	}
-	if err = transState.Database().TrieDB().Commit(transRoot, false); err != nil {
+	if err = transNdb.Commit(transRoot, false); err != nil {
 		t.Errorf("can not commit trie %v to persistent database", transRoot.Hex())
 	}
 
@@ -128,7 +138,7 @@ func TestIntermediateLeaks(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to commit final state: %v", err)
 	}
-	if err = finalState.Database().TrieDB().Commit(finalRoot, false); err != nil {
+	if err = finalNdb.Commit(finalRoot, false); err != nil {
 		t.Errorf("can not commit trie %v to persistent database", finalRoot.Hex())
 	}
 
@@ -450,6 +460,43 @@ func (test *snapshotTest) run() bool {
 	return true
 }
 
+func forEachStorage(s *StateDB, addr common.Address, cb func(key, value common.Hash) bool) error {
+	so := s.getStateObject(addr)
+	if so == nil {
+		return nil
+	}
+	tr, err := so.getTrie()
+	if err != nil {
+		return err
+	}
+	trieIt, err := tr.NodeIterator(nil)
+	if err != nil {
+		return err
+	}
+	it := trie.NewIterator(trieIt)
+
+	for it.Next() {
+		key := common.BytesToHash(s.trie.GetKey(it.Key))
+		if value, dirty := so.dirtyStorage[key]; dirty {
+			if !cb(key, value) {
+				return nil
+			}
+			continue
+		}
+
+		if len(it.Value) > 0 {
+			_, content, _, err := rlp.Split(it.Value)
+			if err != nil {
+				return err
+			}
+			if !cb(key, common.BytesToHash(content)) {
+				return nil
+			}
+		}
+	}
+	return nil
+}
+
 // checkEqual checks that methods of state and checkstate return the same values.
 func (test *snapshotTest) checkEqual(state, checkstate *StateDB) error {
 	for _, addr := range test.addrs {
@@ -471,10 +518,10 @@ func (test *snapshotTest) checkEqual(state, checkstate *StateDB) error {
 		checkeq("GetCodeSize", state.GetCodeSize(addr), checkstate.GetCodeSize(addr))
 		// Check storage.
 		if obj := state.getStateObject(addr); obj != nil {
-			state.ForEachStorage(addr, func(key, value common.Hash) bool {
+			forEachStorage(state, addr, func(key, value common.Hash) bool {
 				return checkeq("GetState("+key.Hex()+")", checkstate.GetState(addr, key), value)
 			})
-			checkstate.ForEachStorage(addr, func(key, value common.Hash) bool {
+			forEachStorage(checkstate, addr, func(key, value common.Hash) bool {
 				return checkeq("GetState("+key.Hex()+")", checkstate.GetState(addr, key), value)
 			})
 		}
@@ -757,9 +804,28 @@ func TestDeleteCreateRevert(t *testing.T) {
 // the Commit operation fails with an error
 // If we are missing trie nodes, we should not continue writing to the trie
 func TestMissingTrieNodes(t *testing.T) {
+	testMissingTrieNodes(t, rawdb.HashScheme)
+	testMissingTrieNodes(t, rawdb.PathScheme)
+}
+
+func testMissingTrieNodes(t *testing.T, scheme string) {
 	// Create an initial state with a few accounts
-	memDb := rawdb.NewMemoryDatabase()
-	db := NewDatabase(memDb)
+	var (
+		triedb *trie.Database
+		memDb  = rawdb.NewMemoryDatabase()
+	)
+	if scheme == rawdb.PathScheme {
+		triedb = trie.NewDatabase(memDb, &trie.Config{PathDB: &pathdb.Config{
+			CleanCacheSize: 0,
+			DirtyCacheSize: 0,
+		}}) // disable caching
+	} else {
+		triedb = trie.NewDatabase(memDb, &trie.Config{HashDB: &hashdb.Config{
+			CleanCacheSize: 0,
+		}}) // disable caching
+	}
+	db := NewDatabaseWithNodeDB(memDb, triedb)
+
 	var root common.Hash
 	state, _ := New(types.EmptyRootHash, db, nil)
 	addr := common.BytesToAddress([]byte("so"))
@@ -772,7 +838,7 @@ func TestMissingTrieNodes(t *testing.T) {
 		root, _ = state.Commit(0, false, false)
 		t.Logf("root: %x", root)
 		// force-flush
-		state.Database().TrieDB().Cap(0)
+		triedb.Commit(root, false)
 	}
 	// Create a new state on the old root
 	state, _ = New(root, db, nil)
@@ -979,7 +1045,8 @@ func TestFlushOrderDataLoss(t *testing.T) {
 	// Create a state trie with many accounts and slots
 	var (
 		memdb    = rawdb.NewMemoryDatabase()
-		statedb  = NewDatabase(memdb)
+		triedb   = trie.NewDatabase(memdb, nil)
+		statedb  = NewDatabaseWithNodeDB(memdb, triedb)
 		state, _ = New(types.EmptyRootHash, statedb, nil)
 	)
 	for a := byte(0); a < 10; a++ {
@@ -992,11 +1059,11 @@ func TestFlushOrderDataLoss(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to commit state trie: %v", err)
 	}
-	statedb.TrieDB().Reference(root, common.Hash{})
-	if err := statedb.TrieDB().Cap(1024); err != nil {
+	triedb.Reference(root, common.Hash{})
+	if err := triedb.Cap(1024); err != nil {
 		t.Fatalf("failed to cap trie dirty cache: %v", err)
 	}
-	if err := statedb.TrieDB().Commit(root, false); err != nil {
+	if err := triedb.Commit(root, false); err != nil {
 		t.Fatalf("failed to commit state trie: %v", err)
 	}
 	// Reopen the state trie from flushed disk and verify it
@@ -1050,7 +1117,7 @@ func TestStateDBTransientStorage(t *testing.T) {
 func TestResetObject(t *testing.T) {
 	var (
 		disk     = rawdb.NewMemoryDatabase()
-		tdb      = trie.NewDatabase(disk)
+		tdb      = trie.NewDatabase(disk, nil)
 		db       = NewDatabaseWithNodeDB(disk, tdb)
 		snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, common.Hash{}, types.EmptyRootHash)
 		state, _ = New(types.EmptyRootHash, db, snaps)
@@ -1080,3 +1147,57 @@ func TestResetObject(t *testing.T) {
 		t.Fatalf("Unexpected storage slot value %v", slot)
 	}
 }
+
+func TestDeleteStorage(t *testing.T) {
+	var (
+		disk     = rawdb.NewMemoryDatabase()
+		tdb      = trie.NewDatabase(disk, nil)
+		db       = NewDatabaseWithNodeDB(disk, tdb)
+		snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, common.Hash{}, types.EmptyRootHash)
+		state, _ = New(types.EmptyRootHash, db, snaps)
+		addr     = common.HexToAddress("0x1")
+	)
+	// Initialize account and populate storage
+	state.SetBalance(addr, big.NewInt(1))
+	state.CreateAccount(addr)
+	for i := 0; i < 1000; i++ {
+		slot := common.Hash(uint256.NewInt(uint64(i)).Bytes32())
+		value := common.Hash(uint256.NewInt(uint64(10 * i)).Bytes32())
+		state.SetState(addr, slot, value)
+	}
+	root, _ := state.CommitWithSnap(0, true, snaps, common.Hash{}, common.Hash{}, false)
+	// Init phase done, create two states, one with snap and one without
+	fastState, _ := New(root, db, snaps)
+	slowState, _ := New(root, db, nil)
+
+	obj := fastState.GetOrNewStateObject(addr)
+	storageRoot := obj.data.Root
+
+	_, _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot)
+	if err != nil {
+		t.Fatal(err)
+	}
+	check := func(set *trienode.NodeSet) string {
+		var a []string
+		set.ForEachWithOrder(func(path string, n *trienode.Node) {
+			if n.Hash != (common.Hash{}) {
+				t.Fatal("delete should have empty hashes")
+			}
+			if len(n.Blob) != 0 {
+				t.Fatal("delete should have have empty blobs")
+			}
+			a = append(a, fmt.Sprintf("%x", path))
+		})
+		return strings.Join(a, ",")
+	}
+	slowRes := check(slowNodes)
+	fastRes := check(fastNodes)
+	if slowRes != fastRes {
+		t.Fatalf("difference found:\nfast: %v\nslow: %v\n", fastRes, slowRes)
+	}
+}

core/state/sync_test.go

@@ -22,6 +22,8 @@ import (
 	"github.com/ava-labs/subnet-evm/core/rawdb"
 	"github.com/ava-labs/subnet-evm/core/types"
 	"github.com/ava-labs/subnet-evm/trie"
+	"github.com/ava-labs/subnet-evm/trie/triedb/hashdb"
+	"github.com/ava-labs/subnet-evm/trie/triedb/pathdb"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -36,10 +38,17 @@ type testAccount struct {
 }
 
 // makeTestState create a sample test state to test node-wise reconstruction.
-func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) {
+func makeTestState(scheme string) (ethdb.Database, Database, *trie.Database, common.Hash, []*testAccount) {
 	// Create an empty state
+	config := &trie.Config{Preimages: true}
+	if scheme == rawdb.PathScheme {
+		config.PathDB = pathdb.Defaults
+	} else {
+		config.HashDB = hashdb.Defaults
+	}
 	db := rawdb.NewMemoryDatabase()
-	sdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true})
+	nodeDb := trie.NewDatabase(db, config)
+	sdb := NewDatabaseWithNodeDB(db, nodeDb)
 	state, _ := New(types.EmptyRootHash, sdb, nil)
 
 	// Fill it with some arbitrary data
@@ -64,11 +73,10 @@ func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) {
 				obj.SetState(hash, hash)
 			}
 		}
-		state.updateStateObject(obj)
 		accounts = append(accounts, acc)
 	}
 	root, _ := state.Commit(0, false, false)
 
 	// Return the generated state
-	return db, sdb, root, accounts
+	return db, sdb, nodeDb, root, accounts
 }

core/state_manager.go

@@ -67,7 +67,7 @@ type TrieWriter interface {
 type TrieDB interface {
 	Dereference(root common.Hash) error
 	Commit(root common.Hash, report bool) error
-	Size() (common.StorageSize, common.StorageSize)
+	Size() (common.StorageSize, common.StorageSize, common.StorageSize)
 	Cap(limit common.StorageSize) error
 }
 
@@ -126,7 +126,7 @@ type cappedMemoryTrieWriter struct {
 func (cm *cappedMemoryTrieWriter) InsertTrie(block *types.Block) error {
 	// The use of [Cap] in [InsertTrie] prevents exceeding the configured memory
 	// limit (and OOM) in case there is a large backlog of processing (unaccepted) blocks.
-	nodes, imgs := cm.TrieDB.Size()
+	_, nodes, imgs := cm.TrieDB.Size() // all memory is contained within the nodes return for hashdb
 	if nodes <= cm.memoryCap && imgs <= cm.imageCap {
 		return nil
 	}
@@ -174,7 +174,7 @@ func (cm *cappedMemoryTrieWriter) AcceptTrie(block *types.Block) error {
 		return nil
 	}
 	targetMemory := cm.targetCommitSize + cm.flushStepSize*common.StorageSize(distanceFromCommit)
-	nodes, _ := cm.TrieDB.Size()
+	_, nodes, _ := cm.TrieDB.Size()
 	if nodes <= targetMemory {
 		return nil
 	}

core/state_manager_test.go

@@ -26,8 +26,8 @@ func (t *MockTrieDB) Commit(root common.Hash, report bool) error {
 	t.LastCommit = root
 	return nil
 }
-func (t *MockTrieDB) Size() (common.StorageSize, common.StorageSize) {
-	return 0, 0
+func (t *MockTrieDB) Size() (common.StorageSize, common.StorageSize, common.StorageSize) {
+	return 0, 0, 0
 }
 func (t *MockTrieDB) Cap(limit common.StorageSize) error {
 	return nil

core/state_processor.go

@@ -32,6 +32,7 @@ import (
 	"math/big"
 
 	"github.com/ava-labs/subnet-evm/consensus"
+	"github.com/ava-labs/subnet-evm/consensus/misc/eip4844"
 	"github.com/ava-labs/subnet-evm/core/state"
 	"github.com/ava-labs/subnet-evm/core/types"
 	"github.com/ava-labs/subnet-evm/core/vm"
@@ -93,6 +94,9 @@ func (p *StateProcessor) Process(block *types.Block, parent *types.Header, state
 		vmenv   = vm.NewEVM(context, vm.TxContext{}, statedb, p.config, cfg)
 		signer  = types.MakeSigner(p.config, header.Number, header.Time)
 	)
+	if beaconRoot := block.BeaconRoot(); beaconRoot != nil {
+		ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb)
+	}
 	// Iterate over and process the individual transactions
 	for i, tx := range block.Transactions() {
 		msg, err := TransactionToMessage(tx, signer, header.BaseFee)
@@ -146,6 +150,11 @@ func applyTransaction(msg *Message, config *params.ChainConfig, gp *GasPool, sta
 	receipt.TxHash = tx.Hash()
 	receipt.GasUsed = result.UsedGas
 
+	if tx.Type() == types.BlobTxType {
+		receipt.BlobGasUsed = uint64(len(tx.BlobHashes()) * params.BlobTxBlobGasPerBlob)
+		receipt.BlobGasPrice = eip4844.CalcBlobFee(*evm.Context.ExcessBlobGas)
+	}
+
 	// If the transaction created a contract, store the creation address in the receipt.
 	if msg.To == nil {
 		receipt.ContractAddress = crypto.CreateAddress(evm.TxContext.Origin, tx.Nonce())
@@ -174,6 +183,26 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, blockContext
 	return applyTransaction(msg, config, gp, statedb, header.Number, header.Hash(), tx, usedGas, vmenv)
 }
 
+// ProcessBeaconBlockRoot applies the EIP-4788 system call to the beacon block root
+// contract. This method is exported to be used in tests.
+func ProcessBeaconBlockRoot(beaconRoot common.Hash, vmenv *vm.EVM, statedb *state.StateDB) {
+	// If EIP-4788 is enabled, we need to invoke the beaconroot storage contract with
+	// the new root
+	msg := &Message{
+		From:      params.SystemAddress,
+		GasLimit:  30_000_000,
+		GasPrice:  common.Big0,
+		GasFeeCap: common.Big0,
+		GasTipCap: common.Big0,
+		To:        &params.BeaconRootsStorageAddress,
+		Data:      beaconRoot[:],
+	}
+	vmenv.Reset(NewEVMTxContext(msg), statedb)
+	statedb.AddAddressToAccessList(params.BeaconRootsStorageAddress)
+	_, _, _ = vmenv.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.Big0)
+	statedb.Finalise(true)
+}
+
 // ApplyPrecompileActivations checks if any of the precompiles specified by the chain config are enabled or disabled by the block
 // transition from [parentTimestamp] to the timestamp set in [blockContext]. If this is the case, it calls [Configure]
 // to apply the necessary state transitions for the upgrade.

core/state_processor_test.go

@@ -471,6 +471,9 @@ func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Tr
 		used := uint64(nBlobs * params.BlobTxBlobGasPerBlob)
 		header.ExcessBlobGas = &excess
 		header.BlobGasUsed = &used
+
+		beaconRoot := common.HexToHash("0xbeac00")
+		header.ParentBeaconRoot = &beaconRoot
 	}
 	// Assemble and return the final block for sealing
 	return types.NewBlock(header, txs, nil, receipts, trie.NewStackTrie(nil))

core/txpool/blobpool/blobpool.go

@@ -29,6 +29,7 @@ package blobpool
 
 import (
 	"container/heap"
+	"errors"
 	"fmt"
 	"math"
 	"math/big"
@@ -47,7 +48,6 @@ import (
 	"github.com/ava-labs/subnet-evm/metrics"
 	"github.com/ava-labs/subnet-evm/params"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto/kzg4844"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -63,7 +63,7 @@ const (
 	// maxBlobsPerTransaction is the maximum number of blobs a single transaction
 	// is allowed to contain. Whilst the spec states it's unlimited, the block
 	// data slots are protocol bound, which implicitly also limit this.
-	maxBlobsPerTransaction = params.BlobTxMaxBlobGasPerBlock / params.BlobTxBlobGasPerBlob
+	maxBlobsPerTransaction = params.MaxBlobGasPerBlock / params.BlobTxBlobGasPerBlob
 
 	// txAvgSize is an approximate byte size of a transaction metadata to avoid
 	// tiny overflows causing all txs to move a shelf higher, wasting disk space.
@@ -93,16 +93,6 @@ const (
 	limboedTransactionStore = "limbo"
 )
 
-// blobTx is a wrapper around types.BlobTx which also contains the literal blob
-// data along with all the transaction metadata.
-type blobTx struct {
-	Tx *types.Transaction
-
-	Blobs   []kzg4844.Blob
-	Commits []kzg4844.Commitment
-	Proofs  []kzg4844.Proof
-}
-
 // blobTxMeta is the minimal subset of types.BlobTx necessary to validate and
 // schedule the blob transactions into the following blocks. Only ever add the
 // bare minimum needed fields to keep the size down (and thus number of entries
@@ -375,7 +365,13 @@ func (p *BlobPool) Init(gasTip *big.Int, head *types.Header, reserve txpool.Addr
 			return err
 		}
 	}
+	// Initialize the state with head block, or fallback to empty one in
+	// case the head state is not available(might occur when node is not
+	// fully synced).
 	state, err := p.chain.StateAt(head.Root)
+	if err != nil {
+		state, err = p.chain.StateAt(types.EmptyRootHash)
+	}
 	if err != nil {
 		return err
 	}
@@ -483,22 +479,27 @@ func (p *BlobPool) Close() error {
 // parseTransaction is a callback method on pool creation that gets called for
 // each transaction on disk to create the in-memory metadata index.
 func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error {
-	item := new(blobTx)
-	if err := rlp.DecodeBytes(blob, item); err != nil {
+	tx := new(types.Transaction)
+	if err := rlp.DecodeBytes(blob, tx); err != nil {
 		// This path is impossible unless the disk data representation changes
 		// across restarts. For that ever unprobable case, recover gracefully
 		// by ignoring this data entry.
 		log.Error("Failed to decode blob pool entry", "id", id, "err", err)
 		return err
 	}
-	meta := newBlobTxMeta(id, size, item.Tx)
+	if tx.BlobTxSidecar() == nil {
+		log.Error("Missing sidecar in blob pool entry", "id", id, "hash", tx.Hash())
+		return errors.New("missing blob sidecar")
+	}
 
-	sender, err := p.signer.Sender(item.Tx)
+	meta := newBlobTxMeta(id, size, tx)
+
+	sender, err := p.signer.Sender(tx)
 	if err != nil {
 		// This path is impossible unless the signature validity changes across
 		// restarts. For that ever unprobable case, recover gracefully by ignoring
 		// this data entry.
-		log.Error("Failed to recover blob tx sender", "id", id, "hash", item.Tx.Hash(), "err", err)
+		log.Error("Failed to recover blob tx sender", "id", id, "hash", tx.Hash(), "err", err)
 		return err
 	}
 	if _, ok := p.index[sender]; !ok {
@@ -746,17 +747,17 @@ func (p *BlobPool) offload(addr common.Address, nonce uint64, id uint64, inclusi
 		log.Error("Blobs missing for included transaction", "from", addr, "nonce", nonce, "id", id, "err", err)
 		return
 	}
-	item := new(blobTx)
-	if err = rlp.DecodeBytes(data, item); err != nil {
+	var tx types.Transaction
+	if err = rlp.DecodeBytes(data, &tx); err != nil {
 		log.Error("Blobs corrupted for included transaction", "from", addr, "nonce", nonce, "id", id, "err", err)
 		return
 	}
-	block, ok := inclusions[item.Tx.Hash()]
+	block, ok := inclusions[tx.Hash()]
 	if !ok {
 		log.Warn("Blob transaction swapped out by signer", "from", addr, "nonce", nonce, "id", id)
 		return
 	}
-	if err := p.limbo.push(item.Tx.Hash(), block, item.Blobs, item.Commits, item.Proofs); err != nil {
+	if err := p.limbo.push(&tx, block); err != nil {
 		log.Warn("Failed to offload blob tx into limbo", "err", err)
 		return
 	}
@@ -788,7 +789,7 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) {
 		for addr, txs := range reinject {
 			// Blindly push all the lost transactions back into the pool
 			for _, tx := range txs {
-				p.reinject(addr, tx)
+				p.reinject(addr, tx.Hash())
 			}
 			// Recheck the account's pooled transactions to drop included and
 			// invalidated one
@@ -964,16 +965,19 @@ func (p *BlobPool) reorg(oldHead, newHead *types.Header) (map[common.Address][]*
 // Note, the method will not initialize the eviction cache values as those will
 // be done once for all transactions belonging to an account after all individual
 // transactions are injected back into the pool.
-func (p *BlobPool) reinject(addr common.Address, tx *types.Transaction) {
+func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) {
 	// Retrieve the associated blob from the limbo. Without the blobs, we cannot
 	// add the transaction back into the pool as it is not mineable.
-	blobs, commits, proofs, err := p.limbo.pull(tx.Hash())
+	tx, err := p.limbo.pull(txhash)
 	if err != nil {
 		log.Error("Blobs unavailable, dropping reorged tx", "err", err)
 		return
 	}
-	// Serialize the transaction back into the primary datastore
-	blob, err := rlp.EncodeToBytes(&blobTx{Tx: tx, Blobs: blobs, Commits: commits, Proofs: proofs})
+	// TODO: seems like an easy optimization here would be getting the serialized tx
+	// from limbo instead of re-serializing it here.
+
+	// Serialize the transaction back into the primary datastore.
+	blob, err := rlp.EncodeToBytes(tx)
 	if err != nil {
 		log.Error("Failed to encode transaction for storage", "hash", tx.Hash(), "err", err)
 		return
@@ -983,9 +987,9 @@ func (p *BlobPool) reinject(addr common.Address, tx *types.Transaction) {
 		log.Error("Failed to write transaction into storage", "hash", tx.Hash(), "err", err)
 		return
 	}
+
 	// Update the indixes and metrics
 	meta := newBlobTxMeta(id, p.store.Size(id), tx)
-
 	if _, ok := p.index[addr]; !ok {
 		if err := p.reserve(addr, true); err != nil {
 			log.Warn("Failed to reserve account for blob pool", "tx", tx.Hash(), "from", addr, "err", err)
@@ -1067,7 +1071,7 @@ func (p *BlobPool) SetGasTip(tip *big.Int) {
 
 // validateTx checks whether a transaction is valid according to the consensus
 // rules and adheres to some heuristic limits of the local node (price and size).
-func (p *BlobPool) validateTx(tx *types.Transaction, blobs []kzg4844.Blob, commits []kzg4844.Commitment, proofs []kzg4844.Proof) error {
+func (p *BlobPool) validateTx(tx *types.Transaction) error {
 	// Ensure the transaction adheres to basic pool filters (type, size, tip) and
 	// consensus rules
 	baseOpts := &txpool.ValidationOptions{
@@ -1076,7 +1080,7 @@ func (p *BlobPool) validateTx(tx *types.Transaction, blobs []kzg4844.Blob, commi
 		MaxSize: txMaxSize,
 		MinTip:  p.gasTip.ToBig(),
 	}
-	if err := txpool.ValidateTransaction(tx, blobs, commits, proofs, p.head, p.signer, baseOpts); err != nil {
+	if err := txpool.ValidateTransaction(tx, p.head, p.signer, baseOpts); err != nil {
 		return err
 	}
 	// Ensure the transaction adheres to the stateful pool filters (nonce, balance)
@@ -1166,7 +1170,7 @@ func (p *BlobPool) HasLocal(hash common.Hash) bool {
 }
 
 // Get returns a transaction if it is contained in the pool, or nil otherwise.
-func (p *BlobPool) Get(hash common.Hash) *txpool.Transaction {
+func (p *BlobPool) Get(hash common.Hash) *types.Transaction {
 	// Track the amount of time waiting to retrieve a fully resolved blob tx from
 	// the pool and the amount of time actually spent on pulling the data from disk.
 	getStart := time.Now()
@@ -1188,32 +1192,27 @@ func (p *BlobPool) Get(hash common.Hash) *txpool.Transaction {
 		log.Error("Tracked blob transaction missing from store", "hash", hash, "id", id, "err", err)
 		return nil
 	}
-	item := new(blobTx)
+	item := new(types.Transaction)
 	if err = rlp.DecodeBytes(data, item); err != nil {
 		log.Error("Blobs corrupted for traced transaction", "hash", hash, "id", id, "err", err)
 		return nil
 	}
-	return &txpool.Transaction{
-		Tx:            item.Tx,
-		BlobTxBlobs:   item.Blobs,
-		BlobTxCommits: item.Commits,
-		BlobTxProofs:  item.Proofs,
-	}
+	return item
 }
 
 // Add inserts a set of blob transactions into the pool if they pass validation (both
 // consensus validity and pool restictions).
-func (p *BlobPool) Add(txs []*txpool.Transaction, local bool, sync bool) []error {
+func (p *BlobPool) Add(txs []*types.Transaction, local bool, sync bool) []error {
 	errs := make([]error, len(txs))
 	for i, tx := range txs {
-		errs[i] = p.add(tx.Tx, tx.BlobTxBlobs, tx.BlobTxCommits, tx.BlobTxProofs)
+		errs[i] = p.add(tx)
 	}
 	return errs
 }
 
 // Add inserts a new blob transaction into the pool if it passes validation (both
 // consensus validity and pool restictions).
-func (p *BlobPool) add(tx *types.Transaction, blobs []kzg4844.Blob, commits []kzg4844.Commitment, proofs []kzg4844.Proof) (err error) {
+func (p *BlobPool) add(tx *types.Transaction) (err error) {
 	// The blob pool blocks on adding a transaction. This is because blob txs are
 	// only even pulled form the network, so this method will act as the overload
 	// protection for fetches.
@@ -1227,7 +1226,7 @@ func (p *BlobPool) add(tx *types.Transaction, blobs []kzg4844.Blob, commits []kz
 	}(time.Now())
 
 	// Ensure the transaction is valid from all perspectives
-	if err := p.validateTx(tx, blobs, commits, proofs); err != nil {
+	if err := p.validateTx(tx); err != nil {
 		log.Trace("Transaction validation failed", "hash", tx.Hash(), "err", err)
 		return err
 	}
@@ -1252,7 +1251,7 @@ func (p *BlobPool) add(tx *types.Transaction, blobs []kzg4844.Blob, commits []kz
 	}
 	// Transaction permitted into the pool from a nonce and cost perspective,
 	// insert it into the database and update the indices
-	blob, err := rlp.EncodeToBytes(&blobTx{Tx: tx, Blobs: blobs, Commits: commits, Proofs: proofs})
+	blob, err := rlp.EncodeToBytes(tx)
 	if err != nil {
 		log.Error("Failed to encode transaction for storage", "hash", tx.Hash(), "err", err)
 		return err
@@ -1488,7 +1487,7 @@ func (p *BlobPool) PendingFrom(addrs []common.Address, enforceTips bool) map[com
 
 // IteratePending iterates over [pool.pending] until [f] returns false.
 // The caller must not modify [tx]. Returns false if iteration was interrupted.
-func (pool *BlobPool) IteratePending(f func(tx *txpool.Transaction) bool) bool {
+func (pool *BlobPool) IteratePending(f func(tx *types.Transaction) bool) bool {
 	pool.lock.RLock()
 	defer pool.lock.RUnlock()
 

core/txpool/blobpool/blobpool_test.go

@@ -218,8 +218,8 @@ func makeAddressReserver() txpool.AddressReserver {
 // with a valid key, only setting the interesting fields from the perspective of
 // the blob pool.
 func makeTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64, key *ecdsa.PrivateKey) *types.Transaction {
-	tx, _ := types.SignNewTx(key, types.LatestSigner(testChainConfig), makeUnsignedTx(nonce, gasTipCap, gasFeeCap, blobFeeCap))
-	return tx
+	blobtx := makeUnsignedTx(nonce, gasTipCap, gasFeeCap, blobFeeCap)
+	return types.MustSignNewTx(key, types.LatestSigner(testChainConfig), blobtx)
 }
 
 // makeUnsignedTx is a utility method to construct a random blob tranasaction
@@ -234,6 +234,11 @@ func makeUnsignedTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap
 		BlobFeeCap: uint256.NewInt(blobFeeCap),
 		BlobHashes: []common.Hash{emptyBlobVHash},
 		Value:      uint256.NewInt(100),
+		Sidecar: &types.BlobTxSidecar{
+			Blobs:       []kzg4844.Blob{emptyBlob},
+			Commitments: []kzg4844.Commitment{emptyBlobCommit},
+			Proofs:      []kzg4844.Proof{emptyBlobProof},
+		},
 	}
 }
 
@@ -366,7 +371,7 @@ func TestOpenDrops(t *testing.T) {
 		R:          new(uint256.Int),
 		S:          new(uint256.Int),
 	})
-	blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx})
+	blob, _ := rlp.EncodeToBytes(tx)
 	badsig, _ := store.Put(blob)
 
 	// Insert a sequence of transactions with a nonce gap in between to verify
@@ -379,7 +384,7 @@ func TestOpenDrops(t *testing.T) {
 	)
 	for _, nonce := range []uint64{0, 1, 3, 4, 6, 7} { // first gap at #2, another at #5
 		tx := makeTx(nonce, 1, 1, 1, gapper)
-		blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx})
+		blob, _ := rlp.EncodeToBytes(tx)
 
 		id, _ := store.Put(blob)
 		if nonce < 2 {
@@ -396,7 +401,7 @@ func TestOpenDrops(t *testing.T) {
 	)
 	for _, nonce := range []uint64{1, 2, 3} { // first gap at #0, all set dangling
 		tx := makeTx(nonce, 1, 1, 1, dangler)
-		blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx})
+		blob, _ := rlp.EncodeToBytes(tx)
 
 		id, _ := store.Put(blob)
 		dangling[id] = struct{}{}
@@ -409,7 +414,7 @@ func TestOpenDrops(t *testing.T) {
 	)
 	for _, nonce := range []uint64{0, 1, 2} { // account nonce at 3, all set filled
 		tx := makeTx(nonce, 1, 1, 1, filler)
-		blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx})
+		blob, _ := rlp.EncodeToBytes(tx)
 
 		id, _ := store.Put(blob)
 		filled[id] = struct{}{}
@@ -422,7 +427,7 @@ func TestOpenDrops(t *testing.T) {
 	)
 	for _, nonce := range []uint64{0, 1, 2, 3} { // account nonce at 2, half filled
 		tx := makeTx(nonce, 1, 1, 1, overlapper)
-		blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx})
+		blob, _ := rlp.EncodeToBytes(tx)
 
 		id, _ := store.Put(blob)
 		if nonce >= 2 {
@@ -444,7 +449,7 @@ func TestOpenDrops(t *testing.T) {
 		} else {
 			tx = makeTx(uint64(i), 1, 1, 1, underpayer)
 		}
-		blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx})
+		blob, _ := rlp.EncodeToBytes(tx)
 
 		id, _ := store.Put(blob)
 		underpaid[id] = struct{}{}
@@ -463,7 +468,7 @@ func TestOpenDrops(t *testing.T) {
 		} else {
 			tx = makeTx(uint64(i), 1, 1, 1, outpricer)
 		}
-		blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx})
+		blob, _ := rlp.EncodeToBytes(tx)
 
 		id, _ := store.Put(blob)
 		if i < 2 {
@@ -485,7 +490,7 @@ func TestOpenDrops(t *testing.T) {
 		} else {
 			tx = makeTx(nonce, 1, 1, 1, exceeder)
 		}
-		blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx})
+		blob, _ := rlp.EncodeToBytes(tx)
 
 		id, _ := store.Put(blob)
 		exceeded[id] = struct{}{}
@@ -503,7 +508,7 @@ func TestOpenDrops(t *testing.T) {
 		} else {
 			tx = makeTx(nonce, 1, 1, 1, overdrafter)
 		}
-		blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx})
+		blob, _ := rlp.EncodeToBytes(tx)
 
 		id, _ := store.Put(blob)
 		if nonce < 1 {
@@ -519,7 +524,7 @@ func TestOpenDrops(t *testing.T) {
 		overcapped    = make(map[uint64]struct{})
 	)
 	for nonce := uint64(0); nonce < maxTxsPerAccount+3; nonce++ {
-		blob, _ := rlp.EncodeToBytes(&blobTx{Tx: makeTx(nonce, 1, 1, 1, overcapper)})
+		blob, _ := rlp.EncodeToBytes(makeTx(nonce, 1, 1, 1, overcapper))
 
 		id, _ := store.Put(blob)
 		if nonce < maxTxsPerAccount {
@@ -650,7 +655,7 @@ func TestOpenIndex(t *testing.T) {
 	)
 	for _, i := range []int{5, 3, 4, 2, 0, 1} { // Randomize the tx insertion order to force sorting on load
 		tx := makeTx(uint64(i), txExecTipCaps[i], txExecFeeCaps[i], txBlobFeeCaps[i], key)
-		blob, _ := rlp.EncodeToBytes(&blobTx{Tx: tx})
+		blob, _ := rlp.EncodeToBytes(tx)
 		store.Put(blob)
 	}
 	store.Close()
@@ -743,9 +748,9 @@ func TestOpenHeap(t *testing.T) {
 		tx2 = makeTx(0, 1, 800, 70, key2)
 		tx3 = makeTx(0, 1, 1500, 110, key3)
 
-		blob1, _ = rlp.EncodeToBytes(&blobTx{Tx: tx1})
-		blob2, _ = rlp.EncodeToBytes(&blobTx{Tx: tx2})
-		blob3, _ = rlp.EncodeToBytes(&blobTx{Tx: tx3})
+		blob1, _ = rlp.EncodeToBytes(tx1)
+		blob2, _ = rlp.EncodeToBytes(tx2)
+		blob3, _ = rlp.EncodeToBytes(tx3)
 
 		heapOrder = []common.Address{addr2, addr1, addr3}
 		heapIndex = map[common.Address]int{addr2: 0, addr1: 1, addr3: 2}
@@ -819,9 +824,9 @@ func TestOpenCap(t *testing.T) {
 		tx2 = makeTx(0, 1, 800, 70, key2)
 		tx3 = makeTx(0, 1, 1500, 110, key3)
 
-		blob1, _ = rlp.EncodeToBytes(&blobTx{Tx: tx1, Blobs: []kzg4844.Blob{emptyBlob}, Commits: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}})
-		blob2, _ = rlp.EncodeToBytes(&blobTx{Tx: tx2, Blobs: []kzg4844.Blob{emptyBlob}, Commits: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}})
-		blob3, _ = rlp.EncodeToBytes(&blobTx{Tx: tx3, Blobs: []kzg4844.Blob{emptyBlob}, Commits: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}})
+		blob1, _ = rlp.EncodeToBytes(tx1)
+		blob2, _ = rlp.EncodeToBytes(tx2)
+		blob3, _ = rlp.EncodeToBytes(tx3)
 
 		keep = []common.Address{addr1, addr3}
 		drop = []common.Address{addr2}
@@ -1235,10 +1240,8 @@ func TestAdd(t *testing.T) {
 
 			// Sign the seed transactions and store them in the data store
 			for _, tx := range seed.txs {
-				var (
-					signed, _ = types.SignNewTx(keys[acc], types.LatestSigner(testChainConfig), tx)
-					blob, _   = rlp.EncodeToBytes(&blobTx{Tx: signed, Blobs: []kzg4844.Blob{emptyBlob}, Commits: []kzg4844.Commitment{emptyBlobCommit}, Proofs: []kzg4844.Proof{emptyBlobProof}})
-				)
+				signed := types.MustSignNewTx(keys[acc], types.LatestSigner(testChainConfig), tx)
+				blob, _ := rlp.EncodeToBytes(signed)
 				store.Put(blob)
 			}
 		}
@@ -1261,7 +1264,7 @@ func TestAdd(t *testing.T) {
 		// Add each transaction one by one, verifying the pool internals in between
 		for j, add := range tt.adds {
 			signed, _ := types.SignNewTx(keys[add.from], types.LatestSigner(testChainConfig), add.tx)
-			if err := pool.add(signed, []kzg4844.Blob{emptyBlob}, []kzg4844.Commitment{emptyBlobCommit}, []kzg4844.Proof{emptyBlobProof}); !errors.Is(err, add.err) {
+			if err := pool.add(signed); !errors.Is(err, add.err) {
 				t.Errorf("test %d, tx %d: adding transaction error mismatch: have %v, want %v", i, j, err, add.err)
 			}
 			verifyPoolInternals(t, pool)

core/txpool/blobpool/evictheap.go

@@ -54,7 +54,7 @@ type evictHeap struct {
 	index map[common.Address]int // Indices into the heap for replacements
 }
 
-// newPriceHeap creates a new heap of cheapets accounts in the blob pool to evict
+// newPriceHeap creates a new heap of cheapest accounts in the blob pool to evict
 // from in case of over saturation.
 func newPriceHeap(basefee *uint256.Int, blobfee *uint256.Int, index *map[common.Address][]*blobTxMeta) *evictHeap {
 	heap := &evictHeap{

core/txpool/blobpool/limbo.go

@@ -31,7 +31,6 @@ import (
 
 	"github.com/ava-labs/subnet-evm/core/types"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto/kzg4844"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/holiman/billy"
@@ -41,12 +40,9 @@ import (
 // to which it belongs as well as the block number in which it was included for
 // finality eviction.
 type limboBlob struct {
-	Owner common.Hash // Owner transaction's hash to support resurrecting reorged txs
-	Block uint64      // Block in which the blob transaction was included
-
-	Blobs   []kzg4844.Blob       // The opaque blobs originally part of the transaction
-	Commits []kzg4844.Commitment // The commitments for the original blobs
-	Proofs  []kzg4844.Proof      // The proofs verifying the commitments
+	TxHash common.Hash // Owner transaction's hash to support resurrecting reorged txs
+	Block  uint64      // Block in which the blob transaction was included
+	Tx     *types.Transaction
 }
 
 // limbo is a light, indexed database to temporarily store recently included
@@ -108,19 +104,19 @@ func (l *limbo) parseBlob(id uint64, data []byte) error {
 		log.Error("Failed to decode blob limbo entry", "id", id, "err", err)
 		return err
 	}
-	if _, ok := l.index[item.Owner]; ok {
+	if _, ok := l.index[item.TxHash]; ok {
 		// This path is impossible, unless due to a programming error a blob gets
 		// inserted into the limbo which was already part of if. Recover gracefully
 		// by ignoring this data entry.
-		log.Error("Dropping duplicate blob limbo entry", "owner", item.Owner, "id", id)
+		log.Error("Dropping duplicate blob limbo entry", "owner", item.TxHash, "id", id)
 		return errors.New("duplicate blob")
 	}
-	l.index[item.Owner] = id
+	l.index[item.TxHash] = id
 
 	if _, ok := l.groups[item.Block]; !ok {
 		l.groups[item.Block] = make(map[uint64]common.Hash)
 	}
-	l.groups[item.Block][id] = item.Owner
+	l.groups[item.Block][id] = item.TxHash
 
 	return nil
 }
@@ -149,15 +145,15 @@ func (l *limbo) finalize(final *types.Header) {
 
 // push stores a new blob transaction into the limbo, waiting until finality for
 // it to be automatically evicted.
-func (l *limbo) push(tx common.Hash, block uint64, blobs []kzg4844.Blob, commits []kzg4844.Commitment, proofs []kzg4844.Proof) error {
+func (l *limbo) push(tx *types.Transaction, block uint64) error {
 	// If the blobs are already tracked by the limbo, consider it a programming
 	// error. There's not much to do against it, but be loud.
-	if _, ok := l.index[tx]; ok {
+	if _, ok := l.index[tx.Hash()]; ok {
 		log.Error("Limbo cannot push already tracked blobs", "tx", tx)
 		return errors.New("already tracked blob transaction")
 	}
-	if err := l.setAndIndex(tx, block, blobs, commits, proofs); err != nil {
-		log.Error("Failed to set and index liboed blobs", "tx", tx, "err", err)
+	if err := l.setAndIndex(tx, block); err != nil {
+		log.Error("Failed to set and index limboed blobs", "tx", tx, "err", err)
 		return err
 	}
 	return nil
@@ -166,21 +162,21 @@ func (l *limbo) push(tx common.Hash, block uint64, blobs []kzg4844.Blob, commits
 // pull retrieves a previously pushed set of blobs back from the limbo, removing
 // it at the same time. This method should be used when a previously included blob
 // transaction gets reorged out.
-func (l *limbo) pull(tx common.Hash) ([]kzg4844.Blob, []kzg4844.Commitment, []kzg4844.Proof, error) {
+func (l *limbo) pull(tx common.Hash) (*types.Transaction, error) {
 	// If the blobs are not tracked by the limbo, there's not much to do. This
 	// can happen for example if a blob transaction is mined without pushing it
 	// into the network first.
 	id, ok := l.index[tx]
 	if !ok {
 		log.Trace("Limbo cannot pull non-tracked blobs", "tx", tx)
-		return nil, nil, nil, errors.New("unseen blob transaction")
+		return nil, errors.New("unseen blob transaction")
 	}
 	item, err := l.getAndDrop(id)
 	if err != nil {
 		log.Error("Failed to get and drop limboed blobs", "tx", tx, "id", id, "err", err)
-		return nil, nil, nil, err
+		return nil, err
 	}
-	return item.Blobs, item.Commits, item.Proofs, nil
+	return item.Tx, nil
 }
 
 // update changes the block number under which a blob transaction is tracked. This
@@ -190,33 +186,33 @@ func (l *limbo) pull(tx common.Hash) ([]kzg4844.Blob, []kzg4844.Commitment, []kz
 // any of it since there's no clear error case. Some errors may be due to coding
 // issues, others caused by signers mining MEV stuff or swapping transactions. In
 // all cases, the pool needs to continue operating.
-func (l *limbo) update(tx common.Hash, block uint64) {
+func (l *limbo) update(txhash common.Hash, block uint64) {
 	// If the blobs are not tracked by the limbo, there's not much to do. This
 	// can happen for example if a blob transaction is mined without pushing it
 	// into the network first.
-	id, ok := l.index[tx]
+	id, ok := l.index[txhash]
 	if !ok {
-		log.Trace("Limbo cannot update non-tracked blobs", "tx", tx)
+		log.Trace("Limbo cannot update non-tracked blobs", "tx", txhash)
 		return
 	}
 	// If there was no change in the blob's inclusion block, don't mess around
 	// with heavy database operations.
 	if _, ok := l.groups[block][id]; ok {
-		log.Trace("Blob transaction unchanged in limbo", "tx", tx, "block", block)
+		log.Trace("Blob transaction unchanged in limbo", "tx", txhash, "block", block)
 		return
 	}
-	// Retrieve the old blobs from the data store and write tehm back with a new
+	// Retrieve the old blobs from the data store and write them back with a new
 	// block number. IF anything fails, there's not much to do, go on.
 	item, err := l.getAndDrop(id)
 	if err != nil {
-		log.Error("Failed to get and drop limboed blobs", "tx", tx, "id", id, "err", err)
+		log.Error("Failed to get and drop limboed blobs", "tx", txhash, "id", id, "err", err)
 		return
 	}
-	if err := l.setAndIndex(tx, block, item.Blobs, item.Commits, item.Proofs); err != nil {
-		log.Error("Failed to set and index limboed blobs", "tx", tx, "err", err)
+	if err := l.setAndIndex(item.Tx, block); err != nil {
+		log.Error("Failed to set and index limboed blobs", "tx", txhash, "err", err)
 		return
 	}
-	log.Trace("Blob transaction updated in limbo", "tx", tx, "old-block", item.Block, "new-block", block)
+	log.Trace("Blob transaction updated in limbo", "tx", txhash, "old-block", item.Block, "new-block", block)
 }
 
 // getAndDrop retrieves a blob item from the limbo store and deletes it both from
@@ -230,7 +226,7 @@ func (l *limbo) getAndDrop(id uint64) (*limboBlob, error) {
 	if err = rlp.DecodeBytes(data, item); err != nil {
 		return nil, err
 	}
-	delete(l.index, item.Owner)
+	delete(l.index, item.TxHash)
 	delete(l.groups[item.Block], id)
 	if len(l.groups[item.Block]) == 0 {
 		delete(l.groups, item.Block)
@@ -243,13 +239,12 @@ func (l *limbo) getAndDrop(id uint64) (*limboBlob, error) {
 
 // setAndIndex assembles a limbo blob database entry and stores it, also updating
 // the in-memory indices.
-func (l *limbo) setAndIndex(tx common.Hash, block uint64, blobs []kzg4844.Blob, commits []kzg4844.Commitment, proofs []kzg4844.Proof) error {
+func (l *limbo) setAndIndex(tx *types.Transaction, block uint64) error {
+	txhash := tx.Hash()
 	item := &limboBlob{
-		Owner:   tx,
-		Block:   block,
-		Blobs:   blobs,
-		Commits: commits,
-		Proofs:  proofs,
+		TxHash: txhash,
+		Block:  block,
+		Tx:     tx,
 	}
 	data, err := rlp.EncodeToBytes(item)
 	if err != nil {
@@ -259,10 +254,10 @@ func (l *limbo) setAndIndex(tx common.Hash, block uint64, blobs []kzg4844.Blob,
 	if err != nil {
 		return err
 	}
-	l.index[tx] = id
+	l.index[txhash] = id
 	if _, ok := l.groups[block]; !ok {
 		l.groups[block] = make(map[uint64]common.Hash)
 	}
-	l.groups[block][id] = tx
+	l.groups[block][id] = txhash
 	return nil
 }

core/txpool/blobpool/priority.go

@@ -37,7 +37,7 @@ import (
 var log2_1_125 = math.Log2(1.125)
 
 // evictionPriority calculates the eviction priority based on the algorithm
-// described in the BlobPool docs for a both fee components.
+// described in the BlobPool docs for both fee components.
 //
 // This method takes about 8ns on a very recent laptop CPU, recalculating about
 // 125 million transaction priority values per second.

core/txpool/blobpool/slotter_test.go

@@ -48,14 +48,16 @@ func TestNewSlotter(t *testing.T) {
 		2*blobSize + txAvgSize,  // 2 blob + some expected tx infos (could be fewer blobs and more tx data)
 		3*blobSize + txAvgSize,  // 3 blob + some expected tx infos (could be fewer blobs and more tx data)
 		4*blobSize + txAvgSize,  // 4 blob + some expected tx infos (could be fewer blobs and more tx data)
-		5*blobSize + txAvgSize,  // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
-		6*blobSize + txAvgSize,  // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
-		7*blobSize + txAvgSize,  // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
-		8*blobSize + txAvgSize,  // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
-		9*blobSize + txAvgSize,  // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
-		10*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
-		11*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
-		12*blobSize + txAvgSize, // 1-4 blobs + unexpectedly large tx infos >= 4 blobs + max tx metadata size
+		5*blobSize + txAvgSize,  // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
+		6*blobSize + txAvgSize,  // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
+		7*blobSize + txAvgSize,  // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
+		8*blobSize + txAvgSize,  // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
+		9*blobSize + txAvgSize,  // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
+		10*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
+		11*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
+		12*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
+		13*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos < 4 blobs + max tx metadata size
+		14*blobSize + txAvgSize, // 1-6 blobs + unexpectedly large tx infos >= 4 blobs + max tx metadata size
 	}
 	if len(shelves) != len(want) {
 		t.Errorf("shelves count mismatch: have %d, want %d", len(shelves), len(want))

core/txpool/errors.go

@@ -62,6 +62,6 @@ var (
 	ErrOversizedData = errors.New("oversized data")
 
 	// ErrFutureReplacePending is returned if a future transaction replaces a pending
-	// transaction. Future transactions should only be able to replace other future transactions.
+	// one. Future transactions should only be able to replace other future transactions.
 	ErrFutureReplacePending = errors.New("future transaction tries to replace pending")
 )

core/txpool/legacypool/legacypool.go

@@ -328,7 +328,20 @@ func (pool *LegacyPool) Init(gasTip *big.Int, head *types.Header, reserve txpool
 
 	// Set the basic pool parameters
 	pool.gasTip.Store(gasTip)
-	pool.reset(nil, head)
+
+	// Initialize the state with head block, or fallback to empty one in
+	// case the head state is not available(might occur when node is not
+	// fully synced).
+	statedb, err := pool.chain.StateAt(head.Root)
+	if err != nil {
+		statedb, err = pool.chain.StateAt(types.EmptyRootHash)
+	}
+	if err != nil {
+		return err
+	}
+	pool.currentHead.Store(head)
+	pool.currentState = statedb
+	pool.pendingNonces = newNoncer(statedb)
 
 	// Start the reorg loop early, so it can handle requests generated during
 	// journal loading.
@@ -441,7 +454,7 @@ func (pool *LegacyPool) Close() error {
 }
 
 // Reset implements txpool.SubPool, allowing the legacy pool's internal state to be
-// kept in sync with the main transacion pool's internal state.
+// kept in sync with the main transaction pool's internal state.
 func (pool *LegacyPool) Reset(oldHead, newHead *types.Header) {
 	wait := pool.requestReset(oldHead, newHead)
 	<-wait
@@ -586,7 +599,7 @@ func (pool *LegacyPool) PendingWithBaseFee(enforceTips bool, baseFee *big.Int) m
 				lazies[i] = &txpool.LazyTransaction{
 					Pool:      pool,
 					Hash:      txs[i].Hash(),
-					Tx:        &txpool.Transaction{Tx: txs[i]},
+					Tx:        txs[i].WithoutBlobTxSidecar(),
 					Time:      txs[i].Time(),
 					GasFeeCap: txs[i].GasFeeCap(),
 					GasTipCap: txs[i].GasTipCap(),
@@ -627,7 +640,7 @@ func (pool *LegacyPool) PendingFrom(addrs []common.Address, enforceTips bool) ma
 				lazies[i] = &txpool.LazyTransaction{
 					Pool:      pool,
 					Hash:      txs[i].Hash(),
-					Tx:        &txpool.Transaction{Tx: txs[i]},
+					Tx:        txs[i],
 					Time:      txs[i].Time(),
 					GasFeeCap: txs[i].GasFeeCap(),
 					GasTipCap: txs[i].GasTipCap(),
@@ -641,13 +654,13 @@ func (pool *LegacyPool) PendingFrom(addrs []common.Address, enforceTips bool) ma
 
 // IteratePending iterates over [pool.pending] until [f] returns false.
 // The caller must not modify [tx]. Returns false if iteration was interrupted.
-func (pool *LegacyPool) IteratePending(f func(tx *txpool.Transaction) bool) bool {
+func (pool *LegacyPool) IteratePending(f func(tx *types.Transaction) bool) bool {
 	pool.mu.RLock()
 	defer pool.mu.RUnlock()
 
 	for _, list := range pool.pending {
 		for _, tx := range list.txs.items {
-			if !f(&txpool.Transaction{Tx: tx}) {
+			if !f(tx) {
 				return false
 			}
 		}
@@ -696,7 +709,7 @@ func (pool *LegacyPool) validateTxBasics(tx *types.Transaction, local bool) erro
 	if local {
 		opts.MinTip = new(big.Int)
 	}
-	if err := txpool.ValidateTransaction(tx, nil, nil, nil, pool.currentHead.Load(), pool.signer, opts); err != nil {
+	if err := txpool.ValidateTransaction(tx, pool.currentHead.Load(), pool.signer, opts); err != nil {
 		return err
 	}
 	return nil
@@ -753,7 +766,7 @@ func (pool *LegacyPool) validateTx(tx *types.Transaction, local bool) error {
 // pending or queued one, it overwrites the previous transaction if its price is higher.
 //
 // If a newly added transaction is marked as local, its sending account will be
-// be added to the allowlist, preventing any associated transaction from being dropped
+// added to the allowlist, preventing any associated transaction from being dropped
 // out of the pool due to pricing constraints.
 func (pool *LegacyPool) add(tx *types.Transaction, local bool) (replaced bool, err error) {
 	// If the transaction is already known, discard it
@@ -1016,26 +1029,13 @@ func (pool *LegacyPool) promoteTx(addr common.Address, hash common.Hash, tx *typ
 	return true
 }
 
-// Add enqueues a batch of transactions into the pool if they are valid. Depending
-// on the local flag, full pricing contraints will or will not be applied.
-//
-// If sync is set, the method will block until all internal maintenance related
-// to the add is finished. Only use this during tests for determinism!
-func (pool *LegacyPool) Add(txs []*txpool.Transaction, local bool, sync bool) []error {
-	unwrapped := make([]*types.Transaction, len(txs))
-	for i, tx := range txs {
-		unwrapped[i] = tx.Tx
-	}
-	return pool.addTxs(unwrapped, local, sync)
-}
-
 // addLocals enqueues a batch of transactions into the pool if they are valid, marking the
-// senders as a local ones, ensuring they go around the local pricing constraints.
+// senders as local ones, ensuring they go around the local pricing constraints.
 //
 // This method is used to add transactions from the RPC API and performs synchronous pool
 // reorganization and event propagation.
 func (pool *LegacyPool) addLocals(txs []*types.Transaction) []error {
-	return pool.addTxs(txs, !pool.config.NoLocals, true)
+	return pool.Add(txs, !pool.config.NoLocals, true)
 }
 
 // addLocal enqueues a single local transaction into the pool if it is valid. This is
@@ -1051,7 +1051,7 @@ func (pool *LegacyPool) addLocal(tx *types.Transaction) error {
 // This method is used to add transactions from the p2p network and does not wait for pool
 // reorganization and internal event propagation.
 func (pool *LegacyPool) addRemotes(txs []*types.Transaction) []error {
-	return pool.addTxs(txs, false, false)
+	return pool.Add(txs, false, false)
 }
 
 // addRemote enqueues a single transaction into the pool if it is valid. This is a convenience
@@ -1063,16 +1063,20 @@ func (pool *LegacyPool) addRemote(tx *types.Transaction) error {
 
 // addRemotesSync is like addRemotes, but waits for pool reorganization. Tests use this method.
 func (pool *LegacyPool) addRemotesSync(txs []*types.Transaction) []error {
-	return pool.addTxs(txs, false, true)
+	return pool.Add(txs, false, true)
 }
 
 // This is like addRemotes with a single transaction, but waits for pool reorganization. Tests use this method.
 func (pool *LegacyPool) addRemoteSync(tx *types.Transaction) error {
-	return pool.addTxs([]*types.Transaction{tx}, false, true)[0]
+	return pool.Add([]*types.Transaction{tx}, false, true)[0]
 }
 
-// addTxs attempts to queue a batch of transactions if they are valid.
-func (pool *LegacyPool) addTxs(txs []*types.Transaction, local, sync bool) []error {
+// Add enqueues a batch of transactions into the pool if they are valid. Depending
+// on the local flag, full pricing constraints will or will not be applied.
+//
+// If sync is set, the method will block until all internal maintenance related
+// to the add is finished. Only use this during tests for determinism!
+func (pool *LegacyPool) Add(txs []*types.Transaction, local, sync bool) []error {
 	// Filter out known ones without obtaining the pool lock or recovering signatures
 	var (
 		errs = make([]error, len(txs))
@@ -1158,12 +1162,12 @@ func (pool *LegacyPool) Status(hash common.Hash) txpool.TxStatus {
 }
 
 // Get returns a transaction if it is contained in the pool and nil otherwise.
-func (pool *LegacyPool) Get(hash common.Hash) *txpool.Transaction {
+func (pool *LegacyPool) Get(hash common.Hash) *types.Transaction {
 	tx := pool.get(hash)
 	if tx == nil {
 		return nil
 	}
-	return &txpool.Transaction{Tx: tx}
+	return tx
 }
 
 // get returns a transaction if it is contained in the pool and nil otherwise.

core/txpool/legacypool/list.go

@@ -63,9 +63,10 @@ func (h *nonceHeap) Pop() interface{} {
 // sortedMap is a nonce->transaction hash map with a heap based index to allow
 // iterating over the contents in a nonce-incrementing way.
 type sortedMap struct {
-	items map[uint64]*types.Transaction // Hash map storing the transaction data
-	index *nonceHeap                    // Heap of nonces of all the stored transactions (non-strict mode)
-	cache types.Transactions            // Cache of the transactions already sorted
+	items   map[uint64]*types.Transaction // Hash map storing the transaction data
+	index   *nonceHeap                    // Heap of nonces of all the stored transactions (non-strict mode)
+	cache   types.Transactions            // Cache of the transactions already sorted
+	cacheMu sync.Mutex                    // Mutex covering the cache
 }
 
 // newSortedMap creates a new nonce-sorted transaction map.
@@ -88,7 +89,9 @@ func (m *sortedMap) Put(tx *types.Transaction) {
 	if m.items[nonce] == nil {
 		heap.Push(m.index, nonce)
 	}
+	m.cacheMu.Lock()
 	m.items[nonce], m.cache = tx, nil
+	m.cacheMu.Unlock()
 }
 
 // Forward removes all transactions from the map with a nonce lower than the
@@ -104,9 +107,11 @@ func (m *sortedMap) Forward(threshold uint64) types.Transactions {
 		delete(m.items, nonce)
 	}
 	// If we had a cached order, shift the front
+	m.cacheMu.Lock()
 	if m.cache != nil {
 		m.cache = m.cache[len(removed):]
 	}
+	m.cacheMu.Unlock()
 	return removed
 }
 
@@ -130,7 +135,9 @@ func (m *sortedMap) reheap() {
 		*m.index = append(*m.index, nonce)
 	}
 	heap.Init(m.index)
+	m.cacheMu.Lock()
 	m.cache = nil
+	m.cacheMu.Unlock()
 }
 
 // filter is identical to Filter, but **does not** regenerate the heap. This method
@@ -146,7 +153,9 @@ func (m *sortedMap) filter(filter func(*types.Transaction) bool) types.Transacti
 		}
 	}
 	if len(removed) > 0 {
+		m.cacheMu.Lock()
 		m.cache = nil
+		m.cacheMu.Unlock()
 	}
 	return removed
 }
@@ -170,9 +179,11 @@ func (m *sortedMap) Cap(threshold int) types.Transactions {
 	heap.Init(m.index)
 
 	// If we had a cache, shift the back
+	m.cacheMu.Lock()
 	if m.cache != nil {
 		m.cache = m.cache[:len(m.cache)-len(drops)]
 	}
+	m.cacheMu.Unlock()
 	return drops
 }
 
@@ -192,7 +203,9 @@ func (m *sortedMap) Remove(nonce uint64) bool {
 		}
 	}
 	delete(m.items, nonce)
+	m.cacheMu.Lock()
 	m.cache = nil
+	m.cacheMu.Unlock()
 
 	return true
 }
@@ -216,7 +229,9 @@ func (m *sortedMap) Ready(start uint64) types.Transactions {
 		delete(m.items, next)
 		heap.Pop(m.index)
 	}
+	m.cacheMu.Lock()
 	m.cache = nil
+	m.cacheMu.Unlock()
 
 	return ready
 }
@@ -227,6 +242,8 @@ func (m *sortedMap) Len() int {
 }
 
 func (m *sortedMap) flatten() types.Transactions {
+	m.cacheMu.Lock()
+	defer m.cacheMu.Unlock()
 	// If the sorting was not cached yet, create and cache it
 	if m.cache == nil {
 		m.cache = make(types.Transactions, 0, len(m.items))
@@ -242,8 +259,8 @@ func (m *sortedMap) flatten() types.Transactions {
 // sorted internal representation. The result of the sorting is cached in case
 // it's requested again before any modifications are made to the contents.
 func (m *sortedMap) Flatten() types.Transactions {
-	// Copy the cache to prevent accidental modifications
 	cache := m.flatten()
+	// Copy the cache to prevent accidental modification
 	txs := make(types.Transactions, len(cache))
 	copy(txs, cache)
 	return txs

core/txpool/subpool.go

@@ -33,27 +33,16 @@ import (
 	"github.com/ava-labs/subnet-evm/core"
 	"github.com/ava-labs/subnet-evm/core/types"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto/kzg4844"
 	"github.com/ethereum/go-ethereum/event"
 )
 
-// Transaction is a helper struct to group together a canonical transaction with
-// satellite data items that are needed by the pool but are not part of the chain.
-type Transaction struct {
-	Tx *types.Transaction // Canonical transaction
-
-	BlobTxBlobs   []kzg4844.Blob       // Blobs needed by the blob pool
-	BlobTxCommits []kzg4844.Commitment // Commitments needed by the blob pool
-	BlobTxProofs  []kzg4844.Proof      // Proofs needed by the blob pool
-}
-
 // LazyTransaction contains a small subset of the transaction properties that is
 // enough for the miner and other APIs to handle large batches of transactions;
 // and supports pulling up the entire transaction when really needed.
 type LazyTransaction struct {
-	Pool SubPool      // Transaction subpool to pull the real transaction up
-	Hash common.Hash  // Transaction hash to pull up if needed
-	Tx   *Transaction // Transaction if already resolved
+	Pool SubPool            // Transaction subpool to pull the real transaction up
+	Hash common.Hash        // Transaction hash to pull up if needed
+	Tx   *types.Transaction // Transaction if already resolved
 
 	Time      time.Time // Time when the transaction was first seen
 	GasFeeCap *big.Int  // Maximum fee per gas the transaction may consume
@@ -62,7 +51,7 @@ type LazyTransaction struct {
 
 // Resolve retrieves the full transaction belonging to a lazy handle if it is still
 // maintained by the transaction pool.
-func (ltx *LazyTransaction) Resolve() *Transaction {
+func (ltx *LazyTransaction) Resolve() *types.Transaction {
 	if ltx.Tx == nil {
 		ltx.Tx = ltx.Pool.Get(ltx.Hash)
 	}
@@ -111,19 +100,19 @@ type SubPool interface {
 	HasLocal(hash common.Hash) bool
 
 	// Get returns a transaction if it is contained in the pool, or nil otherwise.
-	Get(hash common.Hash) *Transaction
+	Get(hash common.Hash) *types.Transaction
 
 	// Add enqueues a batch of transactions into the pool if they are valid. Due
 	// to the large transaction churn, add may postpone fully integrating the tx
 	// to a later point to batch multiple ones together.
-	Add(txs []*Transaction, local bool, sync bool) []error
+	Add(txs []*types.Transaction, local bool, sync bool) []error
 
 	// Pending retrieves all currently processable transactions, grouped by origin
 	// account and sorted by nonce.
 	Pending(enforceTips bool) map[common.Address][]*LazyTransaction
 	PendingWithBaseFee(enforceTips bool, baseFee *big.Int) map[common.Address][]*LazyTransaction
 	PendingFrom(addrs []common.Address, enforceTips bool) map[common.Address][]*LazyTransaction
-	IteratePending(f func(tx *Transaction) bool) bool // Returns false if iteration was interrupted.
+	IteratePending(f func(tx *types.Transaction) bool) bool // Returns false if iteration was interrupted.
 
 	// SubscribeTransactions subscribes to new transaction events.
 	SubscribeTransactions(ch chan<- core.NewTxsEvent) event.Subscription

core/txpool/txpool.go

@@ -86,7 +86,7 @@ type TxPool struct {
 	reservations map[common.Address]SubPool // Map with the account to pool reservations
 	reserveLock  sync.Mutex                 // Lock protecting the account reservations
 
-	subs event.SubscriptionScope // Subscription scope to unscubscribe all on shutdown
+	subs event.SubscriptionScope // Subscription scope to unsubscribe all on shutdown
 	quit chan chan error         // Quit channel to tear down the head updater
 
 	gasTip    atomic.Pointer[big.Int] // Remember last value set so it can be retrieved
@@ -299,7 +299,7 @@ func (p *TxPool) HasLocal(hash common.Hash) bool {
 }
 
 // Get returns a transaction if it is contained in the pool, or nil otherwise.
-func (p *TxPool) Get(hash common.Hash) *Transaction {
+func (p *TxPool) Get(hash common.Hash) *types.Transaction {
 	for _, subpool := range p.subpools {
 		if tx := subpool.Get(hash); tx != nil {
 			return tx
@@ -311,14 +311,14 @@ func (p *TxPool) Get(hash common.Hash) *Transaction {
 // Add enqueues a batch of transactions into the pool if they are valid. Due
 // to the large transaction churn, add may postpone fully integrating the tx
 // to a later point to batch multiple ones together.
-func (p *TxPool) Add(txs []*Transaction, local bool, sync bool) []error {
+func (p *TxPool) Add(txs []*types.Transaction, local bool, sync bool) []error {
 	// Split the input transactions between the subpools. It shouldn't really
 	// happen that we receive merged batches, but better graceful than strange
 	// errors.
 	//
 	// We also need to track how the transactions were split across the subpools,
 	// so we can piece back the returned errors into the original order.
-	txsets := make([][]*Transaction, len(p.subpools))
+	txsets := make([][]*types.Transaction, len(p.subpools))
 	splits := make([]int, len(txs))
 
 	for i, tx := range txs {
@@ -327,7 +327,7 @@ func (p *TxPool) Add(txs []*Transaction, local bool, sync bool) []error {
 
 		// Try to find a subpool that accepts the transaction
 		for j, subpool := range p.subpools {
-			if subpool.Filter(tx.Tx) {
+			if subpool.Filter(tx) {
 				txsets[j] = append(txsets[j], tx)
 				splits[i] = j
 				break
@@ -355,11 +355,7 @@ func (p *TxPool) Add(txs []*Transaction, local bool, sync bool) []error {
 }
 
 func (p *TxPool) AddRemotesSync(txs []*types.Transaction) []error {
-	wrapped := make([]*Transaction, len(txs))
-	for i, tx := range txs {
-		wrapped[i] = &Transaction{Tx: tx}
-	}
-	return p.Add(wrapped, false, true)
+	return p.Add(txs, false, true)
 }
 
 // Pending retrieves all currently processable transactions, grouped by origin
@@ -414,7 +410,7 @@ func (p *TxPool) PendingFrom(addrs []common.Address, enforceTips bool) map[commo
 
 // IteratePending iterates over [pool.pending] until [f] returns false.
 // The caller must not modify [tx].
-func (p *TxPool) IteratePending(f func(tx *Transaction) bool) {
+func (p *TxPool) IteratePending(f func(tx *types.Transaction) bool) {
 	for _, subpool := range p.subpools {
 		if !subpool.IteratePending(f) {
 			return
@@ -520,7 +516,7 @@ func (p *TxPool) Locals() []common.Address {
 }
 
 // Status returns the known status (unknown/pending/queued) of a transaction
-// identified by their hashes.
+// identified by its hash.
 func (p *TxPool) Status(hash common.Hash) TxStatus {
 	for _, subpool := range p.subpools {
 		if status := subpool.Status(hash); status != TxStatusUnknown {

core/txpool/validation.go

@@ -58,7 +58,7 @@ type ValidationOptions struct {
 //
 // This check is public to allow different transaction pools to check the basic
 // rules without duplicating code and running the risk of missed updates.
-func ValidateTransaction(tx *types.Transaction, blobs []kzg4844.Blob, commits []kzg4844.Commitment, proofs []kzg4844.Proof, head *types.Header, signer types.Signer, opts *ValidationOptions) error {
+func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types.Signer, opts *ValidationOptions) error {
 	// Ensure transactions not implemented by the calling pool are rejected
 	if opts.Accept&(1<<tx.Type()) == 0 {
 		return fmt.Errorf("%w: tx type %v not supported by this pool", core.ErrTxTypeNotSupported, tx.Type())
@@ -128,46 +128,57 @@ func ValidateTransaction(tx *types.Transaction, blobs []kzg4844.Blob, commits []
 	}
 	// Ensure blob transactions have valid commitments
 	if tx.Type() == types.BlobTxType {
-		// Ensure the number of items in the blob transaction and vairous side
+		sidecar := tx.BlobTxSidecar()
+		if sidecar == nil {
+			return fmt.Errorf("missing sidecar in blob transaction")
+		}
+		// Ensure the number of items in the blob transaction and various side
 		// data match up before doing any expensive validations
 		hashes := tx.BlobHashes()
 		if len(hashes) == 0 {
 			return fmt.Errorf("blobless blob transaction")
 		}
-		if len(hashes) > params.BlobTxMaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob {
-			return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.BlobTxMaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob)
+		if len(hashes) > params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob {
+			return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob)
 		}
-		if len(blobs) != len(hashes) {
-			return fmt.Errorf("invalid number of %d blobs compared to %d blob hashes", len(blobs), len(hashes))
+		if err := validateBlobSidecar(hashes, sidecar); err != nil {
+			return err
 		}
-		if len(commits) != len(hashes) {
-			return fmt.Errorf("invalid number of %d blob commitments compared to %d blob hashes", len(commits), len(hashes))
-		}
-		if len(proofs) != len(hashes) {
-			return fmt.Errorf("invalid number of %d blob proofs compared to %d blob hashes", len(proofs), len(hashes))
-		}
-		// Blob quantities match up, validate that the provers match with the
-		// transaction hash before getting to the cryptography
-		hasher := sha256.New()
-		for i, want := range hashes {
-			hasher.Write(commits[i][:])
-			hash := hasher.Sum(nil)
-			hasher.Reset()
+	}
+	return nil
+}
 
-			var vhash common.Hash
-			vhash[0] = params.BlobTxHashVersion
-			copy(vhash[1:], hash[1:])
+func validateBlobSidecar(hashes []common.Hash, sidecar *types.BlobTxSidecar) error {
+	if len(sidecar.Blobs) != len(hashes) {
+		return fmt.Errorf("invalid number of %d blobs compared to %d blob hashes", len(sidecar.Blobs), len(hashes))
+	}
+	if len(sidecar.Commitments) != len(hashes) {
+		return fmt.Errorf("invalid number of %d blob commitments compared to %d blob hashes", len(sidecar.Commitments), len(hashes))
+	}
+	if len(sidecar.Proofs) != len(hashes) {
+		return fmt.Errorf("invalid number of %d blob proofs compared to %d blob hashes", len(sidecar.Proofs), len(hashes))
+	}
+	// Blob quantities match up, validate that the provers match with the
+	// transaction hash before getting to the cryptography
+	hasher := sha256.New()
+	for i, want := range hashes {
+		hasher.Write(sidecar.Commitments[i][:])
+		hash := hasher.Sum(nil)
+		hasher.Reset()
 
-			if vhash != want {
-				return fmt.Errorf("blob %d: computed hash %#x mismatches transaction one %#x", i, vhash, want)
-			}
+		var vhash common.Hash
+		vhash[0] = params.BlobTxHashVersion
+		copy(vhash[1:], hash[1:])
+
+		if vhash != want {
+			return fmt.Errorf("blob %d: computed hash %#x mismatches transaction one %#x", i, vhash, want)
 		}
-		// Blob commitments match with the hashes in the transaction, verify the
-		// blobs themselves via KZG
-		for i := range blobs {
-			if err := kzg4844.VerifyBlobProof(blobs[i], commits[i], proofs[i]); err != nil {
-				return fmt.Errorf("invalid blob %d: %v", i, err)
-			}
+	}
+	// Blob commitments match with the hashes in the transaction, verify the
+	// blobs themselves via KZG
+	for i := range sidecar.Blobs {
+		if err := kzg4844.VerifyBlobProof(sidecar.Blobs[i], sidecar.Commitments[i], sidecar.Proofs[i]); err != nil {
+			return fmt.Errorf("invalid blob %d: %v", i, err)
 		}
 	}
 	return nil
@@ -189,7 +200,7 @@ type ValidationOptionsWithState struct {
 	// be rejected once the number of remaining slots reaches zero.
 	UsedAndLeftSlots func(addr common.Address) (int, int)
 
-	// ExistingExpenditure is a mandatory callback to retrieve the cummulative
+	// ExistingExpenditure is a mandatory callback to retrieve the cumulative
 	// cost of the already pooled transactions to check for overdrafts.
 	ExistingExpenditure func(addr common.Address) *big.Int
 
@@ -253,7 +264,7 @@ func ValidateTransactionWithState(tx *types.Transaction, signer types.Signer, op
 			return fmt.Errorf("%w: balance %v, queued cost %v, tx cost %v, overshot %v", core.ErrInsufficientFunds, balance, spent, cost, new(big.Int).Sub(need, balance))
 		}
 		// Transaction takes a new nonce value out of the pool. Ensure it doesn't
-		// overflow the number of permitted transactions from a single accoun
+		// overflow the number of permitted transactions from a single account
 		// (i.e. max cancellable via out-of-bound transaction).
 		if used, left := opts.UsedAndLeftSlots(from); left <= 0 {
 			return fmt.Errorf("%w: pooled %d txs", ErrAccountLimitExceeded, used)

core/types/block.go

@@ -99,6 +99,9 @@ type Header struct {
 
 	// ExcessBlobGas was added by EIP-4844 and is ignored in legacy headers.
 	ExcessBlobGas *uint64 `json:"excessBlobGas" rlp:"optional"`
+
+	// ParentBeaconRoot was added by EIP-4788 and is ignored in legacy headers.
+	ParentBeaconRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"`
 }
 
 // field type overrides for gencodec
@@ -253,6 +256,10 @@ func CopyHeader(h *Header) *Header {
 		cpy.BlobGasUsed = new(uint64)
 		*cpy.BlobGasUsed = *h.BlobGasUsed
 	}
+	if h.ParentBeaconRoot != nil {
+		cpy.ParentBeaconRoot = new(common.Hash)
+		*cpy.ParentBeaconRoot = *h.ParentBeaconRoot
+	}
 	return &cpy
 }
 
@@ -331,12 +338,7 @@ func (b *Block) BaseFee() *big.Int {
 	return new(big.Int).Set(b.header.BaseFee)
 }
 
-func (b *Block) BlockGasCost() *big.Int {
-	if b.header.BlockGasCost == nil {
-		return nil
-	}
-	return new(big.Int).Set(b.header.BlockGasCost)
-}
+func (b *Block) BeaconRoot() *common.Hash { return b.header.ParentBeaconRoot }
 
 func (b *Block) ExcessBlobGas() *uint64 {
 	var excessBlobGas *uint64
@@ -356,6 +358,13 @@ func (b *Block) BlobGasUsed() *uint64 {
 	return blobGasUsed
 }
 
+func (b *Block) BlockGasCost() *big.Int {
+	if b.header.BlockGasCost == nil {
+		return nil
+	}
+	return new(big.Int).Set(b.header.BlockGasCost)
+}
+
 // Size returns the true RLP encoded storage size of the block, either by encoding
 // and returning it, or returning a previously cached value.
 func (b *Block) Size() uint64 {

core/types/gen_account_rlp.go

@@ -1,8 +1,5 @@
 // Code generated by rlpgen. DO NOT EDIT.
 
-//go:build !norlpgen
-// +build !norlpgen
-
 package types
 
 import (

core/types/gen_header_json.go

@@ -16,26 +16,27 @@ var _ = (*headerMarshaling)(nil)
 // MarshalJSON marshals as JSON.
 func (h Header) MarshalJSON() ([]byte, error) {
 	type Header struct {
-		ParentHash    common.Hash     `json:"parentHash"       gencodec:"required"`
-		UncleHash     common.Hash     `json:"sha3Uncles"       gencodec:"required"`
-		Coinbase      common.Address  `json:"miner"            gencodec:"required"`
-		Root          common.Hash     `json:"stateRoot"        gencodec:"required"`
-		TxHash        common.Hash     `json:"transactionsRoot" gencodec:"required"`
-		ReceiptHash   common.Hash     `json:"receiptsRoot"     gencodec:"required"`
-		Bloom         Bloom           `json:"logsBloom"        gencodec:"required"`
-		Difficulty    *hexutil.Big    `json:"difficulty"       gencodec:"required"`
-		Number        *hexutil.Big    `json:"number"           gencodec:"required"`
-		GasLimit      hexutil.Uint64  `json:"gasLimit"         gencodec:"required"`
-		GasUsed       hexutil.Uint64  `json:"gasUsed"          gencodec:"required"`
-		Time          hexutil.Uint64  `json:"timestamp"        gencodec:"required"`
-		Extra         hexutil.Bytes   `json:"extraData"        gencodec:"required"`
-		MixDigest     common.Hash     `json:"mixHash"`
-		Nonce         BlockNonce      `json:"nonce"`
-		BaseFee       *hexutil.Big    `json:"baseFeePerGas" rlp:"optional"`
-		BlockGasCost  *hexutil.Big    `json:"blockGasCost" rlp:"optional"`
-		BlobGasUsed   *hexutil.Uint64 `json:"blobGasUsed" rlp:"optional"`
-		ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"`
-		Hash          common.Hash     `json:"hash"`
+		ParentHash       common.Hash     `json:"parentHash"       gencodec:"required"`
+		UncleHash        common.Hash     `json:"sha3Uncles"       gencodec:"required"`
+		Coinbase         common.Address  `json:"miner"            gencodec:"required"`
+		Root             common.Hash     `json:"stateRoot"        gencodec:"required"`
+		TxHash           common.Hash     `json:"transactionsRoot" gencodec:"required"`
+		ReceiptHash      common.Hash     `json:"receiptsRoot"     gencodec:"required"`
+		Bloom            Bloom           `json:"logsBloom"        gencodec:"required"`
+		Difficulty       *hexutil.Big    `json:"difficulty"       gencodec:"required"`
+		Number           *hexutil.Big    `json:"number"           gencodec:"required"`
+		GasLimit         hexutil.Uint64  `json:"gasLimit"         gencodec:"required"`
+		GasUsed          hexutil.Uint64  `json:"gasUsed"          gencodec:"required"`
+		Time             hexutil.Uint64  `json:"timestamp"        gencodec:"required"`
+		Extra            hexutil.Bytes   `json:"extraData"        gencodec:"required"`
+		MixDigest        common.Hash     `json:"mixHash"`
+		Nonce            BlockNonce      `json:"nonce"`
+		BaseFee          *hexutil.Big    `json:"baseFeePerGas" rlp:"optional"`
+		BlockGasCost     *hexutil.Big    `json:"blockGasCost" rlp:"optional"`
+		BlobGasUsed      *hexutil.Uint64 `json:"blobGasUsed" rlp:"optional"`
+		ExcessBlobGas    *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"`
+		ParentBeaconRoot *common.Hash    `json:"parentBeaconBlockRoot" rlp:"optional"`
+		Hash             common.Hash     `json:"hash"`
 	}
 	var enc Header
 	enc.ParentHash = h.ParentHash
@@ -57,6 +58,7 @@ func (h Header) MarshalJSON() ([]byte, error) {
 	enc.BlockGasCost = (*hexutil.Big)(h.BlockGasCost)
 	enc.BlobGasUsed = (*hexutil.Uint64)(h.BlobGasUsed)
 	enc.ExcessBlobGas = (*hexutil.Uint64)(h.ExcessBlobGas)
+	enc.ParentBeaconRoot = h.ParentBeaconRoot
 	enc.Hash = h.Hash()
 	return json.Marshal(&enc)
 }
@@ -64,25 +66,26 @@ func (h Header) MarshalJSON() ([]byte, error) {
 // UnmarshalJSON unmarshals from JSON.
 func (h *Header) UnmarshalJSON(input []byte) error {
 	type Header struct {
-		ParentHash    *common.Hash    `json:"parentHash"       gencodec:"required"`
-		UncleHash     *common.Hash    `json:"sha3Uncles"       gencodec:"required"`
-		Coinbase      *common.Address `json:"miner"            gencodec:"required"`
-		Root          *common.Hash    `json:"stateRoot"        gencodec:"required"`
-		TxHash        *common.Hash    `json:"transactionsRoot" gencodec:"required"`
-		ReceiptHash   *common.Hash    `json:"receiptsRoot"     gencodec:"required"`
-		Bloom         *Bloom          `json:"logsBloom"        gencodec:"required"`
-		Difficulty    *hexutil.Big    `json:"difficulty"       gencodec:"required"`
-		Number        *hexutil.Big    `json:"number"           gencodec:"required"`
-		GasLimit      *hexutil.Uint64 `json:"gasLimit"         gencodec:"required"`
-		GasUsed       *hexutil.Uint64 `json:"gasUsed"          gencodec:"required"`
-		Time          *hexutil.Uint64 `json:"timestamp"        gencodec:"required"`
-		Extra         *hexutil.Bytes  `json:"extraData"        gencodec:"required"`
-		MixDigest     *common.Hash    `json:"mixHash"`
-		Nonce         *BlockNonce     `json:"nonce"`
-		BaseFee       *hexutil.Big    `json:"baseFeePerGas" rlp:"optional"`
-		BlockGasCost  *hexutil.Big    `json:"blockGasCost" rlp:"optional"`
-		BlobGasUsed   *hexutil.Uint64 `json:"blobGasUsed" rlp:"optional"`
-		ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"`
+		ParentHash       *common.Hash    `json:"parentHash"       gencodec:"required"`
+		UncleHash        *common.Hash    `json:"sha3Uncles"       gencodec:"required"`
+		Coinbase         *common.Address `json:"miner"            gencodec:"required"`
+		Root             *common.Hash    `json:"stateRoot"        gencodec:"required"`
+		TxHash           *common.Hash    `json:"transactionsRoot" gencodec:"required"`
+		ReceiptHash      *common.Hash    `json:"receiptsRoot"     gencodec:"required"`
+		Bloom            *Bloom          `json:"logsBloom"        gencodec:"required"`
+		Difficulty       *hexutil.Big    `json:"difficulty"       gencodec:"required"`
+		Number           *hexutil.Big    `json:"number"           gencodec:"required"`
+		GasLimit         *hexutil.Uint64 `json:"gasLimit"         gencodec:"required"`
+		GasUsed          *hexutil.Uint64 `json:"gasUsed"          gencodec:"required"`
+		Time             *hexutil.Uint64 `json:"timestamp"        gencodec:"required"`
+		Extra            *hexutil.Bytes  `json:"extraData"        gencodec:"required"`
+		MixDigest        *common.Hash    `json:"mixHash"`
+		Nonce            *BlockNonce     `json:"nonce"`
+		BaseFee          *hexutil.Big    `json:"baseFeePerGas" rlp:"optional"`
+		BlockGasCost     *hexutil.Big    `json:"blockGasCost" rlp:"optional"`
+		BlobGasUsed      *hexutil.Uint64 `json:"blobGasUsed" rlp:"optional"`
+		ExcessBlobGas    *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"`
+		ParentBeaconRoot *common.Hash    `json:"parentBeaconBlockRoot" rlp:"optional"`
 	}
 	var dec Header
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -158,5 +161,8 @@ func (h *Header) UnmarshalJSON(input []byte) error {
 	if dec.ExcessBlobGas != nil {
 		h.ExcessBlobGas = (*uint64)(dec.ExcessBlobGas)
 	}
+	if dec.ParentBeaconRoot != nil {
+		h.ParentBeaconRoot = dec.ParentBeaconRoot
+	}
 	return nil
 }

core/types/gen_header_rlp.go

@@ -1,8 +1,5 @@
 // Code generated by rlpgen. DO NOT EDIT.
 
-//go:build !norlpgen
-// +build !norlpgen
-
 package types
 
 import "github.com/ethereum/go-ethereum/rlp"
@@ -44,7 +41,8 @@ func (obj *Header) EncodeRLP(_w io.Writer) error {
 	_tmp2 := obj.BlockGasCost != nil
 	_tmp3 := obj.BlobGasUsed != nil
 	_tmp4 := obj.ExcessBlobGas != nil
-	if _tmp1 || _tmp2 || _tmp3 || _tmp4 {
+	_tmp5 := obj.ParentBeaconRoot != nil
+	if _tmp1 || _tmp2 || _tmp3 || _tmp4 || _tmp5 {
 		if obj.BaseFee == nil {
 			w.Write(rlp.EmptyString)
 		} else {
@@ -54,7 +52,7 @@ func (obj *Header) EncodeRLP(_w io.Writer) error {
 			w.WriteBigInt(obj.BaseFee)
 		}
 	}
-	if _tmp2 || _tmp3 || _tmp4 {
+	if _tmp2 || _tmp3 || _tmp4 || _tmp5 {
 		if obj.BlockGasCost == nil {
 			w.Write(rlp.EmptyString)
 		} else {
@@ -64,20 +62,27 @@ func (obj *Header) EncodeRLP(_w io.Writer) error {
 			w.WriteBigInt(obj.BlockGasCost)
 		}
 	}
-	if _tmp3 || _tmp4 {
+	if _tmp3 || _tmp4 || _tmp5 {
 		if obj.BlobGasUsed == nil {
 			w.Write([]byte{0x80})
 		} else {
 			w.WriteUint64((*obj.BlobGasUsed))
 		}
 	}
-	if _tmp4 {
+	if _tmp4 || _tmp5 {
 		if obj.ExcessBlobGas == nil {
 			w.Write([]byte{0x80})
 		} else {
 			w.WriteUint64((*obj.ExcessBlobGas))
 		}
 	}
+	if _tmp5 {
+		if obj.ParentBeaconRoot == nil {
+			w.Write([]byte{0x80})
+		} else {
+			w.WriteBytes(obj.ParentBeaconRoot[:])
+		}
+	}
 	w.ListEnd(_tmp0)
 	return w.Flush()
 }

core/types/gen_log_json.go

@@ -18,12 +18,12 @@ func (l Log) MarshalJSON() ([]byte, error) {
 		Address     common.Address `json:"address" gencodec:"required"`
 		Topics      []common.Hash  `json:"topics" gencodec:"required"`
 		Data        hexutil.Bytes  `json:"data" gencodec:"required"`
-		BlockNumber hexutil.Uint64 `json:"blockNumber"`
-		TxHash      common.Hash    `json:"transactionHash" gencodec:"required"`
-		TxIndex     hexutil.Uint   `json:"transactionIndex"`
-		BlockHash   common.Hash    `json:"blockHash"`
-		Index       hexutil.Uint   `json:"logIndex"`
-		Removed     bool           `json:"removed"`
+		BlockNumber hexutil.Uint64 `json:"blockNumber" rlp:"-"`
+		TxHash      common.Hash    `json:"transactionHash" gencodec:"required" rlp:"-"`
+		TxIndex     hexutil.Uint   `json:"transactionIndex" rlp:"-"`
+		BlockHash   common.Hash    `json:"blockHash" rlp:"-"`
+		Index       hexutil.Uint   `json:"logIndex" rlp:"-"`
+		Removed     bool           `json:"removed" rlp:"-"`
 	}
 	var enc Log
 	enc.Address = l.Address
@@ -44,12 +44,12 @@ func (l *Log) UnmarshalJSON(input []byte) error {
 		Address     *common.Address `json:"address" gencodec:"required"`
 		Topics      []common.Hash   `json:"topics" gencodec:"required"`
 		Data        *hexutil.Bytes  `json:"data" gencodec:"required"`
-		BlockNumber *hexutil.Uint64 `json:"blockNumber"`
-		TxHash      *common.Hash    `json:"transactionHash" gencodec:"required"`
-		TxIndex     *hexutil.Uint   `json:"transactionIndex"`
-		BlockHash   *common.Hash    `json:"blockHash"`
-		Index       *hexutil.Uint   `json:"logIndex"`
-		Removed     *bool           `json:"removed"`
+		BlockNumber *hexutil.Uint64 `json:"blockNumber" rlp:"-"`
+		TxHash      *common.Hash    `json:"transactionHash" gencodec:"required" rlp:"-"`
+		TxIndex     *hexutil.Uint   `json:"transactionIndex" rlp:"-"`
+		BlockHash   *common.Hash    `json:"blockHash" rlp:"-"`
+		Index       *hexutil.Uint   `json:"logIndex" rlp:"-"`
+		Removed     *bool           `json:"removed" rlp:"-"`
 	}
 	var dec Log
 	if err := json.Unmarshal(input, &dec); err != nil {

core/types/gen_log_rlp.go

@@ -1,8 +1,5 @@
 // Code generated by rlpgen. DO NOT EDIT.
 
-//go:build !norlpgen
-// +build !norlpgen
-
 package types
 
 import (
@@ -11,7 +8,7 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
-func (obj *rlpLog) EncodeRLP(_w io.Writer) error {
+func (obj *Log) EncodeRLP(_w io.Writer) error {
 	w := rlp.NewEncoderBuffer(_w)
 	_tmp0 := w.List()
 	w.WriteBytes(obj.Address[:])

core/types/hashing.go

@@ -28,6 +28,8 @@ package types
 
 import (
 	"bytes"
+	"fmt"
+	"math"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -46,6 +48,22 @@ var encodeBufferPool = sync.Pool{
 	New: func() interface{} { return new(bytes.Buffer) },
 }
 
+// getPooledBuffer retrieves a buffer from the pool and creates a byte slice of the
+// requested size from it.
+//
+// The caller should return the *bytes.Buffer object back into encodeBufferPool after use!
+// The returned byte slice must not be used after returning the buffer.
+func getPooledBuffer(size uint64) ([]byte, *bytes.Buffer, error) {
+	if size > math.MaxInt {
+		return nil, nil, fmt.Errorf("can't get buffer of size %d", size)
+	}
+	buf := encodeBufferPool.Get().(*bytes.Buffer)
+	buf.Reset()
+	buf.Grow(int(size))
+	b := buf.Bytes()[:int(size)]
+	return b, buf, nil
+}
+
 // rlpHash encodes x and hashes the encoded bytes.
 func rlpHash(x interface{}) (h common.Hash) {
 	sha := hasherPool.Get().(crypto.KeccakState)

core/types/hashing_test.go

@@ -49,7 +49,7 @@ func TestDeriveSha(t *testing.T) {
 		t.Fatal(err)
 	}
 	for len(txs) < 1000 {
-		exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
+		exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)))
 		got := types.DeriveSha(txs, trie.NewStackTrie(nil))
 		if !bytes.Equal(got[:], exp[:]) {
 			t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp)
@@ -96,7 +96,7 @@ func BenchmarkDeriveSha200(b *testing.B) {
 		b.ResetTimer()
 		b.ReportAllocs()
 		for i := 0; i < b.N; i++ {
-			exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
+			exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)))
 		}
 	})
 
@@ -117,7 +117,7 @@ func TestFuzzDeriveSha(t *testing.T) {
 	rndSeed := mrand.Int()
 	for i := 0; i < 10; i++ {
 		seed := rndSeed + i
-		exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
+		exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)))
 		got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil))
 		if !bytes.Equal(got[:], exp[:]) {
 			printList(newDummy(seed))
@@ -145,7 +145,7 @@ func TestDerivableList(t *testing.T) {
 		},
 	}
 	for i, tc := range tcs[1:] {
-		exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())))
+		exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase(), nil)))
 		got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil))
 		if !bytes.Equal(got[:], exp[:]) {
 			t.Fatalf("case %d: got %x exp %x", i, got, exp)

core/types/log.go

@@ -27,13 +27,11 @@
 package types
 
 import (
-	"io"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/rlp"
 )
 
+//go:generate go run github.com/ethereum/go-ethereum/rlp/rlpgen -type Log -out gen_log_rlp.go
 //go:generate go run github.com/fjl/gencodec -type Log -field-override logMarshaling -out gen_log_json.go
 
 // Log represents a contract log event. These events are generated by the LOG opcode and
@@ -50,19 +48,19 @@ type Log struct {
 	// Derived fields. These fields are filled in by the node
 	// but not secured by consensus.
 	// block in which the transaction was included
-	BlockNumber uint64 `json:"blockNumber"`
+	BlockNumber uint64 `json:"blockNumber" rlp:"-"`
 	// hash of the transaction
-	TxHash common.Hash `json:"transactionHash" gencodec:"required"`
+	TxHash common.Hash `json:"transactionHash" gencodec:"required" rlp:"-"`
 	// index of the transaction in the block
-	TxIndex uint `json:"transactionIndex"`
+	TxIndex uint `json:"transactionIndex" rlp:"-"`
 	// hash of the block in which the transaction was included
-	BlockHash common.Hash `json:"blockHash"`
+	BlockHash common.Hash `json:"blockHash" rlp:"-"`
 	// index of the log in the block
-	Index uint `json:"logIndex"`
+	Index uint `json:"logIndex" rlp:"-"`
 
 	// The Removed field is true if this log was reverted due to a chain reorganisation.
 	// You must pay attention to this field if you receive logs through a filter query.
-	Removed bool `json:"removed"`
+	Removed bool `json:"removed" rlp:"-"`
 }
 
 type logMarshaling struct {
@@ -72,31 +70,6 @@ type logMarshaling struct {
 	Index       hexutil.Uint
 }
 
-//go:generate go run github.com/ethereum/go-ethereum/rlp/rlpgen -type rlpLog -out gen_log_rlp.go
-
-// rlpLog is used to RLP-encode both the consensus and storage formats.
-type rlpLog struct {
-	Address common.Address
-	Topics  []common.Hash
-	Data    []byte
-}
-
-// EncodeRLP implements rlp.Encoder.
-func (l *Log) EncodeRLP(w io.Writer) error {
-	rl := rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data}
-	return rlp.Encode(w, &rl)
-}
-
-// DecodeRLP implements rlp.Decoder.
-func (l *Log) DecodeRLP(s *rlp.Stream) error {
-	var dec rlpLog
-	err := s.Decode(&dec)
-	if err == nil {
-		l.Address, l.Topics, l.Data = dec.Address, dec.Topics, dec.Data
-	}
-	return err
-}
-
 // FlattenLogs converts a nested array of logs to a single array of logs.
 func FlattenLogs(list [][]*Log) []*Log {
 	var flat []*Log

core/types/receipt.go

@@ -163,7 +163,7 @@ func (r *Receipt) MarshalBinary() ([]byte, error) {
 // DecodeRLP implements rlp.Decoder, and loads the consensus fields of a receipt
 // from an RLP stream.
 func (r *Receipt) DecodeRLP(s *rlp.Stream) error {
-	kind, _, err := s.Kind()
+	kind, size, err := s.Kind()
 	switch {
 	case err != nil:
 		return err
@@ -175,12 +175,18 @@ func (r *Receipt) DecodeRLP(s *rlp.Stream) error {
 		}
 		r.Type = LegacyTxType
 		return r.setFromRLP(dec)
+	case kind == rlp.Byte:
+		return errShortTypedReceipt
 	default:
 		// It's an EIP-2718 typed tx receipt.
-		b, err := s.Bytes()
+		b, buf, err := getPooledBuffer(size)
 		if err != nil {
 			return err
 		}
+		defer encodeBufferPool.Put(buf)
+		if err := s.ReadBytes(b); err != nil {
+			return err
+		}
 		return r.decodeTyped(b)
 	}
 }
@@ -274,7 +280,7 @@ func (r *ReceiptForStorage) EncodeRLP(_w io.Writer) error {
 	w.WriteUint64(r.CumulativeGasUsed)
 	logList := w.List()
 	for _, log := range r.Logs {
-		if err := rlp.Encode(w, log); err != nil {
+		if err := log.EncodeRLP(w); err != nil {
 			return err
 		}
 	}

core/types/transaction.go

@@ -92,9 +92,6 @@ type TxData interface {
 	value() *big.Int
 	nonce() uint64
 	to() *common.Address
-	blobGas() uint64
-	blobGasFeeCap() *big.Int
-	blobHashes() []common.Hash
 
 	rawSignatureValues() (v, r, s *big.Int)
 	setSignatureValues(chainID, v, r, s *big.Int)
@@ -106,6 +103,9 @@ type TxData interface {
 	// copy of the computed value, i.e. callers are allowed to mutate the result.
 	// Method implementations can use 'dst' to store the result.
 	effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int
+
+	encode(*bytes.Buffer) error
+	decode([]byte) error
 }
 
 // EncodeRLP implements rlp.Encoder
@@ -126,7 +126,7 @@ func (tx *Transaction) EncodeRLP(w io.Writer) error {
 // encodeTyped writes the canonical encoding of a typed transaction to w.
 func (tx *Transaction) encodeTyped(w *bytes.Buffer) error {
 	w.WriteByte(tx.Type())
-	return rlp.Encode(w, tx.inner)
+	return tx.inner.encode(w)
 }
 
 // MarshalBinary returns the canonical encoding of the transaction.
@@ -155,15 +155,23 @@ func (tx *Transaction) DecodeRLP(s *rlp.Stream) error {
 			tx.setDecoded(&inner, rlp.ListSize(size))
 		}
 		return err
+	case kind == rlp.Byte:
+		return errShortTypedTx
 	default:
 		// It's an EIP-2718 typed TX envelope.
-		var b []byte
-		if b, err = s.Bytes(); err != nil {
+		// First read the tx payload bytes into a temporary buffer.
+		b, buf, err := getPooledBuffer(size)
+		if err != nil {
 			return err
 		}
+		defer encodeBufferPool.Put(buf)
+		if err := s.ReadBytes(b); err != nil {
+			return err
+		}
+		// Now decode the inner transaction.
 		inner, err := tx.decodeTyped(b)
 		if err == nil {
-			tx.setDecoded(inner, uint64(len(b)))
+			tx.setDecoded(inner, size)
 		}
 		return err
 	}
@@ -196,22 +204,19 @@ func (tx *Transaction) decodeTyped(b []byte) (TxData, error) {
 	if len(b) <= 1 {
 		return nil, errShortTypedTx
 	}
+	var inner TxData
 	switch b[0] {
 	case AccessListTxType:
-		var inner AccessListTx
-		err := rlp.DecodeBytes(b[1:], &inner)
-		return &inner, err
+		inner = new(AccessListTx)
 	case DynamicFeeTxType:
-		var inner DynamicFeeTx
-		err := rlp.DecodeBytes(b[1:], &inner)
-		return &inner, err
+		inner = new(DynamicFeeTx)
 	case BlobTxType:
-		var inner BlobTx
-		err := rlp.DecodeBytes(b[1:], &inner)
-		return &inner, err
+		inner = new(BlobTx)
 	default:
 		return nil, ErrTxTypeNotSupported
 	}
+	err := inner.decode(b[1:])
+	return inner, err
 }
 
 // setDecoded sets the inner transaction and size after decoding.
@@ -298,15 +303,6 @@ func (tx *Transaction) GasTipCap() *big.Int { return new(big.Int).Set(tx.inner.g
 // GasFeeCap returns the fee cap per gas of the transaction.
 func (tx *Transaction) GasFeeCap() *big.Int { return new(big.Int).Set(tx.inner.gasFeeCap()) }
 
-// BlobGas returns the blob gas limit of the transaction for blob transactions, 0 otherwise.
-func (tx *Transaction) BlobGas() uint64 { return tx.inner.blobGas() }
-
-// BlobGasFeeCap returns the blob gas fee cap per blob gas of the transaction for blob transactions, nil otherwise.
-func (tx *Transaction) BlobGasFeeCap() *big.Int { return tx.inner.blobGasFeeCap() }
-
-// BlobHashes returns the hases of the blob commitments for blob transactions, nil otherwise.
-func (tx *Transaction) BlobHashes() []common.Hash { return tx.inner.blobHashes() }
-
 // Value returns the ether amount of the transaction.
 func (tx *Transaction) Value() *big.Int { return new(big.Int).Set(tx.inner.value()) }
 
@@ -393,14 +389,66 @@ func (tx *Transaction) EffectiveGasTipIntCmp(other *big.Int, baseFee *big.Int) i
 	return tx.EffectiveGasTipValue(baseFee).Cmp(other)
 }
 
+// BlobGas returns the blob gas limit of the transaction for blob transactions, 0 otherwise.
+func (tx *Transaction) BlobGas() uint64 {
+	if blobtx, ok := tx.inner.(*BlobTx); ok {
+		return blobtx.blobGas()
+	}
+	return 0
+}
+
+// BlobGasFeeCap returns the blob gas fee cap per blob gas of the transaction for blob transactions, nil otherwise.
+func (tx *Transaction) BlobGasFeeCap() *big.Int {
+	if blobtx, ok := tx.inner.(*BlobTx); ok {
+		return blobtx.BlobFeeCap.ToBig()
+	}
+	return nil
+}
+
+// BlobHashes returns the hases of the blob commitments for blob transactions, nil otherwise.
+func (tx *Transaction) BlobHashes() []common.Hash {
+	if blobtx, ok := tx.inner.(*BlobTx); ok {
+		return blobtx.BlobHashes
+	}
+	return nil
+}
+
+// BlobTxSidecar returns the sidecar of a blob transaction, nil otherwise.
+func (tx *Transaction) BlobTxSidecar() *BlobTxSidecar {
+	if blobtx, ok := tx.inner.(*BlobTx); ok {
+		return blobtx.Sidecar
+	}
+	return nil
+}
+
 // BlobGasFeeCapCmp compares the blob fee cap of two transactions.
 func (tx *Transaction) BlobGasFeeCapCmp(other *Transaction) int {
-	return tx.inner.blobGasFeeCap().Cmp(other.inner.blobGasFeeCap())
+	return tx.BlobGasFeeCap().Cmp(other.BlobGasFeeCap())
 }
 
 // BlobGasFeeCapIntCmp compares the blob fee cap of the transaction against the given blob fee cap.
 func (tx *Transaction) BlobGasFeeCapIntCmp(other *big.Int) int {
-	return tx.inner.blobGasFeeCap().Cmp(other)
+	return tx.BlobGasFeeCap().Cmp(other)
+}
+
+// WithoutBlobTxSidecar returns a copy of tx with the blob sidecar removed.
+func (tx *Transaction) WithoutBlobTxSidecar() *Transaction {
+	blobtx, ok := tx.inner.(*BlobTx)
+	if !ok {
+		return tx
+	}
+	cpy := &Transaction{
+		inner: blobtx.withoutSidecar(),
+		time:  tx.time,
+	}
+	// Note: tx.size cache not carried over because the sidecar is included in size!
+	if h := tx.hash.Load(); h != nil {
+		cpy.hash.Store(h)
+	}
+	if f := tx.from.Load(); f != nil {
+		cpy.from.Store(f)
+	}
+	return cpy
 }
 
 // SetTime sets the decoding time of a transaction. This is used by tests to set
@@ -438,13 +486,24 @@ func (tx *Transaction) Size() uint64 {
 	if size := tx.size.Load(); size != nil {
 		return size.(uint64)
 	}
+
+	// Cache miss, encode and cache.
+	// Note we rely on the assumption that all tx.inner values are RLP-encoded!
 	c := writeCounter(0)
 	rlp.Encode(&c, &tx.inner)
-
 	size := uint64(c)
-	if tx.Type() != LegacyTxType {
-		size += 1 // type byte
+
+	// For blob transactions, add the size of the blob content and the outer list of the
+	// tx + sidecar encoding.
+	if sc := tx.BlobTxSidecar(); sc != nil {
+		size += rlp.ListSize(sc.encodedSize())
 	}
+
+	// For typed transactions, the encoding also includes the leading type byte.
+	if tx.Type() != LegacyTxType {
+		size += 1
+	}
+
 	tx.size.Store(size)
 	return size
 }

core/types/transaction_marshalling.go

@@ -383,20 +383,20 @@ func (tx *Transaction) UnmarshalJSON(input []byte) error {
 		itx.BlobHashes = dec.BlobVersionedHashes
 
 		// signature R
-		var ok bool
+		var overflow bool
 		if dec.R == nil {
 			return errors.New("missing required field 'r' in transaction")
 		}
-		itx.R, ok = uint256.FromBig((*big.Int)(dec.R))
-		if !ok {
+		itx.R, overflow = uint256.FromBig((*big.Int)(dec.R))
+		if overflow {
 			return errors.New("'r' value overflows uint256")
 		}
 		// signature S
 		if dec.S == nil {
 			return errors.New("missing required field 's' in transaction")
 		}
-		itx.S, ok = uint256.FromBig((*big.Int)(dec.S))
-		if !ok {
+		itx.S, overflow = uint256.FromBig((*big.Int)(dec.S))
+		if overflow {
 			return errors.New("'s' value overflows uint256")
 		}
 		// signature V
@@ -404,8 +404,8 @@ func (tx *Transaction) UnmarshalJSON(input []byte) error {
 		if err != nil {
 			return err
 		}
-		itx.V, ok = uint256.FromBig(vbig)
-		if !ok {
+		itx.V, overflow = uint256.FromBig(vbig)
+		if overflow {
 			return errors.New("'v' value overflows uint256")
 		}
 		if itx.V.Sign() != 0 || itx.R.Sign() != 0 || itx.S.Sign() != 0 {

core/types/transaction_signing.go

@@ -334,11 +334,7 @@ func (s eip2930Signer) Sender(tx *Transaction) (common.Address, error) {
 	V, R, S := tx.RawSignatureValues()
 	switch tx.Type() {
 	case LegacyTxType:
-		if !tx.Protected() {
-			return HomesteadSigner{}.Sender(tx)
-		}
-		V = new(big.Int).Sub(V, s.chainIdMul)
-		V.Sub(V, big8)
+		return s.EIP155Signer.Sender(tx)
 	case AccessListTxType:
 		// AL txs are defined to use 0 and 1 as their recovery
 		// id, add 27 to become equivalent to unprotected Homestead signatures.
@@ -375,15 +371,7 @@ func (s eip2930Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *bi
 func (s eip2930Signer) Hash(tx *Transaction) common.Hash {
 	switch tx.Type() {
 	case LegacyTxType:
-		return rlpHash([]interface{}{
-			tx.Nonce(),
-			tx.GasPrice(),
-			tx.Gas(),
-			tx.To(),
-			tx.Value(),
-			tx.Data(),
-			s.chainId, uint(0), uint(0),
-		})
+		return s.EIP155Signer.Hash(tx)
 	case AccessListTxType:
 		return prefixedRlpHash(
 			tx.Type(),

core/types/transaction_test.go

@@ -224,7 +224,7 @@ func TestEIP2718TransactionEncode(t *testing.T) {
 
 func decodeTx(data []byte) (*Transaction, error) {
 	var tx Transaction
-	t, err := &tx, rlp.Decode(bytes.NewReader(data), &tx)
+	t, err := &tx, rlp.DecodeBytes(data, &tx)
 	return t, err
 }
 

core/types/tx_access_list.go

@@ -27,9 +27,11 @@
 package types
 
 import (
+	"bytes"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
 //go:generate go run github.com/fjl/gencodec -type AccessTuple -out gen_access_tuple.go
@@ -104,20 +106,17 @@ func (tx *AccessListTx) copy() TxData {
 }
 
 // accessors for innerTx.
-func (tx *AccessListTx) txType() byte              { return AccessListTxType }
-func (tx *AccessListTx) chainID() *big.Int         { return tx.ChainID }
-func (tx *AccessListTx) accessList() AccessList    { return tx.AccessList }
-func (tx *AccessListTx) data() []byte              { return tx.Data }
-func (tx *AccessListTx) gas() uint64               { return tx.Gas }
-func (tx *AccessListTx) gasPrice() *big.Int        { return tx.GasPrice }
-func (tx *AccessListTx) gasTipCap() *big.Int       { return tx.GasPrice }
-func (tx *AccessListTx) gasFeeCap() *big.Int       { return tx.GasPrice }
-func (tx *AccessListTx) value() *big.Int           { return tx.Value }
-func (tx *AccessListTx) nonce() uint64             { return tx.Nonce }
-func (tx *AccessListTx) to() *common.Address       { return tx.To }
-func (tx *AccessListTx) blobGas() uint64           { return 0 }
-func (tx *AccessListTx) blobGasFeeCap() *big.Int   { return nil }
-func (tx *AccessListTx) blobHashes() []common.Hash { return nil }
+func (tx *AccessListTx) txType() byte           { return AccessListTxType }
+func (tx *AccessListTx) chainID() *big.Int      { return tx.ChainID }
+func (tx *AccessListTx) accessList() AccessList { return tx.AccessList }
+func (tx *AccessListTx) data() []byte           { return tx.Data }
+func (tx *AccessListTx) gas() uint64            { return tx.Gas }
+func (tx *AccessListTx) gasPrice() *big.Int     { return tx.GasPrice }
+func (tx *AccessListTx) gasTipCap() *big.Int    { return tx.GasPrice }
+func (tx *AccessListTx) gasFeeCap() *big.Int    { return tx.GasPrice }
+func (tx *AccessListTx) value() *big.Int        { return tx.Value }
+func (tx *AccessListTx) nonce() uint64          { return tx.Nonce }
+func (tx *AccessListTx) to() *common.Address    { return tx.To }
 
 func (tx *AccessListTx) effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int {
 	return dst.Set(tx.GasPrice)
@@ -130,3 +129,11 @@ func (tx *AccessListTx) rawSignatureValues() (v, r, s *big.Int) {
 func (tx *AccessListTx) setSignatureValues(chainID, v, r, s *big.Int) {
 	tx.ChainID, tx.V, tx.R, tx.S = chainID, v, r, s
 }
+
+func (tx *AccessListTx) encode(b *bytes.Buffer) error {
+	return rlp.Encode(b, tx)
+}
+
+func (tx *AccessListTx) decode(input []byte) error {
+	return rlp.DecodeBytes(input, tx)
+}

core/types/tx_blob.go

@@ -17,10 +17,14 @@
 package types
 
 import (
+	"bytes"
+	"crypto/sha256"
 	"math/big"
 
 	"github.com/ava-labs/subnet-evm/params"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto/kzg4844"
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/holiman/uint256"
 )
 
@@ -38,12 +42,56 @@ type BlobTx struct {
 	BlobFeeCap *uint256.Int // a.k.a. maxFeePerBlobGas
 	BlobHashes []common.Hash
 
+	// A blob transaction can optionally contain blobs. This field must be set when BlobTx
+	// is used to create a transaction for sigining.
+	Sidecar *BlobTxSidecar `rlp:"-"`
+
 	// Signature values
 	V *uint256.Int `json:"v" gencodec:"required"`
 	R *uint256.Int `json:"r" gencodec:"required"`
 	S *uint256.Int `json:"s" gencodec:"required"`
 }
 
+// BlobTxSidecar contains the blobs of a blob transaction.
+type BlobTxSidecar struct {
+	Blobs       []kzg4844.Blob       // Blobs needed by the blob pool
+	Commitments []kzg4844.Commitment // Commitments needed by the blob pool
+	Proofs      []kzg4844.Proof      // Proofs needed by the blob pool
+}
+
+// BlobHashes computes the blob hashes of the given blobs.
+func (sc *BlobTxSidecar) BlobHashes() []common.Hash {
+	h := make([]common.Hash, len(sc.Commitments))
+	for i := range sc.Blobs {
+		h[i] = blobHash(&sc.Commitments[i])
+	}
+	return h
+}
+
+// encodedSize computes the RLP size of the sidecar elements. This does NOT return the
+// encoded size of the BlobTxSidecar, it's just a helper for tx.Size().
+func (sc *BlobTxSidecar) encodedSize() uint64 {
+	var blobs, commitments, proofs uint64
+	for i := range sc.Blobs {
+		blobs += rlp.BytesSize(sc.Blobs[i][:])
+	}
+	for i := range sc.Commitments {
+		commitments += rlp.BytesSize(sc.Commitments[i][:])
+	}
+	for i := range sc.Proofs {
+		proofs += rlp.BytesSize(sc.Proofs[i][:])
+	}
+	return rlp.ListSize(blobs) + rlp.ListSize(commitments) + rlp.ListSize(proofs)
+}
+
+// blobTxWithBlobs is used for encoding of transactions when blobs are present.
+type blobTxWithBlobs struct {
+	BlobTx      *BlobTx
+	Blobs       []kzg4844.Blob
+	Commitments []kzg4844.Commitment
+	Proofs      []kzg4844.Proof
+}
+
 // copy creates a deep copy of the transaction data and initializes all fields.
 func (tx *BlobTx) copy() TxData {
 	cpy := &BlobTx{
@@ -90,24 +138,29 @@ func (tx *BlobTx) copy() TxData {
 	if tx.S != nil {
 		cpy.S.Set(tx.S)
 	}
+	if tx.Sidecar != nil {
+		cpy.Sidecar = &BlobTxSidecar{
+			Blobs:       append([]kzg4844.Blob(nil), tx.Sidecar.Blobs...),
+			Commitments: append([]kzg4844.Commitment(nil), tx.Sidecar.Commitments...),
+			Proofs:      append([]kzg4844.Proof(nil), tx.Sidecar.Proofs...),
+		}
+	}
 	return cpy
 }
 
 // accessors for innerTx.
-func (tx *BlobTx) txType() byte              { return BlobTxType }
-func (tx *BlobTx) chainID() *big.Int         { return tx.ChainID.ToBig() }
-func (tx *BlobTx) accessList() AccessList    { return tx.AccessList }
-func (tx *BlobTx) data() []byte              { return tx.Data }
-func (tx *BlobTx) gas() uint64               { return tx.Gas }
-func (tx *BlobTx) gasFeeCap() *big.Int       { return tx.GasFeeCap.ToBig() }
-func (tx *BlobTx) gasTipCap() *big.Int       { return tx.GasTipCap.ToBig() }
-func (tx *BlobTx) gasPrice() *big.Int        { return tx.GasFeeCap.ToBig() }
-func (tx *BlobTx) value() *big.Int           { return tx.Value.ToBig() }
-func (tx *BlobTx) nonce() uint64             { return tx.Nonce }
-func (tx *BlobTx) to() *common.Address       { tmp := tx.To; return &tmp }
-func (tx *BlobTx) blobGas() uint64           { return params.BlobTxBlobGasPerBlob * uint64(len(tx.BlobHashes)) }
-func (tx *BlobTx) blobGasFeeCap() *big.Int   { return tx.BlobFeeCap.ToBig() }
-func (tx *BlobTx) blobHashes() []common.Hash { return tx.BlobHashes }
+func (tx *BlobTx) txType() byte           { return BlobTxType }
+func (tx *BlobTx) chainID() *big.Int      { return tx.ChainID.ToBig() }
+func (tx *BlobTx) accessList() AccessList { return tx.AccessList }
+func (tx *BlobTx) data() []byte           { return tx.Data }
+func (tx *BlobTx) gas() uint64            { return tx.Gas }
+func (tx *BlobTx) gasFeeCap() *big.Int    { return tx.GasFeeCap.ToBig() }
+func (tx *BlobTx) gasTipCap() *big.Int    { return tx.GasTipCap.ToBig() }
+func (tx *BlobTx) gasPrice() *big.Int     { return tx.GasFeeCap.ToBig() }
+func (tx *BlobTx) value() *big.Int        { return tx.Value.ToBig() }
+func (tx *BlobTx) nonce() uint64          { return tx.Nonce }
+func (tx *BlobTx) to() *common.Address    { tmp := tx.To; return &tmp }
+func (tx *BlobTx) blobGas() uint64        { return params.BlobTxBlobGasPerBlob * uint64(len(tx.BlobHashes)) }
 
 func (tx *BlobTx) effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int {
 	if baseFee == nil {
@@ -130,3 +183,64 @@ func (tx *BlobTx) setSignatureValues(chainID, v, r, s *big.Int) {
 	tx.R.SetFromBig(r)
 	tx.S.SetFromBig(s)
 }
+
+func (tx *BlobTx) withoutSidecar() *BlobTx {
+	cpy := *tx
+	cpy.Sidecar = nil
+	return &cpy
+}
+
+func (tx *BlobTx) encode(b *bytes.Buffer) error {
+	if tx.Sidecar == nil {
+		return rlp.Encode(b, tx)
+	}
+	inner := &blobTxWithBlobs{
+		BlobTx:      tx,
+		Blobs:       tx.Sidecar.Blobs,
+		Commitments: tx.Sidecar.Commitments,
+		Proofs:      tx.Sidecar.Proofs,
+	}
+	return rlp.Encode(b, inner)
+}
+
+func (tx *BlobTx) decode(input []byte) error {
+	// Here we need to support two formats: the network protocol encoding of the tx (with
+	// blobs) or the canonical encoding without blobs.
+	//
+	// The two encodings can be distinguished by checking whether the first element of the
+	// input list is itself a list.
+
+	outerList, _, err := rlp.SplitList(input)
+	if err != nil {
+		return err
+	}
+	firstElemKind, _, _, err := rlp.Split(outerList)
+	if err != nil {
+		return err
+	}
+
+	if firstElemKind != rlp.List {
+		return rlp.DecodeBytes(input, tx)
+	}
+	// It's a tx with blobs.
+	var inner blobTxWithBlobs
+	if err := rlp.DecodeBytes(input, &inner); err != nil {
+		return err
+	}
+	*tx = *inner.BlobTx
+	tx.Sidecar = &BlobTxSidecar{
+		Blobs:       inner.Blobs,
+		Commitments: inner.Commitments,
+		Proofs:      inner.Proofs,
+	}
+	return nil
+}
+
+func blobHash(commit *kzg4844.Commitment) common.Hash {
+	hasher := sha256.New()
+	hasher.Write(commit[:])
+	var vhash common.Hash
+	hasher.Sum(vhash[:0])
+	vhash[0] = params.BlobTxHashVersion
+	return vhash
+}

core/types/tx_blob_test.go

@@ -0,0 +1,90 @@
+package types
+
+import (
+	"crypto/ecdsa"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/crypto/kzg4844"
+	"github.com/holiman/uint256"
+)
+
+// This test verifies that tx.Hash() is not affected by presence of a BlobTxSidecar.
+func TestBlobTxHashing(t *testing.T) {
+	key, _ := crypto.GenerateKey()
+	withBlobs := createEmptyBlobTx(key, true)
+	withBlobsStripped := withBlobs.WithoutBlobTxSidecar()
+	withoutBlobs := createEmptyBlobTx(key, false)
+
+	hash := withBlobs.Hash()
+	t.Log("tx hash:", hash)
+
+	if h := withBlobsStripped.Hash(); h != hash {
+		t.Fatal("wrong tx hash after WithoutBlobTxSidecar:", h)
+	}
+	if h := withoutBlobs.Hash(); h != hash {
+		t.Fatal("wrong tx hash on tx created without sidecar:", h)
+	}
+}
+
+// This test verifies that tx.Size() takes BlobTxSidecar into account.
+func TestBlobTxSize(t *testing.T) {
+	key, _ := crypto.GenerateKey()
+	withBlobs := createEmptyBlobTx(key, true)
+	withBlobsStripped := withBlobs.WithoutBlobTxSidecar()
+	withoutBlobs := createEmptyBlobTx(key, false)
+
+	withBlobsEnc, _ := withBlobs.MarshalBinary()
+	withoutBlobsEnc, _ := withoutBlobs.MarshalBinary()
+
+	size := withBlobs.Size()
+	t.Log("size with blobs:", size)
+
+	sizeNoBlobs := withoutBlobs.Size()
+	t.Log("size without blobs:", sizeNoBlobs)
+
+	if size != uint64(len(withBlobsEnc)) {
+		t.Error("wrong size with blobs:", size, "encoded length:", len(withBlobsEnc))
+	}
+	if sizeNoBlobs != uint64(len(withoutBlobsEnc)) {
+		t.Error("wrong size without blobs:", sizeNoBlobs, "encoded length:", len(withoutBlobsEnc))
+	}
+	if sizeNoBlobs >= size {
+		t.Error("size without blobs >= size with blobs")
+	}
+	if sz := withBlobsStripped.Size(); sz != sizeNoBlobs {
+		t.Fatal("wrong size on tx after WithoutBlobTxSidecar:", sz)
+	}
+}
+
+var (
+	emptyBlob          = kzg4844.Blob{}
+	emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob)
+	emptyBlobProof, _  = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit)
+)
+
+func createEmptyBlobTx(key *ecdsa.PrivateKey, withSidecar bool) *Transaction {
+	sidecar := &BlobTxSidecar{
+		Blobs:       []kzg4844.Blob{emptyBlob},
+		Commitments: []kzg4844.Commitment{emptyBlobCommit},
+		Proofs:      []kzg4844.Proof{emptyBlobProof},
+	}
+	blobtx := &BlobTx{
+		ChainID:    uint256.NewInt(1),
+		Nonce:      5,
+		GasTipCap:  uint256.NewInt(22),
+		GasFeeCap:  uint256.NewInt(5),
+		Gas:        25000,
+		To:         common.Address{0x03, 0x04, 0x05},
+		Value:      uint256.NewInt(99),
+		Data:       make([]byte, 50),
+		BlobFeeCap: uint256.NewInt(15),
+		BlobHashes: sidecar.BlobHashes(),
+	}
+	if withSidecar {
+		blobtx.Sidecar = sidecar
+	}
+	signer := NewCancunSigner(blobtx.ChainID.ToBig())
+	return MustSignNewTx(key, signer, blobtx)
+}

core/types/tx_dynamic_fee.go

@@ -27,9 +27,11 @@
 package types
 
 import (
+	"bytes"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // DynamicFeeTx represents an EIP-1559 transaction.
@@ -93,20 +95,17 @@ func (tx *DynamicFeeTx) copy() TxData {
 }
 
 // accessors for innerTx.
-func (tx *DynamicFeeTx) txType() byte              { return DynamicFeeTxType }
-func (tx *DynamicFeeTx) chainID() *big.Int         { return tx.ChainID }
-func (tx *DynamicFeeTx) accessList() AccessList    { return tx.AccessList }
-func (tx *DynamicFeeTx) data() []byte              { return tx.Data }
-func (tx *DynamicFeeTx) gas() uint64               { return tx.Gas }
-func (tx *DynamicFeeTx) gasFeeCap() *big.Int       { return tx.GasFeeCap }
-func (tx *DynamicFeeTx) gasTipCap() *big.Int       { return tx.GasTipCap }
-func (tx *DynamicFeeTx) gasPrice() *big.Int        { return tx.GasFeeCap }
-func (tx *DynamicFeeTx) value() *big.Int           { return tx.Value }
-func (tx *DynamicFeeTx) nonce() uint64             { return tx.Nonce }
-func (tx *DynamicFeeTx) to() *common.Address       { return tx.To }
-func (tx *DynamicFeeTx) blobGas() uint64           { return 0 }
-func (tx *DynamicFeeTx) blobGasFeeCap() *big.Int   { return nil }
-func (tx *DynamicFeeTx) blobHashes() []common.Hash { return nil }
+func (tx *DynamicFeeTx) txType() byte           { return DynamicFeeTxType }
+func (tx *DynamicFeeTx) chainID() *big.Int      { return tx.ChainID }
+func (tx *DynamicFeeTx) accessList() AccessList { return tx.AccessList }
+func (tx *DynamicFeeTx) data() []byte           { return tx.Data }
+func (tx *DynamicFeeTx) gas() uint64            { return tx.Gas }
+func (tx *DynamicFeeTx) gasFeeCap() *big.Int    { return tx.GasFeeCap }
+func (tx *DynamicFeeTx) gasTipCap() *big.Int    { return tx.GasTipCap }
+func (tx *DynamicFeeTx) gasPrice() *big.Int     { return tx.GasFeeCap }
+func (tx *DynamicFeeTx) value() *big.Int        { return tx.Value }
+func (tx *DynamicFeeTx) nonce() uint64          { return tx.Nonce }
+func (tx *DynamicFeeTx) to() *common.Address    { return tx.To }
 
 func (tx *DynamicFeeTx) effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int {
 	if baseFee == nil {
@@ -126,3 +125,11 @@ func (tx *DynamicFeeTx) rawSignatureValues() (v, r, s *big.Int) {
 func (tx *DynamicFeeTx) setSignatureValues(chainID, v, r, s *big.Int) {
 	tx.ChainID, tx.V, tx.R, tx.S = chainID, v, r, s
 }
+
+func (tx *DynamicFeeTx) encode(b *bytes.Buffer) error {
+	return rlp.Encode(b, tx)
+}
+
+func (tx *DynamicFeeTx) decode(input []byte) error {
+	return rlp.DecodeBytes(input, tx)
+}

core/types/tx_legacy.go

@@ -27,6 +27,7 @@
 package types
 
 import (
+	"bytes"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -101,20 +102,17 @@ func (tx *LegacyTx) copy() TxData {
 }
 
 // accessors for innerTx.
-func (tx *LegacyTx) txType() byte              { return LegacyTxType }
-func (tx *LegacyTx) chainID() *big.Int         { return deriveChainId(tx.V) }
-func (tx *LegacyTx) accessList() AccessList    { return nil }
-func (tx *LegacyTx) data() []byte              { return tx.Data }
-func (tx *LegacyTx) gas() uint64               { return tx.Gas }
-func (tx *LegacyTx) gasPrice() *big.Int        { return tx.GasPrice }
-func (tx *LegacyTx) gasTipCap() *big.Int       { return tx.GasPrice }
-func (tx *LegacyTx) gasFeeCap() *big.Int       { return tx.GasPrice }
-func (tx *LegacyTx) value() *big.Int           { return tx.Value }
-func (tx *LegacyTx) nonce() uint64             { return tx.Nonce }
-func (tx *LegacyTx) to() *common.Address       { return tx.To }
-func (tx *LegacyTx) blobGas() uint64           { return 0 }
-func (tx *LegacyTx) blobGasFeeCap() *big.Int   { return nil }
-func (tx *LegacyTx) blobHashes() []common.Hash { return nil }
+func (tx *LegacyTx) txType() byte           { return LegacyTxType }
+func (tx *LegacyTx) chainID() *big.Int      { return deriveChainId(tx.V) }
+func (tx *LegacyTx) accessList() AccessList { return nil }
+func (tx *LegacyTx) data() []byte           { return tx.Data }
+func (tx *LegacyTx) gas() uint64            { return tx.Gas }
+func (tx *LegacyTx) gasPrice() *big.Int     { return tx.GasPrice }
+func (tx *LegacyTx) gasTipCap() *big.Int    { return tx.GasPrice }
+func (tx *LegacyTx) gasFeeCap() *big.Int    { return tx.GasPrice }
+func (tx *LegacyTx) value() *big.Int        { return tx.Value }
+func (tx *LegacyTx) nonce() uint64          { return tx.Nonce }
+func (tx *LegacyTx) to() *common.Address    { return tx.To }
 
 func (tx *LegacyTx) effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int {
 	return dst.Set(tx.GasPrice)
@@ -127,3 +125,11 @@ func (tx *LegacyTx) rawSignatureValues() (v, r, s *big.Int) {
 func (tx *LegacyTx) setSignatureValues(chainID, v, r, s *big.Int) {
 	tx.V, tx.R, tx.S = v, r, s
 }
+
+func (tx *LegacyTx) encode(*bytes.Buffer) error {
+	panic("encode called on LegacyTx")
+}
+
+func (tx *LegacyTx) decode([]byte) error {
+	panic("decode called on LegacyTx)")
+}

core/vm/instructions.go

@@ -261,6 +261,7 @@ func opKeccak256(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) (
 	size.SetBytes(interpreter.hasherBuf[:])
 	return nil, nil
 }
+
 func opAddress(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) {
 	scope.Stack.push(new(uint256.Int).SetBytes(scope.Contract.Address().Bytes()))
 	return nil, nil
@@ -277,6 +278,7 @@ func opOrigin(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]b
 	scope.Stack.push(new(uint256.Int).SetBytes(interpreter.evm.Origin.Bytes()))
 	return nil, nil
 }
+
 func opCaller(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) {
 	scope.Stack.push(new(uint256.Int).SetBytes(scope.Contract.Caller().Bytes()))
 	return nil, nil

core/vm/runtime/runtime.go

@@ -55,6 +55,7 @@ type Config struct {
 	EVMConfig   vm.Config
 	BaseFee     *big.Int
 	BlobHashes  []common.Hash
+	Random      *common.Hash
 
 	State     *state.StateDB
 	GetHashFn func(n uint64) common.Hash

eth/api_admin.go

@@ -128,6 +128,10 @@ func (api *AdminAPI) ImportChain(file string) (bool, error) {
 			} else if err != nil {
 				return false, fmt.Errorf("block %d: failed to parse: %v", index, err)
 			}
+			// ignore the genesis block when importing blocks
+			if block.NumberU64() == 0 {
+				continue
+			}
 			blocks = append(blocks, block)
 			index++
 		}

eth/api_backend.go

@@ -40,7 +40,6 @@ import (
 	"github.com/ava-labs/subnet-evm/core/bloombits"
 	"github.com/ava-labs/subnet-evm/core/rawdb"
 	"github.com/ava-labs/subnet-evm/core/state"
-	"github.com/ava-labs/subnet-evm/core/txpool"
 	"github.com/ava-labs/subnet-evm/core/types"
 	"github.com/ava-labs/subnet-evm/core/vm"
 	"github.com/ava-labs/subnet-evm/eth/gasprice"
@@ -245,7 +244,10 @@ func (b *EthAPIBackend) StateAndHeaderByNumber(ctx context.Context, number rpc.B
 		return nil, nil, errors.New("header not found")
 	}
 	stateDb, err := b.eth.BlockChain().StateAt(header.Root)
-	return stateDb, header, err
+	if err != nil {
+		return nil, nil, err
+	}
+	return stateDb, header, nil
 }
 
 func (b *EthAPIBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) {
@@ -264,7 +266,10 @@ func (b *EthAPIBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockN
 			return nil, nil, errors.New("header for hash not found")
 		}
 		stateDb, err := b.eth.BlockChain().StateAt(header.Root)
-		return stateDb, header, err
+		if err != nil {
+			return nil, nil, err
+		}
+		return stateDb, header, nil
 	}
 	return nil, nil, errors.New("invalid arguments; neither block nor hash specified")
 }
@@ -337,7 +342,7 @@ func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction)
 	if err := ctx.Err(); err != nil {
 		return err
 	}
-	if err := b.eth.txPool.Add([]*txpool.Transaction{{Tx: signedTx}}, true, false)[0]; err != nil {
+	if err := b.eth.txPool.Add([]*types.Transaction{signedTx}, true, false)[0]; err != nil {
 		return err
 	}
 
@@ -353,7 +358,7 @@ func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) {
 	for _, batch := range pending {
 		for _, lazy := range batch {
 			if tx := lazy.Resolve(); tx != nil {
-				txs = append(txs, tx.Tx)
+				txs = append(txs, tx)
 			}
 		}
 	}
@@ -361,10 +366,7 @@ func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) {
 }
 
 func (b *EthAPIBackend) GetPoolTransaction(hash common.Hash) *types.Transaction {
-	if tx := b.eth.txPool.Get(hash); tx != nil {
-		return tx.Tx
-	}
-	return nil
+	return b.eth.txPool.Get(hash)
 }
 
 func (b *EthAPIBackend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) {
@@ -494,7 +496,7 @@ func (b *EthAPIBackend) GetMaxBlocksPerRequest() int64 {
 }
 
 func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) {
-	return b.eth.StateAtBlock(ctx, block, reexec, base, readOnly, preferDisk)
+	return b.eth.stateAtBlock(ctx, block, reexec, base, readOnly, preferDisk)
 }
 
 func (b *EthAPIBackend) StateAtNextBlock(ctx context.Context, parent, nextBlock *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) {

eth/api_debug.go

@@ -40,6 +40,7 @@ import (
 	"github.com/ava-labs/subnet-evm/trie"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 )
@@ -170,11 +171,15 @@ type storageEntry struct {
 }
 
 // StorageRangeAt returns the storage at the given block height and transaction index.
-func (api *DebugAPI) StorageRangeAt(ctx context.Context, blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) {
-	// Retrieve the block
-	block := api.eth.blockchain.GetBlockByHash(blockHash)
+func (api *DebugAPI) StorageRangeAt(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) {
+	var block *types.Block
+
+	block, err := api.eth.APIBackend.BlockByNumberOrHash(ctx, blockNrOrHash)
+	if err != nil {
+		return StorageRangeResult{}, err
+	}
 	if block == nil {
-		return StorageRangeResult{}, fmt.Errorf("block %#x not found", blockHash)
+		return StorageRangeResult{}, fmt.Errorf("block %v not found", blockNrOrHash)
 	}
 	_, _, statedb, release, err := api.eth.stateAtTransaction(ctx, block, txIndex, 0)
 	if err != nil {
@@ -182,18 +187,20 @@ func (api *DebugAPI) StorageRangeAt(ctx context.Context, blockHash common.Hash,
 	}
 	defer release()
 
-	st, err := statedb.StorageTrie(contractAddress)
+	return storageRangeAt(statedb, block.Root(), contractAddress, keyStart, maxResult)
+}
+
+func storageRangeAt(statedb *state.StateDB, root common.Hash, address common.Address, start []byte, maxResult int) (StorageRangeResult, error) {
+	storageRoot := statedb.GetStorageRoot(address)
+	if storageRoot == types.EmptyRootHash || storageRoot == (common.Hash{}) {
+		return StorageRangeResult{}, nil // empty storage
+	}
+	id := trie.StorageTrieID(root, crypto.Keccak256Hash(address.Bytes()), storageRoot)
+	tr, err := trie.NewStateTrie(id, statedb.Database().TrieDB())
 	if err != nil {
 		return StorageRangeResult{}, err
 	}
-	if st == nil {
-		return StorageRangeResult{}, fmt.Errorf("account %x doesn't exist", contractAddress)
-	}
-	return storageRangeAt(st, keyStart, maxResult)
-}
-
-func storageRangeAt(st state.Trie, start []byte, maxResult int) (StorageRangeResult, error) {
-	trieIt, err := st.NodeIterator(start)
+	trieIt, err := tr.NodeIterator(start)
 	if err != nil {
 		return StorageRangeResult{}, err
 	}
@@ -205,7 +212,7 @@ func storageRangeAt(st state.Trie, start []byte, maxResult int) (StorageRangeRes
 			return StorageRangeResult{}, err
 		}
 		e := storageEntry{Value: common.BytesToHash(content)}
-		if preimage := st.GetKey(it.Key); preimage != nil {
+		if preimage := tr.GetKey(it.Key); preimage != nil {
 			preimage := common.BytesToHash(preimage)
 			e.Key = &preimage
 		}
@@ -278,7 +285,7 @@ func (api *DebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]c
 	if startBlock.Number().Uint64() >= endBlock.Number().Uint64() {
 		return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64())
 	}
-	triedb := api.eth.BlockChain().StateCache().TrieDB()
+	triedb := api.eth.BlockChain().TrieDB()
 
 	oldTrie, err := trie.NewStateTrie(trie.StateTrieID(startBlock.Root()), triedb)
 	if err != nil {
@@ -316,6 +323,9 @@ func (api *DebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]c
 // The (from, to) parameters are the sequence of blocks to search, which can go
 // either forwards or backwards
 func (api *DebugAPI) GetAccessibleState(from, to rpc.BlockNumber) (uint64, error) {
+	if api.eth.blockchain.TrieDB().Scheme() == rawdb.PathScheme {
+		return 0, errors.New("state history is not yet available in path-based scheme")
+	}
 	var resolveNum = func(num rpc.BlockNumber) (uint64, error) {
 		// We don't have state for pending (-2), so treat it as latest
 		if num.Int64() < 0 {

eth/backend.go

@@ -150,14 +150,17 @@ func New(
 		"snapshot clean", common.StorageSize(config.SnapshotCache)*1024*1024,
 	)
 
-	// Note: RecoverPruning must be called to handle the case that we are midway through offline pruning.
-	// If the data directory is changed in between runs preventing RecoverPruning from performing its job correctly,
-	// it may cause DB corruption.
-	// Since RecoverPruning will only continue a pruning run that already began, we do not need to ensure that
-	// reprocessState has already been called and completed successfully. To ensure this, we must maintain
-	// that Prune is only run after reprocessState has finished successfully.
-	if err := pruner.RecoverPruning(config.OfflinePruningDataDirectory, chainDb); err != nil {
-		log.Error("Failed to recover state", "error", err)
+	// Try to recover offline state pruning only in hash-based.
+	if config.StateScheme == rawdb.HashScheme {
+		// Note: RecoverPruning must be called to handle the case that we are midway through offline pruning.
+		// If the data directory is changed in between runs preventing RecoverPruning from performing its job correctly,
+		// it may cause DB corruption.
+		// Since RecoverPruning will only continue a pruning run that already began, we do not need to ensure that
+		// reprocessState has already been called and completed successfully. To ensure this, we must maintain
+		// that Prune is only run after reprocessState has finished successfully.
+		if err := pruner.RecoverPruning(config.OfflinePruningDataDirectory, chainDb); err != nil {
+			log.Error("Failed to recover state", "error", err)
+		}
 	}
 
 	eth := &Ethereum{
@@ -175,7 +178,6 @@ func New(
 		settings:          settings,
 		shutdownTracker:   shutdowncheck.NewShutdownTracker(chainDb),
 	}
-
 	bcVersion := rawdb.ReadDatabaseVersion(chainDb)
 	dbVer := "<nil>"
 	if bcVersion != nil {
@@ -215,6 +217,8 @@ func New(
 			AcceptedCacheSize:               config.AcceptedCacheSize,
 			TxLookupLimit:                   config.TxLookupLimit,
 			SkipTxIndexing:                  config.SkipTxIndexing,
+			StateHistory:                    config.StateHistory,
+			StateScheme:                     config.StateScheme,
 		}
 	)
 

eth/ethconfig/config.go

@@ -30,10 +30,12 @@ import (
 	"time"
 
 	"github.com/ava-labs/subnet-evm/core"
+	"github.com/ava-labs/subnet-evm/core/rawdb"
 	"github.com/ava-labs/subnet-evm/core/txpool/blobpool"
 	"github.com/ava-labs/subnet-evm/core/txpool/legacypool"
 	"github.com/ava-labs/subnet-evm/eth/gasprice"
 	"github.com/ava-labs/subnet-evm/miner"
+	"github.com/ava-labs/subnet-evm/params"
 	"github.com/ethereum/go-ethereum/common"
 )
 
@@ -55,6 +57,8 @@ var DefaultConfig = NewDefaultConfig()
 func NewDefaultConfig() Config {
 	return Config{
 		NetworkId:                 1,
+		StateHistory:              params.FullImmutabilityThreshold,
+		StateScheme:               rawdb.HashScheme,
 		TrieCleanCache:            512,
 		TrieDirtyCache:            256,
 		TrieDirtyCommitTarget:     20,
@@ -158,7 +162,11 @@ type Config struct {
 	// are reserved:
 	//  * 0:   means no limit
 	//  * N:   means N block limit [HEAD-N+1, HEAD] and delete extra indexes
-	TxLookupLimit uint64
+	// Deprecated, use 'TransactionHistory' instead.
+	TxLookupLimit      uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved.
+	TransactionHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved.
+	StateHistory       uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved.
+	StateScheme        string `toml:",omitempty"` // State scheme used to store ethereum state and merkle trie nodes on top
 
 	// SkipTxIndexing skips indexing transactions.
 	// This is useful for validators that don't need to index transactions.

eth/filters/filter.go

@@ -364,48 +364,47 @@ func (f *Filter) checkMatches(ctx context.Context, header *types.Header) ([]*typ
 	return logs, nil
 }
 
-func includes(addresses []common.Address, a common.Address) bool {
-	for _, addr := range addresses {
-		if addr == a {
+// includes returns true if the element is present in the list.
+func includes[T comparable](things []T, element T) bool {
+	for _, thing := range things {
+		if thing == element {
 			return true
 		}
 	}
-
 	return false
 }
 
 // filterLogs creates a slice of logs matching the given criteria.
 func filterLogs(logs []*types.Log, fromBlock, toBlock *big.Int, addresses []common.Address, topics [][]common.Hash) []*types.Log {
-	var ret []*types.Log
-Logs:
-	for _, log := range logs {
+	var check = func(log *types.Log) bool {
 		if fromBlock != nil && fromBlock.Int64() >= 0 && fromBlock.Uint64() > log.BlockNumber {
-			continue
+			return false
 		}
 		if toBlock != nil && toBlock.Int64() >= 0 && toBlock.Uint64() < log.BlockNumber {
-			continue
+			return false
 		}
-
 		if len(addresses) > 0 && !includes(addresses, log.Address) {
-			continue
+			return false
 		}
 		// If the to filtered topics is greater than the amount of topics in logs, skip.
 		if len(topics) > len(log.Topics) {
-			continue
+			return false
 		}
 		for i, sub := range topics {
-			match := len(sub) == 0 // empty rule set == wildcard
-			for _, topic := range sub {
-				if log.Topics[i] == topic {
-					match = true
-					break
-				}
+			if len(sub) == 0 {
+				continue // empty rule set == wildcard
 			}
-			if !match {
-				continue Logs
+			if !includes(sub, log.Topics[i]) {
+				return false
 			}
 		}
-		ret = append(ret, log)
+		return true
+	}
+	var ret []*types.Log
+	for _, log := range logs {
+		if check(log) {
+			ret = append(ret, log)
+		}
 	}
 	return ret
 }

eth/filters/filter_test.go

@@ -98,7 +98,7 @@ func BenchmarkFilters(b *testing.B) {
 	// The test txs are not properly signed, can't simply create a chain
 	// and then import blocks. TODO(rjl493456442) try to get rid of the
 	// manual database writes.
-	gspec.MustCommit(db)
+	gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults))
 
 	for i, block := range chain {
 		rawdb.WriteBlock(db, block)
@@ -192,7 +192,7 @@ func TestFilters(t *testing.T) {
 
 	// Hack: GenerateChainWithGenesis creates a new db.
 	// Commit the genesis manually and use GenerateChain.
-	_, err = gspec.Commit(db, trie.NewDatabase(db))
+	_, err = gspec.Commit(db, trie.NewDatabase(db, nil))
 	if err != nil {
 		t.Fatal(err)
 	}

eth/state_accessor.go

@@ -33,6 +33,7 @@ import (
 	"time"
 
 	"github.com/ava-labs/subnet-evm/core"
+	"github.com/ava-labs/subnet-evm/core/rawdb"
 	"github.com/ava-labs/subnet-evm/core/state"
 	"github.com/ava-labs/subnet-evm/core/types"
 	"github.com/ava-labs/subnet-evm/core/vm"
@@ -46,32 +47,12 @@ import (
 // for releasing state.
 var noopReleaser = tracers.StateReleaseFunc(func() {})
 
-// StateAtBlock retrieves the state database associated with a certain block.
-// If no state is locally available for the given block, a number of blocks
-// are attempted to be reexecuted to generate the desired state. The optional
-// base layer statedb can be provided which is regarded as the statedb of the
-// parent block.
-//
-// An additional release function will be returned if the requested state is
-// available. Release is expected to be invoked when the returned state is no longer needed.
-// Its purpose is to prevent resource leaking. Though it can be noop in some cases.
-//
-// Parameters:
-//   - block:      The block for which we want the state(state = block.Root)
-//   - reexec:     The maximum number of blocks to reprocess trying to obtain the desired state
-//   - base:       If the caller is tracing multiple blocks, the caller can provide the parent
-//     state continuously from the callsite.
-//   - readOnly:   If true, then the live 'blockchain' state database is used. No mutation should
-//     be made from caller, e.g. perform Commit or other 'save-to-disk' changes.
-//     Otherwise, the trash generated by caller may be persisted permanently.
-//   - preferDisk: this arg can be used by the caller to signal that even though the 'base' is
-//     provided, it would be preferable to start from a fresh state, if we have it
-//     on disk.
-func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) {
+func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) {
 	reexec = 0 // Do not support re-executing historical blocks to grab state
 	var (
 		current  *types.Block
 		database state.Database
+		triedb   *trie.Database
 		report   = true
 		origin   = block.NumberU64()
 	)
@@ -82,9 +63,9 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe
 		// on top to prevent garbage collection and return a release
 		// function to deref it.
 		if statedb, err = eth.blockchain.StateAt(block.Root()); err == nil {
-			statedb.Database().TrieDB().Reference(block.Root(), common.Hash{})
+			eth.blockchain.TrieDB().Reference(block.Root(), common.Hash{})
 			return statedb, func() {
-				statedb.Database().TrieDB().Dereference(block.Root())
+				eth.blockchain.TrieDB().Dereference(block.Root())
 			}, nil
 		}
 	}
@@ -95,14 +76,16 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe
 		if preferDisk {
 			// Create an ephemeral trie.Database for isolating the live one. Otherwise
 			// the internal junks created by tracing will be persisted into the disk.
-			database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16})
+			// TODO(rjl493456442), clean cache is disabled to prevent memory leak,
+			// please re-enable it for better performance.
+			database = state.NewDatabaseWithConfig(eth.chainDb, trie.HashDefaults)
 			if statedb, err = state.New(block.Root(), database, nil); err == nil {
 				log.Info("Found disk backend for state trie", "root", block.Root(), "number", block.Number())
 				return statedb, noopReleaser, nil
 			}
 		}
 		// The optional base statedb is given, mark the start point as parent block
-		statedb, database, report = base, base.Database(), false
+		statedb, database, triedb, report = base, base.Database(), base.Database().TrieDB(), false
 		current = eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1)
 	} else {
 		// Otherwise, try to reexec blocks until we find a state or reach our limit
@@ -110,7 +93,10 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe
 
 		// Create an ephemeral trie.Database for isolating the live one. Otherwise
 		// the internal junks created by tracing will be persisted into the disk.
-		database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16})
+		// TODO(rjl493456442), clean cache is disabled to prevent memory leak,
+		// please re-enable it for better performance.
+		triedb = trie.NewDatabase(eth.chainDb, trie.HashDefaults)
+		database = state.NewDatabaseWithNodeDB(eth.chainDb, triedb)
 
 		// If we didn't check the live database, do check state over ephemeral database,
 		// otherwise we would rewind past a persisted block (specific corner case is
@@ -188,15 +174,56 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe
 		// Note: In coreth, the state reference is held by passing true to [statedb.Commit].
 		// Drop the parent state to prevent accumulating too many nodes in memory.
 		if parent != (common.Hash{}) {
-			database.TrieDB().Dereference(parent)
+			triedb.Dereference(parent)
 		}
 		parent = root
 	}
 	if report {
-		nodes, imgs := database.TrieDB().Size()
+		_, nodes, imgs := triedb.Size() // all memory is contained within the nodes return in hashdb
 		log.Info("Historical state regenerated", "block", current.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs)
 	}
-	return statedb, func() { database.TrieDB().Dereference(block.Root()) }, nil
+	return statedb, func() { triedb.Dereference(block.Root()) }, nil
+}
+
+func (eth *Ethereum) pathState(block *types.Block) (*state.StateDB, func(), error) {
+	// Check if the requested state is available in the live chain.
+	statedb, err := eth.blockchain.StateAt(block.Root())
+	if err == nil {
+		return statedb, noopReleaser, nil
+	}
+	// TODO historic state is not supported in path-based scheme.
+	// Fully archive node in pbss will be implemented by relying
+	// on state history, but needs more work on top.
+	return nil, nil, errors.New("historical state not available in path scheme yet")
+}
+
+// stateAtBlock retrieves the state database associated with a certain block.
+// If no state is locally available for the given block, a number of blocks
+// are attempted to be reexecuted to generate the desired state. The optional
+// base layer statedb can be provided which is regarded as the statedb of the
+// parent block.
+//
+// An additional release function will be returned if the requested state is
+// available. Release is expected to be invoked when the returned state is no
+// longer needed. Its purpose is to prevent resource leaking. Though it can be
+// noop in some cases.
+//
+// Parameters:
+//   - block:      The block for which we want the state(state = block.Root)
+//   - reexec:     The maximum number of blocks to reprocess trying to obtain the desired state
+//   - base:       If the caller is tracing multiple blocks, the caller can provide the parent
+//     state continuously from the callsite.
+//   - readOnly:   If true, then the live 'blockchain' state database is used. No mutation should
+//     be made from caller, e.g. perform Commit or other 'save-to-disk' changes.
+//     Otherwise, the trash generated by caller may be persisted permanently.
+//   - preferDisk: This arg can be used by the caller to signal that even though the 'base' is
+//     provided, it would be preferable to start from a fresh state, if we have it
+//     on disk.
+func (eth *Ethereum) stateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) {
+	if eth.blockchain.TrieDB().Scheme() == rawdb.HashScheme {
+		return eth.hashState(ctx, block, reexec, base, readOnly, preferDisk)
+	}
+	return eth.pathState(block)
 }
 
 // stateAtTransaction returns the execution environment of a certain transaction.
@@ -249,7 +276,7 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block,
 // apply the upgrades to the [parent] state before returning it.
 func (eth *Ethereum) StateAtNextBlock(ctx context.Context, parent *types.Block, nextBlock *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) {
 	// Get state for [parent]
-	statedb, release, err := eth.StateAtBlock(ctx, parent, reexec, base, readOnly, preferDisk)
+	statedb, release, err := eth.stateAtBlock(ctx, parent, reexec, base, readOnly, preferDisk)
 	if err != nil {
 		return nil, nil, err
 	}

eth/tracers/api.go

@@ -28,7 +28,6 @@ package tracers
 
 import (
 	"bufio"
-	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -401,8 +400,8 @@ func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed
 			// if the relevant state is available in disk.
 			var preferDisk bool
 			if statedb != nil {
-				s1, s2 := statedb.Database().TrieDB().Size()
-				preferDisk = s1+s2 > defaultTracechainMemLimit
+				s1, s2, s3 := statedb.Database().TrieDB().Size()
+				preferDisk = s1+s2+s3 > defaultTracechainMemLimit
 			}
 			statedb, release, err = api.backend.StateAtNextBlock(ctx, block, next, reexec, statedb, false, preferDisk)
 			if err != nil {
@@ -485,7 +484,7 @@ func (api *API) TraceBlockByHash(ctx context.Context, hash common.Hash, config *
 // and returns them as a JSON object.
 func (api *baseAPI) TraceBlock(ctx context.Context, blob hexutil.Bytes, config *TraceConfig) ([]*txTraceResult, error) {
 	block := new(types.Block)
-	if err := rlp.Decode(bytes.NewReader(blob), block); err != nil {
+	if err := rlp.DecodeBytes(blob, block); err != nil {
 		return nil, fmt.Errorf("could not decode block: %v", err)
 	}
 	return api.traceBlock(ctx, block, config)

eth/tracers/internal/tracetest/calltrace_test.go

@@ -147,8 +147,10 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) {
 					GasLimit:    uint64(test.Context.GasLimit),
 					BaseFee:     test.Genesis.BaseFee,
 				}
-				_, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false)
+				triedb, _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme)
 			)
+			triedb.Close()
+
 			tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig)
 			if err != nil {
 				t.Fatalf("failed to create call tracer: %v", err)
@@ -247,7 +249,8 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) {
 		Difficulty:  (*big.Int)(test.Context.Difficulty),
 		GasLimit:    uint64(test.Context.GasLimit),
 	}
-	_, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false)
+	triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme)
+	defer triedb.Close()
 
 	b.ReportAllocs()
 	b.ResetTimer()
@@ -373,7 +376,7 @@ func TestInternals(t *testing.T) {
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			_, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(),
+			triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(),
 				core.GenesisAlloc{
 					to: core.GenesisAccount{
 						Code: tc.code,
@@ -381,7 +384,9 @@ func TestInternals(t *testing.T) {
 					origin: core.GenesisAccount{
 						Balance: big.NewInt(500000000000000),
 					},
-				}, false)
+				}, false, rawdb.HashScheme)
+			defer triedb.Close()
+
 			evm := vm.NewEVM(context, txContext, statedb, params.TestPreSubnetEVMConfig, vm.Config{Tracer: tc.tracer})
 			msg := &core.Message{
 				To:                &to,

eth/tracers/internal/tracetest/flat_calltrace_test.go

@@ -110,7 +110,8 @@ func flatCallTracerTestRunner(tracerName string, filename string, dirPath string
 		Difficulty:  (*big.Int)(test.Context.Difficulty),
 		GasLimit:    uint64(test.Context.GasLimit),
 	}
-	_, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false)
+	triedb, _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false, rawdb.HashScheme)
+	defer triedb.Close()
 
 	// Create the tracer, the EVM environment and run it
 	tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), test.TracerConfig)