chore: sync with node requirements, add bytecache package

bloom/bloom_filter_test.go

@@ -0,0 +1,32 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package bloom
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNew(t *testing.T) {
+	var (
+		require = require.New(t)
+		count   = 10000
+		p       = 0.1
+	)
+
+	numHashes, numEntries := OptimalParameters(count, p)
+	f, err := New(numHashes, numEntries)
+	require.NoError(err)
+	require.NotNil(f)
+
+	salt := []byte("test salt")
+	Add(f, []byte("hello"), salt)
+
+	contains := Contains(f, []byte("hello"), salt)
+	require.True(contains, "should have contained the key")
+
+	contains = Contains(f, []byte("bye"), salt)
+	require.False(contains, "shouldn't have contained the key")
+}

bloom/bloom_interface.go

@@ -0,0 +1,14 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package bloom
+
+// BloomFilter is the interface for bloom filter implementations that support
+// adding and checking raw byte slices
+type BloomFilter interface {
+	// Add adds to filter, assumed thread safe
+	Add(...[]byte)
+
+	// Check checks filter, assumed thread safe
+	Check([]byte) bool
+}

bloom/filter.go

@@ -0,0 +1,147 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package bloom
+
+import (
+	"crypto/rand"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"math/bits"
+	"sync"
+)
+
+const (
+	minHashes  = 1
+	maxHashes  = 16 // Supports a false positive probability of 2^-16 when using optimal size values
+	minEntries = 1
+
+	bitsPerByte    = 8
+	bytesPerUint64 = 8
+	hashRotation   = 17
+)
+
+var (
+	errInvalidNumHashes = errors.New("invalid num hashes")
+	errTooFewHashes     = errors.New("too few hashes")
+	errTooManyHashes    = errors.New("too many hashes")
+	errTooFewEntries    = errors.New("too few entries")
+)
+
+type Filter struct {
+	// numBits is always equal to [bitsPerByte * len(entries)]
+	numBits uint64
+
+	lock      sync.RWMutex
+	hashSeeds []uint64
+	entries   []byte
+	count     int
+}
+
+// New creates a new Filter with the specified number of hashes and bytes for
+// entries. The returned bloom filter is safe for concurrent usage.
+func New(numHashes, numEntries int) (*Filter, error) {
+	if numEntries < minEntries {
+		return nil, errTooFewEntries
+	}
+
+	hashSeeds, err := newHashSeeds(numHashes)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Filter{
+		numBits:   uint64(numEntries * bitsPerByte),
+		hashSeeds: hashSeeds,
+		entries:   make([]byte, numEntries),
+		count:     0,
+	}, nil
+}
+
+func (f *Filter) Add(hash uint64) {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	_ = 1 % f.numBits // hint to the compiler that numBits is not 0
+	for _, seed := range f.hashSeeds {
+		hash = bits.RotateLeft64(hash, hashRotation) ^ seed
+		index := hash % f.numBits
+		byteIndex := index / bitsPerByte
+		bitIndex := index % bitsPerByte
+		f.entries[byteIndex] |= 1 << bitIndex
+	}
+	f.count++
+}
+
+// Count returns the number of elements that have been added to the bloom
+// filter.
+func (f *Filter) Count() int {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+
+	return f.count
+}
+
+func (f *Filter) Contains(hash uint64) bool {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+
+	return contains(f.hashSeeds, f.entries, hash)
+}
+
+func (f *Filter) Marshal() []byte {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+
+	return marshal(f.hashSeeds, f.entries)
+}
+
+func newHashSeeds(count int) ([]uint64, error) {
+	switch {
+	case count < minHashes:
+		return nil, fmt.Errorf("%w: %d < %d", errTooFewHashes, count, minHashes)
+	case count > maxHashes:
+		return nil, fmt.Errorf("%w: %d > %d", errTooManyHashes, count, maxHashes)
+	}
+
+	bytes := make([]byte, count*bytesPerUint64)
+	if _, err := rand.Reader.Read(bytes); err != nil {
+		return nil, err
+	}
+
+	seeds := make([]uint64, count)
+	for i := range seeds {
+		seeds[i] = binary.BigEndian.Uint64(bytes[i*bytesPerUint64:])
+	}
+	return seeds, nil
+}
+
+func contains(hashSeeds []uint64, entries []byte, hash uint64) bool {
+	var (
+		numBits          = bitsPerByte * uint64(len(entries))
+		_                = 1 % numBits // hint to the compiler that numBits is not 0
+		accumulator byte = 1
+	)
+	for seedIndex := 0; seedIndex < len(hashSeeds) && accumulator != 0; seedIndex++ {
+		hash = bits.RotateLeft64(hash, hashRotation) ^ hashSeeds[seedIndex]
+		index := hash % numBits
+		byteIndex := index / bitsPerByte
+		bitIndex := index % bitsPerByte
+		accumulator &= entries[byteIndex] >> bitIndex
+	}
+	return accumulator != 0
+}
+
+func marshal(hashSeeds []uint64, entries []byte) []byte {
+	numHashes := len(hashSeeds)
+	entriesOffset := 1 + numHashes*bytesPerUint64
+
+	bytes := make([]byte, entriesOffset+len(entries))
+	bytes[0] = byte(numHashes)
+	for i, seed := range hashSeeds {
+		binary.BigEndian.PutUint64(bytes[1+i*bytesPerUint64:], seed)
+	}
+	copy(bytes[entriesOffset:], entries)
+	return bytes
+}

bloom/filter_common.go

@@ -0,0 +1,71 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package bloom
+
+import (
+	"encoding/binary"
+
+	"github.com/spaolacci/murmur3"
+)
+
+// marshalCommon serializes [hashSeeds] and [entries] into a byte slice.
+func marshalCommon(hashSeeds []uint64, entries []byte) []byte {
+	bytes := make([]byte, 1+len(hashSeeds)*bytesPerUint64+len(entries))
+	bytes[0] = byte(len(hashSeeds))
+	offset := 1
+	for _, seed := range hashSeeds {
+		binary.BigEndian.PutUint64(bytes[offset:], seed)
+		offset += bytesPerUint64
+	}
+	copy(bytes[offset:], entries)
+	return bytes
+}
+
+// containsCommon returns true if [hash] is in the filter defined by [hashSeeds] and [entries].
+func containsCommon(hashSeeds []uint64, entries []byte, hash uint64) bool {
+	for _, seed := range hashSeeds {
+		if !containsWithSeed(entries, hash, seed) {
+			return false
+		}
+	}
+	return true
+}
+
+func containsWithSeed(entries []byte, hash, seed uint64) bool {
+	index := getIndex(entries, hash, seed)
+	byteIndex := index / bitsPerByte
+	bitIndex := index % bitsPerByte
+	return entries[byteIndex]&(1<<bitIndex) != 0
+}
+
+func getIndex(entries []byte, hash, seed uint64) uint64 {
+	// If the filter has L entries, we only want to use hashes in the range
+	// [0, L). We achieve this by incrementally shifting the hash to use
+	// bits that we haven't used before. If we run out of bits, we
+	// perform a "hash extension" by rehashing the original hash and the
+	// seed.
+	//
+	// If the size of the bloom filter approaches MaxUint64, the hash may
+	// overflow back to the beginning of the range. This means that the
+	// same index can be used more than once for a single value.
+	// This is OK, it just means the filter isn't as good as it could be
+	// if we had more bits in the hash.
+	numEntries := uint64(len(entries))
+	entriesMask := numEntries*bitsPerByte - 1
+	hash += seed
+	// Note: It may be significantly more performant to use:
+	// https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
+	// over the modulo operation.
+	index := hash & entriesMask
+	for index >= numEntries*bitsPerByte {
+		hash = hash >> hashRotation
+		// If the hash is zero, we have run out of bits and need to
+		// rehash.
+		if hash == 0 {
+			hash = uint64(murmur3.Sum64(binary.BigEndian.AppendUint64(binary.BigEndian.AppendUint64(nil, hash), seed)))
+		}
+		index = hash & entriesMask
+	}
+	return index
+}

bloom/filter_test.go

@@ -0,0 +1,96 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package bloom
+
+import (
+	"math/rand"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/luxfi/constants"
+)
+
+func TestNewErrors(t *testing.T) {
+	tests := []struct {
+		numHashes  int
+		numEntries int
+		err        error
+	}{
+		{
+			numHashes:  0,
+			numEntries: 1,
+			err:        errTooFewHashes,
+		},
+		{
+			numHashes:  17,
+			numEntries: 1,
+			err:        errTooManyHashes,
+		},
+		{
+			numHashes:  8,
+			numEntries: 0,
+			err:        errTooFewEntries,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.err.Error(), func(t *testing.T) {
+			_, err := New(test.numHashes, test.numEntries)
+			require.ErrorIs(t, err, test.err)
+		})
+	}
+}
+
+func TestNormalUsage(t *testing.T) {
+	require := require.New(t)
+
+	toAdd := make([]uint64, 1024)
+	for i := range toAdd {
+		toAdd[i] = rand.Uint64() //#nosec G404
+	}
+
+	initialNumHashes, initialNumBytes := OptimalParameters(1024, 0.01)
+	filter, err := New(initialNumHashes, initialNumBytes)
+	require.NoError(err)
+
+	for i, elem := range toAdd {
+		filter.Add(elem)
+		for _, elem := range toAdd[:i] {
+			require.True(filter.Contains(elem))
+		}
+	}
+
+	require.Equal(len(toAdd), filter.Count())
+
+	filterBytes := filter.Marshal()
+	parsedFilter, err := Parse(filterBytes)
+	require.NoError(err)
+
+	for _, elem := range toAdd {
+		require.True(parsedFilter.Contains(elem))
+	}
+
+	parsedFilterBytes := parsedFilter.Marshal()
+	require.Equal(filterBytes, parsedFilterBytes)
+}
+
+func BenchmarkAdd(b *testing.B) {
+	f, err := New(8, 16*constants.KiB)
+	require.NoError(b, err)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		f.Add(1)
+	}
+}
+
+func BenchmarkMarshal(b *testing.B) {
+	f, err := New(OptimalParameters(10_000, .01))
+	require.NoError(b, err)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		f.Marshal()
+	}
+}

bloom/hasher.go

@@ -0,0 +1,31 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package bloom
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+)
+
+func Add(f *Filter, key, salt []byte) {
+	f.Add(Hash(key, salt))
+}
+
+func Contains(c Checker, key, salt []byte) bool {
+	return c.Contains(Hash(key, salt))
+}
+
+type Checker interface {
+	Contains(hash uint64) bool
+}
+
+func Hash(key, salt []byte) uint64 {
+	hash := sha256.New()
+	// sha256.Write never returns errors
+	_, _ = hash.Write(key)
+	_, _ = hash.Write(salt)
+
+	output := make([]byte, 0, sha256.Size)
+	return binary.BigEndian.Uint64(hash.Sum(output))
+}

bloom/hasher_test.go

@@ -0,0 +1,34 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package bloom
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/luxfi/constants"
+	"github.com/luxfi/ids"
+)
+
+func TestCollisionResistance(t *testing.T) {
+	require := require.New(t)
+
+	f, err := New(8, 16*constants.KiB)
+	require.NoError(err)
+
+	Add(f, []byte("hello world?"), []byte("so salty"))
+	collision := Contains(f, []byte("hello world!"), []byte("so salty"))
+	require.False(collision)
+}
+
+func BenchmarkHash(b *testing.B) {
+	key := ids.GenerateTestID()
+	salt := ids.GenerateTestID()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		Hash(key[:], salt[:])
+	}
+}

bloom/map_filter.go

@@ -0,0 +1,35 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package bloom
+
+import (
+	"sync"
+
+	"github.com/luxfi/math/set"
+)
+
+type mapFilter struct {
+	lock   sync.RWMutex
+	values set.Set[string]
+}
+
+func NewMap() BloomFilter {
+	return &mapFilter{}
+}
+
+func (m *mapFilter) Add(bl ...[]byte) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	for _, b := range bl {
+		m.values.Add(string(b))
+	}
+}
+
+func (m *mapFilter) Check(b []byte) bool {
+	m.lock.RLock()
+	defer m.lock.RUnlock()
+
+	return m.values.Contains(string(b))
+}

bloom/metrics.go

@@ -0,0 +1,56 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package bloom
+
+import "github.com/luxfi/metric"
+
+// Metrics is a collection of commonly useful metrics when using a long-lived
+// bloom filter.
+type Metrics struct {
+	Count      metric.Gauge
+	NumHashes  metric.Gauge
+	NumEntries metric.Gauge
+	MaxCount   metric.Gauge
+	ResetCount metric.Counter
+}
+
+func NewMetrics(
+	namespace string,
+	registry metric.Registry,
+) (*Metrics, error) {
+	metricsInstance := metric.NewWithRegistry(namespace, registry)
+
+	m := &Metrics{
+		Count: metricsInstance.NewGauge(
+			"count",
+			"Number of additions that have been performed to the bloom",
+		),
+		NumHashes: metricsInstance.NewGauge(
+			"hashes",
+			"Number of hashes in the bloom",
+		),
+		NumEntries: metricsInstance.NewGauge(
+			"entries",
+			"Number of bytes allocated to slots in the bloom",
+		),
+		MaxCount: metricsInstance.NewGauge(
+			"max_count",
+			"Maximum number of additions that should be performed to the bloom before resetting",
+		),
+		ResetCount: metricsInstance.NewCounter(
+			"reset_count",
+			"Number times the bloom has been reset",
+		),
+	}
+	return m, nil
+}
+
+// Reset the metrics to align with the provided bloom filter and max count.
+func (m *Metrics) Reset(newFilter *Filter, maxCount int) {
+	m.Count.Set(float64(newFilter.Count()))
+	m.NumHashes.Set(float64(len(newFilter.hashSeeds)))
+	m.NumEntries.Set(float64(len(newFilter.entries)))
+	m.MaxCount.Set(float64(maxCount))
+	m.ResetCount.Inc()
+}

bloom/optimal.go

@@ -0,0 +1,111 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package bloom
+
+import "math"
+
+const ln2Squared = math.Ln2 * math.Ln2
+
+// OptimalParameters calculates the optimal [numHashes] and [numEntries] that
+// should be allocated for a bloom filter which will contain [count] and target
+// [falsePositiveProbability].
+func OptimalParameters(count int, falsePositiveProbability float64) (int, int) {
+	numEntries := OptimalEntries(count, falsePositiveProbability)
+	numHashes := OptimalHashes(numEntries, count)
+	return numHashes, numEntries
+}
+
+// OptimalHashes calculates the number of hashes which will minimize the false
+// positive probability of a bloom filter with [numEntries] after [count]
+// additions.
+//
+// It is guaranteed to return a value in the range [minHashes, maxHashes].
+//
+// ref: https://en.wikipedia.org/wiki/Bloom_filter
+func OptimalHashes(numEntries, count int) int {
+	switch {
+	case numEntries < minEntries:
+		return minHashes
+	case count <= 0:
+		return maxHashes
+	}
+
+	numHashes := math.Ceil(float64(numEntries) * bitsPerByte * math.Ln2 / float64(count))
+	// Converting a floating-point value to an int produces an undefined value
+	// if the floating-point value cannot be represented as an int. To avoid
+	// this undefined behavior, we explicitly check against MaxInt here.
+	//
+	// ref: https://go.dev/ref/spec#Conversions
+	if numHashes >= maxHashes {
+		return maxHashes
+	}
+	return max(int(numHashes), minHashes)
+}
+
+// OptimalEntries calculates the optimal number of entries to use when creating
+// a new Bloom filter when targenting a size of [count] with
+// [falsePositiveProbability] assuming that the optimal number of hashes is
+// used.
+//
+// It is guaranteed to return a value in the range [minEntries, MaxInt].
+//
+// ref: https://en.wikipedia.org/wiki/Bloom_filter
+func OptimalEntries(count int, falsePositiveProbability float64) int {
+	switch {
+	case count <= 0:
+		return minEntries
+	case falsePositiveProbability >= 1:
+		return minEntries
+	case falsePositiveProbability <= 0:
+		return math.MaxInt
+	}
+
+	entriesInBits := -float64(count) * math.Log(falsePositiveProbability) / ln2Squared
+	entries := (entriesInBits + bitsPerByte - 1) / bitsPerByte
+	// Converting a floating-point value to an int produces an undefined value
+	// if the floating-point value cannot be represented as an int. To avoid
+	// this undefined behavior, we explicitly check against MaxInt here.
+	//
+	// ref: https://go.dev/ref/spec#Conversions
+	if entries >= math.MaxInt {
+		return math.MaxInt
+	}
+	return max(int(entries), minEntries)
+}
+
+// EstimateCount estimates the number of additions a bloom filter with
+// [numHashes] and [numEntries] must have to reach [falsePositiveProbability].
+// This is derived by inversing a lower-bound on the probability of false
+// positives. For values where numBits >> numHashes, the predicted probability
+// is fairly accurate.
+//
+// It is guaranteed to return a value in the range [0, MaxInt].
+//
+// ref: https://tsapps.nist.gov/publication/get_pdf.cfm?pub_id=903775
+func EstimateCount(numHashes, numEntries int, falsePositiveProbability float64) int {
+	switch {
+	case numHashes < minHashes:
+		return 0
+	case numEntries < minEntries:
+		return 0
+	case falsePositiveProbability <= 0:
+		return 0
+	case falsePositiveProbability >= 1:
+		return math.MaxInt
+	}
+
+	invNumHashes := 1 / float64(numHashes)
+	numBits := float64(numEntries * 8)
+	exp := 1 - math.Pow(falsePositiveProbability, invNumHashes)
+	count := math.Ceil(-math.Log(exp) * numBits * invNumHashes)
+	// Converting a floating-point value to an int produces an undefined value
+	// if the floating-point value cannot be represented as an int. To avoid
+	// this undefined behavior, we explicitly check against MaxInt here.
+	//
+	// ref: https://go.dev/ref/spec#Conversions
+	if count >= math.MaxInt {
+		return math.MaxInt
+	}
+	return int(count)
+}

bloom/optimal_test.go

@@ -0,0 +1,203 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package bloom
+
+import (
+	"fmt"
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+const largestFloat64LessThan1 float64 = 1 - 1e-16
+
+func TestOptimalHashes(t *testing.T) {
+	tests := []struct {
+		numEntries     int
+		count          int
+		expectedHashes int
+	}{
+		{ // invalid params
+			numEntries:     0,
+			count:          1024,
+			expectedHashes: minHashes,
+		},
+		{ // invalid params
+			numEntries:     1024,
+			count:          0,
+			expectedHashes: maxHashes,
+		},
+		{
+			numEntries:     math.MaxInt,
+			count:          1,
+			expectedHashes: maxHashes,
+		},
+		{
+			numEntries:     1,
+			count:          math.MaxInt,
+			expectedHashes: minHashes,
+		},
+		{
+			numEntries:     1024,
+			count:          1024,
+			expectedHashes: 6,
+		},
+	}
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%d_%d", test.numEntries, test.count), func(t *testing.T) {
+			hashes := OptimalHashes(test.numEntries, test.count)
+			require.Equal(t, test.expectedHashes, hashes)
+		})
+	}
+}
+
+func TestOptimalEntries(t *testing.T) {
+	tests := []struct {
+		count                    int
+		falsePositiveProbability float64
+		expectedEntries          int
+	}{
+		{ // invalid params
+			count:                    0,
+			falsePositiveProbability: .5,
+			expectedEntries:          minEntries,
+		},
+		{ // invalid params
+			count:                    1,
+			falsePositiveProbability: 0,
+			expectedEntries:          math.MaxInt,
+		},
+		{ // invalid params
+			count:                    1,
+			falsePositiveProbability: 1,
+			expectedEntries:          minEntries,
+		},
+		{
+			count:                    math.MaxInt,
+			falsePositiveProbability: math.SmallestNonzeroFloat64,
+			expectedEntries:          math.MaxInt,
+		},
+		{
+			count:                    1024,
+			falsePositiveProbability: largestFloat64LessThan1,
+			expectedEntries:          minEntries,
+		},
+		{
+			count:                    1024,
+			falsePositiveProbability: .01,
+			expectedEntries:          1227,
+		},
+	}
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%d_%f", test.count, test.falsePositiveProbability), func(t *testing.T) {
+			entries := OptimalEntries(test.count, test.falsePositiveProbability)
+			require.Equal(t, test.expectedEntries, entries)
+		})
+	}
+}
+
+func TestEstimateEntries(t *testing.T) {
+	tests := []struct {
+		numHashes                int
+		numEntries               int
+		falsePositiveProbability float64
+		expectedEntries          int
+	}{
+		{ // invalid params
+			numHashes:                0,
+			numEntries:               2_048,
+			falsePositiveProbability: .5,
+			expectedEntries:          0,
+		},
+		{ // invalid params
+			numHashes:                1,
+			numEntries:               0,
+			falsePositiveProbability: .5,
+			expectedEntries:          0,
+		},
+		{ // invalid params
+			numHashes:                1,
+			numEntries:               1,
+			falsePositiveProbability: 2,
+			expectedEntries:          math.MaxInt,
+		},
+		{ // invalid params
+			numHashes:                1,
+			numEntries:               1,
+			falsePositiveProbability: -1,
+			expectedEntries:          0,
+		},
+		{
+			numHashes:                8,
+			numEntries:               2_048,
+			falsePositiveProbability: 0,
+			expectedEntries:          0,
+		},
+		{ // params from OptimalParameters(10_000, .01)
+			numHashes:                7,
+			numEntries:               11_982,
+			falsePositiveProbability: .01,
+			expectedEntries:          9_993,
+		},
+		{ // params from OptimalParameters(100_000, .001)
+			numHashes:                10,
+			numEntries:               179_720,
+			falsePositiveProbability: .001,
+			expectedEntries:          100_000,
+		},
+		{ // params from OptimalParameters(10_000, .01)
+			numHashes:                7,
+			numEntries:               11_982,
+			falsePositiveProbability: .05,
+			expectedEntries:          14_449,
+		},
+		{ // params from OptimalParameters(10_000, .01)
+			numHashes:                7,
+			numEntries:               11_982,
+			falsePositiveProbability: 1,
+			expectedEntries:          math.MaxInt,
+		},
+		{ // params from OptimalParameters(10_000, .01)
+			numHashes:                7,
+			numEntries:               11_982,
+			falsePositiveProbability: math.SmallestNonzeroFloat64,
+			expectedEntries:          0,
+		},
+		{ // params from OptimalParameters(10_000, .01)
+			numHashes:                7,
+			numEntries:               11_982,
+			falsePositiveProbability: largestFloat64LessThan1,
+			expectedEntries:          math.MaxInt,
+		},
+	}
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%d_%d_%f", test.numHashes, test.numEntries, test.falsePositiveProbability), func(t *testing.T) {
+			entries := EstimateCount(test.numHashes, test.numEntries, test.falsePositiveProbability)
+			require.Equal(t, test.expectedEntries, entries)
+		})
+	}
+}
+
+func FuzzOptimalHashes(f *testing.F) {
+	f.Fuzz(func(t *testing.T, numEntries, count int) {
+		hashes := OptimalHashes(numEntries, count)
+		require.GreaterOrEqual(t, hashes, minHashes)
+		require.LessOrEqual(t, hashes, maxHashes)
+	})
+}
+
+func FuzzOptimalEntries(f *testing.F) {
+	f.Fuzz(func(t *testing.T, count int, falsePositiveProbability float64) {
+		entries := OptimalEntries(count, falsePositiveProbability)
+		require.GreaterOrEqual(t, entries, minEntries)
+	})
+}
+
+func FuzzEstimateEntries(f *testing.F) {
+	f.Fuzz(func(t *testing.T, numHashes, numEntries int, falsePositiveProbability float64) {
+		entries := EstimateCount(numHashes, numEntries, falsePositiveProbability)
+		require.GreaterOrEqual(t, entries, 0)
+	})
+}

bloom/read_filter.go

@@ -0,0 +1,65 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package bloom
+
+import (
+	"encoding/binary"
+	"fmt"
+)
+
+var (
+	EmptyFilter = &ReadFilter{
+		hashSeeds: make([]uint64, minHashes),
+		entries:   make([]byte, minEntries),
+	}
+	FullFilter = &ReadFilter{
+		hashSeeds: make([]uint64, minHashes),
+		entries:   make([]byte, minEntries),
+	}
+)
+
+func init() {
+	for i := range FullFilter.entries {
+		FullFilter.entries[i] = 0xFF
+	}
+}
+
+type ReadFilter struct {
+	hashSeeds []uint64
+	entries   []byte
+}
+
+// Parse [bytes] into a read-only bloom filter.
+func Parse(bytes []byte) (*ReadFilter, error) {
+	if len(bytes) == 0 {
+		return nil, errInvalidNumHashes
+	}
+	numHashes := bytes[0]
+	entriesOffset := 1 + int(numHashes)*bytesPerUint64
+	switch {
+	case numHashes < minHashes:
+		return nil, fmt.Errorf("%w: %d < %d", errTooFewHashes, numHashes, minHashes)
+	case numHashes > maxHashes:
+		return nil, fmt.Errorf("%w: %d > %d", errTooManyHashes, numHashes, maxHashes)
+	case len(bytes) < entriesOffset+minEntries: // numEntries = len(bytes) - entriesOffset
+		return nil, errTooFewEntries
+	}
+
+	f := &ReadFilter{
+		hashSeeds: make([]uint64, numHashes),
+		entries:   bytes[entriesOffset:],
+	}
+	for i := range f.hashSeeds {
+		f.hashSeeds[i] = binary.BigEndian.Uint64(bytes[1+i*bytesPerUint64:])
+	}
+	return f, nil
+}
+
+func (f *ReadFilter) Contains(hash uint64) bool {
+	return contains(f.hashSeeds, f.entries, hash)
+}
+
+func (f *ReadFilter) Marshal() []byte {
+	return marshal(f.hashSeeds, f.entries)
+}

bloom/read_filter_test.go

@@ -0,0 +1,112 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package bloom
+
+import (
+	"math"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func NewMaliciousFilter(numHashes, numEntries int) *Filter {
+	f := &Filter{
+		numBits:   uint64(numEntries * bitsPerByte),
+		hashSeeds: make([]uint64, numHashes),
+		entries:   make([]byte, numEntries),
+		count:     0,
+	}
+	for i := range f.entries {
+		f.entries[i] = math.MaxUint8
+	}
+	return f
+}
+
+func TestParseErrors(t *testing.T) {
+	tests := []struct {
+		bytes []byte
+		err   error
+	}{
+		{
+			bytes: nil,
+			err:   errInvalidNumHashes,
+		},
+		{
+			bytes: NewMaliciousFilter(0, 1).Marshal(),
+			err:   errTooFewHashes,
+		},
+		{
+			bytes: NewMaliciousFilter(17, 1).Marshal(),
+			err:   errTooManyHashes,
+		},
+		{
+			bytes: NewMaliciousFilter(1, 0).Marshal(),
+			err:   errTooFewEntries,
+		},
+		{
+			bytes: []byte{
+				0x01, // num hashes = 1
+			},
+			err: errTooFewEntries,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.err.Error(), func(t *testing.T) {
+			_, err := Parse(test.bytes)
+			require.ErrorIs(t, err, test.err)
+		})
+	}
+}
+
+func BenchmarkParse(b *testing.B) {
+	f, err := New(OptimalParameters(10_000, .01))
+	require.NoError(b, err)
+	bytes := f.Marshal()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, _ = Parse(bytes)
+	}
+}
+
+func BenchmarkContains(b *testing.B) {
+	f := NewMaliciousFilter(maxHashes, 1)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		f.Contains(1)
+	}
+}
+
+func FuzzParseThenMarshal(f *testing.F) {
+	f.Fuzz(func(t *testing.T, bytes []byte) {
+		f, err := Parse(bytes)
+		if err != nil {
+			return
+		}
+
+		marshalledBytes := marshal(f.hashSeeds, f.entries)
+		require.Equal(t, bytes, marshalledBytes)
+	})
+}
+
+func FuzzMarshalThenParse(f *testing.F) {
+	f.Fuzz(func(t *testing.T, numHashes int, entries []byte) {
+		require := require.New(t)
+
+		hashSeeds, err := newHashSeeds(numHashes)
+		if err != nil {
+			return
+		}
+		if len(entries) < minEntries {
+			return
+		}
+
+		marshalledBytes := marshal(hashSeeds, entries)
+		rf, err := Parse(marshalledBytes)
+		require.NoError(err)
+		require.Equal(hashSeeds, rf.hashSeeds)
+		require.Equal(entries, rf.entries)
+	})
+}

bytecache/cache.go

@@ -0,0 +1,279 @@
+// Copyright (C) 2019-2025, Lux Industries, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package bytecache
+
+import (
+	"sync"
+	"sync/atomic"
+)
+
+const (
+	numShards    = 256
+	shardMask    = numShards - 1
+)
+
+// Stats contains cache performance metrics.
+type Stats struct {
+	EntriesCount uint64
+	BytesSize    uint64
+	Collisions   uint64
+	GetCalls     uint64
+	SetCalls     uint64
+	Misses       uint64
+}
+
+// Cache is a high-performance sharded LRU byte cache.
+// It provides O(1) lookups with minimal lock contention.
+type Cache struct {
+	shards      [numShards]*byteShard
+	maxBytes    int64
+	getCalls    uint64
+	setCalls    uint64
+	misses      uint64
+}
+
+type byteShard struct {
+	mu          sync.RWMutex
+	items       map[string]*byteEntry
+	head, tail  *byteEntry
+	currentSize int64
+	maxSize     int64
+}
+
+type byteEntry struct {
+	key        string
+	value      []byte
+	size       int
+	prev, next *byteEntry
+}
+
+// New creates a new byte cache with the given max size in bytes.
+func New(maxBytes int) *Cache {
+	if maxBytes <= 0 {
+		maxBytes = 1
+	}
+	c := &Cache{maxBytes: int64(maxBytes)}
+	perShard := int64(maxBytes) / numShards
+	if perShard < 1 {
+		perShard = 1
+	}
+	for i := range c.shards {
+		c.shards[i] = &byteShard{
+			items:   make(map[string]*byteEntry),
+			maxSize: perShard,
+		}
+	}
+	return c
+}
+
+func (c *Cache) shard(key []byte) *byteShard {
+	h := uint8(0)
+	for _, b := range key {
+		h ^= b
+	}
+	return c.shards[h&shardMask]
+}
+
+// Reset clears all cached entries.
+func (c *Cache) Reset() {
+	for _, s := range c.shards {
+		s.mu.Lock()
+		s.items = make(map[string]*byteEntry)
+		s.head, s.tail = nil, nil
+		s.currentSize = 0
+		s.mu.Unlock()
+	}
+}
+
+// Del removes a key from the cache.
+func (c *Cache) Del(key []byte) {
+	s := c.shard(key)
+	k := string(key)
+	s.mu.Lock()
+	if e, ok := s.items[k]; ok {
+		s.unlink(e)
+		s.currentSize -= int64(e.size)
+		delete(s.items, k)
+	}
+	s.mu.Unlock()
+}
+
+// Has reports whether a key exists.
+func (c *Cache) Has(key []byte) bool {
+	s := c.shard(key)
+	k := string(key)
+	s.mu.RLock()
+	_, ok := s.items[k]
+	s.mu.RUnlock()
+	return ok
+}
+
+// HasGet returns the value and whether it exists.
+func (c *Cache) HasGet(dst, key []byte) ([]byte, bool) {
+	atomic.AddUint64(&c.getCalls, 1)
+	s := c.shard(key)
+	k := string(key)
+
+	s.mu.Lock()
+	e, ok := s.items[k]
+	if ok {
+		s.moveToFront(e)
+		val := e.value
+		s.mu.Unlock()
+		if dst == nil {
+			return append([]byte(nil), val...), true
+		}
+		return append(dst[:0], val...), true
+	}
+	s.mu.Unlock()
+
+	atomic.AddUint64(&c.misses, 1)
+	if dst == nil {
+		return nil, false
+	}
+	return dst[:0], false
+}
+
+// Get looks up a value by key, copying into dst if provided.
+func (c *Cache) Get(dst, key []byte) []byte {
+	atomic.AddUint64(&c.getCalls, 1)
+	s := c.shard(key)
+	k := string(key)
+
+	s.mu.Lock()
+	e, ok := s.items[k]
+	if ok {
+		s.moveToFront(e)
+		val := e.value
+		s.mu.Unlock()
+		if dst == nil {
+			return append([]byte(nil), val...)
+		}
+		return append(dst[:0], val...)
+	}
+	s.mu.Unlock()
+
+	atomic.AddUint64(&c.misses, 1)
+	if dst == nil {
+		return nil
+	}
+	return dst[:0]
+}
+
+// GetBig is an alias for Get (compatibility).
+func (c *Cache) GetBig(dst, key []byte) []byte {
+	return c.Get(dst, key)
+}
+
+// Set stores a key/value pair.
+func (c *Cache) Set(key, value []byte) {
+	atomic.AddUint64(&c.setCalls, 1)
+	s := c.shard(key)
+	k := string(key)
+	v := append([]byte(nil), value...)
+	entrySize := len(k) + len(v)
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// Entry too large for shard
+	if int64(entrySize) > s.maxSize {
+		return
+	}
+
+	// Update existing
+	if e, ok := s.items[k]; ok {
+		s.currentSize -= int64(e.size)
+		e.value = v
+		e.size = entrySize
+		s.currentSize += int64(entrySize)
+		s.moveToFront(e)
+		return
+	}
+
+	// Evict until we have space
+	for s.currentSize+int64(entrySize) > s.maxSize && s.tail != nil {
+		old := s.tail
+		s.unlink(old)
+		s.currentSize -= int64(old.size)
+		delete(s.items, old.key)
+	}
+
+	// Insert new entry
+	e := &byteEntry{key: k, value: v, size: entrySize}
+	s.items[k] = e
+	s.pushFront(e)
+	s.currentSize += int64(entrySize)
+}
+
+// SetBig is an alias for Set (compatibility).
+func (c *Cache) SetBig(key, value []byte) {
+	c.Set(key, value)
+}
+
+// UpdateStats populates the provided stats struct.
+func (c *Cache) UpdateStats(s *Stats) {
+	if s == nil {
+		return
+	}
+	var entries, size uint64
+	for _, sh := range c.shards {
+		sh.mu.RLock()
+		entries += uint64(len(sh.items))
+		size += uint64(sh.currentSize)
+		sh.mu.RUnlock()
+	}
+	s.EntriesCount = entries
+	s.BytesSize = size
+	s.Collisions = 0
+	s.GetCalls = atomic.LoadUint64(&c.getCalls)
+	s.SetCalls = atomic.LoadUint64(&c.setCalls)
+	s.Misses = atomic.LoadUint64(&c.misses)
+}
+
+// Doubly-linked list operations for LRU
+
+func (s *byteShard) pushFront(e *byteEntry) {
+	e.prev = nil
+	e.next = s.head
+	if s.head != nil {
+		s.head.prev = e
+	}
+	s.head = e
+	if s.tail == nil {
+		s.tail = e
+	}
+}
+
+func (s *byteShard) unlink(e *byteEntry) {
+	if e.prev != nil {
+		e.prev.next = e.next
+	} else {
+		s.head = e.next
+	}
+	if e.next != nil {
+		e.next.prev = e.prev
+	} else {
+		s.tail = e.prev
+	}
+	e.prev, e.next = nil, nil
+}
+
+func (s *byteShard) moveToFront(e *byteEntry) {
+	if s.head == e {
+		return
+	}
+	s.unlink(e)
+	s.pushFront(e)
+}
+
+// SaveToFileConcurrent is a no-op for compatibility with fastcache API.
+func (c *Cache) SaveToFileConcurrent(filePath string, concurrency int) error {
+	return nil
+}
+
+// LoadFromFile is a no-op for compatibility with fastcache API.
+func (c *Cache) LoadFromFile(filePath string) error {
+	return nil
+}

dualmap_cache.go

@@ -17,7 +17,7 @@ type DualMapCache[K comparable, V any] struct {
 }
 
 // NewDualMapCache creates a new DualMapCache. Metrics are optional.
-func NewDualMapCache[K comparable, V any](_ *metric.MetricsRegistry) *DualMapCache[K, V] {
+func NewDualMapCache[K comparable, V any](_ metric.Registry) *DualMapCache[K, V] {
 	return &DualMapCache[K, V]{
 		items: make(map[K]V),
 	}

go.mod

@@ -4,8 +4,10 @@ go 1.25.5
 
 require (
 	github.com/luxfi/container v0.0.2
+	github.com/luxfi/ids v1.2.7
 	github.com/luxfi/math v1.2.3
 	github.com/luxfi/metric v1.4.10
+	github.com/spaolacci/murmur3 v1.1.0
 	github.com/stretchr/testify v1.11.1
 )
 
@@ -14,11 +16,9 @@ require (
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/gorilla/rpc v1.2.1 // indirect
-	github.com/klauspost/compress v1.18.2 // indirect
-	github.com/kr/text v0.2.0 // indirect
 	github.com/luxfi/crypto v1.17.36 // indirect
-	github.com/luxfi/ids v1.2.7 // indirect
 	github.com/luxfi/math/big v0.1.0 // indirect
+	github.com/luxfi/sampler v1.0.0 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
@@ -30,6 +30,7 @@ require (
 	golang.org/x/crypto v0.46.0 // indirect
 	golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect
 	golang.org/x/sys v0.39.0 // indirect
+	gonum.org/v1/gonum v0.16.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -2,7 +2,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
@@ -27,10 +26,9 @@ github.com/luxfi/math v1.2.3 h1:BgvIFw/srPXFLbcqtoDhLJOfmBsn86GPA1iWgsoyUb4=
 github.com/luxfi/math v1.2.3/go.mod h1:C8STnF2H+D6rqBPt248CiWY2TGuJgdtv/+4UqrT15iM=
 github.com/luxfi/math/big v0.1.0 h1:Vz4c0RsZVPdIKPsHPgAJChH/R3p15WHRUz7LkLf+NIQ=
 github.com/luxfi/math/big v0.1.0/go.mod h1:BuxSu22RbO93xBLk5Eam5nldFponoJ73xDFz4uJ3Huk=
-github.com/luxfi/metric v1.4.5 h1:fD6gtpzB5ebGB06/m4K7e5+tOV5RiVHSZCV+Zb8ZDxk=
-github.com/luxfi/metric v1.4.5/go.mod h1:AUQ7NSNz9WndAcr/SKnOkP7XSFFnBXOa+ihtJYfDaQY=
 github.com/luxfi/metric v1.4.10 h1:uWeLupMLxK7YK1c99gY23cT+hornWg7OmZgHNK4HNbw=
 github.com/luxfi/metric v1.4.10/go.mod h1:3OIQ+e6mciesm3cik3Q4CSOAvTlBy4JA0ZoKzCB9APo=
+github.com/luxfi/sampler v1.0.0 h1:k8Sf6otW83w4pQp0jXLA+g3J/joB7w7SqXQsWmNTOV0=
 github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
 github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
@@ -47,6 +45,7 @@ github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4
 github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@@ -59,6 +58,7 @@ golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1i
 golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU=
 golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
 golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
 google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

lru/cache.go

@@ -105,7 +105,7 @@ func (c *Cache[K, V]) Flush() {
 func (c *Cache[K, V]) PortionFilled() float64 {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	current := float64(c.Len())
+	current := float64(c.containerCache.Len())
 	capacity := float64(c.capacity)
 	if capacity == 0 {
 		return 0