mirror of
https://github.com/luxfi/database.git
synced 2026-10-30 15:01:18 +00:00
zeekayandHanzo Dev
fb16ece6a3
Replicates a zapdb to S3 via hanzoai/vfs with no keyspace scan in steady state and post-quantum encryption out of the box. - CDC incrementals via db.Subscribe (the WAL tail), not Backup(since)'s O(total) scan — a 100-key delta out of 200k keys: 2.76us/19KB vs 20ms/1GB (~7000x). - Physical SST-copy snapshots (sparse, hole-aware): 134ms/25MB vs logical 158ms/1.25GB/2.2M allocs. Hydrated before Badger opens; incrementals replay on top. - Streaming codec: Backup -> zstd -> age into a pooled buffer (peak mem = compressed). - Post-quantum (ML-KEM-768 + X25519 via luxfi/age v1.5.0), client-side before S3. Out of the box via REPLICATE_AGE_KEY_FILE (generate+persist, recipient derived). - Per-DB streams (REPLICATE_AUTO_ON_OPEN) for subprocess VM plugins; backer/restorer split (REPLICATE_BACKUP / REPLICATE_SOURCE_ORDINAL) for multi-node. - Public imports only (hanzoai/vfs v0.4.1, no replace). Verified live on an EVM C-chain end-to-end incl. disaster-recovery exact-match. Canonical KV schema (zapdb/zapdb.zap) + zero-copy codec (zapkv.go). Renames engine alias badger->zdb. Co-authored-by: Hanzo Dev <[email protected]>
91 lines
2.9 KiB
Go
91 lines
2.9 KiB
Go
package zapdb
|
|
|
|
import (
|
|
"testing"
|
|
"unsafe"
|
|
)
|
|
|
|
// TestZapKVRoundTrip proves the canonical ZAP KV codec (zap-spec/zapdb.zap)
|
|
// round-trips every field byte-for-byte.
|
|
func TestZapKVRoundTrip(t *testing.T) {
|
|
key := []byte("account/0x35d64ff3/balance")
|
|
val := []byte{0x19, 0xd9, 0x71, 0xe4, 0xfe, 0x84, 0x01, 0xe7}
|
|
um := []byte{0x01}
|
|
meta := []byte("trie")
|
|
|
|
enc := encodeZapKV(424242, 9999, key, val, um, meta)
|
|
if len(enc) != kvHeaderSize+len(key)+len(val)+len(um)+len(meta) {
|
|
t.Fatalf("size %d unexpected", len(enc))
|
|
}
|
|
kv, err := decodeZapKV(enc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if kv.Version() != 424242 || kv.ExpiresAt() != 9999 {
|
|
t.Fatalf("scalars: ver=%d exp=%d", kv.Version(), kv.ExpiresAt())
|
|
}
|
|
if string(kv.Key()) != string(key) || string(kv.Value()) != string(val) ||
|
|
string(kv.UserMeta()) != string(um) || string(kv.Meta()) != string(meta) {
|
|
t.Fatal("field mismatch")
|
|
}
|
|
}
|
|
|
|
// TestZapKVZeroCopy proves the accessors alias the source buffer — no allocation,
|
|
// no copy. The Key() slice must point INTO enc, which is the whole point of ZAP
|
|
// vs the legacy sequential marshal_zap (which has no in-place reads).
|
|
func TestZapKVZeroCopy(t *testing.T) {
|
|
key := []byte("the-key-bytes")
|
|
enc := encodeZapKV(1, 0, key, []byte("v"), nil, nil)
|
|
kv, err := decodeZapKV(enc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := kv.Key()
|
|
// The returned slice's backing array must be the same as enc's — i.e. its
|
|
// data pointer falls within enc. That's zero-copy.
|
|
base := uintptr(unsafe.Pointer(&enc[0]))
|
|
end := base + uintptr(len(enc))
|
|
p := uintptr(unsafe.Pointer(&got[0]))
|
|
if p < base || p >= end {
|
|
t.Fatalf("Key() did not alias the message buffer (copy detected)")
|
|
}
|
|
// Mutating the message is visible through the view → confirms aliasing.
|
|
enc[int(p-base)] = 'X'
|
|
if kv.Key()[0] != 'X' {
|
|
t.Fatal("view did not reflect buffer mutation — not zero-copy")
|
|
}
|
|
}
|
|
|
|
// TestZapKVBoundsCheck proves a truncated/forged record can't read out of range.
|
|
func TestZapKVBoundsCheck(t *testing.T) {
|
|
enc := encodeZapKV(1, 0, []byte("key"), []byte("value"), nil, nil)
|
|
if _, err := decodeZapKV(enc[:kvHeaderSize-1]); err == nil {
|
|
t.Fatal("expected short-header error")
|
|
}
|
|
// Corrupt Value length to overflow the buffer.
|
|
bad := make([]byte, len(enc))
|
|
copy(bad, enc)
|
|
bad[28] = 0xff
|
|
bad[29] = 0xff
|
|
if _, err := decodeZapKV(bad); err == nil {
|
|
t.Fatal("expected out-of-range pointer error")
|
|
}
|
|
}
|
|
|
|
// BenchmarkZapKVDecodeRead contrasts the cost of accessing a field. The canonical
|
|
// ZAP decode is a bounds-check then O(1) pointer reads with no allocation.
|
|
func BenchmarkZapKVDecodeRead(b *testing.B) {
|
|
enc := encodeZapKV(1, 0, []byte("account/0x35d64ff3/balance"), make([]byte, 64), []byte{1}, nil)
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
var n int
|
|
for i := 0; i < b.N; i++ {
|
|
kv, err := decodeZapKV(enc)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
n += len(kv.Key()) + len(kv.Value()) + int(kv.Version())
|
|
}
|
|
_ = n
|
|
}
|