Files
zeekayandHanzo Dev fb16ece6a3 feat(zapdb): native ZAP replication — CDC, physical snapshots, PQ, streaming
Replicates a zapdb to S3 via hanzoai/vfs with no keyspace scan in steady state
and post-quantum encryption out of the box.

- CDC incrementals via db.Subscribe (the WAL tail), not Backup(since)'s O(total)
  scan — a 100-key delta out of 200k keys: 2.76us/19KB vs 20ms/1GB (~7000x).
- Physical SST-copy snapshots (sparse, hole-aware): 134ms/25MB vs logical
  158ms/1.25GB/2.2M allocs. Hydrated before Badger opens; incrementals replay on top.
- Streaming codec: Backup -> zstd -> age into a pooled buffer (peak mem = compressed).
- Post-quantum (ML-KEM-768 + X25519 via luxfi/age v1.5.0), client-side before S3.
  Out of the box via REPLICATE_AGE_KEY_FILE (generate+persist, recipient derived).
- Per-DB streams (REPLICATE_AUTO_ON_OPEN) for subprocess VM plugins; backer/restorer
  split (REPLICATE_BACKUP / REPLICATE_SOURCE_ORDINAL) for multi-node.
- Public imports only (hanzoai/vfs v0.4.1, no replace). Verified live on an EVM
  C-chain end-to-end incl. disaster-recovery exact-match. Canonical KV schema
  (zapdb/zapdb.zap) + zero-copy codec (zapkv.go). Renames engine alias badger->zdb.

Co-authored-by: Hanzo Dev <[email protected]>
2026-06-16 02:00:03 +00:00

184 lines
4.0 KiB
Go

// Copyright (C) 2020-2026, Lux Industries Inc. All rights reserved.
// See the file LICENSE for licensing terms.
package zapdb
import (
"bytes"
"time"
"github.com/luxfi/database"
zdb "github.com/luxfi/zapdb"
)
// txnWrap adapts a zdb.Txn to database.Txn.
type txnWrap struct {
tx *zdb.Txn
readOnly bool
}
// keyFor maps a user-visible key to the on-disk key, applying the empty-key
// placeholder so badger (which rejects empty keys) sees a non-empty value.
func keyFor(k []byte) []byte {
if len(k) == 0 {
return emptyKeyPlaceholder
}
return k
}
// userKey reverses keyFor for iteration output.
func userKey(k []byte) []byte {
if bytes.Equal(k, emptyKeyPlaceholder) {
return []byte{}
}
return k
}
func (t *txnWrap) Has(key []byte) (bool, error) {
_, err := t.tx.Get(keyFor(key))
if err == zdb.ErrKeyNotFound {
return false, nil
}
if err != nil {
return false, err
}
return true, nil
}
func (t *txnWrap) Get(key []byte) ([]byte, error) {
item, err := t.tx.Get(keyFor(key))
if err == zdb.ErrKeyNotFound {
return nil, database.ErrNotFound
}
if err != nil {
return nil, err
}
return item.ValueCopy(nil)
}
func (t *txnWrap) Put(key, value []byte) error {
if t.readOnly {
return database.ErrReadOnlyTxn
}
return t.tx.Set(keyFor(key), value)
}
func (t *txnWrap) PutWithTTL(key, value []byte, ttl time.Duration) error {
if t.readOnly {
return database.ErrReadOnlyTxn
}
e := zdb.NewEntry(keyFor(key), value)
if ttl > 0 {
e = e.WithTTL(ttl)
}
return t.tx.SetEntry(e)
}
func (t *txnWrap) Delete(key []byte) error {
if t.readOnly {
return database.ErrReadOnlyTxn
}
return t.tx.Delete(keyFor(key))
}
func (t *txnWrap) NewIterator() database.Iterator {
return t.newIterator(nil, nil)
}
func (t *txnWrap) NewIteratorWithPrefix(prefix []byte) database.Iterator {
return t.newIterator(nil, prefix)
}
func (t *txnWrap) NewIteratorWithStart(start []byte) database.Iterator {
return t.newIterator(start, nil)
}
func (t *txnWrap) newIterator(start, prefix []byte) database.Iterator {
opts := zdb.DefaultIteratorOptions
opts.PrefetchSize = 10
it := t.tx.NewIterator(opts)
iter := &iterator{
txn: t.tx, // shared txn; iterator must NOT close it
iter: it,
prefix: prefix,
start: start,
closed: false,
shared: true,
}
if len(start) > 0 {
it.Seek(start)
} else if len(prefix) > 0 {
it.Seek(prefix)
} else {
it.Rewind()
}
if len(prefix) > 0 && it.Valid() && !bytes.HasPrefix(it.Item().Key(), prefix) {
iter.Next()
}
return iter
}
// Update runs fn in a writable badger transaction. On non-nil error from fn
// the transaction is rolled back; on nil the transaction is committed.
// badger handles conflict retries internally via View/Update — we surface
// any commit error to the caller.
func (d *Database) Update(fn func(txn database.Txn) error) error {
d.closeMu.RLock()
defer d.closeMu.RUnlock()
if d.closed {
return database.ErrClosed
}
tx := d.db.NewTransaction(true)
defer tx.Discard()
wrap := &txnWrap{tx: tx, readOnly: false}
if err := fn(wrap); err != nil {
return err
}
return tx.Commit()
}
// View runs fn in a read-only badger transaction.
func (d *Database) View(fn func(txn database.Txn) error) error {
d.closeMu.RLock()
defer d.closeMu.RUnlock()
if d.closed {
return database.ErrClosed
}
tx := d.db.NewTransaction(false)
defer tx.Discard()
wrap := &txnWrap{tx: tx, readOnly: true}
return fn(wrap)
}
// PutWithTTL implements database.TTLWriter outside of an explicit transaction.
// It opens a write txn, sets the entry with the TTL, and commits.
func (d *Database) PutWithTTL(key, value []byte, ttl time.Duration) error {
d.closeMu.RLock()
defer d.closeMu.RUnlock()
if d.closed {
return database.ErrClosed
}
return d.db.Update(func(tx *zdb.Txn) error {
e := zdb.NewEntry(keyFor(key), value)
if ttl > 0 {
e = e.WithTTL(ttl)
}
return tx.SetEntry(e)
})
}
// Compile-time interface checks.
var (
_ database.Transactional = (*Database)(nil)
_ database.TTLWriter = (*Database)(nil)
_ database.Txn = (*txnWrap)(nil)
)