mirror of
https://github.com/luxfi/database.git
synced 2026-10-30 15:01:18 +00:00
zeekayandHanzo Dev
fb16ece6a3
Replicates a zapdb to S3 via hanzoai/vfs with no keyspace scan in steady state and post-quantum encryption out of the box. - CDC incrementals via db.Subscribe (the WAL tail), not Backup(since)'s O(total) scan — a 100-key delta out of 200k keys: 2.76us/19KB vs 20ms/1GB (~7000x). - Physical SST-copy snapshots (sparse, hole-aware): 134ms/25MB vs logical 158ms/1.25GB/2.2M allocs. Hydrated before Badger opens; incrementals replay on top. - Streaming codec: Backup -> zstd -> age into a pooled buffer (peak mem = compressed). - Post-quantum (ML-KEM-768 + X25519 via luxfi/age v1.5.0), client-side before S3. Out of the box via REPLICATE_AGE_KEY_FILE (generate+persist, recipient derived). - Per-DB streams (REPLICATE_AUTO_ON_OPEN) for subprocess VM plugins; backer/restorer split (REPLICATE_BACKUP / REPLICATE_SOURCE_ORDINAL) for multi-node. - Public imports only (hanzoai/vfs v0.4.1, no replace). Verified live on an EVM C-chain end-to-end incl. disaster-recovery exact-match. Canonical KV schema (zapdb/zapdb.zap) + zero-copy codec (zapkv.go). Renames engine alias badger->zdb. Co-authored-by: Hanzo Dev <[email protected]>
51 lines
1.7 KiB
Go
51 lines
1.7 KiB
Go
package zapdb
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"io"
|
||
"testing"
|
||
)
|
||
|
||
// TestPeakBackupBuffer quantifies what the streaming pipeline actually buys:
|
||
// peak in-memory buffer. The old path did `var buf bytes.Buffer; Backup(&buf)`,
|
||
// so it held the ENTIRE uncompressed dump, then allocated a separate compressed
|
||
// copy on top. The streaming path never materializes the uncompressed dump — the
|
||
// only buffer is the compressed output. This isn't a CPU win (the profile shows
|
||
// db.Backup internals dominate); it's the difference between fitting a multi-GB
|
||
// chain backup in RAM or OOMing.
|
||
func TestPeakBackupBuffer(t *testing.T) {
|
||
r, done := benchReplicator(t, 200_000, true)
|
||
defer done()
|
||
disc := r.be.(*discardBackend)
|
||
|
||
// Uncompressed dump size: what the OLD path buffered in full.
|
||
var uncompressed int64
|
||
if mv, err := r.db.Backup(&countSink{n: &uncompressed}, 0); err != nil || mv == 0 {
|
||
t.Fatalf("backup: mv=%d err=%v", mv, err)
|
||
}
|
||
|
||
// New path: the only buffer held is the compressed output (disc.lastPut).
|
||
r.lastSnapVersion = 0
|
||
if err := r.backupAndUpload(context.Background(), 0, true); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
compressed := disc.lastPut
|
||
|
||
t.Logf("200k keys: OLD peak buffer ≈ uncompressed %d + compressed %d = %d bytes; NEW peak buffer = compressed %d bytes (%.1f× less)",
|
||
uncompressed, compressed, uncompressed+compressed, compressed,
|
||
float64(uncompressed+compressed)/float64(compressed))
|
||
if compressed >= uncompressed {
|
||
t.Fatalf("expected compressed (%d) << uncompressed (%d)", compressed, uncompressed)
|
||
}
|
||
}
|
||
|
||
type countSink struct{ n *int64 }
|
||
|
||
func (c *countSink) Write(p []byte) (int, error) { *c.n += int64(len(p)); return len(p), nil }
|
||
|
||
var (
|
||
_ io.Writer = (*countSink)(nil)
|
||
_ = fmt.Sprintf
|
||
)
|