mirror of
https://github.com/hanzoai/base.git
synced 2026-10-21 03:41:50 +00:00
Phase 1 of the Base Network archive tier: witness-only validator
appends quasar-finalised WAL frames to object storage as per-shard
.lbn segments. PITR replay recovers SQLite state at any txseq.
Scope:
- network/archive.go Archive interface + URL dispatch
(s3://…|gs://…|off)
- network/segment.go .lbn encoder/decoder. Magic "LBN1",
CRC32-IEEE footer. Forwards-compat via
magic bump on any format change.
- network/archive_writer.go Goroutine-per-shard buffer with
size+time flush, exponential-backoff
retry against RetryDeadline, and
never-drop re-buffering on timeout.
- network/archive_s3.go minio-go/v7 backend. Works against AWS
S3 and any MinIO-protocol endpoint —
hanzoai/s3 self-hosted is one such.
Static creds from AWS_ACCESS_KEY_ID /
SECRET, else IAM/IRSA via
credentials.NewIAM().
- network/archive_gcs.go cloud.google.com/go/storage backend.
ADC by default; Workload Identity in
k8s, GOOGLE_APPLICATION_CREDENTIALS
for local dev.
- hack/pitr-restore.go CLI replay tool. Usage:
pitr-restore --archive gs://…/svc \
--shard <id> --to-seq <n> \
--out file.db
Writes a SQLite file with one row per
frame; Base's real apply path replays
into the schema.
Segment layout: "LBN1" | shardIDLen u16 | shardID | startSeq u64 |
frameCount u32 | (frameLen u32 | frameBytes){N} | crc32 footer.
PQ signature on each frame is preserved byte-for-byte — the
archive never re-signs.
Storage key layout per docs/NETWORK.md:
<svc>/<shard>/<16-digit-seq-prefix>/<20-digit-start-seq>.lbn
Config knobs (defaults): SegmentTargetBytes=8MiB, FlushInterval=10s,
RetryDeadline=5m. lagBytes gauge reflects backlog awaiting flush.
Metrics binding: BindArchiveMetrics(*Metrics) wires archive callbacks
onto the core Metrics.ArchiveLagBytes gauge + ApplyErrors counter.
Archive stays decoupled from prometheus/client_golang.
Tests: 12 archive-specific tests pass alongside the 20 network tests
from Phase 1 (32/32 total). Integration tests (`//go:build integration`)
cover 1000-frame S3 round-trip against local MinIO and PITR CLI
restore of 100 frames at cut-off seq 50 producing a SQLite file
with exactly 50 rows.
128 lines
3.4 KiB
Go
128 lines
3.4 KiB
Go
//go:build integration
|
|
|
|
package network
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"runtime"
|
|
"testing"
|
|
"time"
|
|
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
// TestPITRRestore smoke-tests the hack/pitr-restore CLI by writing
|
|
// 100 frames to an S3 archive, running the CLI to restore up to
|
|
// seq 50, and asserting the resulting SQLite file has exactly 50
|
|
// rows.
|
|
//
|
|
// Requires the same MinIO container as TestS3IntegrationRoundTrip —
|
|
// AWS_ENDPOINT_URL, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY env.
|
|
func TestPITRRestore(t *testing.T) {
|
|
bucket := skipWithoutMinIO(t)
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
|
defer cancel()
|
|
|
|
const svc = "svc-pitr"
|
|
const shard = "shard-pitr"
|
|
|
|
a, err := NewArchive(ctx, ArchiveConfig{
|
|
URL: fmt.Sprintf("s3://%s", bucket),
|
|
SegmentTargetBytes: 1024,
|
|
FlushInterval: 50 * time.Millisecond,
|
|
RetryDeadline: 30 * time.Second,
|
|
}, svc, nil)
|
|
if err != nil {
|
|
t.Fatalf("open archive: %v", err)
|
|
}
|
|
for i := uint64(1); i <= 100; i++ {
|
|
f := newFrame(shard, i, i-1, []byte(fmt.Sprintf("payload-%d", i)))
|
|
if err := a.Append(ctx, shard, i, f.encode()); err != nil {
|
|
t.Fatalf("append %d: %v", i, err)
|
|
}
|
|
}
|
|
if err := a.Close(); err != nil {
|
|
t.Fatalf("close: %v", err)
|
|
}
|
|
|
|
// Build the CLI.
|
|
tmpDir := t.TempDir()
|
|
bin := filepath.Join(tmpDir, "pitr-restore")
|
|
if runtime.GOOS == "windows" {
|
|
bin += ".exe"
|
|
}
|
|
build := exec.CommandContext(ctx, "go", "build", "-o", bin, "./hack")
|
|
build.Dir = repoRootFromCaller(t)
|
|
var bstderr bytes.Buffer
|
|
build.Stderr = &bstderr
|
|
if err := build.Run(); err != nil {
|
|
t.Fatalf("build cli: %v; stderr: %s", err, bstderr.String())
|
|
}
|
|
|
|
// Run the CLI with --to-seq=50.
|
|
outFile := filepath.Join(tmpDir, "restored.db")
|
|
cmd := exec.CommandContext(ctx, bin,
|
|
"--archive", fmt.Sprintf("s3://%s/%s", bucket, svc),
|
|
"--shard", shard,
|
|
"--to-seq", "50",
|
|
"--out", outFile,
|
|
)
|
|
cmd.Env = append(os.Environ(),
|
|
"AWS_ENDPOINT_URL="+os.Getenv("AWS_ENDPOINT_URL"),
|
|
"AWS_ACCESS_KEY_ID="+os.Getenv("AWS_ACCESS_KEY_ID"),
|
|
"AWS_SECRET_ACCESS_KEY="+os.Getenv("AWS_SECRET_ACCESS_KEY"),
|
|
"AWS_REGION="+envOrDefault("AWS_REGION", "us-east-1"),
|
|
)
|
|
var stderr bytes.Buffer
|
|
cmd.Stderr = &stderr
|
|
out, err := cmd.Output()
|
|
if err != nil {
|
|
t.Fatalf("cli failed: %v\nstdout: %s\nstderr: %s", err, out, stderr.String())
|
|
}
|
|
t.Logf("cli: %s", bytes.TrimSpace(out))
|
|
|
|
// Verify the SQLite file has exactly 50 rows with seqs 1..50.
|
|
db, err := sql.Open("sqlite", outFile)
|
|
if err != nil {
|
|
t.Fatalf("open restored db: %v", err)
|
|
}
|
|
defer db.Close()
|
|
var count, minSeq, maxSeq int
|
|
if err := db.QueryRowContext(ctx, "SELECT COUNT(*), IFNULL(MIN(seq),0), IFNULL(MAX(seq),0) FROM wal_frames").Scan(&count, &minSeq, &maxSeq); err != nil {
|
|
t.Fatalf("query: %v", err)
|
|
}
|
|
if count != 50 {
|
|
t.Fatalf("want 50 rows, got %d", count)
|
|
}
|
|
if minSeq != 1 || maxSeq != 50 {
|
|
t.Fatalf("seq bounds: want [1,50], got [%d,%d]", minSeq, maxSeq)
|
|
}
|
|
}
|
|
|
|
func envOrDefault(k, def string) string {
|
|
if v := os.Getenv(k); v != "" {
|
|
return v
|
|
}
|
|
return def
|
|
}
|
|
|
|
// repoRootFromCaller walks up from this test file to the base/ root so
|
|
// the CLI `go build` invocation runs at the module root where `./hack`
|
|
// resolves correctly.
|
|
func repoRootFromCaller(t *testing.T) string {
|
|
t.Helper()
|
|
_, file, _, ok := runtime.Caller(0)
|
|
if !ok {
|
|
t.Fatalf("cannot locate caller file")
|
|
}
|
|
// file = .../base/network/pitr_test.go → ../ = .../base
|
|
return filepath.Dir(filepath.Dir(file))
|
|
}
|