mirror of
https://github.com/hanzoai/replicate.git
synced 2026-10-30 18:23:20 +00:00
* fix(replicate): fail-closed age encryption (no plaintext LTX to S3)
Replicate silently wrote plaintext LTX1 to the destination whenever an age
recipient was unset -- fleet-wide plaintext SQLite backups. This makes
encryption fail-closed: when a replica is told it must encrypt, the absence
of a recipient REFUSES the write (ErrEncryptionRequired) instead of
downgrading to plaintext.
- Replica.RequireEncryption + guards in Start(), WriteLTXFile(), and Compactor
.Compact() -- every write path funnels through a fail-closed check.
- DB.init() fails closed at the earliest point (before the "no db file yet"
early return) so a misconfigured deployment surfaces the error on its first
Sync, loudly, every interval -- never silently no-ops until data appears.
- DB.Snapshot() now routes through Replica.WriteLTXFile (was raw Client), so
snapshots are age-encrypted like L0 sync; previously a plaintext bypass.
- internal.ExtractLTXTimestamp: sniff the age v1 intro so encrypted objects
fall back to write time instead of every backend failing to parse an LTX
header out of ciphertext. All 8 replica clients use it.
- AutoReplicate / CLI: RequireEncryption defaults on; opt out only via explicit
REPLICATE_ALLOW_PLAINTEXT / age.allow-plaintext for non-sensitive dev targets.
Tests: replica_failclosed_test.go (unit: write/start/compact fail closed;
encrypt+round-trip with recipient) and replicate_e2e_encrypt_test.go (full DB
stack: every destination object is age ciphertext across L0 sync AND the
snapshot path; live-writer restore round-trip; DB sync fails closed).
Re-applies the fix from #6 onto a fresh branch off main (#6's branch shares no
git history with main and cannot merge). age bumped 1.4.0->1.6.0 to match the
platform (cloud) and resolve main's broken [email protected] go.sum checksum; go mod
tidy prunes the orphaned prometheus indirect deps left by the metric migration.
* test(replicate): define testutil shim so the package test binary compiles
The luxfi/metric migration (commit 7c8ef84) dropped prometheus but left
db_internal_test.go referencing an undefined `testutil`, so `go test ./` did
not compile at all on main. This adds the minimal in-tree ToFloat64 shim
(luxfi/metric Counter/Gauge expose Get() float64) and drops the now-unused
metric import, unblocking the package test binary.
NOTE: this only fixes COMPILATION. The metric value-assertion tests
(TestDB_*Metrics) remain pre-existing RED because luxfi/metric's
WithLabelValues/With are non-memoizing (proven: vec.WithLabelValues("x").Set(5)
then vec.WithLabelValues("x").Get() == 0). That is an orthogonal, incomplete
metric-migration defect (also affects runtime aggregation at db.go checkpoint
Inc) and must be completed in a dedicated metric PR -- not folded into the
fail-closed security fix. Identical failures occur with or without encryption.
* docs(deps): justify luxfi/age v1.4.0->v1.6.0 bump (forced by broken main go.sum)
* fix(ci): unpoison luxfi/zap + run root metric tests under -tags metrics
Two pre-existing main breakages surfaced by CI (not by the fail-closed
change):
1. luxfi/zap v0.8.1's tag was re-published upstream, so its go.sum hash
no longer verifies (SECURITY ERROR on every job). Bump to the latest
patch in-minor, v0.8.12 (sumdb-verified) — same remedy as the age bump,
no lazy major.
2. luxfi/metric defaults to a NO-OP registry (//go:build !metrics); the
real in-process registry is behind `metrics`. `go test .` ran without
it, so every WithLabelValues returned a throwaway gauge that reads 0 —
the 8 TestDB_*Metrics failures. Root cause is the build tag (not
'non-memoizing' as first guessed, nor a prod defect): the DB's cached
gauge holds the value; only re-derivations under the no-op build miss.
Run the root suite with -tags metrics; passes 8/8 (27s, full package).
* revert(ci): drop -tags metrics test kludge
Prod builds cmd/replicate without -tags metrics (Dockerfile), so the
metric registry is no-op in production regardless; adding the tag only to
CI tests validated a mode prod never runs. The 8 TestDB_*Metrics failures
are a pre-existing main issue (luxfi/metric's no-op default), tracked
separately — not papered over with a test-only build tag. The essential
fix in this PR stands: luxfi/zap v0.8.1->v0.8.12 (re-published-tag go.sum
poison). One way, no kludge.
171 lines
5.7 KiB
Go
171 lines
5.7 KiB
Go
package replicate_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/hanzoai/ltx"
|
|
"github.com/luxfi/age"
|
|
|
|
"github.com/hanzoai/replicate"
|
|
"github.com/hanzoai/replicate/mock"
|
|
)
|
|
|
|
// ageIntro is the fixed header every age v1 ciphertext begins with. A plaintext
|
|
// Litestream LTX object begins with the "LTX1" magic instead; asserting on both
|
|
// proves the destination holds ciphertext, not clear-text WAL.
|
|
const ageIntro = "age-encryption.org/v1"
|
|
|
|
// TestWriteLTXFileFailsClosedWithoutRecipient is the core regression test for the
|
|
// fleet-wide plaintext exposure: with RequireEncryption set but no age recipient
|
|
// configured, WriteLTXFile MUST return ErrEncryptionRequired and MUST NOT hand
|
|
// any bytes to the underlying client. Previously it silently wrote plaintext.
|
|
func TestWriteLTXFileFailsClosedWithoutRecipient(t *testing.T) {
|
|
var writeCalled bool
|
|
client := &mock.ReplicaClient{
|
|
WriteLTXFileFunc: func(_ context.Context, _ int, _, _ ltx.TXID, r io.Reader) (*ltx.FileInfo, error) {
|
|
writeCalled = true
|
|
_, _ = io.Copy(io.Discard, r)
|
|
return <x.FileInfo{MinTXID: 1, MaxTXID: 1}, nil
|
|
},
|
|
}
|
|
|
|
r := replicate.NewReplica(nil)
|
|
r.Client = client
|
|
r.RequireEncryption = true
|
|
// No AgeRecipients — the misconfiguration under test.
|
|
|
|
_, err := r.WriteLTXFile(context.Background(), 0, 1, 1, strings.NewReader("money ledger row: debit 100.00"))
|
|
if !errors.Is(err, replicate.ErrEncryptionRequired) {
|
|
t.Fatalf("expected ErrEncryptionRequired, got %v", err)
|
|
}
|
|
if writeCalled {
|
|
t.Fatal("client.WriteLTXFile was called — plaintext would have reached the destination despite fail-closed policy")
|
|
}
|
|
}
|
|
|
|
// TestStartFailsClosedWithoutRecipient verifies a monitored replica refuses to
|
|
// begin the write loop when encryption is required but unconfigured, so a
|
|
// misconfigured sidecar crash-loops instead of streaming plaintext WAL.
|
|
func TestStartFailsClosedWithoutRecipient(t *testing.T) {
|
|
r := replicate.NewReplica(nil)
|
|
r.Client = &mock.ReplicaClient{}
|
|
r.MonitorEnabled = true
|
|
r.RequireEncryption = true
|
|
// No AgeRecipients.
|
|
|
|
if err := r.Start(context.Background()); !errors.Is(err, replicate.ErrEncryptionRequired) {
|
|
t.Fatalf("expected ErrEncryptionRequired from Start, got %v", err)
|
|
}
|
|
}
|
|
|
|
// TestCompactFailsClosedWithoutRecipient verifies the compaction write path also
|
|
// fails closed: it must not decrypt source files and re-emit a plaintext
|
|
// compacted result. The guard runs before any source listing, so the client is
|
|
// never touched.
|
|
func TestCompactFailsClosedWithoutRecipient(t *testing.T) {
|
|
var touched bool
|
|
client := &mock.ReplicaClient{
|
|
LTXFilesFunc: func(_ context.Context, _ int, _ ltx.TXID, _ bool) (ltx.FileIterator, error) {
|
|
touched = true
|
|
return nil, nil
|
|
},
|
|
WriteLTXFileFunc: func(_ context.Context, _ int, _, _ ltx.TXID, r io.Reader) (*ltx.FileInfo, error) {
|
|
touched = true
|
|
_, _ = io.Copy(io.Discard, r)
|
|
return <x.FileInfo{}, nil
|
|
},
|
|
}
|
|
|
|
c := replicate.NewCompactor(client, nil)
|
|
c.RequireEncryption = true
|
|
// No AgeRecipients.
|
|
|
|
if _, err := c.Compact(context.Background(), 1); !errors.Is(err, replicate.ErrEncryptionRequired) {
|
|
t.Fatalf("expected ErrEncryptionRequired from Compact, got %v", err)
|
|
}
|
|
if touched {
|
|
t.Fatal("compactor read/wrote via the client despite fail-closed policy")
|
|
}
|
|
}
|
|
|
|
// TestWriteLTXFileEncryptsWhenRequiredWithRecipient verifies the happy path:
|
|
// with RequireEncryption AND a recipient, the write succeeds, the stored object
|
|
// is age ciphertext (age header, not LTX1), and it round-trips back to the
|
|
// original plaintext through the decrypt path.
|
|
func TestWriteLTXFileEncryptsWhenRequiredWithRecipient(t *testing.T) {
|
|
identity, err := age.GenerateX25519Identity()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
var stored bytes.Buffer
|
|
client := &mock.ReplicaClient{
|
|
WriteLTXFileFunc: func(_ context.Context, _ int, _, _ ltx.TXID, r io.Reader) (*ltx.FileInfo, error) {
|
|
if _, err := io.Copy(&stored, r); err != nil {
|
|
return nil, err
|
|
}
|
|
return <x.FileInfo{MinTXID: 1, MaxTXID: 1}, nil
|
|
},
|
|
OpenLTXFileFunc: func(_ context.Context, _ int, _, _ ltx.TXID, _, _ int64) (io.ReadCloser, error) {
|
|
return io.NopCloser(bytes.NewReader(stored.Bytes())), nil
|
|
},
|
|
}
|
|
|
|
r := replicate.NewReplica(nil)
|
|
r.Client = client
|
|
r.RequireEncryption = true
|
|
r.AgeRecipients = []age.Recipient{identity.Recipient()}
|
|
r.AgeIdentities = []age.Identity{identity}
|
|
|
|
plaintext := []byte("money ledger row: debit 100.00")
|
|
if _, err := r.WriteLTXFile(context.Background(), 0, 1, 1, bytes.NewReader(plaintext)); err != nil {
|
|
t.Fatalf("WriteLTXFile: %v", err)
|
|
}
|
|
|
|
assertAgeCiphertext(t, stored.Bytes(), plaintext)
|
|
|
|
// Round-trip: decrypt path returns the original bytes.
|
|
rc, err := r.OpenLTXFile(context.Background(), 0, 1, 1, 0, 0)
|
|
if err != nil {
|
|
t.Fatalf("OpenLTXFile: %v", err)
|
|
}
|
|
defer rc.Close()
|
|
got, err := io.ReadAll(rc)
|
|
if err != nil {
|
|
t.Fatalf("ReadAll: %v", err)
|
|
}
|
|
if !bytes.Equal(got, plaintext) {
|
|
t.Fatalf("round-trip mismatch:\n want: %q\n got: %q", plaintext, got)
|
|
}
|
|
}
|
|
|
|
// assertAgeCiphertext asserts that b is age ciphertext and NOT a plaintext
|
|
// LTX/SQLite object that contains the source data.
|
|
func assertAgeCiphertext(t *testing.T, b, plaintext []byte) {
|
|
t.Helper()
|
|
if !bytes.HasPrefix(b, []byte(ageIntro)) {
|
|
t.Fatalf("stored object is not age ciphertext: header=%q", firstBytes(b, 24))
|
|
}
|
|
if bytes.HasPrefix(b, []byte(ltx.Magic)) {
|
|
t.Fatalf("stored object begins with LTX plaintext magic %q", ltx.Magic)
|
|
}
|
|
if bytes.HasPrefix(b, []byte("SQLite format 3\x00")) {
|
|
t.Fatal("stored object is a raw SQLite database")
|
|
}
|
|
if bytes.Contains(b, plaintext) {
|
|
t.Fatal("plaintext found inside stored object — encryption did not protect the data")
|
|
}
|
|
}
|
|
|
|
func firstBytes(b []byte, n int) []byte {
|
|
if len(b) < n {
|
|
return b
|
|
}
|
|
return b[:n]
|
|
}
|