mirror of
https://github.com/hanzoai/replicate.git
synced 2026-10-30 18:23:20 +00:00
zooqueenandHanzo Dev
8afa54d0fe
_replicate_seq and _replicate_lock were created once, in init(). We do not own the file we keep them in, and an application that reconciles its whole schema against that same file drops every table its schema does not declare -- `prisma db push`, `drizzle-kit push`, atlas. Ours are never declared. dataroom does exactly this at boot, and said so in its own log: You are about to drop the `_replicate_seq` table, which is not empty (1 rows) one second after we created it. The drop replicates like any other transaction and costs nothing; what it cost was our bookkeeping. Every checkpoint from then on released the read lock, checkpointed, and could not reacquire -- "no such table: _replicate_seq" -- and init() had already run, so nothing ever recreated it. 776 consecutive failures over three days with a dead replica and no other symptom. Assert the schema on every read-lock acquisition instead. That is the one place both call sites already pass through, it makes init's copy redundant rather than adding a second one, and it costs two no-op statements per checkpoint on the happy path. Co-authored-by: Hanzo Dev <[email protected]>
100 lines
3.1 KiB
Go
100 lines
3.1 KiB
Go
package replicate
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
_ "github.com/hanzoai/sqlite"
|
|
)
|
|
|
|
// TestDB_Checkpoint_AfterApplicationDropsOurTables reproduces the dataroom
|
|
// outage of 2026-08-02.
|
|
//
|
|
// dataroom boots `prisma db push --accept-data-loss`, which reconciles the
|
|
// whole file against the application's schema and drops every table that schema
|
|
// does not declare -- including _replicate_seq and _replicate_lock, which we
|
|
// had created one second earlier. The drop itself is harmless and replicates
|
|
// like any other transaction, but the next checkpoint releases the read lock,
|
|
// checkpoints, and then cannot reacquire:
|
|
//
|
|
// checkpoint: reacquire read lock: SQL logic error: no such table: _replicate_seq (1)
|
|
//
|
|
// init() had already run, so nothing recreated the table and every sync from
|
|
// then on failed the same way -- 776 consecutive errors over three days with a
|
|
// dead replica and no other symptom.
|
|
func TestDB_Checkpoint_AfterApplicationDropsOurTables(t *testing.T) {
|
|
dir := t.TempDir()
|
|
dbPath := filepath.Join(dir, "db")
|
|
|
|
db := NewDB(dbPath)
|
|
db.MonitorInterval = 0 // disable background goroutine
|
|
db.Replica = NewReplica(db)
|
|
db.Replica.Client = &testReplicaClient{dir: t.TempDir()}
|
|
db.Replica.MonitorEnabled = false
|
|
if err := db.Open(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() {
|
|
if err := db.Close(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}()
|
|
|
|
// The application, on its own connection, as dataroom's Next.js process is.
|
|
app, err := sql.Open("sqlite", dbPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer app.Close()
|
|
|
|
if _, err := app.Exec(`PRAGMA journal_mode = wal;`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := app.Exec(`CREATE TABLE t (id INT, data TEXT)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := app.Exec(`INSERT INTO t VALUES (1, 'before')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
ctx := context.Background()
|
|
if err := db.Sync(ctx); err != nil {
|
|
t.Fatalf("sync before drop: %v", err)
|
|
}
|
|
|
|
// `prisma db push` drops what its schema does not declare.
|
|
for _, table := range []string{"_replicate_seq", "_replicate_lock"} {
|
|
if _, err := app.Exec(`DROP TABLE ` + table); err != nil {
|
|
t.Fatalf("drop %s: %v", table, err)
|
|
}
|
|
}
|
|
|
|
// The checkpoint that used to wedge replication for good.
|
|
if err := db.Checkpoint(ctx, CheckpointModeRestart); err != nil {
|
|
t.Fatalf("checkpoint after application dropped our tables: %v", err)
|
|
}
|
|
|
|
// Both tables are back, so the next checkpoint works too.
|
|
for _, table := range []string{"_replicate_seq", "_replicate_lock"} {
|
|
var n int
|
|
if err := app.QueryRow(`SELECT COUNT(1) FROM sqlite_master WHERE type='table' AND name=?`, table).Scan(&n); err != nil {
|
|
t.Fatal(err)
|
|
} else if n != 1 {
|
|
t.Fatalf("%s was not recreated", table)
|
|
}
|
|
}
|
|
|
|
// And replication carries on: writes made after the drop still sync.
|
|
if _, err := app.Exec(`INSERT INTO t VALUES (2, 'after')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.Sync(ctx); err != nil {
|
|
t.Fatalf("sync after drop: %v", err)
|
|
}
|
|
if err := db.Checkpoint(ctx, CheckpointModeRestart); err != nil {
|
|
t.Fatalf("second checkpoint after drop: %v", err)
|
|
}
|
|
}
|