mirror of
https://github.com/hanzoai/cloud.git
synced 2026-11-17 16:36:24 +00:00
205 lines
8.6 KiB
Go
205 lines
8.6 KiB
Go
// Copyright 2025 Hanzo AI Inc.
|
|
// SPDX-License-Identifier: Apache-2.0 OR MIT
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// Package idem is exactly-once request execution over a per-org SQLite database:
|
|
// a request named by a caller-supplied idempotency key runs AT MOST ONCE, and any
|
|
// retry — including one re-routed to a different replica after a rolling upgrade —
|
|
// returns the first run's recorded result instead of executing the effect again.
|
|
//
|
|
// It is the request-layer generalization of visor's insert-once MeterLease
|
|
// (object/meter_lease.go): the key is the PRIMARY KEY of a row written IN THE SAME
|
|
// TRANSACTION as the effect, so the dedup record and the effect commit atomically —
|
|
// either both land or neither. There is no window in which the effect is applied
|
|
// but the key is unrecorded (which would allow a re-run), nor one in which the key
|
|
// is recorded but the effect is missing (which would wrongly suppress a run).
|
|
//
|
|
// # Exactly-once across a writer handoff
|
|
//
|
|
// The row lives in the per-org SQLite the HA substrate snapshots to the object
|
|
// store, so the dedup record SHIPS WITH THE DATA. A successor that hydrates the
|
|
// latest fenced snapshot before it serves (hydrate-before-write) therefore sees
|
|
// every acknowledged request's key and refuses to re-run it. Exactly-once across a
|
|
// handoff is two composed properties, neither sufficient alone:
|
|
//
|
|
// fenced ship-before-ack : a request is 'done' only once its effect is durably
|
|
// in the fenced object store; a FENCED ship means the
|
|
// request FAILED and the caller retries (never acks).
|
|
// dedup key in the WAL : the successor hydrates the shipped snapshot, so a
|
|
// retry of an acknowledged request finds its key present.
|
|
//
|
|
// This package owns only the first-writer-wins dedup + result recall. The fenced
|
|
// ship (github.com/hanzoai/vfs/replica.FencedStore) and the single-writer gate +
|
|
// hydrate (internal/org) are its composition partners, each in its own lane.
|
|
package idem
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// ErrAlreadyApplied reports that key was executed by a prior call. It is the
|
|
// 'fail if already done' signal: Once returns it (with the stored result) instead
|
|
// of running the effect a second time. Callers that want idempotent success return
|
|
// the result and swallow this; callers that want a hard 'duplicate' error surface
|
|
// it. errors.Is(err, ErrAlreadyApplied) distinguishes it from a real failure.
|
|
var ErrAlreadyApplied = errors.New("idem: request already applied")
|
|
|
|
// maxBusyRetries bounds retries when a concurrent writer holds the SQLite write
|
|
// lock longer than busy_timeout. The per-org DB opens with busy_timeout(10000), so
|
|
// this is a rarely-reached backstop, not the primary concurrency mechanism.
|
|
const maxBusyRetries = 5
|
|
|
|
// Once runs apply EXACTLY ONCE for key against db and records the outcome.
|
|
//
|
|
// On the FIRST call for key: it opens a transaction, inserts the key row, runs
|
|
// apply WITHIN that same transaction, stores apply's result bytes, and commits —
|
|
// so the dedup row and every effect apply performed (which apply MUST perform
|
|
// through the provided *sql.Tx, or atomicity is lost) land together. It returns
|
|
// (result, true, nil).
|
|
//
|
|
// On any LATER call for the same key: apply is NOT run; Once returns the stored
|
|
// result with applied=false and ErrAlreadyApplied wrapped in err. round is
|
|
// recorded alongside the row (the fencing round under which the request was first
|
|
// applied) for audit; it does not affect dedup.
|
|
func Once(ctx context.Context, db *sql.DB, key string, round uint64, apply func(context.Context, *sql.Tx) ([]byte, error)) (result []byte, applied bool, err error) {
|
|
if db == nil {
|
|
return nil, false, fmt.Errorf("idem: nil db")
|
|
}
|
|
if key == "" {
|
|
return nil, false, fmt.Errorf("idem: empty idempotency key")
|
|
}
|
|
if err := EnsureSchema(ctx, db); err != nil {
|
|
return nil, false, err
|
|
}
|
|
for attempt := 0; attempt < maxBusyRetries; attempt++ {
|
|
result, applied, err = once(ctx, db, key, round, apply)
|
|
if isBusy(err) {
|
|
continue // busy_timeout was exceeded under contention; retry the whole tx.
|
|
}
|
|
return result, applied, err
|
|
}
|
|
return nil, false, fmt.Errorf("idem: %q contended beyond %d retries", key, maxBusyRetries)
|
|
}
|
|
|
|
func once(ctx context.Context, db *sql.DB, key string, round uint64, apply func(context.Context, *sql.Tx) ([]byte, error)) ([]byte, bool, error) {
|
|
tx, err := db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
// The INSERT is the FIRST statement, so the transaction takes the write lock
|
|
// here; a concurrent Once for the same key either blocks (busy_timeout) then
|
|
// fails the unique key, or fails it outright — both routes converge on 'load
|
|
// the winner's result'.
|
|
_, err = tx.ExecContext(ctx,
|
|
`INSERT INTO idempotency(key, round, result, created_at) VALUES(?, ?, NULL, ?)`,
|
|
key, round, time.Now().UTC().Format(time.RFC3339Nano))
|
|
if err != nil {
|
|
_ = tx.Rollback()
|
|
if isBusy(err) {
|
|
return nil, false, err // surfaced to the retry loop.
|
|
}
|
|
// Any non-busy insert failure most likely means the key already exists
|
|
// (the winner committed). Confirm by loading it; a present row is a
|
|
// definitive duplicate, so we never misreport a genuine error as one.
|
|
if prior, ok, lerr := load(ctx, db, key); lerr != nil {
|
|
return nil, false, lerr
|
|
} else if ok {
|
|
return prior, false, ErrAlreadyApplied
|
|
}
|
|
return nil, false, fmt.Errorf("idem: insert %q: %w", key, err)
|
|
}
|
|
|
|
res, err := apply(ctx, tx)
|
|
if err != nil {
|
|
_ = tx.Rollback() // undoes the key row AND every effect apply wrote — retryable.
|
|
return nil, false, err
|
|
}
|
|
if _, err := tx.ExecContext(ctx, `UPDATE idempotency SET result = ? WHERE key = ?`, res, key); err != nil {
|
|
_ = tx.Rollback()
|
|
return nil, false, fmt.Errorf("idem: record result %q: %w", key, err)
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
if isBusy(err) {
|
|
return nil, false, err
|
|
}
|
|
// A commit that lost to a peer leaves the peer's row committed; treat a
|
|
// now-present row as the duplicate it is.
|
|
if prior, ok, lerr := load(ctx, db, key); lerr == nil && ok {
|
|
return prior, false, ErrAlreadyApplied
|
|
}
|
|
return nil, false, fmt.Errorf("idem: commit %q: %w", key, err)
|
|
}
|
|
return res, true, nil
|
|
}
|
|
|
|
// load reads a recorded result. ok=false means the key has never been applied.
|
|
func load(ctx context.Context, db *sql.DB, key string) (result []byte, ok bool, err error) {
|
|
var res []byte
|
|
err = db.QueryRowContext(ctx, `SELECT result FROM idempotency WHERE key = ?`, key).Scan(&res)
|
|
switch {
|
|
case errors.Is(err, sql.ErrNoRows):
|
|
return nil, false, nil
|
|
case err != nil:
|
|
return nil, false, fmt.Errorf("idem: load %q: %w", key, err)
|
|
}
|
|
return res, true, nil
|
|
}
|
|
|
|
// Applied reports whether key has already been executed, returning the stored
|
|
// result — the read-only 'fail if already done' check for a caller that wants to
|
|
// short-circuit before doing any work.
|
|
func Applied(ctx context.Context, db *sql.DB, key string) (result []byte, done bool, err error) {
|
|
if err := EnsureSchema(ctx, db); err != nil {
|
|
return nil, false, err
|
|
}
|
|
return load(ctx, db, key)
|
|
}
|
|
|
|
// EnsureSchema creates the idempotency table if absent. Safe to call repeatedly;
|
|
// Once calls it, so callers rarely need to.
|
|
func EnsureSchema(ctx context.Context, db *sql.DB) error {
|
|
_, err := db.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS idempotency(
|
|
key TEXT PRIMARY KEY,
|
|
round INTEGER NOT NULL,
|
|
result BLOB,
|
|
created_at TEXT NOT NULL
|
|
)`)
|
|
if err != nil {
|
|
return fmt.Errorf("idem: ensure schema: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Prune deletes dedup rows first applied before cutoff. Best-effort retention
|
|
// control: prune only keys older than the longest window in which a retry can
|
|
// still arrive, or a legitimately-retried old request would re-execute. Returns
|
|
// the number of rows removed.
|
|
func Prune(ctx context.Context, db *sql.DB, cutoff time.Time) (int64, error) {
|
|
res, err := db.ExecContext(ctx, `DELETE FROM idempotency WHERE created_at < ?`, cutoff.UTC().Format(time.RFC3339Nano))
|
|
if err != nil {
|
|
return 0, fmt.Errorf("idem: prune: %w", err)
|
|
}
|
|
n, _ := res.RowsAffected()
|
|
return n, nil
|
|
}
|
|
|
|
// isBusy reports whether err is a SQLite lock-contention error (busy_timeout
|
|
// exceeded). Matched by message so this package stays driver-agnostic (no
|
|
// modernc/mattn import): both drivers surface these exact substrings.
|
|
func isBusy(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
s := err.Error()
|
|
return strings.Contains(s, "database is locked") ||
|
|
strings.Contains(s, "database table is locked") ||
|
|
strings.Contains(s, "SQLITE_BUSY")
|
|
}
|