base/store/multitenant_test.go

package store_test

import (
	"context"
	"crypto/rand"
	"errors"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"sync"
	"testing"
	"time"

	"github.com/hanzoai/base/store"
	"github.com/hanzoai/base/tools/claims"
	"github.com/hanzoai/base/tools/filesystem"
)

// newTestStore constructs a store backed by a fileblob bucket in a temp
// directory. Returns the store and the path to the bucket so tests can peek
// at object keys directly.
func newTestStore(t *testing.T, opts ...func(*store.Options)) (*store.OrgStore, string) {
	t.Helper()
	bucketDir := filepath.Join(t.TempDir(), "bucket")
	cacheDir := filepath.Join(t.TempDir(), "cache")

	fs, err := filesystem.NewLocal(bucketDir)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { fs.Close() })

	kr, err := store.NewKeyring(newMemRoot(), fs)
	if err != nil {
		t.Fatal(err)
	}

	o := store.Options{
		ObjectStore: fs,
		Keys:        kr,
		CacheRoot:   cacheDir,
		LRUSize:     4, // tiny on purpose to exercise eviction
		// IdleTTL is long by default so standard tests don't race the
		// reaper while operating on a DB handle. Tests that need to
		// exercise reap behavior pass a small IdleTTL via the opts
		// variadic.
		IdleTTL:            5 * time.Second,
		CheckpointWrites:   2,
		CheckpointInterval: 10 * time.Millisecond,
	}
	for _, fn := range opts {
		fn(&o)
	}
	s, err := store.New(o)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() {
		_ = s.Close(context.Background())
	})
	return s, bucketDir
}

// ctxWithClaims builds a request context like claims.Inject would.
func ctxWithClaims(orgID, userID string, roles ...string) context.Context {
	r := httptest.NewRequest(http.MethodGet, "/", nil)
	r.Header.Set(claims.HeaderOrgID, orgID)
	r.Header.Set(claims.HeaderUserID, userID)
	if len(roles) > 0 {
		r.Header.Set(claims.HeaderRoles, joinRoles(roles))
	}

	var ctx context.Context
	claims.Inject(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ctx = r.Context()
	})).ServeHTTP(httptest.NewRecorder(), r)
	return ctx
}

func joinRoles(roles []string) string {
	out := ""
	for i, r := range roles {
		if i > 0 {
			out += ","
		}
		out += r
	}
	return out
}

func TestKey_ObjectKey(t *testing.T) {
	cases := []struct {
		k    store.Key
		want string
	}{
		{store.Key{OrgID: "acme", UserID: "alice", Scope: store.ScopeUser}, "acme/users/alice.db"},
		{store.Key{OrgID: "acme", Scope: store.ScopeOrg}, "acme/org.db"},
		{store.Key{OrgID: "acme", App: "fleet", Scope: store.ScopeApp}, "acme/apps/fleet.db"},
		{store.Key{OrgID: "acme", App: "fleet", Project: "east", Scope: store.ScopeProject}, "acme/apps/fleet/projects/east.db"},
	}
	for _, tc := range cases {
		if got := tc.k.ObjectKey(); got != tc.want {
			t.Errorf("ObjectKey(%v) = %q, want %q", tc.k, got, tc.want)
		}
	}
}

// TestKey_LocalPath verifies the on-disk cache path mirrors ObjectKey under
// the cache root for every tier — one layout, no divergence.
func TestKey_LocalPath(t *testing.T) {
	root := filepath.FromSlash("/data/cache")
	cases := []struct {
		k    store.Key
		want string
	}{
		{store.Key{OrgID: "acme", Scope: store.ScopeOrg}, filepath.Join(root, "acme", "org.db")},
		{store.Key{OrgID: "acme", App: "fleet", Scope: store.ScopeApp}, filepath.Join(root, "acme", "apps", "fleet.db")},
		{store.Key{OrgID: "acme", App: "fleet", Project: "east", Scope: store.ScopeProject}, filepath.Join(root, "acme", "apps", "fleet", "projects", "east.db")},
		{store.Key{OrgID: "acme", UserID: "alice", Scope: store.ScopeUser}, filepath.Join(root, "acme", "users", "alice.db")},
	}
	for _, tc := range cases {
		if got := tc.k.LocalPath(root); got != tc.want {
			t.Errorf("LocalPath(%s) = %q, want %q", tc.k, got, tc.want)
		}
	}
}

func TestKey_Valid_RejectsTraversal(t *testing.T) {
	bad := []store.Key{
		{OrgID: "../etc", UserID: "x", Scope: store.ScopeUser},
		{OrgID: "acme", UserID: "../passwd", Scope: store.ScopeUser},
		{OrgID: "UPPER", UserID: "x", Scope: store.ScopeUser},
		{OrgID: "", UserID: "x", Scope: store.ScopeUser},
		{OrgID: "acme", UserID: "", Scope: store.ScopeUser},
		{OrgID: "acme has space", UserID: "x", Scope: store.ScopeUser},
		// App / project tiers pass the SAME guard as org / user.
		{OrgID: "acme", App: "../etc", Scope: store.ScopeApp},
		{OrgID: "acme", App: "bad/slash", Scope: store.ScopeApp},
		{OrgID: "acme", App: "", Scope: store.ScopeApp},
		{OrgID: "acme", App: "fleet", Project: "../../secret", Scope: store.ScopeProject},
		{OrgID: "acme", App: "fleet", Project: "", Scope: store.ScopeProject},
		{OrgID: "acme", App: "", Project: "east", Scope: store.ScopeProject}, // app must also be valid
	}
	for _, k := range bad {
		if err := k.Valid(); err == nil {
			t.Errorf("expected Valid()=err for %+v", k)
		}
	}
}

func TestKey_Valid_AcceptsSlugs(t *testing.T) {
	good := []store.Key{
		{OrgID: "acme", UserID: "alice", Scope: store.ScopeUser},
		{OrgID: "org-123", UserID: "user_42", Scope: store.ScopeUser},
		{OrgID: "a", UserID: "b", Scope: store.ScopeUser},
		{OrgID: "acme", Scope: store.ScopeOrg},
		{OrgID: "acme", App: "fleet", Scope: store.ScopeApp},
		{OrgID: "acme", App: "fleet-1", Project: "east_2", Scope: store.ScopeProject},
	}
	for _, k := range good {
		if err := k.Valid(); err != nil {
			t.Errorf("Valid(%+v) = %v, want nil", k, err)
		}
	}
}

// TestHydrate_MissCreatesLocalAndOpens verifies that a first Get for a
// brand-new base creates the local file and opens a working SQLite.
func TestHydrate_MissCreatesLocalAndOpens(t *testing.T) {
	s, _ := newTestStore(t)
	ctx := ctxWithClaims("acme", "alice")

	db, err := s.ForCtx(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if _, err := db.NewQuery("CREATE TABLE t(id INTEGER PRIMARY KEY, v TEXT)").Execute(); err != nil {
		t.Fatal(err)
	}
	if _, err := db.NewQuery("INSERT INTO t(v) VALUES ('hello')").Execute(); err != nil {
		t.Fatal(err)
	}
	var v string
	if err := db.NewQuery("SELECT v FROM t LIMIT 1").Row(&v); err != nil {
		t.Fatal(err)
	}
	if v != "hello" {
		t.Fatalf("got %q", v)
	}
}

// TestCacheHit verifies that the second Get returns the same *dbx.DB pointer
// as the first (cache hit, no hydration).
func TestCacheHit(t *testing.T) {
	s, _ := newTestStore(t)
	ctx := ctxWithClaims("acme", "alice")

	db1, err := s.ForCtx(ctx)
	if err != nil {
		t.Fatal(err)
	}
	db2, err := s.ForCtx(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if db1 != db2 {
		t.Fatal("expected cache hit to return same DB pointer")
	}
}

// TestIsolationBetweenTenants verifies that two (org, user) pairs see
// independent SQLite databases.
func TestIsolationBetweenTenants(t *testing.T) {
	s, _ := newTestStore(t)

	alice := ctxWithClaims("acme", "alice")
	bob := ctxWithClaims("acme", "bob")
	carol := ctxWithClaims("globex", "carol")

	dbAlice, err := s.ForCtx(alice)
	if err != nil {
		t.Fatal(err)
	}
	dbBob, err := s.ForCtx(bob)
	if err != nil {
		t.Fatal(err)
	}
	dbCarol, err := s.ForCtx(carol)
	if err != nil {
		t.Fatal(err)
	}

	if _, err := dbAlice.NewQuery(`CREATE TABLE t(id INTEGER, v TEXT)`).Execute(); err != nil {
		t.Fatal(err)
	}
	if _, err := dbBob.NewQuery(`CREATE TABLE t(id INTEGER, v TEXT)`).Execute(); err != nil {
		t.Fatal(err)
	}
	if _, err := dbCarol.NewQuery(`CREATE TABLE t(id INTEGER, v TEXT)`).Execute(); err != nil {
		t.Fatal(err)
	}
	if _, err := dbAlice.NewQuery(`INSERT INTO t VALUES (1, 'from-alice')`).Execute(); err != nil {
		t.Fatal(err)
	}
	if _, err := dbBob.NewQuery(`INSERT INTO t VALUES (1, 'from-bob')`).Execute(); err != nil {
		t.Fatal(err)
	}

	var count int
	if err := dbCarol.NewQuery(`SELECT count(*) FROM t`).Row(&count); err != nil {
		t.Fatal(err)
	}
	if count != 0 {
		t.Fatalf("carol leaked data: count=%d", count)
	}

	var v string
	if err := dbAlice.NewQuery(`SELECT v FROM t WHERE id=1`).Row(&v); err != nil {
		t.Fatal(err)
	}
	if v != "from-alice" {
		t.Fatalf("alice saw %q, expected from-alice", v)
	}
}

// TestForCtx_GatewayBypass returns claims.ErrGatewayBypass when the context
// has no claims attached (identity stripped or never injected).
func TestForCtx_GatewayBypass(t *testing.T) {
	s, _ := newTestStore(t)
	// Bare context, no Inject.
	_, err := s.ForCtx(context.Background())
	if !errors.Is(err, claims.ErrGatewayBypass) {
		t.Fatalf("expected ErrGatewayBypass, got %v", err)
	}
}

// TestCheckpoint_WritesToObjectStorage ensures that explicit Checkpoint
// uploads the DB into the backing bucket.
func TestCheckpoint_WritesToObjectStorage(t *testing.T) {
	s, bucketDir := newTestStore(t)
	ctx := ctxWithClaims("acme", "alice")

	db, err := s.ForCtx(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if _, err := db.NewQuery("CREATE TABLE t(v TEXT)").Execute(); err != nil {
		t.Fatal(err)
	}
	if _, err := db.NewQuery("INSERT INTO t VALUES ('persist-me')").Execute(); err != nil {
		t.Fatal(err)
	}

	k := store.Key{OrgID: "acme", UserID: "alice", Scope: store.ScopeUser}
	if err := s.Checkpoint(ctx, k); err != nil {
		t.Fatalf("Checkpoint: %v", err)
	}

	// Object should exist in the bucket on disk.
	objPath := filepath.Join(bucketDir, "acme", "users", "alice.db")
	if !fileExists(t, objPath) {
		t.Fatalf("expected object at %s", objPath)
	}
}

// TestEvictionPersistsAndRehydrates: fill the LRU past its cap, confirm the
// coldest is evicted (and its state is recoverable via re-hydrate from the
// bucket).
func TestEvictionPersistsAndRehydrates(t *testing.T) {
	s, _ := newTestStore(t)

	// LRUSize = 4. Open 5 distinct bases; the first should be evicted.
	bases := []struct{ org, user string }{
		{"o1", "u1"}, {"o2", "u2"}, {"o3", "u3"}, {"o4", "u4"}, {"o5", "u5"},
	}
	for _, tn := range bases {
		ctx := ctxWithClaims(tn.org, tn.user)
		db, err := s.ForCtx(ctx)
		if err != nil {
			t.Fatal(err)
		}
		if _, err := db.NewQuery("CREATE TABLE t(v TEXT)").Execute(); err != nil {
			t.Fatal(err)
		}
		if _, err := db.NewQuery("INSERT INTO t VALUES ('" + tn.user + "')").Execute(); err != nil {
			t.Fatal(err)
		}
		// Force explicit checkpoint so the bucket has the data before LRU
		// eviction triggers best-effort flush.
		k := store.Key{OrgID: tn.org, UserID: tn.user, Scope: store.ScopeUser}
		if err := s.Checkpoint(ctx, k); err != nil {
			t.Fatalf("checkpoint %v: %v", k, err)
		}
	}

	// Re-hydrate the first (evicted) base from the bucket.
	ctx := ctxWithClaims("o1", "u1")
	db, err := s.ForCtx(ctx)
	if err != nil {
		t.Fatalf("rehydrate o1/u1: %v", err)
	}
	var v string
	if err := db.NewQuery("SELECT v FROM t").Row(&v); err != nil {
		t.Fatalf("read after rehydrate: %v", err)
	}
	if v != "u1" {
		t.Fatalf("expected u1, got %q", v)
	}
}

// TestClose_FlushesDirty: confirm that Close snapshots every resident handle
// to the bucket before returning.
func TestClose_FlushesDirty(t *testing.T) {
	s, bucketDir := newTestStore(t)

	for _, u := range []string{"a", "b", "c"} {
		ctx := ctxWithClaims("acme", u)
		db, err := s.ForCtx(ctx)
		if err != nil {
			t.Fatal(err)
		}
		if _, err := db.NewQuery("CREATE TABLE t(v TEXT)").Execute(); err != nil {
			t.Fatal(err)
		}
		if _, err := db.NewQuery("INSERT INTO t VALUES ('" + u + "')").Execute(); err != nil {
			t.Fatal(err)
		}
	}

	if err := s.Close(context.Background()); err != nil {
		t.Fatal(err)
	}

	for _, u := range []string{"a", "b", "c"} {
		obj := filepath.Join(bucketDir, "acme", "users", u+".db")
		if !fileExists(t, obj) {
			t.Errorf("expected flushed object for %s at %s", u, obj)
		}
	}
}

// TestConcurrentReaders_OnSameTenant: multiple goroutines hitting ForCtx on
// the same key hydrate exactly once.
func TestConcurrentReaders_OnSameTenant(t *testing.T) {
	s, _ := newTestStore(t)
	ctx := ctxWithClaims("acme", "alice")

	var wg sync.WaitGroup
	seen := make([]interface{}, 16)
	for i := range seen {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			db, err := s.ForCtx(ctx)
			if err != nil {
				t.Error(err)
				return
			}
			seen[i] = db
		}()
	}
	wg.Wait()
	for i := 1; i < len(seen); i++ {
		if seen[i] != seen[0] {
			t.Fatalf("goroutine %d got a different *dbx.DB", i)
		}
	}
}

// TestOrgScope: ForOrg resolves the org-wide DB (distinct from any user DB).
func TestOrgScope(t *testing.T) {
	s, _ := newTestStore(t)
	ctx := ctxWithClaims("acme", "alice")

	orgDB, err := s.ForOrg(ctx)
	if err != nil {
		t.Fatal(err)
	}
	userDB, err := s.ForCtx(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if orgDB == userDB {
		t.Fatal("ForOrg must not alias ForCtx — they are different DBs")
	}
}

// TestForApp_ResolvesAndFallsBack: with an app in context ForApp resolves a
// distinct app-tier DB; without an app it composably falls back to the org
// tier and persists at {org}/apps/{app}.db.
func TestForApp_ResolvesAndFallsBack(t *testing.T) {
	s, bucketDir := newTestStore(t)
	base := ctxWithClaims("acme", "alice")

	orgDB, err := s.ForOrg(base)
	if err != nil {
		t.Fatal(err)
	}
	// No app in context → app tier falls back to the org DB.
	fbDB, err := s.ForApp(base)
	if err != nil {
		t.Fatal(err)
	}
	if fbDB != orgDB {
		t.Fatal("ForApp with no app must fall back to the org DB")
	}
	// App in context → distinct app-tier DB.
	appDB, err := s.ForApp(store.WithApp(base, "fleet"))
	if err != nil {
		t.Fatal(err)
	}
	if appDB == orgDB {
		t.Fatal("ForApp must not alias ForOrg when an app is present")
	}
	if _, err := appDB.NewQuery("CREATE TABLE t(v TEXT)").Execute(); err != nil {
		t.Fatal(err)
	}
	k := store.Key{OrgID: "acme", App: "fleet", Scope: store.ScopeApp}
	if err := s.Checkpoint(base, k); err != nil {
		t.Fatal(err)
	}
	if !fileExists(t, filepath.Join(bucketDir, "acme", "apps", "fleet.db")) {
		t.Fatal("expected app-tier object at acme/apps/fleet.db")
	}
}

// TestForProject_ResolvesAndFallsBack: project resolves the deepest tier;
// absent project falls to app; absent app falls to org — composable.
func TestForProject_ResolvesAndFallsBack(t *testing.T) {
	s, bucketDir := newTestStore(t)
	base := ctxWithClaims("acme", "alice")

	orgDB, err := s.ForOrg(base)
	if err != nil {
		t.Fatal(err)
	}
	// No app → org tier.
	if db, err := s.ForProject(base); err != nil || db != orgDB {
		t.Fatalf("ForProject with no app must fall back to org DB (equal=%v err=%v)", db == orgDB, err)
	}
	// App only → app tier.
	appCtx := store.WithApp(base, "fleet")
	appDB, err := s.ForApp(appCtx)
	if err != nil {
		t.Fatal(err)
	}
	if db, err := s.ForProject(appCtx); err != nil || db != appDB {
		t.Fatalf("ForProject with app but no project must fall back to app DB (equal=%v err=%v)", db == appDB, err)
	}
	// App + project → distinct project tier.
	projCtx := store.WithProject(appCtx, "east")
	projDB, err := s.ForProject(projCtx)
	if err != nil {
		t.Fatal(err)
	}
	if projDB == appDB || projDB == orgDB {
		t.Fatal("ForProject must resolve a distinct project-tier DB")
	}
	if _, err := projDB.NewQuery("CREATE TABLE t(v TEXT)").Execute(); err != nil {
		t.Fatal(err)
	}
	k := store.Key{OrgID: "acme", App: "fleet", Project: "east", Scope: store.ScopeProject}
	if err := s.Checkpoint(projCtx, k); err != nil {
		t.Fatal(err)
	}
	if !fileExists(t, filepath.Join(bucketDir, "acme", "apps", "fleet", "projects", "east.db")) {
		t.Fatal("expected project-tier object at acme/apps/fleet/projects/east.db")
	}
}

// TestForApp_GatewayBypass / project variant: an app/project scope with no
// IAM identity still fails closed with ErrGatewayBypass.
func TestForApp_GatewayBypass(t *testing.T) {
	s, _ := newTestStore(t)
	if _, err := s.ForApp(store.WithApp(context.Background(), "fleet")); !errors.Is(err, claims.ErrGatewayBypass) {
		t.Fatalf("ForApp: expected ErrGatewayBypass, got %v", err)
	}
	ctx := store.WithProject(store.WithApp(context.Background(), "fleet"), "east")
	if _, err := s.ForProject(ctx); !errors.Is(err, claims.ErrGatewayBypass) {
		t.Fatalf("ForProject: expected ErrGatewayBypass, got %v", err)
	}
}

// TestForApp_RejectsTraversalApp: a hostile app slug injected into the
// request context is rejected by the store's Valid guard before any FS
// access — a crafted app can never escape cacheRoot.
func TestForApp_RejectsTraversalApp(t *testing.T) {
	s, _ := newTestStore(t)
	ctx := store.WithApp(ctxWithClaims("acme", "alice"), "../../../etc/passwd")
	if _, err := s.ForApp(ctx); err == nil {
		t.Fatal("expected ForApp to reject a traversal app slug")
	}
}

// TestForProject_RejectsTraversalProject: same guard on the project slug.
func TestForProject_RejectsTraversalProject(t *testing.T) {
	s, _ := newTestStore(t)
	ctx := store.WithProject(store.WithApp(ctxWithClaims("acme", "alice"), "fleet"), "../../secret")
	if _, err := s.ForProject(ctx); err == nil {
		t.Fatal("expected ForProject to reject a traversal project slug")
	}
}

// TestDoubleCloseIdempotent: calling Close twice is safe.
func TestDoubleCloseIdempotent(t *testing.T) {
	s, _ := newTestStore(t)
	if err := s.Close(context.Background()); err != nil {
		t.Fatal(err)
	}
	if err := s.Close(context.Background()); err != nil {
		t.Fatal(err)
	}
}

// ============================================================
// P7 — Red re-review coverage. Each test maps to one Red finding.
// ============================================================

// TestHydrate_RejectsPoisonedBlob (P7-C2) — a non-SQLite blob planted in
// the bucket is rejected on hydrate and the pure-Go driver never touches
// the bytes.
func TestHydrate_RejectsPoisonedBlob(t *testing.T) {
	s, bucketDir := newTestStore(t)

	// Plant a malicious blob at the expected object path.
	if err := os.MkdirAll(filepath.Join(bucketDir, "acme", "users"), 0o700); err != nil {
		t.Fatal(err)
	}
	poison := []byte("--evil DROP TABLE users; SELECT * FROM sqlite_master;--")
	if err := os.WriteFile(filepath.Join(bucketDir, "acme", "users", "alice.db"), poison, 0o600); err != nil {
		t.Fatal(err)
	}

	ctx := ctxWithClaims("acme", "alice")
	db, err := s.ForCtx(ctx)
	if !errors.Is(err, store.ErrCorruptDB) {
		t.Fatalf("ForCtx: got err=%v, want ErrCorruptDB (db=%v)", err, db)
	}
	if db != nil {
		t.Fatalf("ForCtx returned a non-nil db for a poisoned blob: %v", db)
	}
}

// TestValidateSQLiteMagic checks boundary cases on the magic-header check.
func TestValidateSQLiteMagic_AcceptsEmpty(t *testing.T) {
	// This exercises store.verifySQLiteFile indirectly through a fresh
	// base — empty local file must NOT be rejected (a new base's
	// localPath is 0 bytes on first hydrate).
	s, _ := newTestStore(t)
	ctx := ctxWithClaims("acme", "alice")
	if _, err := s.ForCtx(ctx); err != nil {
		t.Fatalf("fresh base (empty file) must not be rejected: %v", err)
	}
}

// TestEvict_SurfacesUploadError (P7-H1) — when object-store Upload fails,
// Evict returns an error wrapping ErrUploadFailed and the local data is
// retained (caller can retry, data not silently lost).
func TestEvict_SurfacesUploadError(t *testing.T) {
	s, bucketDir := newTestStore(t)
	ctx := ctxWithClaims("acme", "alice")

	db, err := s.ForCtx(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if _, err := db.NewQuery("CREATE TABLE t(v TEXT)").Execute(); err != nil {
		t.Fatal(err)
	}
	if _, err := db.NewQuery("INSERT INTO t VALUES ('keep-me')").Execute(); err != nil {
		t.Fatal(err)
	}

	// Break the bucket so Upload fails: fileblob has to create the
	// directory that holds the target, and a regular file sitting at that
	// path makes it ENOTDIR.
	poisonUploadPath(t, bucketDir, "acme", "users")

	k := store.Key{OrgID: "acme", UserID: "alice", Scope: store.ScopeUser}
	err = s.Evict(context.Background(), k)
	if !errors.Is(err, store.ErrUploadFailed) {
		t.Fatalf("Evict: err=%v, want ErrUploadFailed", err)
	}
}

// TestClose_AggregatesUploadFailures (P7-H1) — Close loudly surfaces how
// many bases failed to flush so ops can intervene.
func TestClose_AggregatesUploadFailures(t *testing.T) {
	s, bucketDir := newTestStore(t)

	for _, u := range []string{"alice", "bob", "carol"} {
		ctx := ctxWithClaims("acme", u)
		db, err := s.ForCtx(ctx)
		if err != nil {
			t.Fatal(err)
		}
		if _, err := db.NewQuery("CREATE TABLE t(v TEXT)").Execute(); err != nil {
			t.Fatal(err)
		}
		if _, err := db.NewQuery("INSERT INTO t VALUES ('data')").Execute(); err != nil {
			t.Fatal(err)
		}
	}

	// Poison every upload target.
	poisonUploadPath(t, bucketDir, "acme", "users")

	err := s.Close(context.Background())
	if err == nil {
		t.Fatalf("Close: expected aggregated upload-failure error, got nil")
	}
	if !errors.Is(err, store.ErrUploadFailed) {
		t.Fatalf("Close: err=%v, want Is(ErrUploadFailed)", err)
	}
}

// TestReaper_LockHoldTimeChunked (P7-H2) — with 100 resident handles, a
// Get call for a fresh base made while the reaper is flushing must
// return fast (under 500ms), proving the reaper releases s.mu per-handle.
func TestReaper_LockHoldTimeChunked(t *testing.T) {
	// Large LRU so filling 100 handles doesn't trigger cap eviction; tiny
	// idle TTL so every handle is eligible for reap.
	s, _ := newTestStore(t, func(o *store.Options) {
		o.LRUSize = 200
		o.IdleTTL = 1 * time.Millisecond
		o.CheckpointInterval = 10 * time.Minute // disable time-based flush
	})

	// Fill 100 handles.
	for i := 0; i < 100; i++ {
		ctx := ctxWithClaims("acme", "u"+itoa(i))
		if _, err := s.ForCtx(ctx); err != nil {
			t.Fatalf("warmup %d: %v", i, err)
		}
	}

	// Force every handle's lastAccess beyond the TTL cutoff.
	time.Sleep(5 * time.Millisecond)

	// Kick the reaper by sleeping past one tick interval. The default
	// reaperLoop tick is IdleTTL/2 = 500µs; definitely fired by 5ms.
	done := make(chan struct{})
	start := time.Now()
	go func() {
		// While the reaper is mid-flush, issue a cold Get for a NEW base.
		// Under the old (O(N)) reaper this Get blocks for the entire reap
		// cycle; under the chunked reaper it blocks at most one handle.
		ctx := ctxWithClaims("acme", "probe")
		_, _ = s.ForCtx(ctx)
		close(done)
	}()
	<-done
	elapsed := time.Since(start)
	if elapsed > 500*time.Millisecond {
		t.Fatalf("cold Get during reap blocked for %v (>500ms)", elapsed)
	}
}

// TestEvictionReturnsError_RecoversOnRetry (P7-H1) — after an upload-fail
// Evict, clearing the failure lets the next Evict succeed and drops the
// handle cleanly. No data is lost in the local cache until durable write
// is confirmed.
func TestEvictionReturnsError_RecoversOnRetry(t *testing.T) {
	s, bucketDir := newTestStore(t)
	ctx := ctxWithClaims("acme", "alice")

	db, err := s.ForCtx(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if _, err := db.NewQuery("CREATE TABLE t(v TEXT)").Execute(); err != nil {
		t.Fatal(err)
	}
	if _, err := db.NewQuery("INSERT INTO t VALUES ('x')").Execute(); err != nil {
		t.Fatal(err)
	}

	targetDir := filepath.Join(bucketDir, "acme", "users")
	clearPoison := poisonUploadPath(t, bucketDir, "acme", "users")

	k := store.Key{OrgID: "acme", UserID: "alice", Scope: store.ScopeUser}
	if err := s.Evict(context.Background(), k); !errors.Is(err, store.ErrUploadFailed) {
		t.Fatalf("first Evict should fail: %v", err)
	}

	clearPoison()

	// A retry succeeds; data is now durable.
	if err := s.Evict(context.Background(), k); err != nil {
		t.Fatalf("second Evict should succeed: %v", err)
	}
	if !fileExists(t, filepath.Join(targetDir, "alice.db")) {
		t.Fatalf("bucket should have the flushed DB now")
	}
}

// TestCAS_ConstantExposed (P7-C1 option B) — consumers that want
// generation-checked writes must feature-gate on store.CAS. In v1 it's
// false; a CAS-requiring service can refuse to boot.
func TestCAS_ConstantExposed(t *testing.T) {
	if store.CAS {
		t.Fatal("store.CAS must be false in v1 until ETag upload path lands")
	}
}

// TestKey_Valid_RejectsLongSlug (P7-M1/H additional) — slugs over the cap
// are rejected by validateSlug before any FS call. No log-injection
// surface through ENAMETOOLONG.
func TestKey_Valid_RejectsLongSlug(t *testing.T) {
	long := make([]byte, store.MaxSlugLen+1)
	for i := range long {
		long[i] = 'a'
	}
	k := store.Key{OrgID: string(long), UserID: "alice", Scope: store.ScopeUser}
	if err := k.Valid(); err == nil {
		t.Fatalf("expected Valid()=err for %d-byte slug", len(long))
	}
}

// helpers

// poisonUploadPath makes every upload under bucketDir/parts… fail, for any
// uid, by putting a regular FILE where fileblob needs a directory: its
// os.MkdirAll(filepath.Dir(path)) then returns ENOTDIR.
//
// The obvious alternative — chmod 0500 on the target directory — does
// NOTHING when the tests run as root, which is exactly how CI runs them (the
// runner's GOPATH is /root/go). Root bypasses the DAC write check, the upload
// succeeds, and the test fails asserting on an error that never came. ENOTDIR
// is enforced by the kernel against root as well, so the injected failure is
// the same one everywhere.
//
// Returns the clear func; calling it lets the next upload succeed.
func poisonUploadPath(t *testing.T, bucketDir string, parts ...string) (clear func()) {
	t.Helper()
	p := filepath.Join(append([]string{bucketDir}, parts...)...)
	if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil {
		t.Fatal(err)
	}
	if err := os.RemoveAll(p); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(p, []byte("not a directory"), 0o600); err != nil {
		t.Fatal(err)
	}
	return func() {
		t.Helper()
		if err := os.Remove(p); err != nil {
			t.Fatal(err)
		}
	}
}

func fileExists(t *testing.T, path string) bool {
	t.Helper()
	info, err := os.Stat(path)
	return err == nil && info != nil
}

// itoa is a tiny int→string for test IDs; avoids strconv just to keep
// the test file focused on the testing DSL.
func itoa(i int) string {
	if i == 0 {
		return "0"
	}
	var buf [20]byte
	pos := len(buf)
	for i > 0 {
		pos--
		buf[pos] = byte('0' + i%10)
		i /= 10
	}
	return string(buf[pos:])
}

// memRoot is an in-memory RootSource for tests: a stable random 32-byte KEK
// per org. It exercises the full envelope crypto path (HKDF + AES-256-GCM +
// age) without a live KMS. OrgRoot returns a COPY each call so the store may
// safely zero the returned KEK.
type memRoot struct {
	mu   sync.Mutex
	keys map[string][]byte
}

func newMemRoot() *memRoot { return &memRoot{keys: make(map[string][]byte)} }

func (m *memRoot) OrgRoot(_ context.Context, orgID string) ([]byte, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	k, ok := m.keys[orgID]
	if !ok {
		k = make([]byte, 32)
		if _, err := rand.Read(k); err != nil {
			return nil, err
		}
		m.keys[orgID] = k
	}
	out := make([]byte, len(k))
	copy(out, k)
	return out, nil
}

// TestGet_DeniesCrossTenant proves the IDOR guard: an authenticated caller
// cannot resolve a well-formed Key for another org.
func TestGet_DeniesCrossTenant(t *testing.T) {
	s, _ := newTestStore(t)
	ctx := ctxWithClaims("orga", "alice")

	if _, err := s.Get(ctx, store.Key{OrgID: "orga", UserID: "alice", Scope: store.ScopeUser}); err != nil {
		t.Fatalf("same-org Get must succeed: %v", err)
	}
	_, err := s.Get(ctx, store.Key{OrgID: "orgb", UserID: "bob", Scope: store.ScopeUser})
	if !errors.Is(err, store.ErrCrossOrg) {
		t.Fatalf("cross-base Get must be denied with ErrCrossOrg, got %v", err)
	}
}

// TestGet_ClaimlessInternalCallerAllowed proves trusted internal callers
// (no claims) can still resolve any org's Key (migrations, background jobs).
func TestGet_ClaimlessInternalCallerAllowed(t *testing.T) {
	s, _ := newTestStore(t)
	if _, err := s.Get(context.Background(), store.Key{OrgID: "orgb", UserID: "bob", Scope: store.ScopeUser}); err != nil {
		t.Fatalf("claim-less internal Get must succeed: %v", err)
	}
}