Files
zeekay bc2c3be4a4 fix(errortracking): harden ingest per red review (HIGH-1/2, MED-1/2/3, LOW-1/2)
HIGH-1 amplification: cap events/envelope (1000); module.Ingest pre-aggregates
occurrences by fingerprint and store.UpsertIssues writes the whole batch in ONE
tx (a flood of identical events => one upsert, count+=N); per-org issue ceiling
(10000) drops NEW fingerprints past the cap; per-org token-bucket rate limit
(50/s burst 100) after DSN verify; retention/TTL sweep (DeleteStale) gated by
O11Y_ERRORTRACKING_RETENTION_DAYS (default 90).

MEDIUM-1: envelope item length is bounds-checked (>=0 && <= remaining) before it
is added to pos — a MaxInt64/negative length no longer overflows the slice bound;
falls back to newline framing.

MEDIUM-2: default-on secret redaction (sk-/AKIA/bearer/JWT/conn-string/PAN, always)
+ PII scrub (email/IP) unless O11Y_ERRORTRACKING_CAPTURE_PII=true (fail-secure,
mirrors llmobs capture-messages). Applied before the value enters the fingerprint.

MEDIUM-3: DSN keys are versioned '<v>:<hmac>' with a per-org revocation watermark
(o11y_ingest_revocations, wholesale-cached); rotate ONE org's min-version to revoke
only its below-min DSNs — no global secret roll.

LOW-1: lifecycle UpdateIssue is optimistic-concurrency guarded (version column,
WHERE version=expected, bump on write; stale => 409 conflict, not clobber). Ingest
never touches version.
LOW-2: orgFromContext uses NewUUID+error (no MustNewUUID panic on a bad claim).
INFO-1: corrected the occurrence-persistence claim + removed the dangling chsink.go
reference (only NoopSink exists; logs sink is an honest fast-follow).

Tests: 67 pass incl amplification-bounded (5000 events -> 1 upsert), event cap,
per-org ceiling, MaxInt64/negative length no-panic, rate-limit, secret/PII scrub,
versioned-key + isolated revocation, optimistic-update conflict, DeleteStale.
Crown-jewel isolation/parity/DSN tests unchanged and still green.
2026-07-10 14:54:25 -07:00

200 lines
7.5 KiB
Go

package implerrortracking
import (
"context"
"path/filepath"
"testing"
"time"
"github.com/hanzoai/o11y/pkg/factory/factorytest"
"github.com/hanzoai/o11y/pkg/modules/errortracking"
"github.com/hanzoai/o11y/pkg/sqlstore"
"github.com/hanzoai/o11y/pkg/sqlstore/sqlitesqlstore"
"github.com/hanzoai/o11y/pkg/types/errortrackingtypes"
"github.com/hanzoai/o11y/pkg/valuer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// newTestStore builds a real sqlite store with the o11y_issues table and its
// (org_id, fingerprint) unique index — the shape the migration ships, so ON CONFLICT
// upserts behave exactly as in production.
func newTestStore(t *testing.T) sqlstore.SQLStore {
t.Helper()
dbPath := filepath.Join(t.TempDir(), "test.db")
store, err := sqlitesqlstore.New(context.Background(), factorytest.NewSettings(), sqlstore.Config{
Provider: "sqlite",
Connection: sqlstore.ConnectionConfig{
MaxOpenConns: 1,
MaxConnLifetime: 0,
},
Sqlite: sqlstore.SqliteConfig{
Path: dbPath,
Mode: "wal",
BusyTimeout: 5 * time.Second,
TransactionMode: "deferred",
},
})
require.NoError(t, err)
_, err = store.BunDB().NewCreateTable().
Model((*errortrackingtypes.Issue)(nil)).
IfNotExists().
Exec(context.Background())
require.NoError(t, err)
_, err = store.BunDB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS uq_o11y_issues_org_fingerprint ON o11y_issues (org_id, fingerprint)`)
require.NoError(t, err)
_, err = store.BunDB().NewCreateTable().
Model((*ingestRevocation)(nil)).
IfNotExists().
Exec(context.Background())
require.NoError(t, err)
return store
}
func newTestModule(t *testing.T) (errortracking.Module, valuer.UUID, valuer.UUID) {
t.Helper()
m := NewModule(NewStore(newTestStore(t)), NewNoopSink())
return m, valuer.GenerateUUID(), valuer.GenerateUUID()
}
func occ(fp, typ, val string, ts time.Time) *errortrackingtypes.Occurrence {
return &errortrackingtypes.Occurrence{
Fingerprint: fp,
Type: typ,
Value: val,
Level: "error",
Timestamp: ts,
EventID: "evt-" + val,
}
}
// mustIngest sends a batch (one or more occurrences) through the module.
func mustIngest(t *testing.T, m errortracking.Module, ctx context.Context, org valuer.UUID, occs ...*errortrackingtypes.Occurrence) {
t.Helper()
_, err := m.Ingest(ctx, org, occs)
require.NoError(t, err)
}
// TestStore_TwoOrgIsolation is the load-bearing tenancy proof: two orgs report the
// SAME fingerprint; each org sees ONLY its own issue, and neither can read/mutate
// the other's issue by id.
func TestStore_TwoOrgIsolation(t *testing.T) {
ctx := context.Background()
mod, orgA, orgB := newTestModule(t)
now := time.Now().UTC()
mustIngest(t, mod, ctx, orgA, occ("fp-shared", "TypeError", "from-A", now))
mustIngest(t, mod, ctx, orgB, occ("fp-shared", "TypeError", "from-B", now))
aList, aTotal, err := mod.ListIssues(ctx, orgA, &errortrackingtypes.IssuesQuery{})
require.NoError(t, err)
require.Equal(t, 1, aTotal)
require.Len(t, aList, 1)
assert.Equal(t, "from-A", aList[0].Value, "org A must see only its own occurrence value")
assert.Equal(t, orgA, aList[0].OrgID)
bList, bTotal, err := mod.ListIssues(ctx, orgB, &errortrackingtypes.IssuesQuery{})
require.NoError(t, err)
require.Equal(t, 1, bTotal)
require.Len(t, bList, 1)
assert.Equal(t, "from-B", bList[0].Value, "org B must see only its own occurrence value")
assert.Equal(t, orgB, bList[0].OrgID)
_, err = mod.GetIssue(ctx, orgB, aList[0].ID)
require.Error(t, err, "org B must NOT read org A's issue by id")
reopen := "resolved"
_, err = mod.UpdateIssue(ctx, orgB, aList[0].ID, &errortrackingtypes.UpdateIssue{Status: &reopen})
require.Error(t, err, "org B must NOT update org A's issue")
got, err := mod.GetIssue(ctx, orgA, aList[0].ID)
require.NoError(t, err)
assert.Equal(t, errortrackingtypes.StatusUnresolved, got.Issue.Status)
}
// TestStore_UpsertGroupsByFingerprint proves the fingerprint bucket: repeated
// occurrences of the same (org, fingerprint) collapse into one issue with a running
// count and advancing last-seen, first-seen preserved.
func TestStore_UpsertGroupsByFingerprint(t *testing.T) {
ctx := context.Background()
mod, orgA, _ := newTestModule(t)
first := time.Now().UTC().Add(-time.Hour)
later := time.Now().UTC()
mustIngest(t, mod, ctx, orgA, occ("fp1", "ValueError", "boom", first))
mustIngest(t, mod, ctx, orgA, occ("fp1", "ValueError", "boom", later))
mustIngest(t, mod, ctx, orgA, occ("fp1", "ValueError", "boom", later))
list, total, err := mod.ListIssues(ctx, orgA, &errortrackingtypes.IssuesQuery{})
require.NoError(t, err)
require.Equal(t, 1, total, "same fingerprint must not create new issues")
require.Len(t, list, 1)
assert.Equal(t, int64(3), list[0].Count, "count must track occurrences")
assert.WithinDuration(t, first, list[0].FirstSeen, time.Second, "first-seen preserved")
assert.WithinDuration(t, later, list[0].LastSeen, time.Second, "last-seen advanced")
mustIngest(t, mod, ctx, orgA, occ("fp2", "KeyError", "missing", later))
_, total2, err := mod.ListIssues(ctx, orgA, &errortrackingtypes.IssuesQuery{})
require.NoError(t, err)
assert.Equal(t, 2, total2)
}
// TestStore_RegressionReopensResolved proves regression detection: a resolved issue
// that recurs flips back to unresolved and is flagged regressed; an ignored issue
// stays muted.
func TestStore_RegressionReopensResolved(t *testing.T) {
ctx := context.Background()
mod, orgA, _ := newTestModule(t)
now := time.Now().UTC()
mustIngest(t, mod, ctx, orgA, occ("fp-reg", "TypeError", "x", now))
list, _, err := mod.ListIssues(ctx, orgA, &errortrackingtypes.IssuesQuery{})
require.NoError(t, err)
id := list[0].ID
resolved := string(errortrackingtypes.StatusResolved)
updated, err := mod.UpdateIssue(ctx, orgA, id, &errortrackingtypes.UpdateIssue{Status: &resolved})
require.NoError(t, err)
require.Equal(t, errortrackingtypes.StatusResolved, updated.Status)
require.NotNil(t, updated.ResolvedAt)
mustIngest(t, mod, ctx, orgA, occ("fp-reg", "TypeError", "x", now.Add(time.Minute)))
got, err := mod.GetIssue(ctx, orgA, id)
require.NoError(t, err)
assert.Equal(t, errortrackingtypes.StatusUnresolved, got.Issue.Status, "resolved issue must reopen on recurrence")
assert.True(t, got.Issue.Regressed, "reopened issue must be flagged a regression")
assert.Equal(t, int64(2), got.Issue.Count)
ignored := string(errortrackingtypes.StatusIgnored)
_, err = mod.UpdateIssue(ctx, orgA, id, &errortrackingtypes.UpdateIssue{Status: &ignored})
require.NoError(t, err)
mustIngest(t, mod, ctx, orgA, occ("fp-reg", "TypeError", "x", now.Add(2*time.Minute)))
got, err = mod.GetIssue(ctx, orgA, id)
require.NoError(t, err)
assert.Equal(t, errortrackingtypes.StatusIgnored, got.Issue.Status, "ignored issue must stay muted")
}
// TestModule_GetIssueParsesLatestEvent proves the detail view is fully served from
// SQL (the stored sample), independent of any occurrence-store read path.
func TestModule_GetIssueParsesLatestEvent(t *testing.T) {
ctx := context.Background()
mod, orgA, _ := newTestModule(t)
now := time.Now().UTC()
o := occ("fp-detail", "RuntimeError", "kaput", now)
o.Culprit = "handler in server.go"
mustIngest(t, mod, ctx, orgA, o)
list, _, err := mod.ListIssues(ctx, orgA, &errortrackingtypes.IssuesQuery{})
require.NoError(t, err)
detail, err := mod.GetIssue(ctx, orgA, list[0].ID)
require.NoError(t, err)
require.NotNil(t, detail.LatestEvent, "detail must carry the latest occurrence sample")
assert.Equal(t, "kaput", detail.LatestEvent.Value)
assert.Equal(t, "handler in server.go", detail.LatestEvent.Culprit)
}