mirror of
https://github.com/hanzoai/o11y.git
synced 2026-10-15 12:58:33 +00:00
HIGH-1 amplification: cap events/envelope (1000); module.Ingest pre-aggregates occurrences by fingerprint and store.UpsertIssues writes the whole batch in ONE tx (a flood of identical events => one upsert, count+=N); per-org issue ceiling (10000) drops NEW fingerprints past the cap; per-org token-bucket rate limit (50/s burst 100) after DSN verify; retention/TTL sweep (DeleteStale) gated by O11Y_ERRORTRACKING_RETENTION_DAYS (default 90). MEDIUM-1: envelope item length is bounds-checked (>=0 && <= remaining) before it is added to pos — a MaxInt64/negative length no longer overflows the slice bound; falls back to newline framing. MEDIUM-2: default-on secret redaction (sk-/AKIA/bearer/JWT/conn-string/PAN, always) + PII scrub (email/IP) unless O11Y_ERRORTRACKING_CAPTURE_PII=true (fail-secure, mirrors llmobs capture-messages). Applied before the value enters the fingerprint. MEDIUM-3: DSN keys are versioned '<v>:<hmac>' with a per-org revocation watermark (o11y_ingest_revocations, wholesale-cached); rotate ONE org's min-version to revoke only its below-min DSNs — no global secret roll. LOW-1: lifecycle UpdateIssue is optimistic-concurrency guarded (version column, WHERE version=expected, bump on write; stale => 409 conflict, not clobber). Ingest never touches version. LOW-2: orgFromContext uses NewUUID+error (no MustNewUUID panic on a bad claim). INFO-1: corrected the occurrence-persistence claim + removed the dangling chsink.go reference (only NoopSink exists; logs sink is an honest fast-follow). Tests: 67 pass incl amplification-bounded (5000 events -> 1 upsert), event cap, per-org ceiling, MaxInt64/negative length no-panic, rate-limit, secret/PII scrub, versioned-key + isolated revocation, optimistic-update conflict, DeleteStale. Crown-jewel isolation/parity/DSN tests unchanged and still green.
200 lines
7.5 KiB
Go
200 lines
7.5 KiB
Go
package implerrortracking
|
|
|
|
import (
|
|
"context"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hanzoai/o11y/pkg/factory/factorytest"
|
|
"github.com/hanzoai/o11y/pkg/modules/errortracking"
|
|
"github.com/hanzoai/o11y/pkg/sqlstore"
|
|
"github.com/hanzoai/o11y/pkg/sqlstore/sqlitesqlstore"
|
|
"github.com/hanzoai/o11y/pkg/types/errortrackingtypes"
|
|
"github.com/hanzoai/o11y/pkg/valuer"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// newTestStore builds a real sqlite store with the o11y_issues table and its
|
|
// (org_id, fingerprint) unique index — the shape the migration ships, so ON CONFLICT
|
|
// upserts behave exactly as in production.
|
|
func newTestStore(t *testing.T) sqlstore.SQLStore {
|
|
t.Helper()
|
|
dbPath := filepath.Join(t.TempDir(), "test.db")
|
|
store, err := sqlitesqlstore.New(context.Background(), factorytest.NewSettings(), sqlstore.Config{
|
|
Provider: "sqlite",
|
|
Connection: sqlstore.ConnectionConfig{
|
|
MaxOpenConns: 1,
|
|
MaxConnLifetime: 0,
|
|
},
|
|
Sqlite: sqlstore.SqliteConfig{
|
|
Path: dbPath,
|
|
Mode: "wal",
|
|
BusyTimeout: 5 * time.Second,
|
|
TransactionMode: "deferred",
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
_, err = store.BunDB().NewCreateTable().
|
|
Model((*errortrackingtypes.Issue)(nil)).
|
|
IfNotExists().
|
|
Exec(context.Background())
|
|
require.NoError(t, err)
|
|
|
|
_, err = store.BunDB().Exec(`CREATE UNIQUE INDEX IF NOT EXISTS uq_o11y_issues_org_fingerprint ON o11y_issues (org_id, fingerprint)`)
|
|
require.NoError(t, err)
|
|
|
|
_, err = store.BunDB().NewCreateTable().
|
|
Model((*ingestRevocation)(nil)).
|
|
IfNotExists().
|
|
Exec(context.Background())
|
|
require.NoError(t, err)
|
|
return store
|
|
}
|
|
|
|
func newTestModule(t *testing.T) (errortracking.Module, valuer.UUID, valuer.UUID) {
|
|
t.Helper()
|
|
m := NewModule(NewStore(newTestStore(t)), NewNoopSink())
|
|
return m, valuer.GenerateUUID(), valuer.GenerateUUID()
|
|
}
|
|
|
|
func occ(fp, typ, val string, ts time.Time) *errortrackingtypes.Occurrence {
|
|
return &errortrackingtypes.Occurrence{
|
|
Fingerprint: fp,
|
|
Type: typ,
|
|
Value: val,
|
|
Level: "error",
|
|
Timestamp: ts,
|
|
EventID: "evt-" + val,
|
|
}
|
|
}
|
|
|
|
// mustIngest sends a batch (one or more occurrences) through the module.
|
|
func mustIngest(t *testing.T, m errortracking.Module, ctx context.Context, org valuer.UUID, occs ...*errortrackingtypes.Occurrence) {
|
|
t.Helper()
|
|
_, err := m.Ingest(ctx, org, occs)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
// TestStore_TwoOrgIsolation is the load-bearing tenancy proof: two orgs report the
|
|
// SAME fingerprint; each org sees ONLY its own issue, and neither can read/mutate
|
|
// the other's issue by id.
|
|
func TestStore_TwoOrgIsolation(t *testing.T) {
|
|
ctx := context.Background()
|
|
mod, orgA, orgB := newTestModule(t)
|
|
now := time.Now().UTC()
|
|
|
|
mustIngest(t, mod, ctx, orgA, occ("fp-shared", "TypeError", "from-A", now))
|
|
mustIngest(t, mod, ctx, orgB, occ("fp-shared", "TypeError", "from-B", now))
|
|
|
|
aList, aTotal, err := mod.ListIssues(ctx, orgA, &errortrackingtypes.IssuesQuery{})
|
|
require.NoError(t, err)
|
|
require.Equal(t, 1, aTotal)
|
|
require.Len(t, aList, 1)
|
|
assert.Equal(t, "from-A", aList[0].Value, "org A must see only its own occurrence value")
|
|
assert.Equal(t, orgA, aList[0].OrgID)
|
|
|
|
bList, bTotal, err := mod.ListIssues(ctx, orgB, &errortrackingtypes.IssuesQuery{})
|
|
require.NoError(t, err)
|
|
require.Equal(t, 1, bTotal)
|
|
require.Len(t, bList, 1)
|
|
assert.Equal(t, "from-B", bList[0].Value, "org B must see only its own occurrence value")
|
|
assert.Equal(t, orgB, bList[0].OrgID)
|
|
|
|
_, err = mod.GetIssue(ctx, orgB, aList[0].ID)
|
|
require.Error(t, err, "org B must NOT read org A's issue by id")
|
|
|
|
reopen := "resolved"
|
|
_, err = mod.UpdateIssue(ctx, orgB, aList[0].ID, &errortrackingtypes.UpdateIssue{Status: &reopen})
|
|
require.Error(t, err, "org B must NOT update org A's issue")
|
|
|
|
got, err := mod.GetIssue(ctx, orgA, aList[0].ID)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, errortrackingtypes.StatusUnresolved, got.Issue.Status)
|
|
}
|
|
|
|
// TestStore_UpsertGroupsByFingerprint proves the fingerprint bucket: repeated
|
|
// occurrences of the same (org, fingerprint) collapse into one issue with a running
|
|
// count and advancing last-seen, first-seen preserved.
|
|
func TestStore_UpsertGroupsByFingerprint(t *testing.T) {
|
|
ctx := context.Background()
|
|
mod, orgA, _ := newTestModule(t)
|
|
|
|
first := time.Now().UTC().Add(-time.Hour)
|
|
later := time.Now().UTC()
|
|
|
|
mustIngest(t, mod, ctx, orgA, occ("fp1", "ValueError", "boom", first))
|
|
mustIngest(t, mod, ctx, orgA, occ("fp1", "ValueError", "boom", later))
|
|
mustIngest(t, mod, ctx, orgA, occ("fp1", "ValueError", "boom", later))
|
|
|
|
list, total, err := mod.ListIssues(ctx, orgA, &errortrackingtypes.IssuesQuery{})
|
|
require.NoError(t, err)
|
|
require.Equal(t, 1, total, "same fingerprint must not create new issues")
|
|
require.Len(t, list, 1)
|
|
assert.Equal(t, int64(3), list[0].Count, "count must track occurrences")
|
|
assert.WithinDuration(t, first, list[0].FirstSeen, time.Second, "first-seen preserved")
|
|
assert.WithinDuration(t, later, list[0].LastSeen, time.Second, "last-seen advanced")
|
|
|
|
mustIngest(t, mod, ctx, orgA, occ("fp2", "KeyError", "missing", later))
|
|
_, total2, err := mod.ListIssues(ctx, orgA, &errortrackingtypes.IssuesQuery{})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 2, total2)
|
|
}
|
|
|
|
// TestStore_RegressionReopensResolved proves regression detection: a resolved issue
|
|
// that recurs flips back to unresolved and is flagged regressed; an ignored issue
|
|
// stays muted.
|
|
func TestStore_RegressionReopensResolved(t *testing.T) {
|
|
ctx := context.Background()
|
|
mod, orgA, _ := newTestModule(t)
|
|
now := time.Now().UTC()
|
|
|
|
mustIngest(t, mod, ctx, orgA, occ("fp-reg", "TypeError", "x", now))
|
|
list, _, err := mod.ListIssues(ctx, orgA, &errortrackingtypes.IssuesQuery{})
|
|
require.NoError(t, err)
|
|
id := list[0].ID
|
|
|
|
resolved := string(errortrackingtypes.StatusResolved)
|
|
updated, err := mod.UpdateIssue(ctx, orgA, id, &errortrackingtypes.UpdateIssue{Status: &resolved})
|
|
require.NoError(t, err)
|
|
require.Equal(t, errortrackingtypes.StatusResolved, updated.Status)
|
|
require.NotNil(t, updated.ResolvedAt)
|
|
|
|
mustIngest(t, mod, ctx, orgA, occ("fp-reg", "TypeError", "x", now.Add(time.Minute)))
|
|
got, err := mod.GetIssue(ctx, orgA, id)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, errortrackingtypes.StatusUnresolved, got.Issue.Status, "resolved issue must reopen on recurrence")
|
|
assert.True(t, got.Issue.Regressed, "reopened issue must be flagged a regression")
|
|
assert.Equal(t, int64(2), got.Issue.Count)
|
|
|
|
ignored := string(errortrackingtypes.StatusIgnored)
|
|
_, err = mod.UpdateIssue(ctx, orgA, id, &errortrackingtypes.UpdateIssue{Status: &ignored})
|
|
require.NoError(t, err)
|
|
mustIngest(t, mod, ctx, orgA, occ("fp-reg", "TypeError", "x", now.Add(2*time.Minute)))
|
|
got, err = mod.GetIssue(ctx, orgA, id)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, errortrackingtypes.StatusIgnored, got.Issue.Status, "ignored issue must stay muted")
|
|
}
|
|
|
|
// TestModule_GetIssueParsesLatestEvent proves the detail view is fully served from
|
|
// SQL (the stored sample), independent of any occurrence-store read path.
|
|
func TestModule_GetIssueParsesLatestEvent(t *testing.T) {
|
|
ctx := context.Background()
|
|
mod, orgA, _ := newTestModule(t)
|
|
now := time.Now().UTC()
|
|
|
|
o := occ("fp-detail", "RuntimeError", "kaput", now)
|
|
o.Culprit = "handler in server.go"
|
|
mustIngest(t, mod, ctx, orgA, o)
|
|
|
|
list, _, err := mod.ListIssues(ctx, orgA, &errortrackingtypes.IssuesQuery{})
|
|
require.NoError(t, err)
|
|
detail, err := mod.GetIssue(ctx, orgA, list[0].ID)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, detail.LatestEvent, "detail must carry the latest occurrence sample")
|
|
assert.Equal(t, "kaput", detail.LatestEvent.Value)
|
|
assert.Equal(t, "handler in server.go", detail.LatestEvent.Culprit)
|
|
}
|