Files
ai/controllers/genai_event_test.go
zeekay ffcc671267 feat(genai): emit $ai_generation so LLM Analytics has something to read
LLM Analytics shows "No LLM generations yet" while the gateway serves real
traffic — 274 calls and 4.5M tokens yesterday alone. The product reads
$ai_generation events from the insights events plane, and nothing has ever
written one there.

The two facts we already emit go elsewhere: zapWriteUsage writes the money row
to cloud_usage, and emitGenAISpan ships a span to o11y. o11y is not a
substitute — its last gen_ai span is from March, so the tracing plane has been
silent for months and the dashboard reads a different plane regardless.

So this is the third consumer of the same call, emitted from recordTrace beside
the other two. One funnel, one attribution, three planes: ledger, trace,
analytics. Adding it at any of the ~15 completion surfaces instead would drift
the moment one grew a new path.

The vocabulary is the product's, not ours — $ai_model, $ai_provider, the token
counts, $ai_total_cost_usd, $ai_latency in seconds — so a customer using the SDK
and a customer served through this gateway land in the same views. Cost comes
from spanMoney, the same trio the ledger and span already report, so the three
carry one number each rather than three roundings of it.

Prompt and completion content is deliberately absent: the span path gates that
behind O11Y_GENAI_CAPTURE_MESSAGES, and an analytics event carrying it unasked
would put customer prompts in a second store under a different retention.

Off unless both INSIGHTS_INGEST_URL and INSIGHTS_INGEST_KEY are set, and never
blocking: the post is a detached goroutine on a 5s timeout with a context that
deliberately does not inherit the request's, since the request is over by then
and its cancellation would drop exactly the calls that succeeded.
2026-08-08 05:17:54 -07:00

119 lines
4.1 KiB
Go

package controllers
import (
"encoding/json"
"testing"
"time"
)
// The LLM Analytics product reads these exact names. A rename here is silent —
// the dashboard keeps rendering, with a column of blanks — so the names are the
// thing worth asserting, not the plumbing around them.
func TestGenAIEventSpeaksTheProductVocabulary(t *testing.T) {
record := &usageRecord{
Model: "claude-opus-5",
Provider: "anthropic",
PromptTokens: 1200,
CompletionTokens: 340,
TotalTokens: 1540,
CacheReadTokens: 800,
Status: "success",
RequestID: "req-abc",
Organization: "hanzo",
User: "z",
}
props := genAIEventProps(record, 2*time.Second)
for _, key := range []string{
"$ai_model", "$ai_provider", "$ai_input_tokens", "$ai_output_tokens",
"$ai_total_cost_usd", "$ai_latency", "$ai_trace_id", "$ai_is_error",
} {
if _, ok := props[key]; !ok {
t.Errorf("%s missing — the product reads it and would render a blank column", key)
}
}
if props["$ai_model"] != "claude-opus-5" || props["$ai_provider"] != "anthropic" {
t.Errorf("model/provider not carried: %v / %v", props["$ai_model"], props["$ai_provider"])
}
if props["$ai_cache_read_input_tokens"] != 800 {
t.Errorf("cache reads dropped: %v", props["$ai_cache_read_input_tokens"])
}
// Seconds, not milliseconds: the product renders this as a duration.
if props["$ai_latency"] != 2.0 {
t.Errorf("latency = %v, want 2 (seconds)", props["$ai_latency"])
}
if props["$ai_is_error"] != false {
t.Errorf("a successful call reported as an error")
}
}
// Prompts and completions are gated on the span path and must not ride along
// here — an analytics event is a different store under a different retention.
func TestGenAIEventCarriesNoPrompts(t *testing.T) {
props := genAIEventProps(&usageRecord{Model: "m", Status: "success"}, time.Second)
for _, key := range []string{"$ai_input", "$ai_output", "$ai_output_choices", "$ai_input_state"} {
if _, ok := props[key]; ok {
t.Errorf("%s present — prompt content must not ride the analytics event", key)
}
}
}
func TestGenAIEventErrorsAreMarked(t *testing.T) {
props := genAIEventProps(&usageRecord{Model: "m", Status: "error", ErrorMsg: "upstream 503"}, time.Second)
if props["$ai_is_error"] != true {
t.Error("a failed call must be marked, or error rate reads zero forever")
}
if props["$ai_error"] != "upstream 503" {
t.Errorf("error message dropped: %v", props["$ai_error"])
}
}
// distinct_id decides who the call is attributed to. A machine credential has no
// User, and an event with an empty distinct_id is attributed to nobody.
func TestGenAIEventAlwaysAttributes(t *testing.T) {
for _, tc := range []struct {
name string
record *usageRecord
want string
}{
{"user wins", &usageRecord{User: "z", Owner: "o", Organization: "hanzo"}, "z"},
{"owner when no user", &usageRecord{Owner: "o", Organization: "hanzo"}, "o"},
{"org as the last resort", &usageRecord{Organization: "hanzo"}, "hanzo"},
} {
t.Run(tc.name, func(t *testing.T) {
raw, err := genAIEventBody("hi_key", tc.record, time.Now(), time.Second)
if err != nil {
t.Fatal(err)
}
var body struct {
Event string `json:"event"`
DistinctID string `json:"distinct_id"`
APIKey string `json:"api_key"`
}
if err := json.Unmarshal(raw, &body); err != nil {
t.Fatal(err)
}
if body.DistinctID != tc.want {
t.Errorf("distinct_id = %q, want %q", body.DistinctID, tc.want)
}
if body.Event != "$ai_generation" {
t.Errorf("event = %q — the product only reads $ai_generation", body.Event)
}
if body.APIKey != "hi_key" {
t.Errorf("api_key = %q", body.APIKey)
}
})
}
}
// Unconfigured must be silent rather than posting to nowhere on every LLM call.
func TestGenAIEventOffWhenUnconfigured(t *testing.T) {
t.Setenv("INSIGHTS_INGEST_URL", "")
t.Setenv("INSIGHTS_INGEST_KEY", "")
url, _ := insightsIngest()
if url != "" {
t.Errorf("resolved a destination from nothing: %q", url)
}
emitGenAIEvent(t.Context(), &usageRecord{Model: "m"}, time.Now()) // must not panic
}