mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
LLM Analytics shows "No LLM generations yet" while the gateway serves real traffic — 274 calls and 4.5M tokens yesterday alone. The product reads $ai_generation events from the insights events plane, and nothing has ever written one there. The two facts we already emit go elsewhere: zapWriteUsage writes the money row to cloud_usage, and emitGenAISpan ships a span to o11y. o11y is not a substitute — its last gen_ai span is from March, so the tracing plane has been silent for months and the dashboard reads a different plane regardless. So this is the third consumer of the same call, emitted from recordTrace beside the other two. One funnel, one attribution, three planes: ledger, trace, analytics. Adding it at any of the ~15 completion surfaces instead would drift the moment one grew a new path. The vocabulary is the product's, not ours — $ai_model, $ai_provider, the token counts, $ai_total_cost_usd, $ai_latency in seconds — so a customer using the SDK and a customer served through this gateway land in the same views. Cost comes from spanMoney, the same trio the ledger and span already report, so the three carry one number each rather than three roundings of it. Prompt and completion content is deliberately absent: the span path gates that behind O11Y_GENAI_CAPTURE_MESSAGES, and an analytics event carrying it unasked would put customer prompts in a second store under a different retention. Off unless both INSIGHTS_INGEST_URL and INSIGHTS_INGEST_KEY are set, and never blocking: the post is a detached goroutine on a 5s timeout with a context that deliberately does not inherit the request's, since the request is over by then and its cancellation would drop exactly the calls that succeeded.
119 lines
4.1 KiB
Go
119 lines
4.1 KiB
Go
package controllers
|
|
|
|
import (
|
|
"encoding/json"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// The LLM Analytics product reads these exact names. A rename here is silent —
|
|
// the dashboard keeps rendering, with a column of blanks — so the names are the
|
|
// thing worth asserting, not the plumbing around them.
|
|
func TestGenAIEventSpeaksTheProductVocabulary(t *testing.T) {
|
|
record := &usageRecord{
|
|
Model: "claude-opus-5",
|
|
Provider: "anthropic",
|
|
PromptTokens: 1200,
|
|
CompletionTokens: 340,
|
|
TotalTokens: 1540,
|
|
CacheReadTokens: 800,
|
|
Status: "success",
|
|
RequestID: "req-abc",
|
|
Organization: "hanzo",
|
|
User: "z",
|
|
}
|
|
props := genAIEventProps(record, 2*time.Second)
|
|
|
|
for _, key := range []string{
|
|
"$ai_model", "$ai_provider", "$ai_input_tokens", "$ai_output_tokens",
|
|
"$ai_total_cost_usd", "$ai_latency", "$ai_trace_id", "$ai_is_error",
|
|
} {
|
|
if _, ok := props[key]; !ok {
|
|
t.Errorf("%s missing — the product reads it and would render a blank column", key)
|
|
}
|
|
}
|
|
if props["$ai_model"] != "claude-opus-5" || props["$ai_provider"] != "anthropic" {
|
|
t.Errorf("model/provider not carried: %v / %v", props["$ai_model"], props["$ai_provider"])
|
|
}
|
|
if props["$ai_cache_read_input_tokens"] != 800 {
|
|
t.Errorf("cache reads dropped: %v", props["$ai_cache_read_input_tokens"])
|
|
}
|
|
// Seconds, not milliseconds: the product renders this as a duration.
|
|
if props["$ai_latency"] != 2.0 {
|
|
t.Errorf("latency = %v, want 2 (seconds)", props["$ai_latency"])
|
|
}
|
|
if props["$ai_is_error"] != false {
|
|
t.Errorf("a successful call reported as an error")
|
|
}
|
|
}
|
|
|
|
// Prompts and completions are gated on the span path and must not ride along
|
|
// here — an analytics event is a different store under a different retention.
|
|
func TestGenAIEventCarriesNoPrompts(t *testing.T) {
|
|
props := genAIEventProps(&usageRecord{Model: "m", Status: "success"}, time.Second)
|
|
for _, key := range []string{"$ai_input", "$ai_output", "$ai_output_choices", "$ai_input_state"} {
|
|
if _, ok := props[key]; ok {
|
|
t.Errorf("%s present — prompt content must not ride the analytics event", key)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGenAIEventErrorsAreMarked(t *testing.T) {
|
|
props := genAIEventProps(&usageRecord{Model: "m", Status: "error", ErrorMsg: "upstream 503"}, time.Second)
|
|
if props["$ai_is_error"] != true {
|
|
t.Error("a failed call must be marked, or error rate reads zero forever")
|
|
}
|
|
if props["$ai_error"] != "upstream 503" {
|
|
t.Errorf("error message dropped: %v", props["$ai_error"])
|
|
}
|
|
}
|
|
|
|
// distinct_id decides who the call is attributed to. A machine credential has no
|
|
// User, and an event with an empty distinct_id is attributed to nobody.
|
|
func TestGenAIEventAlwaysAttributes(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
record *usageRecord
|
|
want string
|
|
}{
|
|
{"user wins", &usageRecord{User: "z", Owner: "o", Organization: "hanzo"}, "z"},
|
|
{"owner when no user", &usageRecord{Owner: "o", Organization: "hanzo"}, "o"},
|
|
{"org as the last resort", &usageRecord{Organization: "hanzo"}, "hanzo"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
raw, err := genAIEventBody("hi_key", tc.record, time.Now(), time.Second)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var body struct {
|
|
Event string `json:"event"`
|
|
DistinctID string `json:"distinct_id"`
|
|
APIKey string `json:"api_key"`
|
|
}
|
|
if err := json.Unmarshal(raw, &body); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if body.DistinctID != tc.want {
|
|
t.Errorf("distinct_id = %q, want %q", body.DistinctID, tc.want)
|
|
}
|
|
if body.Event != "$ai_generation" {
|
|
t.Errorf("event = %q — the product only reads $ai_generation", body.Event)
|
|
}
|
|
if body.APIKey != "hi_key" {
|
|
t.Errorf("api_key = %q", body.APIKey)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Unconfigured must be silent rather than posting to nowhere on every LLM call.
|
|
func TestGenAIEventOffWhenUnconfigured(t *testing.T) {
|
|
t.Setenv("INSIGHTS_INGEST_URL", "")
|
|
t.Setenv("INSIGHTS_INGEST_KEY", "")
|
|
url, _ := insightsIngest()
|
|
if url != "" {
|
|
t.Errorf("resolved a destination from nothing: %q", url)
|
|
}
|
|
emitGenAIEvent(t.Context(), &usageRecord{Model: "m"}, time.Now()) // must not panic
|
|
}
|