Files
ai/controllers/telemetry_test.go
zeekay a3cd3b4309 fee: a call bills what its arms cost and the platform's fee; Enso is quoted from, typical and up to
A call through the gateway bills the price of the arm that answered and the
platform's fee on top, and nothing else: 5.5% on Hanzo's accounts, 1% on the
org's own key, both the host's settings (object.Fee: hanzo, hanzo_byok). A
vendor's fee is part of the cost: a call OpenRouter's credit bought cost its
stated price plus 5.5%, and the platform's fee goes on that.

- Enso answers bill their cost of record (X-Hanzo-Cogs and the buying
  vendor's fee) times the platform's fee; a maker's answer the same; a resold
  OpenRouter SKU its vendor price times both fees. An answer that states no
  cost bills the SKU's listed ceiling. A call billed below its cost writes
  loss_nano and a warning.
- hanzo.cloud_usage gains reasoning, reasoning_tokens, served, vendor, arms
  and loss_nano: each Enso row names the arm that answered, its rate at the
  time and the tokens it was paid for, and each arm that failed first.
- rates.go prices every Enso SKU at each depth: from (its cheapest arm), up to
  (its dearest, a long prompt included), and typical (its measured cost a
  token over the last day, or its last 50 calls), the arms read from the
  engine's admin catalog. /v1/models lists the typical rate with from, up_to,
  fee and window beside it; Prices, Price and Blend serve the rest. The
  board's ledger counts answers, failovers, OpenRouter-bought answers, fee
  and losses by model and vendor.
2026-10-15 12:33:32 -07:00

402 lines
16 KiB
Go

// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
import (
"testing"
"time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
)
// usdPtr is the *float64 the span margin now takes: presence, so an uncomputable
// margin can be absent rather than reported as zero.
//
//go:fix inline
func usdPtr(v float64) *float64 { return new(v) }
func attrMap(attrs []attribute.KeyValue) map[string]attribute.Value {
m := make(map[string]attribute.Value, len(attrs))
for _, a := range attrs {
m[string(a.Key)] = a.Value
}
return m
}
// TestBuildGenAISpanFields_ChatSuccess pins the OTel GenAI contract with o11y:
// span name and every gen_ai.*/attribution/cost/token attribute for a successful
// token-billed chat call, including the cache tokens, per-component cost detail,
// session grouping, and served-by/route attribution.
func TestBuildGenAISpanFields_ChatSuccess(t *testing.T) {
rec := &usageRecord{
Owner: "acme", Organization: "acme", User: "acme/alice",
Model: "zen5", Provider: "digitalocean",
PromptTokens: 120, CompletionTokens: 45, TotalTokens: 165,
CacheReadTokens: 30, CacheWriteTokens: 10,
Status: "success", RequestID: "req-abc",
Session: "conv-42", ServedBy: "byo-gpu", ClusterID: "cl-1", RoutePolicy: "enso:cheap",
}
br := &costBreakdown{Input: 0.0012, Output: 0.0009, CacheRead: 0.00003, CacheWrite: 0.0001, Total: 0.00223}
// Hanzo-served (not BYO): billed == total. Provider COGS 0.0015 < billed ⇒ margin 0.0007.
f := buildGenAISpanFields(rec, 0.0022, 0.0022, new(0.0015), new(0.0007), br, false)
if f.name != "chat zen5" {
t.Fatalf("span name = %q, want %q", f.name, "chat zen5")
}
if f.statusCode != codes.Ok {
t.Errorf("status = %v, want Ok", f.statusCode)
}
m := attrMap(f.attrs)
wantStr := map[string]string{
"gen_ai.system": "hanzo",
"gen_ai.operation.name": "chat",
"gen_ai.request.model": "zen5",
"gen_ai.response.model": "zen5",
"user.id": "acme/alice",
"gen_ai.hanzo.org_id": "acme",
"gen_ai.hanzo.provider": "digitalocean",
"gen_ai.hanzo.request_id": "req-abc",
"session.id": "conv-42",
"gen_ai.conversation.id": "conv-42",
"gen_ai.hanzo.served_by": "byo-gpu",
"gen_ai.hanzo.cluster_id": "cl-1",
"gen_ai.hanzo.route_policy": "enso:cheap",
}
for k, want := range wantStr {
got, ok := m[k]
if !ok {
t.Errorf("missing attribute %q", k)
continue
}
if got.AsString() != want {
t.Errorf("attr %q = %q, want %q", k, got.AsString(), want)
}
}
wantInt := map[string]int64{
"gen_ai.usage.input_tokens": 120,
"gen_ai.usage.output_tokens": 45,
"gen_ai.usage.total_tokens": 165,
"gen_ai.usage.cache_read.input_tokens": 30,
"gen_ai.usage.cache_creation.input_tokens": 10,
}
for k, want := range wantInt {
if got := m[k].AsInt64(); got != want {
t.Errorf("attr %q = %d, want %d", k, got, want)
}
}
wantFloat := map[string]float64{
"_o11y.gen_ai.total_cost": 0.0022,
"_o11y.gen_ai.billed_cost": 0.0022,
"_o11y.gen_ai.provider_cost": 0.0015,
"_o11y.gen_ai.margin": 0.0007,
"_o11y.gen_ai.cost_input": 0.0012,
"_o11y.gen_ai.cost_output": 0.0009,
"_o11y.gen_ai.cost_cache_read": 0.00003,
"_o11y.gen_ai.cost_cache_write": 0.0001,
}
for k, want := range wantFloat {
got, ok := m[k]
if !ok {
t.Errorf("missing cost attribute %q", k)
continue
}
if got.AsFloat64() != want {
t.Errorf("attr %q = %v, want %v", k, got.AsFloat64(), want)
}
}
}
// TestBuildGenAISpanFields_ErrorAndFallbacks covers the error status, the
// org=Owner fallback when Organization is empty, the "unknown" model fallback, the
// served_by default (hanzo), and omission of every empty optional attribute
// (provider, request_id, session, cluster, route, cache tokens, cost breakdown).
func TestBuildGenAISpanFields_ErrorAndFallbacks(t *testing.T) {
rec := &usageRecord{
Owner: "acme", User: "acme/bob",
Model: "", Status: "error", ErrorMsg: "upstream 500",
}
f := buildGenAISpanFields(rec, 0, 0, new(float64(0)), new(float64(0)), nil, false)
if f.name != "chat unknown" {
t.Fatalf("span name = %q, want %q", f.name, "chat unknown")
}
if f.statusCode != codes.Error || f.statusMsg != "upstream 500" {
t.Errorf("status = %v %q, want Error \"upstream 500\"", f.statusCode, f.statusMsg)
}
m := attrMap(f.attrs)
if got := m["gen_ai.hanzo.org_id"].AsString(); got != "acme" {
t.Errorf("gen_ai.hanzo.org_id = %q, want \"acme\" (Owner fallback)", got)
}
// served_by is always present, defaulting to hanzo cloud.
if got := m["gen_ai.hanzo.served_by"].AsString(); got != "hanzo" {
t.Errorf("gen_ai.hanzo.served_by = %q, want \"hanzo\" (default)", got)
}
// total_cost / provider_cost / margin are always present (the columns must exist), 0 here.
for _, k := range []string{"_o11y.gen_ai.total_cost", "_o11y.gen_ai.provider_cost", "_o11y.gen_ai.margin"} {
if got, ok := m[k]; !ok || got.AsFloat64() != 0 {
t.Errorf("%s = %v (present=%v), want 0 present", k, got.AsFloat64(), ok)
}
}
for _, k := range []string{
"gen_ai.hanzo.provider", "gen_ai.hanzo.request_id",
"session.id", "gen_ai.conversation.id",
"gen_ai.hanzo.project", "gen_ai.hanzo.api_key_hash",
"gen_ai.hanzo.cluster_id", "gen_ai.hanzo.route_policy",
"gen_ai.usage.cache_read.input_tokens", "gen_ai.usage.cache_creation.input_tokens",
"_o11y.gen_ai.cost_input", "_o11y.gen_ai.cost_output",
"gen_ai.input.messages", "gen_ai.output.messages",
} {
if _, ok := m[k]; ok {
t.Errorf("attribute %q must be absent when unset", k)
}
}
}
// TestBuildGenAISpanFields_ServedArm pins what a family reports beyond its body:
// response.model is the arm that answered, provider.name the vendor that ran it
// (else our route label), and reasoning, time to first token and the failed arms
// ride beside them. A failed arm is an Error span.
func TestBuildGenAISpanFields_ServedArm(t *testing.T) {
rec := &usageRecord{
Owner: "acme", Model: "zen5", Provider: "zen", Status: "success",
PromptTokens: 200, CompletionTokens: 20, TotalTokens: 220, ReasoningTokens: 18,
Served: "z-ai/glm-5.3", Vendor: "digitalocean", First: 1500 * time.Millisecond,
Failover: "deepseek-v4 (openrouter): upstream status 429",
}
m := attrMap(buildGenAISpanFields(rec, 0, 0, nil, nil, nil, false).attrs)
for k, want := range map[string]string{
"gen_ai.request.model": "zen5",
"gen_ai.response.model": "z-ai/glm-5.3",
"gen_ai.provider.name": "digitalocean",
"gen_ai.hanzo.provider": "zen",
"gen_ai.hanzo.failover": "deepseek-v4 (openrouter): upstream status 429",
} {
if got := m[k].AsString(); got != want {
t.Errorf("attr %q = %q, want %q", k, got, want)
}
}
if got := m["gen_ai.usage.reasoning.output_tokens"].AsInt64(); got != 18 {
t.Errorf("reasoning = %d, want 18", got)
}
if got := m["gen_ai.server.time_to_first_token"].AsFloat64(); got != 1.5 {
t.Errorf("time to first token = %v, want 1.5s", got)
}
// No vendor: provider.name is the route label. No arm: response.model is the SKU.
plain := attrMap(buildGenAISpanFields(&usageRecord{Model: "gpt-5", Provider: "openai", Status: "success"}, 0, 0, nil, nil, nil, false).attrs)
if got := plain["gen_ai.provider.name"].AsString(); got != "openai" {
t.Errorf("provider.name = %q, want the route label", got)
}
if got := plain["gen_ai.response.model"].AsString(); got != "gpt-5" {
t.Errorf("response.model = %q, want the SKU", got)
}
for _, k := range []string{"gen_ai.usage.reasoning.output_tokens", "gen_ai.server.time_to_first_token", "gen_ai.hanzo.failover"} {
if _, ok := plain[k]; ok {
t.Errorf("attribute %q must be absent when unset", k)
}
}
if _, ok := attrMap(buildGenAISpanFields(&usageRecord{Model: "x"}, 0, 0, nil, nil, nil, false).attrs)["gen_ai.provider.name"]; ok {
t.Error("provider.name emitted with no vendor and no provider")
}
f := buildGenAISpanFields(&usageRecord{Model: "zen5", Status: "failover", ErrorMsg: "a: empty answer"}, 0, 0, nil, nil, nil, false)
if f.statusCode != codes.Error || f.statusMsg != "a: empty answer" {
t.Errorf("failover status = %v %q, want Error with the reason", f.statusCode, f.statusMsg)
}
}
// TestBuildGenAISpanFields_OrgSystemFallback pins the MANDATORY org invariant: o11y's
// llmobs views hard-filter on gen_ai.hanzo.org_id and drop empty-org spans, so a span
// must ALWAYS carry an org. When an internal system caller sets neither Organization
// nor Owner, org falls back to the reserved system org — never empty.
func TestBuildGenAISpanFields_OrgSystemFallback(t *testing.T) {
rec := &usageRecord{Model: "zen5", Status: "success"} // no Organization, no Owner
m := attrMap(buildGenAISpanFields(rec, 0, 0, new(float64(0)), new(float64(0)), nil, false).attrs)
got, ok := m["gen_ai.hanzo.org_id"]
if !ok {
t.Fatal("gen_ai.hanzo.org_id must ALWAYS be present (o11y drops empty-org spans)")
}
if got.AsString() != orgSystemDefault {
t.Errorf("gen_ai.hanzo.org_id = %q, want %q (system fallback)", got.AsString(), orgSystemDefault)
}
if got.AsString() == "" {
t.Error("gen_ai.hanzo.org_id must never be empty")
}
}
// TestBuildGenAISpanFields_Environment pins the environment attribution: a record
// carrying the caller's X-Environment emits deployment.environment so Observe narrows
// by environment instead of defaulting to "default"; absent when the caller sent none.
func TestBuildGenAISpanFields_Environment(t *testing.T) {
withEnv := attrMap(buildGenAISpanFields(
&usageRecord{Owner: "acme", Model: "zen5", Status: "success", Environment: "staging"},
0, 0, new(float64(0)), new(float64(0)), nil, false,
).attrs)
if got := withEnv["deployment.environment"].AsString(); got != "staging" {
t.Errorf("deployment.environment = %q, want \"staging\"", got)
}
noEnv := attrMap(buildGenAISpanFields(
&usageRecord{Owner: "acme", Model: "zen5", Status: "success"},
0, 0, new(float64(0)), new(float64(0)), nil, false,
).attrs)
if _, ok := noEnv["deployment.environment"]; ok {
t.Error("deployment.environment must be absent when the caller set no X-Environment")
}
}
// TestBuildGenAISpanFields_ProjectAndKeyHash pins the per-project + credential-ref
// attribution: when the record carries a project and a hashed key ref, the span
// emits gen_ai.hanzo.project + gen_ai.hanzo.api_key_hash — and the hash is a ref,
// never the plaintext key.
func TestBuildGenAISpanFields_ProjectAndKeyHash(t *testing.T) {
rec := &usageRecord{
Owner: "acme", Organization: "acme", Model: "zen5", Status: "success",
Project: "research", APIKeyHash: "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
}
m := attrMap(buildGenAISpanFields(rec, 0, 0, new(float64(0)), new(float64(0)), nil, false).attrs)
if got := m["gen_ai.hanzo.project"].AsString(); got != "research" {
t.Errorf("gen_ai.hanzo.project = %q, want \"research\"", got)
}
if got := m["gen_ai.hanzo.api_key_hash"].AsString(); got != rec.APIKeyHash {
t.Errorf("gen_ai.hanzo.api_key_hash = %q, want the SHA-256 ref", got)
}
}
// TestBuildGenAISpanFields_MessageRedaction proves prompt/completion bodies are
// emitted ONLY when the caller opts in — the privacy gate is fail-secure.
func TestBuildGenAISpanFields_MessageRedaction(t *testing.T) {
rec := &usageRecord{
Owner: "acme", Model: "zen5", Status: "success",
InputMessages: `[{"role":"user","content":"secret"}]`, OutputMessages: `[{"role":"assistant","content":"reply"}]`,
}
// Gate off: bodies must NOT appear.
off := attrMap(buildGenAISpanFields(rec, 0, 0, new(float64(0)), new(float64(0)), nil, false).attrs)
if _, ok := off["gen_ai.input.messages"]; ok {
t.Error("gen_ai.input.messages leaked with capture disabled")
}
if _, ok := off["gen_ai.output.messages"]; ok {
t.Error("gen_ai.output.messages leaked with capture disabled")
}
// Gate on: bodies appear verbatim.
on := attrMap(buildGenAISpanFields(rec, 0, 0, new(float64(0)), new(float64(0)), nil, true).attrs)
if got := on["gen_ai.input.messages"].AsString(); got != rec.InputMessages {
t.Errorf("gen_ai.input.messages = %q, want %q", got, rec.InputMessages)
}
if got := on["gen_ai.output.messages"].AsString(); got != rec.OutputMessages {
t.Errorf("gen_ai.output.messages = %q, want %q", got, rec.OutputMessages)
}
}
// TestGenAICaptureMessages verifies the env gate is fail-secure: only explicit
// truthy values enable message capture.
func TestGenAICaptureMessages(t *testing.T) {
for _, tc := range []struct {
val string
want bool
}{
{"", false},
{"false", false},
{"0", false},
{"no", false},
{"off", false},
{"true", true},
{"1", true},
{"TRUE", true},
{"Yes", true},
{"on", true},
} {
t.Setenv("O11Y_GENAI_CAPTURE_MESSAGES", tc.val)
if got := genAICaptureMessages(); got != tc.want {
t.Errorf("genAICaptureMessages(%q) = %v, want %v", tc.val, got, tc.want)
}
}
}
// TestUsageCostCents_MatchesSpanTotal proves the ledger cost and the span total are
// the ONE value — usageCostCents drives both recordUsage and emitGenAISpan.
func TestUsageCostCents_MatchesSpanTotal(t *testing.T) {
rec := &usageRecord{Model: "zen5", PromptTokens: 1000, CompletionTokens: 500}
cents := usageCostCents(rec)
spanTotalUSD := float64(cents) / 100.0
m := attrMap(buildGenAISpanFields(rec, spanTotalUSD, spanTotalUSD, new(float64(0)), new(float64(0)), nil, false).attrs)
if got := m["_o11y.gen_ai.total_cost"].AsFloat64(); got != spanTotalUSD {
t.Errorf("span total_cost = %v, want %v (== usageCostCents/100)", got, spanTotalUSD)
}
}
// TestUsageBilledCents_BYOReconcilesInvoice proves the span's billed_cost equals what
// the ledger actually debits: full provider cost for a Hanzo-served call, but only the
// ~1% platform fee for a BYO call (where the customer paid the upstream directly). The
// total_cost stays the full provider cost for analytics — so Observe ≠ invoice is fixed.
func TestUsageBilledCents_BYOReconcilesInvoice(t *testing.T) {
fees(t, map[string]string{"hanzo_byok": "1"})
// A call whose full provider cost is > 0.
base := &usageRecord{Model: "zen5", PromptTokens: 100000, CompletionTokens: 50000}
cost := usageCostCents(base)
if cost <= 1 {
t.Fatalf("test needs a cost > 1 cent to distinguish the ~1%% BYO fee, got %d", cost)
}
// Hanzo-served: billed == full cost.
served := *base
served.BYO = false
if got := usageBilledCents(&served, cost); got != cost {
t.Errorf("Hanzo-served billed = %d, want full cost %d", got, cost)
}
// BYO: billed == the ~1% platform fee, strictly less than the full cost.
byo := *base
byo.BYO = true
fee := usageBilledCents(&byo, cost)
if fee != platformFeeCents(cost, true) {
t.Errorf("BYO billed = %d, want platformFeeCents %d", fee, platformFeeCents(cost, true))
}
if fee >= cost {
t.Errorf("BYO billed %d must be < full provider cost %d", fee, cost)
}
// The span reports BOTH: total_cost = full provider cost, billed_cost = the fee.
// BYO provider COGS is 0 (customer paid the upstream), so margin == the billed fee.
totalUSD := float64(cost) / 100.0
billedUSD := float64(fee) / 100.0
m := attrMap(buildGenAISpanFields(&byo, totalUSD, billedUSD, new(float64(0)), new(billedUSD), nil, false).attrs)
if got := m["_o11y.gen_ai.total_cost"].AsFloat64(); got != totalUSD {
t.Errorf("BYO span total_cost = %v, want full %v", got, totalUSD)
}
if got := m["_o11y.gen_ai.billed_cost"].AsFloat64(); got != billedUSD {
t.Errorf("BYO span billed_cost = %v, want fee %v", got, billedUSD)
}
if m["_o11y.gen_ai.billed_cost"].AsFloat64() >= m["_o11y.gen_ai.total_cost"].AsFloat64() {
t.Error("BYO billed_cost must be < total_cost on the span (invoice reconciliation)")
}
if got := m["_o11y.gen_ai.provider_cost"].AsFloat64(); got != 0 {
t.Errorf("BYO span provider_cost = %v, want 0 (customer paid the upstream)", got)
}
if got := m["_o11y.gen_ai.margin"].AsFloat64(); got != billedUSD {
t.Errorf("BYO span margin = %v, want the billed fee %v (0 COGS)", got, billedUSD)
}
}