Files
e55843264b feat(trace): every inference span and usage row names its runtime digest and KV cache connector (#185)
hanzo-inc/cloud#800. A serving node (or the router in front of it) states the
runtime that answered as X-Hanzo-Runtime-Digest and X-Hanzo-Cache-Connector, on
the response head or in the trailer of a stream. ai reads both into the usage
record and writes them on every gen_ai span (gen_ai.hanzo.runtime.digest,
gen_ai.hanzo.kv_cache.connector) and every hanzo.cloud_usage row
(runtime_digest, cache_connector). Neither is relayed to the caller.

The new leaf package provenance is the one rule for what is recorded:
- a digest is `<algo>:<hex>` for sha256/sha384/sha512/blake3, or the digest
  of an image reference pinned by one (`repo@sha256:...`, registry dropped);
  a tag, a path or anything malformed is "unknown";
- a connector is one name from a closed vocabulary (in-memory, cpu-offload,
  disk-offload, external:<lmcache|mooncake|nixl|p2p-nccl|redis|s3|valkey|
  kv-fabric>, external:other); vLLM connector class names map onto it, a URL
  keeps its scheme alone, so no host, path or credential reaches a trace;
- a node that states nothing reads "unknown", on the span and the row alike,
  so both attributes are on every span and a filter for either is one lookup.

The columns are added with DEFAULT 'unknown' (DDL and migration), so rows
written before them, and the subset inserts (DO backfill, cloud's OpenRouter
import), read "unknown" too. ServedUsage carries both for a host that serves a
model in process (TraceServedUsage).

Testing: go test ./provenance ./object ./controllers ./routers (new:
TestTheServingRuntimeIsReadAndNeverRelayed, TestEverySpanAndRowNamesTheRuntime,
plus the provenance package's digest/connector/idempotence tests).
Risk: low. Two String columns with defaults on the usage table (ADD COLUMN IF
NOT EXISTS); values are normalised before any write. Billing is untouched.
How to revert: revert this commit; the columns can stay (they default to
'unknown' and nothing else reads them).

Co-authored-by: Claude Opus 5.5 <[email protected]>
2026-11-18 22:58:35 -05:00

439 lines
20 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
import (
"context"
"os"
"strings"
"time"
"github.com/hanzoai/ai/object"
"github.com/hanzoai/ai/provenance"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
)
// GenAI span attribute keys. These are a WIRE CONTRACT with o11y: they MUST match
// o11y's pkg/types/{llmobstypes,llmpricingruletypes} constants exactly, because the
// o11y llmobs views project spans by these names. Kept as literals here so the ai
// module takes no dependency on the o11y module.
const (
attrGenAISystem = "gen_ai.system"
attrGenAIOperation = "gen_ai.operation.name"
attrGenAIRequestModel = "gen_ai.request.model"
attrGenAIResponseModel = "gen_ai.response.model"
attrGenAIInputTokens = "gen_ai.usage.input_tokens"
attrGenAIOutputTokens = "gen_ai.usage.output_tokens"
attrGenAITotalTokens = "gen_ai.usage.total_tokens"
attrGenAICacheRead = "gen_ai.usage.cache_read.input_tokens"
attrGenAICacheCreation = "gen_ai.usage.cache_creation.input_tokens"
// The part of output_tokens spent reasoning, named after the cache_read and
// cache_creation subsets of input_tokens above.
attrGenAIReasoning = "gen_ai.usage.reasoning.output_tokens"
// The vendor that ran the model (semconv gen_ai.provider.name).
attrGenAIProviderName = "gen_ai.provider.name"
// Time to the first token, in seconds — semconv's name for the same measure.
attrGenAIFirstToken = "gen_ai.server.time_to_first_token"
// _o11y.gen_ai.* cost attributes (dollars) the o11y cost views read.
// total_cost = the FULL provider cost of the call (analytics). billed_cost = what
// Hanzo actually charged the org ledger — equal to total_cost for a Hanzo-served
// call, but only the ~1% platform fee for a BYO call (the customer paid the
// upstream directly), so Observe reconciles with the invoice.
attrO11yTotalCost = "_o11y.gen_ai.total_cost"
attrO11yBilledCost = "_o11y.gen_ai.billed_cost"
attrO11yCostInput = "_o11y.gen_ai.cost_input"
attrO11yCostOutput = "_o11y.gen_ai.cost_output"
attrO11yCostCacheRead = "_o11y.gen_ai.cost_cache_read"
attrO11yCostCacheWrite = "_o11y.gen_ai.cost_cache_write"
// provider_cost = Hanzo's COGS to serve the call (0 for BYO); margin = billed −
// provider_cost. These add the cost/margin view beside the customer-price
// total_cost/billed_cost above (which stay the legacy figures o11y already reads).
attrO11yProviderCost = "_o11y.gen_ai.provider_cost"
attrO11yMargin = "_o11y.gen_ai.margin"
attrSessionID = "session.id"
attrConversationID = "gen_ai.conversation.id"
attrUserID = "user.id"
attrOrgID = "gen_ai.hanzo.org_id"
attrProject = "gen_ai.hanzo.project"
attrAPIKeyHash = "gen_ai.hanzo.api_key_hash"
attrProvider = "gen_ai.hanzo.provider"
attrRequestID = "gen_ai.hanzo.request_id"
attrServedBy = "gen_ai.hanzo.served_by"
attrClusterID = "gen_ai.hanzo.cluster_id"
attrRoutePolicy = "gen_ai.hanzo.route_policy"
attrRouting = "gen_ai.hanzo.routing"
// attrFallback is emitted ONLY when the caller did not get the route they
// named, and it says WHY. Present ⇒ this generation is a downgrade; absent ⇒
// it is not, which is what makes it a filter and not a thing to interpret.
// The two model attributes say what the swap WAS; this one says what caused
// it, because "response.model differs from request.model" will one day have
// more than one cause and a reader should not have to guess which.
attrFallback = "gen_ai.hanzo.fallback"
// attrFailover is the chain of arms a family tried and could not use before
// the one that answered, with why. Absent when the first arm answered.
attrFailover = "gen_ai.hanzo.failover"
// attrVia is the vendor account an answer ran on where a model's own vendor
// served it: the vendor and the key's place in its pool, never a key.
attrVia = "gen_ai.hanzo.via"
// attrPriced is emitted (=false) ONLY when the model had no configured price and
// billed at the default — so o11y can flag the row. Absent ⇒ priced normally.
attrPriced = "gen_ai.hanzo.priced"
attrInputMessages = "gen_ai.input.messages"
attrOutputMessages = "gen_ai.output.messages"
// fallbackSpent is the one cause there is today: the vendor's account with us
// was empty, and it served the request from a route it charges nothing for.
fallbackSpent = "upstream_spent"
// attrDeploymentEnv is the standard OTel key o11y reads for Environment. Emitted
// per-request from record.Environment (the caller's X-Environment) so Observe
// narrows by environment instead of defaulting to "default".
attrDeploymentEnv = "deployment.environment"
// servedByDefault labels a call served by Hanzo cloud (the majority). BYO
// provider/GPU paths override it via record.ServedBy.
servedByDefault = "hanzo"
// orgSystemDefault is the reserved system org stamped on a span when an internal
// system caller (router probe/judge, warmups) carries NEITHER Organization nor
// Owner. o11y's llmobs views apply a mandatory gen_ai.hanzo.org_id filter that
// silently drops empty-org spans, so a last-resort org keeps internal traffic
// visible. Real tenant traffic always resolves its own org before this fallback.
orgSystemDefault = "hanzo"
)
// genAISpanFields is the pure projection of a usageRecord onto OTel GenAI
// semantic-convention span data. It is separated from the side-effecting emit so
// the attribute mapping (the contract with o11y) is unit-testable without any
// tracer/provider global state.
type genAISpanFields struct {
name string
attrs []attribute.KeyValue
statusCode codes.Code
statusMsg string
}
// buildGenAISpanFields maps a completed LLM call to its GenAI span, following the
// OpenTelemetry GenAI semantic conventions (span name "chat {model}",
// gen_ai.* attributes). Attribution (user/org/provider/request-id/session/served-by)
// and cost/token detail ride on span attributes.
//
// Pure: no I/O, no globals, no pricing lookup. Every dollar figure is passed in
// (totalCostUSD is the full customer-price cost; billedCostUSD what the ledger charged;
// providerCostUSD Hanzo's COGS; marginUSD = billed − provider; breakdown is the
// per-component split for token-billed calls, nil for image/video). This keeps the o11y
// attribute contract unit-testable without a pricing table. Message capture is gated by
// the caller (captureMessages) — PII is emitted only on opt-in.
func buildGenAISpanFields(record *usageRecord, totalCostUSD, billedCostUSD float64, providerCostUSD, marginUSD *float64, breakdown *costBreakdown, captureMessages bool) genAISpanFields {
model := record.Model
if model == "" {
model = "unknown"
}
// The two model attributes have always been the same string, because until a
// request could be moved they described one thing. They stop agreeing exactly
// when the caller did not get what they asked for: request.model is the SKU
// they named, response.model the one that answered. Reading them apart is how
// a downgrade shows up in the plane a bad answer is investigated from, and it
// needs no vocabulary of ours — this is what the two attributes are FOR.
requested := model
if record.Requested != "" {
requested = record.Requested
}
// response.model is the model that generated the answer: the family's arm
// when it named one, else the route that answered.
served := model
if record.Served != "" {
served = record.Served
}
attrs := []attribute.KeyValue{
attribute.String(attrGenAISystem, "hanzo"),
attribute.String(attrGenAIOperation, "chat"),
attribute.String(attrGenAIRequestModel, requested),
attribute.String(attrGenAIResponseModel, served),
attribute.Int(attrGenAIInputTokens, record.PromptTokens),
attribute.Int(attrGenAIOutputTokens, record.CompletionTokens),
attribute.Int(attrGenAITotalTokens, record.TotalTokens),
// total_cost = full customer-price cost (analytics). billed_cost = what the
// ledger charged (== total for Hanzo-served, ~1% fee for BYO). Both are always
// known, so both are always emitted; provider_cost and margin are conditional
// below because a cost we were never told is not a cost of zero.
attribute.Float64(attrO11yTotalCost, totalCostUSD),
attribute.Float64(attrO11yBilledCost, billedCostUSD),
}
if providerCostUSD != nil {
attrs = append(attrs, attribute.Float64(attrO11yProviderCost, *providerCostUSD))
}
// Margin, only when it is computable. An unpriced model's COGS is the
// conservative default, so billed − COGS would report a heavy loss that nobody
// took. The priced=false attribute below says why the number is absent.
if marginUSD != nil {
attrs = append(attrs, attribute.Float64(attrO11yMargin, *marginUSD))
}
// Cache tokens (Anthropic-style prompt caching) — omitted when absent.
if record.CacheReadTokens > 0 {
attrs = append(attrs, attribute.Int(attrGenAICacheRead, record.CacheReadTokens))
}
if record.CacheWriteTokens > 0 {
attrs = append(attrs, attribute.Int(attrGenAICacheCreation, record.CacheWriteTokens))
}
if record.ReasoningTokens > 0 {
attrs = append(attrs, attribute.Int(attrGenAIReasoning, record.ReasoningTokens))
}
if record.First > 0 {
attrs = append(attrs, attribute.Float64(attrGenAIFirstToken, record.First.Seconds()))
}
// Per-component cost detail — token-billed calls only (breakdown != nil).
if breakdown != nil {
attrs = append(
attrs,
attribute.Float64(attrO11yCostInput, breakdown.Input),
attribute.Float64(attrO11yCostOutput, breakdown.Output),
attribute.Float64(attrO11yCostCacheRead, breakdown.CacheRead),
attribute.Float64(attrO11yCostCacheWrite, breakdown.CacheWrite),
)
}
// Session turns the o11y sessions/conversations view on for this org. Emitted
// under both the OTel conversation key and o11y's session.id grouping key.
if record.Session != "" {
attrs = append(
attrs,
attribute.String(attrSessionID, record.Session),
attribute.String(attrConversationID, record.Session),
)
}
if record.User != "" {
attrs = append(attrs, attribute.String(attrUserID, record.User))
}
// org_id is MANDATORY: o11y's llmobs views hard-filter on gen_ai.hanzo.org_id and
// drop empty-org spans. Resolve from the record (Organization → Owner) and fall
// back to the reserved system org for internal system callers that carry neither,
// so every ai-emitted span stays visible. Always emitted (never conditionally).
org := firstNonEmptyStr(record.Organization, record.Owner, orgSystemDefault)
attrs = append(attrs, attribute.String(attrOrgID, org))
// Environment (X-Environment) → deployment.environment, when the caller set one.
if record.Environment != "" {
attrs = append(attrs, attribute.String(attrDeploymentEnv, record.Environment))
}
// Project is the org SUB-SCOPE (the caller's X-Project-Id). Emitting it lets the
// o11y views + the cloud metrics board narrow WITHIN an org by project; empty
// (the org's default project) emits no attribute — honest, never a fabricated scope.
if record.Project != "" {
attrs = append(attrs, attribute.String(attrProject, record.Project))
}
// APIKeyHash is a NON-reversible ref (SHA-256 hex) of the caller credential —
// never the plaintext key. It correlates a span to a key without the span store
// ever holding a secret.
if record.APIKeyHash != "" {
attrs = append(attrs, attribute.String(attrAPIKeyHash, record.APIKeyHash))
}
if record.Provider != "" {
attrs = append(attrs, attribute.String(attrProvider, record.Provider))
}
// The vendor that ran the arm, else our route label for who served the call.
if name := firstNonEmptyStr(record.Vendor, record.Provider); name != "" {
attrs = append(attrs, attribute.String(attrGenAIProviderName, name))
}
if record.Requested != "" {
attrs = append(attrs, attribute.String(attrFallback, fallbackSpent))
}
if record.Failover != "" {
attrs = append(attrs, attribute.String(attrFailover, record.Failover))
}
if record.Via != "" {
attrs = append(attrs, attribute.String(attrVia, record.Via))
}
if record.RequestID != "" {
attrs = append(attrs, attribute.String(attrRequestID, record.RequestID))
}
// The runtime that served the call, on EVERY span: a digest or "unknown", a
// connector name or "unknown". Always present, so a filter for either — a
// digest under investigation, or the calls no node could name — is a lookup on
// one attribute rather than a reasoning about its absence (hanzo-inc/cloud#800).
attrs = append(attrs,
attribute.String(provenance.AttrRuntimeDigest, provenance.Digest(record.RuntimeDigest)),
attribute.String(provenance.AttrCacheConnector, provenance.Connector(record.CacheConnector)),
)
// served_by is always present (defaults to hanzo cloud); cluster/route-policy
// only when a producer set them (BYO-GPU / enso routing).
servedBy := record.ServedBy
if servedBy == "" {
servedBy = servedByDefault
}
attrs = append(attrs, attribute.String(attrServedBy, servedBy))
if record.ClusterID != "" {
attrs = append(attrs, attribute.String(attrClusterID, record.ClusterID))
}
if record.RoutePolicy != "" {
attrs = append(attrs, attribute.String(attrRoutePolicy, record.RoutePolicy))
}
if record.Routing != "" {
attrs = append(attrs, attribute.String(attrRouting, record.Routing))
}
// Unpriced call: billed at the conservative default because the model has no
// configured price. Emit priced=false so o11y flags it; omit otherwise.
if record.Unpriced {
attrs = append(attrs, attribute.Bool(attrPriced, false))
}
// Prompt/completion bodies are PII — emitted ONLY on explicit opt-in
// (captureMessages), and only when non-empty. Default is redacted.
if captureMessages {
if record.InputMessages != "" {
attrs = append(attrs, attribute.String(attrInputMessages, record.InputMessages))
}
if record.OutputMessages != "" {
attrs = append(attrs, attribute.String(attrOutputMessages, record.OutputMessages))
}
}
fields := genAISpanFields{name: "chat " + model, attrs: attrs, statusCode: codes.Ok}
// A failed arm ("failover") is a failed call on that arm, the same as an error.
if record.Status == "error" || record.Status == "failover" {
fields.statusCode = codes.Error
fields.statusMsg = record.ErrorMsg
}
return fields
}
// genAICaptureMessages reports whether prompt/completion bodies may be attached to
// the span. Fail-secure: only an explicit opt-in
// (O11Y_GENAI_CAPTURE_MESSAGES=true/1) enables it; anything else redacts.
//
// OPS NOTE: leave O11Y_GENAI_CAPTURE_MESSAGES UNSET in production. Enabling it writes
// raw prompt/completion text (PII, and the customer's proprietary data) into the
// shared o11y_traces span store, where any org member with Observe access reads it and
// it rides the platform's retention/backup. Turn it on only for a scoped debugging
// window with the tenant's consent, then unset it. Default OFF = redacted is the
// safe, compliant posture.
func genAICaptureMessages() bool {
switch strings.ToLower(strings.TrimSpace(os.Getenv("O11Y_GENAI_CAPTURE_MESSAGES"))) {
case "1", "true", "yes", "on":
return true
default:
return false
}
}
// emitGenAISpan emits one OTel GenAI span per LLM call to o11y. It is
// called from recordTrace — the single funnel every completion surface (chat,
// Anthropic, tool-proxy, embeddings, ZAP) passes through — so there is exactly
// one emit site. The span starts from the REQUEST context (ctx), so it nests under
// any in-flight parent span and inherits its trace/baggage rather than orphaning at
// a fresh root. No-op when telemetry is disabled (exporter unset); span export is
// batched/async, so this never blocks the request.
// genAIMoney is the money view of one call as the o11y span reports it, in USD.
type genAIMoney struct {
total float64 // the FULL provider cost of the call (analytics)
billed float64 // what the ledger actually charged
// provider is Hanzo's COGS to serve the call, and NIL where no cost is known —
// see providerCostNano. A span that always carries the attribute has to put a
// number in it, and the only number to hand is the customer price, which reports
// every such call as costing exactly what it earned.
provider *float64
// margin is billed − provider, and NIL when either side is unknown. The span then
// carries no margin attribute at all, beside the priced=false it already emits.
margin *float64
}
// spanMoney derives that view from usageMargin — the ONE billed path, the same trio
// stamped on the cloud_usage row and taken by the debit, so the span, the warehouse
// and the invoice carry one number each.
//
// billed used to be recomputed here from the cent-precision rate table while margin
// beside it came from usageMargin, so a span reported a billed amount and a margin
// derived from two DIFFERENT figures whenever a caller knew its exact one. For a
// model with no configured price the table invents the conservative default, so the
// span could show several times what the invoice charged.
func spanMoney(record *usageRecord) genAIMoney {
mg := usageMargin(record)
money := genAIMoney{
total: float64(usageCostCents(record)) / 100.0,
billed: float64(mg.BilledNano) / 1e9,
}
if mg.CostNano != nil {
provider := float64(*mg.CostNano) / 1e9
money.provider = &provider
}
if mg.MarginNano != nil {
margin := float64(*mg.MarginNano) / 1e9
money.margin = &margin
}
return money
}
func emitGenAISpan(ctx context.Context, record *usageRecord, startTime time.Time) {
if record == nil || !object.TelemetryEnabled() {
return
}
if ctx == nil {
ctx = context.Background()
}
money := spanMoney(record)
totalCostUSD, billedCostUSD := money.total, money.billed
providerCostUSD, marginUSD := money.provider, money.margin
var breakdown *costBreakdown
// A token breakdown describes a token-billed call. Image, video, audio and
// decisions all bill per unit, so computing one would report a cost
// structure the call does not have — and a FREE route has no cost structure at
// all, so pricing its tokens would publish components that sum to more than the
// zero total beside them.
if !record.Free && record.ImageCount == 0 && record.VideoCount == 0 && record.DecisionCount == 0 && !recordIsAudio(record) {
b := modelCostBreakdown(record.Model, record.PromptTokens, record.CompletionTokens, record.CacheReadTokens, record.CacheWriteTokens)
breakdown = &b
}
fields := buildGenAISpanFields(record, totalCostUSD, billedCostUSD, providerCostUSD, marginUSD, breakdown, genAICaptureMessages())
_, span := object.GenAITracer().Start(
ctx, fields.name,
trace.WithSpanKind(trace.SpanKindClient),
trace.WithTimestamp(startTime),
trace.WithAttributes(fields.attrs...),
)
span.SetStatus(fields.statusCode, fields.statusMsg)
// Hand the span's own trace id back to the record so the ledger row and this
// span carry the SAME id and can be joined. Read from the started span rather
// than from ctx: when the request arrives with no parent span this one begins a
// new trace, and ctx has never heard of it.
if sc := span.SpanContext(); sc.HasTraceID() {
record.TraceID = sc.TraceID().String()
}
span.End(trace.WithTimestamp(time.Now().UTC()))
}
// firstNonEmptyStr returns the first argument that is non-empty after trimming
// surrounding whitespace — the single shared helper (finetune, telemetry, …).
func firstNonEmptyStr(vals ...string) string {
for _, v := range vals {
if strings.TrimSpace(v) != "" {
return v
}
}
return ""
}