mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
hanzo-inc/cloud#800. A serving node (or the router in front of it) states the runtime that answered as X-Hanzo-Runtime-Digest and X-Hanzo-Cache-Connector, on the response head or in the trailer of a stream. ai reads both into the usage record and writes them on every gen_ai span (gen_ai.hanzo.runtime.digest, gen_ai.hanzo.kv_cache.connector) and every hanzo.cloud_usage row (runtime_digest, cache_connector). Neither is relayed to the caller. The new leaf package provenance is the one rule for what is recorded: - a digest is `<algo>:<hex>` for sha256/sha384/sha512/blake3, or the digest of an image reference pinned by one (`repo@sha256:...`, registry dropped); a tag, a path or anything malformed is "unknown"; - a connector is one name from a closed vocabulary (in-memory, cpu-offload, disk-offload, external:<lmcache|mooncake|nixl|p2p-nccl|redis|s3|valkey| kv-fabric>, external:other); vLLM connector class names map onto it, a URL keeps its scheme alone, so no host, path or credential reaches a trace; - a node that states nothing reads "unknown", on the span and the row alike, so both attributes are on every span and a filter for either is one lookup. The columns are added with DEFAULT 'unknown' (DDL and migration), so rows written before them, and the subset inserts (DO backfill, cloud's OpenRouter import), read "unknown" too. ServedUsage carries both for a host that serves a model in process (TraceServedUsage). Testing: go test ./provenance ./object ./controllers ./routers (new: TestTheServingRuntimeIsReadAndNeverRelayed, TestEverySpanAndRowNamesTheRuntime, plus the provenance package's digest/connector/idempotence tests). Risk: low. Two String columns with defaults on the usage table (ADD COLUMN IF NOT EXISTS); values are normalised before any write. Billing is untouched. How to revert: revert this commit; the columns can stay (they default to 'unknown' and nothing else reads them). Co-authored-by: Claude Opus 5.5 <[email protected]>
109 lines
4.6 KiB
Go
109 lines
4.6 KiB
Go
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
// ServedUsage is the trace input for a request that was BILLED ELSEWHERE — a
|
|
// co-resident subsystem that debits commerce itself (zen's Meter in the unified
|
|
// cloud binary) but must still land in the ONE usage warehouse + o11y span
|
|
// plane this package owns. Money is exact nano-USD when the biller knows it
|
|
// (zen computes both retail and upstream COGS per served tier); 0 falls back to
|
|
// the rate-table recompute.
|
|
type ServedUsage struct {
|
|
Owner string // billing org (who paid) — the warehouse partition key
|
|
User string // actor, "owner/name" when known
|
|
Model string // the SKU the caller requested (zen5, …)
|
|
Provider string // serving family label ("zen")
|
|
RequestID string
|
|
Status string // "success" | "error" | "failover" (an arm that failed)
|
|
ErrorMsg string
|
|
PromptTokens int // the WHOLE prompt; CachedTokens is a part of it
|
|
CompletionTokens int
|
|
BilledNano int64 // exact retail billed, nano-USD; 0 = recompute from rates
|
|
CostNano int64 // exact upstream COGS, nano-USD; 0 = recompute from rates
|
|
StartTime time.Time
|
|
|
|
Served string // the arm that generated the answer (gen_ai.response.model)
|
|
Vendor string // the provider that ran that arm (gen_ai.provider.name)
|
|
Failover string // the arms that failed before it, with why
|
|
CachedTokens int // of PromptTokens: served from the upstream's prompt cache
|
|
WrittenTokens int // of PromptTokens: written to the upstream's prompt cache
|
|
ReasoningTokens int // of CompletionTokens: spent reasoning
|
|
First time.Duration // time from StartTime to the first token of the answer
|
|
|
|
// The runtime that served the answer, as the serving node stated it
|
|
// (provenance.RuntimeDigestHeader / CacheConnectorHeader): its image or binary
|
|
// digest and its KV cache connector. Recorded through provenance.Digest and
|
|
// Connector, so "" — a node that said nothing — is recorded as "unknown".
|
|
RuntimeDigest string
|
|
CacheConnector string
|
|
}
|
|
|
|
// TraceServedUsage emits the warehouse row (hanzo.cloud_usage) + gen_ai span for
|
|
// a self-billed request — recordTrace WITHOUT recordUsage, so the commerce debit
|
|
// the caller already made is never doubled. One warehouse writer, two billers,
|
|
// zero double-billing. Safe under a nil/zero input: an empty Owner still writes
|
|
// an (unattributed) row so traffic is never silently invisible; StartTime zero
|
|
// anchors at now.
|
|
func TraceServedUsage(ctx context.Context, in ServedUsage) {
|
|
if in.StartTime.IsZero() {
|
|
in.StartTime = time.Now().UTC()
|
|
}
|
|
if in.RequestID == "" {
|
|
in.RequestID = uuid.NewString()
|
|
}
|
|
recordTrace(ctx, servedRecord(in), in.StartTime)
|
|
}
|
|
|
|
// servedRecord projects a ServedUsage onto the usage record recordTrace writes.
|
|
func servedRecord(in ServedUsage) *usageRecord {
|
|
// The record counts the prompt as every path does: PromptTokens is the whole
|
|
// prompt, and CacheReadTokens and CacheWriteTokens the parts of it read from and
|
|
// written to the upstream's cache.
|
|
cached := min(max(in.CachedTokens, 0), max(in.PromptTokens, 0))
|
|
written := min(max(in.WrittenTokens, 0), max(in.PromptTokens, 0)-cached)
|
|
return &usageRecord{
|
|
Owner: in.Owner,
|
|
Organization: in.Owner,
|
|
User: in.User,
|
|
Model: in.Model,
|
|
Provider: in.Provider,
|
|
PromptTokens: max(in.PromptTokens, 0),
|
|
CacheReadTokens: cached,
|
|
CacheWriteTokens: written,
|
|
CompletionTokens: in.CompletionTokens,
|
|
TotalTokens: in.PromptTokens + in.CompletionTokens,
|
|
Served: in.Served,
|
|
Vendor: in.Vendor,
|
|
Failover: in.Failover,
|
|
ReasoningTokens: in.ReasoningTokens,
|
|
First: in.First,
|
|
RuntimeDigest: in.RuntimeDigest,
|
|
CacheConnector: in.CacheConnector,
|
|
Currency: "USD",
|
|
Status: in.Status,
|
|
ErrorMsg: in.ErrorMsg,
|
|
RequestID: in.RequestID,
|
|
BilledNanoExact: &in.BilledNano,
|
|
CostNanoExact: &in.CostNano,
|
|
}
|
|
}
|