Merge pull request #387 from hanzo-inc/fix/o11y-fleet-telemetry

o11y: the product scope is the fleet, and a sub-cent debit is not an outage

apps/billing/usage_coresident.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/hanzoai/cloud"
 	"github.com/hanzoai/cloud/apps/finance"
+	"github.com/hanzoai/cloud/money"
 	"github.com/hanzoai/cloud/plane"
 )
 
@@ -66,11 +67,35 @@ func coResidentUsage(ctx context.Context, org, product, groupBy string) ([]byte,
 		}
 		rows := make([]finance.UsageRow, 0, len(reply.Rows))
 		for _, r := range reply.Rows {
-			cents, cerr := r.Amount.Minor()
+			// Round DOWN, explicitly — the same choice balance.go:126 already made,
+			// for the same reason, on a row read from the same ledger.
+			//
+			// Minor() REFUSES anything finer than a cent rather than round behind
+			// the caller. A per-token AI charge is routinely finer than a cent, so
+			// on this path the refusal was not an edge case: ONE sub-cent row in the
+			// page failed the whole read, and the caller (billing.go:458) tests err
+			// before coResident, so it answered 502 "billing upstream unreachable"
+			// with nothing upstream involved. That is the 2026-08-03 balance bug
+			// verbatim; balance and ai were converted then, this row was missed.
+			// Measured 2026-08-06: /v1/billing/balance 200, /v1/billing/usage 502,
+			// same org, same ledger, same process — only this call differed.
+			cents, cerr := r.Amount.FloorMinor()
 			if cerr != nil {
-				return nil, false, cerr
+				// The peer ANSWERED and the reply did not parse — a real failure,
+				// not an absent ledger. Report it as handled so it surfaces here
+				// rather than falling through to a commerce proxy that is not
+				// configured on this deployment and would mask it as a 501.
+				return nil, true, fmt.Errorf("usage: commerce ledger row %s: %w", r.ID, cerr)
 			}
-			rows = append(rows, finance.UsageRow{ID: r.ID, Model: r.Model, Cents: cents, CreatedAt: r.CreatedAt})
+			// Carry the EXACT debit, not just its rounding. usageEnvelope emits it as
+			// `decimal`, and that field is the whole reason a page of sub-cent calls
+			// totals correctly instead of totalling zero — dropping Amount here would
+			// have traded the 502 for a silently understated bill.
+			exact, perr := money.ParseUSD(r.Amount.Decimal)
+			if perr != nil {
+				return nil, true, fmt.Errorf("usage: commerce ledger row %s amount %q: %w", r.ID, r.Amount.Decimal, perr)
+			}
+			rows = append(rows, finance.UsageRow{ID: r.ID, Model: r.Model, Cents: cents, Amount: exact, CreatedAt: r.CreatedAt})
 		}
 		env := usageEnvelope(org, rows)
 		if out, ok := enrichUsageLedger(env, product, groupBy); ok {

apps/billing/usage_subcent_peer_test.go

@@ -0,0 +1,140 @@
+package billing
+
+// usage_subcent_peer_test.go — a sub-cent debit on the PEER branch must not fail
+// the read.
+//
+// The existing sub-cent test (usage_coresident_test.go) publishes a fake finance
+// and exercises the LOCAL branch, where cents come from finance.UsageRow.Cents and
+// nothing can error. Production runs the other branch: plugin/billing links no
+// ledger, so finance.Current() is nil and the rows arrive over the plane from
+// commerce as plane.Money. That branch called Money.Minor(), which REFUSES an
+// amount finer than a cent instead of rounding behind the caller — and a
+// per-token AI charge is routinely finer than a cent.
+//
+// One such row failed the whole page, and usage() (billing.go:458) tests err
+// before coResident, so the customer got 502 "billing upstream unreachable" with
+// nothing upstream involved. That is the 2026-08-03 balance bug exactly;
+// balance.go and apps/ai were converted to FloorMinor then, this row was missed.
+//
+// Measured on the live fleet 2026-08-06, same org, same ledger, same process:
+//   GET /v1/billing/balance -> 200 {"balance":14989388,...}   (FloorMinor)
+//   GET /v1/billing/usage   -> 502 "billing upstream unreachable"  (Minor)
+// Only the call differed, which is what makes this a code fix and not config.
+
+import (
+	"context"
+	"encoding/json"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/hanzoai/cloud/apps/finance"
+	"github.com/hanzoai/cloud/plane"
+	"github.com/zap-proto/zip"
+)
+
+// subCentCommerce serves the commerce plane socket and answers the usage op with a
+// page whose debits are finer than a cent — the shape a real AI ledger holds.
+func subCentCommerce(t *testing.T, rows []plane.UsageRow) {
+	t.Helper()
+	app := zip.New(zip.Config{AppName: "commerce"})
+	zip.Post[struct{}, plane.UsageRows](app, "/finance/usage",
+		func(context.Context, *struct{}) (*plane.UsageRows, error) {
+			return &plane.UsageRows{Rows: rows}, nil
+		}, zip.WithOperationID(plane.FinanceUsage))
+	go func() { _ = app.Listen(zip.SocketPath("commerce")) }()
+	t.Cleanup(func() { _ = app.Shutdown() })
+
+	path := zip.SocketPath("commerce")
+	for i := 0; i < 400; i++ {
+		if c, err := net.DialTimeout("unix", path, time.Second); err == nil {
+			_ = c.Close()
+			return
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	t.Fatalf("%s never began listening — this test would otherwise pass by never reaching the peer", path)
+}
+
+// TestCoResidentUsage_SubCentPeerRowIsNotAnOutage is the regression. A page of
+// sub-cent debits must come back as a page, not as an error the caller renders
+// as a dead upstream.
+func TestCoResidentUsage_SubCentPeerRowIsNotAnOutage(t *testing.T) {
+	planeDir(t)
+	finance.Publish(nil) // no local ledger — force the peer branch, as in prod
+	t.Cleanup(func() { finance.Publish(nil) })
+
+	rows := []plane.UsageRow{
+		{ID: "tiny-1", Model: "zen-1", Amount: plane.Money{Decimal: "0.0025", Currency: "USD"}, CreatedAt: 1_700_000_000},
+		{ID: "tiny-2", Model: "zen-1", Amount: plane.Money{Decimal: "0.00007", Currency: "USD"}, CreatedAt: 1_700_000_100},
+		{ID: "whole", Model: "gpt-x", Amount: plane.Money{Decimal: "1.50", Currency: "USD"}, CreatedAt: 1_700_000_200},
+	}
+	subCentCommerce(t, rows)
+
+	body, coResident, err := coResidentUsage(context.Background(), "acme", "", "")
+	if err != nil {
+		t.Fatalf("a sub-cent debit failed the usage read: %v\n"+
+			"usage() turns this into 502 \"billing upstream unreachable\" with nothing upstream involved", err)
+	}
+	if !coResident {
+		t.Fatal("the peer answered but the read reported fall-back — usage() would hand this to an " +
+			"unconfigured commerce proxy and answer 501")
+	}
+
+	var env struct {
+		Count int `json:"count"`
+		Usage []struct {
+			TransactionID string `json:"transactionId"`
+			Amount        int64  `json:"amount"`
+			Decimal       string `json:"decimal"`
+		} `json:"usage"`
+	}
+	if uerr := json.Unmarshal(body, &env); uerr != nil {
+		t.Fatalf("envelope not valid JSON: %v\n%s", uerr, body)
+	}
+	// Guard the iteration source before asserting over it: an empty page would make
+	// every assertion below vacuously true, and an empty page is itself the bug
+	// (the customer reads a blank ledger) — so it must fail here, loudly.
+	if len(env.Usage) != len(rows) {
+		t.Fatalf("envelope carries %d rows, want %d — the peer's page did not survive the read; "+
+			"every assertion below would otherwise pass by examining nothing", len(env.Usage), len(rows))
+	}
+
+	byID := map[string]struct {
+		cents   int64
+		decimal string
+	}{}
+	for _, u := range env.Usage {
+		byID[u.TransactionID] = struct {
+			cents   int64
+			decimal string
+		}{u.Amount, u.Decimal}
+	}
+
+	// FloorMinor rounds DOWN: a sub-cent debit is 0 cents. That is correct and is
+	// precisely why `decimal` must carry the exact value beside it.
+	for _, id := range []string{"tiny-1", "tiny-2"} {
+		got, ok := byID[id]
+		if !ok {
+			t.Fatalf("row %q missing from the envelope", id)
+		}
+		if got.cents != 0 {
+			t.Errorf("%s: amount = %d cents, want 0 — the fixture must be sub-cent for this test to mean anything", id, got.cents)
+		}
+		if got.decimal == "" {
+			t.Errorf("%s: decimal is EMPTY — the exact debit was dropped on the peer branch, so a page of "+
+				"sub-cent calls totals ZERO and the customer is billed for work the statement cannot show", id)
+		}
+	}
+	if d := byID["tiny-1"].decimal; d != "" {
+		if !strings.Contains(d, "0.0025") {
+			t.Errorf("tiny-1: decimal = %q, want the exact 0.0025 the ledger holds", d)
+		}
+	}
+	// A whole-cent row must still round to its cents, unchanged.
+	if got := byID["whole"]; got.cents != 150 {
+		t.Errorf("whole: amount = %d cents, want 150", got.cents)
+	}
+	t.Logf("peer page survived: %d rows, sub-cent debits carried as decimal", env.Count)
+}

apps/o11y/fleet_scope_test.go

@@ -0,0 +1,260 @@
+package o11y
+
+// fleet_scope_test.go — the per-product o11y surface must cover the FLEET, not a
+// hand-kept subset of it.
+//
+// Before this gate, knownServices was the only door into resolveService, and it
+// listed 26 k8s workloads. manifest.Apps lists 119 routed apps. The overlap was
+// TWELVE. The other 107 — ai, admin, base, platform, projects, team, usage,
+// tasks, deploy, exec, index, … — answered honest-empty on
+// /v1/o11y/status, /v1/o11y/product/metrics and the scoped log read, not because
+// their telemetry was missing (their request spans have been in event.span the
+// whole time, stamped by cloud's TracingMiddleware) but because this package had
+// never heard of them.
+//
+// The fix was to DERIVE the set instead of keeping a second one. These tests are
+// what stop it from silently becoming a hand list again.
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/hanzoai/cloud/manifest"
+)
+
+// TestEveryRoutedAppHasAProductScope is the fleet direction: an app the host
+// routes must be an app the o11y surface can scope to.
+func TestEveryRoutedAppHasAProductScope(t *testing.T) {
+	// Guard the iteration source before iterating it. Every way this could examine
+	// zero apps — a manifest that failed to load, a package that stopped exporting
+	// Apps — is a defect elsewhere that would otherwise arrive here as a green
+	// tick. A floor rather than a zero-check, because a reader that degrades to
+	// finding 3 of 119 is the same bug wearing a smaller number.
+	if len(manifest.Apps) < 100 {
+		t.Fatalf("manifest.Apps has %d rows — the fleet is 119 apps, so this gate is reading the "+
+			"wrong table and would pass by not looking", len(manifest.Apps))
+	}
+
+	scoped, coresident, fallback := 0, 0, 0
+	for _, a := range manifest.Apps {
+		svc, ok := resolveService(a.Name)
+
+		// The router's FALLBACK claims the API root, so it bounds nothing. Like a
+		// co-resident app it has no product scope, for the mirror-image reason.
+		if claimsAPIRoot(a) {
+			if ok && len(svc.Routes) > 0 {
+				t.Errorf("%s claims the API root %q and still resolved to routes %v — that scope is "+
+					"every request in the fleet", a.Name, apiRoot, svc.Routes)
+			}
+			fallback++
+			continue
+		}
+
+		if a.Coresident {
+			// A co-resident app ROUTES nothing of its own (zen is a Claim middleware
+			// on ai's router). There is no route subtree to attribute spans to, so
+			// there is no honest product scope — and inventing `/v1/zen` would
+			// silently report zero for a service that is genuinely serving.
+			if ok && len(svc.Routes) > 0 && !knownServices[a.Name] {
+				t.Errorf("%s is co-resident but resolved to routes %v — it routes no prefix of its "+
+					"own, so those spans belong to the app it wraps", a.Name, svc.Routes)
+			}
+			coresident++
+			continue
+		}
+
+		if !ok {
+			t.Errorf("routed app %q does not resolve to a product scope — its status, metrics and "+
+				"logs pages answer honest-empty while its spans sit in event.span", a.Name)
+			continue
+		}
+		if len(svc.Routes) == 0 {
+			t.Errorf("routed app %q resolved with NO routes — the RED query would scope to nothing "+
+				"and report a healthy idle service", a.Name)
+			continue
+		}
+		// The routes must be the MANIFEST's answer, not a convention that happens to
+		// agree with it for most rows.
+		want := manifest.PrefixesFor(a.Name)
+		if len(want) > 0 {
+			if len(svc.Routes) != len(want) {
+				t.Errorf("%s: routes = %v, manifest says %v", a.Name, svc.Routes, want)
+				continue
+			}
+			for i := range want {
+				if svc.Routes[i] != want[i] {
+					t.Errorf("%s: routes = %v, manifest says %v", a.Name, svc.Routes, want)
+					break
+				}
+			}
+		}
+		scoped++
+	}
+
+	if scoped == 0 {
+		t.Fatal("no routed app resolved to a product scope — this gate proved nothing")
+	}
+	// The two exemptions are exemptions, not a growing list. If either stops
+	// existing the rule that justifies it has moved and must be re-argued.
+	if fallback != 1 {
+		t.Errorf("%d apps claim the API root, want exactly 1 (ai) — a second fallback means two apps "+
+			"are being handed the same unclaimed traffic", fallback)
+	}
+	if coresident != 1 {
+		t.Errorf("%d co-resident apps, want exactly 1 (zen)", coresident)
+	}
+	t.Logf("%d routed apps scoped, %d co-resident, %d API-root fallback, %d manifest rows",
+		scoped, coresident, fallback, len(manifest.Apps))
+}
+
+// claimsAPIRoot reports whether an app declares the prefix every other app is
+// under — the router's fallback role.
+func claimsAPIRoot(a manifest.App) bool {
+	for _, p := range a.Prefixes {
+		if strings.TrimSuffix(p, "/") == apiRoot {
+			return true
+		}
+	}
+	return false
+}
+
+// TestDerivedRoutesBeatTheSlashV1NameConvention pins the REASON the routes come
+// from the manifest rather than from string concatenation.
+//
+// For ~20 apps `/v1/<name>` is not a path anyone serves: plan serves /v1/plans,
+// storage serves /v1/s3/buckets, account serves /v1/orgs, knowledge serves
+// /v1/kb/*. Scoping their RED series to `/v1/<name>` matches no span at all and
+// renders as a healthy service with no traffic — the worst kind of wrong, because
+// it looks like an answer.
+//
+// If this test ever finds ZERO such apps it must fail, not pass: that would mean
+// either the manifest changed shape or this test stopped reading it, and in both
+// cases the claim it is defending has gone unverified.
+func TestDerivedRoutesBeatTheSlashV1NameConvention(t *testing.T) {
+	var differ []string
+	for _, a := range manifest.Apps {
+		if a.Coresident {
+			continue
+		}
+		svc, ok := resolveService(a.Name)
+		if !ok || len(svc.Routes) == 0 {
+			continue
+		}
+		convention := "/v1/" + a.Name
+		exact := len(svc.Routes) == 1 && svc.Routes[0] == convention
+		if !exact {
+			differ = append(differ, a.Name)
+		}
+	}
+	if len(differ) == 0 {
+		t.Fatal("every app's routes equal \"/v1/\"+name — either the manifest is no longer being " +
+			"read (this gate is asserting nothing) or the derivation has been replaced by the " +
+			"convention it exists to correct")
+	}
+	t.Logf("%d apps whose real prefixes differ from \"/v1/\"+name: %v", len(differ), differ)
+}
+
+// TestTheFallbackDoesNotSwallowTheFleet is the attribution rule.
+//
+// ai declares `/v1` — the catch-all it serves the OpenAI-compatible surface from,
+// so that zen's c.Next() has somewhere to fall through to. A bare
+// startsWith('/v1') scan therefore hands ai every request in the fleet. Over six
+// hours of live spans /v1/kms/* alone was 161,705 of 434,765, so the metrics page
+// would have reported KMS's traffic as inference.
+//
+// manifest.OwnerOf already warns that "anything deciding policy from a bare
+// HasPrefix scan will attribute those paths to the wrong app". A RED query decides
+// policy. ai therefore gets NO route scope at all — the honest answer, and the one
+// that does not cost 15.5s to compute.
+func TestTheFallbackDoesNotSwallowTheFleet(t *testing.T) {
+	// The premise: ai really does claim the API root. If the manifest stops saying
+	// that, this gate is no longer testing anything and must say so rather than pass.
+	if got := manifest.PrefixesFor("ai"); len(got) == 0 {
+		t.Fatal(`manifest has no prefixes for "ai" — this gate cannot check the case it exists for`)
+	} else {
+		root := false
+		for _, p := range got {
+			if strings.TrimSuffix(p, "/") == apiRoot {
+				root = true
+			}
+		}
+		if !root {
+			t.Fatalf(`ai no longer claims %q (prefixes=%v) — the swallow case this gate defends `+
+				`against has moved, so it is asserting nothing`, apiRoot, got)
+		}
+	}
+	if svc, ok := resolveService("ai"); ok && len(svc.Routes) > 0 {
+		t.Errorf("ai resolved to routes %v — every nested app's spans would count as inference", svc.Routes)
+	}
+
+	// A BOUNDED ancestor is different and must still exclude its children: admin
+	// owns /v1/admin and seven other apps live under it.
+	admin, ok := resolveService("admin")
+	if !ok {
+		t.Fatal(`resolveService("admin") refused`)
+	}
+	if len(admin.Excludes) == 0 {
+		t.Error("admin owns /v1/admin and excludes NOTHING — apps nested under it count as admin traffic")
+	}
+	// A leaf app excludes nothing, and must not: over-excluding hides its own spans.
+	kms, ok := resolveService("kms")
+	if !ok {
+		t.Fatal(`resolveService("kms") refused`)
+	}
+	if len(kms.Excludes) != 0 {
+		t.Errorf("kms excludes %v — nothing nests inside /v1/kms", kms.Excludes)
+	}
+	t.Logf("ai: no route scope (fallback); admin excludes %d nested prefixes; kms excludes 0",
+		len(admin.Excludes))
+}
+
+// TestDerivationLosesNoPreviouslyServedProduct is the other direction. Deriving
+// the set must ADD apps, never drop a slug the console already asks for — the 26
+// verified workloads and the 7 console aliases were all serving before.
+func TestDerivationLosesNoPreviouslyServedProduct(t *testing.T) {
+	if len(knownServices) == 0 || len(productAlias) == 0 {
+		t.Fatal("knownServices or productAlias is empty — this gate has nothing to compare against")
+	}
+	checked := 0
+	for name := range knownServices {
+		if _, ok := resolveService(name); !ok {
+			t.Errorf("workload %q resolved before and does not now — the derivation dropped a "+
+				"product the console already asks for", name)
+		}
+		checked++
+	}
+	for slug := range productAlias {
+		if _, ok := resolveService(slug); !ok {
+			t.Errorf("console alias %q no longer resolves", slug)
+		}
+		checked++
+	}
+	if checked == 0 {
+		t.Fatal("compared nothing")
+	}
+	t.Logf("%d previously-served products still resolve", checked)
+}
+
+// TestProductScopeStillRefusesWhatItAlwaysRefused — deriving the ALLOWLIST must
+// not weaken the SHAPE gate. The product param is still placed into a PromQL
+// label, a bound datastore parameter and a hostname, and validProduct is still
+// the one thing standing in front of that.
+func TestProductScopeStillRefusesWhatItAlwaysRefused(t *testing.T) {
+	bad := []string{
+		"", " ", "-leading", "trailing-", "UPPER", "under_score", "dot.dot",
+		`"} or up{`, "../etc/passwd", "a/b", "sp ace",
+		"waytoolongwaytoolongwaytoolongwaytoolongwaytoolongwaytoolongwaytoolong",
+		"definitely-not-an-app-xyz", // well-formed but in neither table
+	}
+	for _, p := range bad {
+		if svc, ok := resolveService(p); ok {
+			t.Errorf("resolveService(%q) = %+v, true — want refused", p, svc)
+		}
+	}
+	// And the gate must still SAY YES to a real one, or the loop above proves only
+	// that everything is refused.
+	if _, ok := resolveService("kms"); !ok {
+		t.Fatal(`resolveService("kms") refused a real routed app — the gate now refuses everything, ` +
+			`which would make every assertion above vacuously true`)
+	}
+}

apps/o11y/metricsread.go

@@ -3,6 +3,7 @@ package o11y
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/hanzoai/cloud/apps/datastore"
@@ -114,7 +115,35 @@ func queryMetrics(ctx context.Context, q metricsQuery) (metricsResponse, error)
 // on the plane (event.span). A non-admin is pinned to its own org; an admin sees
 // the whole product.
 func redSeries(ctx context.Context, q metricsQuery, resp *metricsResponse) error {
-	routePrefix := "/v1/" + q.svc.ID
+	// One product can serve SEVERAL subtrees (account serves /v1/orgs and five
+	// more), so the route predicate is built per prefix and OR'd. Every prefix
+	// still rides as a BOUND PARAMETER — the product param reached here through
+	// validProduct and an allowlist, and the manifest is our own table, but the
+	// rule that no name is interpolated into SQL is the reason neither of those
+	// has to be re-argued at this line.
+	routeSQL := make([]string, 0, len(q.svc.Routes))
+	args := []any{q.stepSec}
+	for _, p := range q.svc.Routes {
+		routeSQL = append(routeSQL, "attributes['http.route'] = ? OR startsWith(attributes['http.route'], ?)")
+		args = append(args, p, strings.TrimSuffix(p, "/")+"/")
+	}
+	routeSQL = append(routeSQL, "service = ?")
+	args = append(args, q.svc.App)
+
+	// Longest prefix wins, exactly as the router decided it. Without this an app
+	// that owns an ancestor path (ai owns /v1) counts every nested app's traffic as
+	// its own. See nestedPrefixes in productmap.go.
+	where := "(" + strings.Join(routeSQL, " OR ") + ")"
+	if len(q.svc.Excludes) > 0 {
+		ex := make([]string, 0, len(q.svc.Excludes))
+		for _, p := range q.svc.Excludes {
+			ex = append(ex, "attributes['http.route'] = ? OR startsWith(attributes['http.route'], ?)")
+			args = append(args, p, p+"/")
+		}
+		where += " AND NOT (" + strings.Join(ex, " OR ") + ")"
+	}
+	args = append(args, q.rangeSec)
+
 	sql := "SELECT toStartOfInterval(time, toIntervalSecond(?)) AS bucket, " +
 		"count() AS reqs, " +
 		// The HTTP status is a span attribute (Map values are strings); coerce
@@ -123,9 +152,8 @@ func redSeries(ctx context.Context, q metricsQuery, resp *metricsResponse) error
 		"quantile(0.5)(duration) AS p50, " +
 		"quantile(0.95)(duration) AS p95 " +
 		"FROM event.span " +
-		"WHERE (attributes['http.route'] = ? OR startsWith(attributes['http.route'], ?) OR service = ?) " +
+		"WHERE " + where + " " +
 		"AND time > now64(9) - toIntervalSecond(?)"
-	args := []any{q.stepSec, routePrefix, routePrefix + "/", q.svc.App, q.rangeSec}
 	// THE tenant gate. A non-admin is pinned to its own org (the plane's first
 	// sort-key column); a validated SuperAdmin sees the whole product (no org
 	// predicate).

apps/o11y/productmap.go

@@ -1,6 +1,11 @@
 package o11y
 
-import "strings"
+import (
+	"sort"
+	"strings"
+
+	"github.com/hanzoai/cloud/manifest"
+)
 
 // productmap is the ONE server-side table that resolves a client-supplied
 // `product` query param into the concrete infra identities the scoped o11y
@@ -71,6 +76,112 @@ var knownServices = map[string]bool{
 	"visor": true,
 }
 
+// knownServices answers "does a WORKLOAD answer at an address"; manifest.Apps
+// answers "does an APP serve routes". They are different questions, and for most
+// of the fleet only the second has a yes.
+//
+// Every routed app's requests are already on the plane: cloud's TracingMiddleware
+// stamps `http.route` on every request span, and manifest.Apps is the table the
+// host builds its router from — so the RED signal for all 119 apps has been in
+// event.span the whole time. The only thing standing between it and a caller was
+// this file's hand-maintained workload list, which had heard of 12 of them. An
+// app that nobody probes still has no address and is still not probed (URL stays
+// empty, status answers honest-empty); what it gains is its own logs and metrics,
+// which never needed an address.
+//
+// This is why the fleet surface is DERIVED rather than a second list to maintain:
+// a new plugin gets metrics/logs/status the moment it has a manifest row, which
+// it must have to be routable at all (TestEveryPluginNameIsInTheManifest).
+func fleetRoutes(name string) []string {
+	ps := manifest.PrefixesFor(name)
+	for _, p := range ps {
+		if strings.TrimSuffix(p, "/") == apiRoot {
+			// An app that declares the API ROOT is the router's FALLBACK — it is
+			// handed whatever no other app claimed. That is a routing role, not a
+			// product boundary, and "every request in the fleet minus everyone
+			// else's" is not a RED series anyone should read as this product's
+			// traffic. ai is the only such app (it serves the OpenAI-compatible
+			// surface at top level so zen's c.Next() has somewhere to fall through
+			// to), and it has no bounded route scope for the same reason zen has
+			// none: zen routes nothing, ai routes everything.
+			//
+			// Measured, in case anyone is tempted to compute it anyway: expressing
+			// it as "/v1 minus the 251 nested prefixes" costs 15.5s against six
+			// hours of live spans, versus a ~4-5s baseline for a bounded app — three
+			// times the cost, for a number that would have counted KMS's 161,705
+			// requests as inference.
+			return nil
+		}
+	}
+	return ps
+}
+
+// apiRoot is the prefix every routed app is under, so declaring it claims
+// everything and bounds nothing.
+const apiRoot = "/v1"
+
+// nestedPrefixes is the LONGEST-PREFIX-WINS half, and it is not optional.
+//
+// manifest.OwnerOf says it plainly: "Anything deciding policy from a bare
+// HasPrefix scan will attribute those paths to the wrong app." A RED query is
+// deciding policy. ai declares `/v1` — the catch-all it serves the
+// OpenAI-compatible surface from — so a bare startsWith('/v1') scan hands ai every
+// request in the fleet. Measured over six hours of live spans: /v1/kms/* alone is
+// 161,705 of 434,765, and ai would have reported all of them as its own.
+//
+// So a product's spans are the ones under its prefixes MINUS the ones a strictly
+// longer prefix belonging to a DIFFERENT app claims — the same rule the router
+// used to route them in the first place. For nearly every app this list is empty
+// (nothing nests inside /v1/kms) and the exclusion costs nothing.
+func nestedPrefixes(own []string) []string {
+	roots := make([]string, 0, len(own))
+	for _, p := range own {
+		if r := strings.TrimSuffix(p, "/"); r != "" {
+			roots = append(roots, r)
+		}
+	}
+	seen := map[string]bool{}
+	var nested []string
+	for _, a := range manifest.Apps {
+		for _, q := range a.Prefixes {
+			qr := strings.TrimSuffix(q, "/")
+			if qr == "" || seen[qr] {
+				continue
+			}
+			for _, pr := range roots {
+				// STRICTLY longer and genuinely nested. An app's own prefixes are
+				// never longer than themselves, so this cannot exclude the product
+				// from its own subtree.
+				if len(qr) > len(pr) && strings.HasPrefix(qr, pr+"/") {
+					seen[qr] = true
+					nested = append(nested, qr)
+					break
+				}
+			}
+		}
+	}
+	// Keep only the MINIMAL set. Excluding /v1/platform already excludes
+	// /v1/platform/fleet, and carrying both puts two predicates in the query where
+	// one decides. For ai this is the difference between 282 exclusions and a set
+	// small enough to read.
+	sort.Slice(nested, func(i, j int) bool { return len(nested[i]) < len(nested[j]) })
+	var out []string
+	for _, q := range nested {
+		covered := false
+		for _, kept := range out {
+			if strings.HasPrefix(q, kept+"/") {
+				covered = true
+				break
+			}
+		}
+		if !covered {
+			out = append(out, q)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
 // service is the resolved infra identity for a product.
 type service struct {
 	// ID is the canonical product id (== the product param, validated).
@@ -84,6 +195,19 @@ type service struct {
 	// addresses (probes.go). Empty when the fleet does not watch this workload,
 	// and the scoped status read then probes nothing.
 	URL string
+	// Excludes is the set of longer prefixes owned by OTHER apps that nest inside
+	// Routes. Without it a product that owns an ancestor path absorbs its
+	// children's traffic — see nestedPrefixes.
+	Excludes []string
+	// Routes is the set of path prefixes whose request spans belong to this
+	// product — the manifest's own answer where there is one.
+	//
+	// It is NOT `/v1/<id>` by convention, because for ~20 apps that convention is
+	// simply wrong: plan serves /v1/plans, storage serves /v1/s3/buckets, account
+	// serves /v1/orgs (and five more), knowledge serves /v1/kb/*. Reading the RED
+	// series off `/v1/<name>` for those scopes the query to a subtree nobody
+	// serves, which returns zero and looks exactly like a healthy idle service.
+	Routes []string
 }
 
 // resolveService validates + resolves a product param. ok=false means the product
@@ -98,9 +222,32 @@ func resolveService(product string) (service, bool) {
 	if a, ok := productAlias[p]; ok {
 		workload = a
 	}
-	if !knownServices[workload] {
+	// Routes come from the PRODUCT id, not the aliased workload, because the alias
+	// answers a different question. productAlias maps a console slug to the k8s
+	// workload that ANSWERS (for probing and the prom `service` label); the routing
+	// table is keyed by APP NAME. For analytics the two disagree in both
+	// directions — the app is `analytics` and serves five prefixes
+	// (/v1/analytics, /v1/errors, /v1/event, /v1/insights/*), while the workload is
+	// `insights-capture` and is not a routed app at all. Looking the routes up by
+	// workload silently lost four of those five prefixes.
+	routes := fleetRoutes(p)
+	if len(routes) == 0 {
+		routes = fleetRoutes(workload)
+	}
+	// EITHER answer admits the product: a verified workload (it has an address, so
+	// status can probe it) or a routed app (it has request spans, so metrics and
+	// logs can read it). Requiring both would keep 107 routed apps dark for want of
+	// a probe they never needed.
+	if !knownServices[workload] && len(routes) == 0 {
 		return service{}, false
 	}
+	if len(routes) == 0 {
+		// A verified workload with no manifest row is not a routed app — it is a
+		// bare k8s service (chat, studio, nats, vector). `/v1/<id>` is what this
+		// file has always assumed for those, and it stays their answer; the
+		// manifest simply has nothing better to say about them.
+		routes = []string{"/v1/" + p}
+	}
 	// A workload the fleet does not watch has no address, and the miss is not an
 	// error: it emits logs and metrics we can still query, we have just never
 	// measured whether it answers.
@@ -110,6 +257,8 @@ func resolveService(product string) (service, bool) {
 		App:         workload,
 		PromService: workload,
 		URL:         url,
+		Routes:      routes,
+		Excludes:    nestedPrefixes(routes),
 	}, true
 }
 

apps/o11y/sessions.go

@@ -34,13 +34,21 @@ func init() {
 // (the embedded runtime serves only the list) and none is registered here.
 //
 // Why an explicit cloud route rather than only the order-70 wildcard: this pins the
-// public flat path to the runtime's internal /api/sessions route SERVER-SIDE (the
-// same discipline query.go uses for the composite query) AND enforces the tenant
-// gate at the cloud boundary — an org-less caller gets a clean 403 here before the
+// public flat path to the runtime's internal /api/sessions route SERVER-SIDE AND
+// enforces the tenant gate at the cloud boundary — an org-less caller gets a clean 403 here before the
 // request reaches the runtime, and the org the runtime binds (gen_ai.hanzo.org_id
 // from X-Org-Id) is the SAME validated tenant this handler refuses to proceed
 // without. Registered by mountScope (order 69), so it precedes the wildcard.
 //
+// This used to cite query.go's composite-query pin as the precedent for the move.
+// That file is GONE — it pinned POST /v1/o11y/query_range to the v3 engine, and when
+// it went the module's v5 querier took the address, whose composite accepts only
+// {queries:[…]}. The console still sent the v3 {queryType,panelType,builderQueries}
+// envelope and every Logs page 400'd on "unknown field \"queryType\" in composite
+// query". Citing a deleted pin as the discipline to follow is how the next route
+// inherits the same break, so the reference is removed rather than reworded: this
+// route stands on its OWN pin, three lines below, which is still here.
+//
 // The list query (?limit=&offset=) rides through unchanged; the runtime returns the
 // llmobstypes.GettableSessions {items,offset,limit} under the {status,data} envelope
 // the console's O11yApi.sessions already unwraps.

apps/o11y/zipdoc_gen.go

@@ -129,7 +129,7 @@ func init() {
 		Example: json.RawMessage(`{"id":"annq_1","status":"PENDING"}`),
 	})
 	zip.Describe("GET /v1/o11y/sessions", zip.Doc{
-		Description: "GET /v1/o11y/sessions — the flat, org-gated public path for the LLM-obs sessions\nlist (traces grouped by session.id on the gen_ai span plane). The console's\nSessionsModule reads this; session DETAIL is composed client-side from this list\n+ the traces list filtered by session, so there is no /sessions/:id backing route\n(the embedded runtime serves only the list) and none is registered here.\n\nWhy an explicit cloud route rather than only the order-70 wildcard: this pins the\npublic flat path to the runtime's internal /api/sessions route SERVER-SIDE (the\nsame discipline query.go uses for the composite query) AND enforces the tenant\ngate at the cloud boundary — an org-less caller gets a clean 403 here before the\nrequest reaches the runtime, and the org the runtime binds (gen_ai.hanzo.org_id\nfrom X-Org-Id) is the SAME validated tenant this handler refuses to proceed\nwithout. Registered by mountScope (order 69), so it precedes the wildcard.\n\nThe list query (?limit=&offset=) rides through unchanged; the runtime returns the\nllmobstypes.GettableSessions {items,offset,limit} under the {status,data} envelope\nthe console's O11yApi.sessions already unwraps.",
+		Description: "GET /v1/o11y/sessions — the flat, org-gated public path for the LLM-obs sessions\nlist (traces grouped by session.id on the gen_ai span plane). The console's\nSessionsModule reads this; session DETAIL is composed client-side from this list\n+ the traces list filtered by session, so there is no /sessions/:id backing route\n(the embedded runtime serves only the list) and none is registered here.\n\nWhy an explicit cloud route rather than only the order-70 wildcard: this pins the\npublic flat path to the runtime's internal /api/sessions route SERVER-SIDE AND\nenforces the tenant gate at the cloud boundary — an org-less caller gets a clean 403 here before the\nrequest reaches the runtime, and the org the runtime binds (gen_ai.hanzo.org_id\nfrom X-Org-Id) is the SAME validated tenant this handler refuses to proceed\nwithout. Registered by mountScope (order 69), so it precedes the wildcard.\n\nThis used to cite query.go's composite-query pin as the precedent for the move.\nThat file is GONE — it pinned POST /v1/o11y/query_range to the v3 engine, and when\nit went the module's v5 querier took the address, whose composite accepts only\n{queries:[…]}. The console still sent the v3 {queryType,panelType,builderQueries}\nenvelope and every Logs page 400'd on \"unknown field \\\"queryType\\\" in composite\nquery\". Citing a deleted pin as the discipline to follow is how the next route\ninherits the same break, so the reference is removed rather than reworded: this\nroute stands on its OWN pin, three lines below, which is still here.\n\nThe list query (?limit=&offset=) rides through unchanged; the runtime returns the\nllmobstypes.GettableSessions {items,offset,limit} under the {status,data} envelope\nthe console's O11yApi.sessions already unwraps.",
 	})
 	zip.Describe("GET /v1/o11y/status", zip.Doc{
 		Description: "Reports whether a product's service is live: an in-cluster\nhealth probe with its measured latency, fused with the per-replica up\ninventory. Infra health is not tenant-partitioned — a service is up or down\nfor everyone — so any validated caller is served, but an unvalidated one is\nrefused. A product with no backing workload answers down/unknown-service\nwithout probing anything; a malformed slug is a 400.",