Compare commits
276
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
70dff81db5 | ||
|
|
7d5123dd3d | ||
|
|
1dccca6b2f | ||
|
|
6f371409c7 | ||
|
|
e6499ac02a | ||
|
|
1eb2d288f0 | ||
|
|
438ff9641b | ||
|
|
83bd636f0f | ||
|
|
5589d05bfd | ||
|
|
124a2a2ffb | ||
|
|
7038607250 | ||
|
|
83b59c509b | ||
|
|
803db8811c | ||
|
|
75395adfa4 | ||
|
|
7a236a2c3a | ||
|
|
4941e0839e | ||
|
|
ac31716ce4 | ||
|
|
0b1ec329ff | ||
|
|
362515eead | ||
|
|
c6a121f096 | ||
|
|
1b54ba50d5 | ||
|
|
4d5083e7b6 | ||
|
|
a182ed755d | ||
|
|
174e7bcd17 | ||
|
|
b0222e0c28 | ||
|
|
8293e9de23 | ||
|
|
af16ada769 | ||
|
|
d7e579ff79 | ||
|
|
ec764e44f9 | ||
|
|
6cb979bbb9 | ||
|
|
f822b772a4 | ||
|
|
e0e16cb319 | ||
|
|
5b503ab749 | ||
|
|
51a35fdcc4 | ||
|
|
4f7fcfe9a4 | ||
|
|
02b090f414 | ||
|
|
6c08ad7339 | ||
|
|
ab21d87764 | ||
|
|
e16fd7a237 | ||
|
|
bc6d0a046b | ||
|
|
072b4e1fde | ||
|
|
09be0200fb | ||
|
|
4d25ca740d | ||
|
|
7a8a1c35e0 | ||
|
|
cccf8f60dd | ||
|
|
c0e06e5cbd | ||
|
|
9a51bffbcb | ||
|
|
b80bc9c37a | ||
|
|
5540308d26 | ||
|
|
ddffb21a3b | ||
|
|
8b3aec0e95 | ||
|
|
e0466a63bf | ||
|
|
7e63dbad36 | ||
|
|
f77706ea52 | ||
|
|
d5c05c8e51 | ||
|
|
48c8e965f0 | ||
|
|
898bc0aa7f | ||
|
|
725f2f808e | ||
|
|
1c9df53bb1 | ||
|
|
541f1b4cf8 | ||
|
|
c47252ee73 | ||
|
|
ba98244759 | ||
|
|
9343cbf63f | ||
|
|
e2684eafad | ||
|
|
db33e46737 | ||
|
|
3d1b9828b5 | ||
|
|
c11058ed05 | ||
|
|
68b1a17903 | ||
|
|
2c88a30872 | ||
|
|
12e3b857ca | ||
|
|
c30eb383db | ||
|
|
30175e876a | ||
|
|
756f995d81 | ||
|
|
4ab24f8884 | ||
|
|
7cc3d501fe | ||
|
|
b47a01e4e1 | ||
|
|
0bee60a961 | ||
|
|
c9702cb422 | ||
|
|
7d47001c55 | ||
|
|
e9dfb16b5e | ||
|
|
7b7c53dcb4 | ||
|
|
6ba7b03d2e | ||
|
|
1b8d6f1d50 | ||
|
|
6c9de45895 | ||
|
|
6355a05279 | ||
|
|
78f832a681 | ||
|
|
f0525629ac | ||
|
|
ba93b55717 | ||
|
|
0ebddafaab | ||
|
|
b048bfcdf1 | ||
|
|
ef454a24cf | ||
|
|
d0529648b9 | ||
|
|
48a14de490 | ||
|
|
a77a938df9 | ||
|
|
133e4a73c7 | ||
|
|
87f9e08c9a | ||
|
|
0e076a61b4 | ||
|
|
7d59226a3b | ||
|
|
3f7254468e | ||
|
|
10d31b6ff5 | ||
|
|
df1e2908ee | ||
|
|
120a74b4b0 | ||
|
|
8724f6231e | ||
|
|
4672e8b89d | ||
|
|
0c153f9c36 | ||
|
|
6fbdfe88d8 | ||
|
|
8ec6d62d16 | ||
|
|
fc8337ebb5 | ||
|
|
ba9330c089 | ||
|
|
33db8b7c2a | ||
|
|
ce5696c182 | ||
|
|
eef518ecdb | ||
|
|
6122473c55 | ||
|
|
43f5656d78 | ||
|
|
6a7333e5d8 | ||
|
|
c3938fed27 | ||
|
|
f05b9a6441 | ||
|
|
c829daeb45 | ||
|
|
2fb9c2e095 | ||
|
|
1f1c5569d5 | ||
|
|
0a1ebbc0b8 | ||
|
|
a36b8d1ba7 | ||
|
|
6caf1826d7 | ||
|
|
b5167139d0 | ||
|
|
3337ddcdd5 | ||
|
|
00dc99d834 | ||
|
|
41b992e7fd | ||
|
|
8b1ba262aa | ||
|
|
2898ae3670 | ||
|
|
6fd497221d | ||
|
|
6d93966ad8 | ||
|
|
a8570e0bd5 | ||
|
|
e4538bcc89 | ||
|
|
082cec000c | ||
|
|
599e063871 | ||
|
|
062e0d4374 | ||
|
|
f88405f6a6 | ||
|
|
35dbdbc589 | ||
|
|
5c8b5c9a08 | ||
|
|
46c0d0e11f | ||
|
|
86b0b79ecc | ||
|
|
12d0081e3b | ||
|
|
1256cda741 | ||
|
|
cf5d807cf0 | ||
|
|
3c4e05632c | ||
|
|
93e23b541d | ||
|
|
1d8de1dda7 | ||
|
|
56dd542879 | ||
|
|
78a8aa3310 | ||
|
|
e06eeb18da | ||
|
|
7f2deef78c | ||
|
|
cf24f0cbfc | ||
|
|
df250b65db | ||
|
|
8bcfb8eff3 | ||
|
|
7f3c381570 | ||
|
|
78a9c1f02e | ||
|
|
abcbbb155b | ||
|
|
48d3b35e67 | ||
|
|
3fef8bccce | ||
|
|
30a566846b | ||
|
|
ff5cefcddd | ||
|
|
aafa89064d | ||
|
|
893e7d7759 | ||
|
|
b6f15cd27c | ||
|
|
31e87b7f1e | ||
|
|
28c4f699d0 | ||
|
|
3649758225 | ||
|
|
e06813a5ed | ||
|
|
f09d98a1a0 | ||
|
|
707b6332ad | ||
|
|
524ffb7cdb | ||
|
|
4bd4feabe4 | ||
|
|
b7b61933bf | ||
|
|
a952bed0b7 | ||
|
|
2f3535d5be | ||
|
|
807bad3f4c | ||
|
|
fc3de9ede2 | ||
|
|
3a417701c8 | ||
|
|
3a464b01c4 | ||
|
|
c90196fa14 | ||
|
|
4264ddeb19 | ||
|
|
69cf880b2f | ||
|
|
1d309c9474 | ||
|
|
db2e9284be | ||
|
|
0a9c727609 | ||
|
|
a63189c83a | ||
|
|
f3843d34b5 | ||
|
|
807cf30294 | ||
|
|
e930c39fd6 | ||
|
|
2c1dd75101 | ||
|
|
a9e6e028b4 | ||
|
|
399b893b09 | ||
|
|
c59dcb2a85 | ||
|
|
4055226f6c | ||
|
|
8408c141eb | ||
|
|
605f565138 | ||
|
|
4a2e098705 | ||
|
|
4500f609ed | ||
|
|
d6f98175df | ||
|
|
5527827b46 | ||
|
|
96232be94e | ||
|
|
096de3d8b1 | ||
|
|
0d8c0df95f | ||
|
|
3b09f22fb1 | ||
|
|
be2243afc8 | ||
|
|
eff0ca6b79 | ||
|
|
17924d81b2 | ||
|
|
55f168324d | ||
|
|
d0f2a64ca9 | ||
|
|
6bdb64d13a | ||
|
|
8f398960bd | ||
|
|
b5292c5639 | ||
|
|
134ff852ef | ||
|
|
fbe07f3e7c | ||
|
|
222f91b898 | ||
|
|
f5a7ccdfc7 | ||
|
|
fdb7a31dd0 | ||
|
|
256b848ece | ||
|
|
643926bc76 | ||
|
|
5019d7a384 | ||
|
|
39b6b12823 | ||
|
|
efbdab87f6 | ||
|
|
ebf96d9851 | ||
|
|
78a8197e49 | ||
|
|
83519a957e | ||
|
|
7b19236966 | ||
|
|
2e3dc7d818 | ||
|
|
906f89a2b6 | ||
|
|
84cea2889a | ||
|
|
fa6cc69a65 | ||
|
|
0063793f12 | ||
|
|
198a025cf9 | ||
|
|
7b204ee0de | ||
|
|
81515ef108 | ||
|
|
320af40b0b | ||
|
|
277ea80a4f | ||
|
|
cd42e5f902 | ||
|
|
c0672a1fa6 | ||
|
|
675d17f29c | ||
|
|
46f42f43e6 | ||
|
|
214b5d2925 | ||
|
|
42f2ed8f52 | ||
|
|
70f8d29447 | ||
|
|
01378dea23 | ||
|
|
694bc4f716 | ||
|
|
184945862f | ||
|
|
b3e058490c | ||
|
|
7c301185ee | ||
|
|
40ca519c51 | ||
|
|
0f86fd4a5b | ||
|
|
5ac8e7a1a5 | ||
|
|
6625b5d41b | ||
|
|
3baee40745 | ||
|
|
2cb2e8e286 | ||
|
|
6dc2b3a355 | ||
|
|
6134408bad | ||
|
|
36fc7c3b74 | ||
|
|
180fd74369 | ||
|
|
86b908e7f5 | ||
|
|
a16db40e16 | ||
|
|
ffe8c05047 | ||
|
|
86100a79c1 | ||
|
|
78e0d35199 | ||
|
|
e80a6fc641 | ||
|
|
27a627bf35 | ||
|
|
ceba27fbbf | ||
|
|
4cdd2f2558 | ||
|
|
4fe61262a9 | ||
|
|
7136fff969 | ||
|
|
62068b2d1e | ||
|
|
cfce8b10d0 | ||
|
|
a4f68617a0 | ||
|
|
e3e87f8b69 | ||
|
|
0df009b329 | ||
|
|
02fab4bc40 | ||
|
|
4aff04accd |
@@ -13,6 +13,14 @@
|
||||
coverage.txt
|
||||
coverage.html
|
||||
|
||||
# cek encryption sidecars are key material — never commit one from a source tree.
|
||||
# Tests seal their stores under t.TempDir(); a sidecar in a package dir (e.g. from a
|
||||
# test that opened ":memory:") is a mistake. The cek testdata fixtures are the one
|
||||
# intentional exception.
|
||||
*.dek
|
||||
*.cek.lock
|
||||
!cek/testdata/**
|
||||
|
||||
# Environment files
|
||||
.env
|
||||
.env.*
|
||||
@@ -34,4 +42,5 @@ Thumbs.db
|
||||
.shots/
|
||||
|
||||
.claude/
|
||||
.worktrees/
|
||||
native/flags/target
|
||||
|
||||
@@ -22,6 +22,7 @@ reverse).
|
||||
| `/v1/compute/bots` | Hosting: `@hanzo/bot` Node containers | `clients/bots` | Shipped |
|
||||
| `/v1/tasks` | Durable engine | `clients/tasks` | Shipped |
|
||||
| `/v1/gpus` + fleet | BYO GPU presence | `clients/fleet` + `clients/visor` | Shipped |
|
||||
| `/v1/cloud` | Cloud accounts: link DO/AWS/GCP/Azure, discover native k8s clusters, fold into the fleet | `clients/venue` (new) | In flight (branch `feat/cloud-account-connectors`; blue-held for red) |
|
||||
| IAM | Identity: users, orgs, roles | IAM | Shipped |
|
||||
| KMS | Secret custody: sealed secrets | `clients/kms` | Shipped |
|
||||
|
||||
@@ -44,6 +45,48 @@ Port roadmap (P1-P15) lives in HIP-0129; do not restate it here. Every claim
|
||||
carries its tier: Shipped (on main, named package/route), In flight (named
|
||||
pre-main branch), Planned (backlog id or named reservation).
|
||||
|
||||
## Build & module graph — standalone module, NOT a go.work member
|
||||
|
||||
`cloud` is a self-contained deploy unit: its own `go.mod`, `Dockerfile`, binary.
|
||||
It is intentionally NOT listed in the parent `~/work/hanzo/go.work` workspace —
|
||||
that workspace deliberately excludes the heavy modules, and merging cloud's
|
||||
k8s/otel dependency tree with `o11y`'s reintroduces `koanf`/`ugorji`
|
||||
monolith-vs-split import ambiguities (the parent workspace is itself red on the
|
||||
koanf split; that is not cloud's bug to fix).
|
||||
|
||||
The catch: `go` auto-discovers that parent `go.work` whenever you run a bare
|
||||
`go build ./...` / `go test ./...` from inside this tree, which puts the build in
|
||||
workspace mode and SILENTLY DROPS cloud's own `go.mod` directives. Those
|
||||
directives are load-bearing and each fixes exactly one graph hazard:
|
||||
|
||||
- `replace github.com/vulcand/oxy/v2 => github.com/traefik/oxy/v2 <pseudo>` — the
|
||||
bare require is a placeholder (`v2.0.0-00010101000000-000000000000`); without
|
||||
the replace it resolves to an invalid version.
|
||||
- `exclude github.com/ugorji/go <old-monolith>` — drops the pre-split monolith so
|
||||
`github.com/ugorji/go/codec` (pulled by gin) is unambiguous.
|
||||
- the `k8s.io/*` staging replace block pins every staging module to the `v0.35.3`
|
||||
line. `k8s.io/kubernetes` is a GRAPH-ONLY transitive require of
|
||||
`hanzoai/deploy/gitops-engine` (clients/deploy uses its `pkg/utils/kube`); NO
|
||||
cloud package imports `k8s.io/kubernetes`, so its staging tree never compiles —
|
||||
do not "drop k8s.io/kubernetes", the pins keep the graph consistent and it is
|
||||
never built. koanf resolves to the split modules; the `koanf v1.5.0` monolith
|
||||
require is a harmless graph leaf, never imported.
|
||||
|
||||
So: build cloud in module mode, never workspace mode. `make build`/`test`/`vet`/
|
||||
`tidy` force `GOWORK=off` (matches CI and the Dockerfile, which check out cloud
|
||||
alone with no parent go.work). For a bare `go` command from this tree, prefix
|
||||
`GOWORK=off`. `GOWORK=off go build ./...` and `go vet ./...` are green; `go mod
|
||||
tidy` is stable. Do NOT commit a `go.work` here — it would flip the Dockerfile
|
||||
(`COPY go.mod go.sum` → `go mod download` → `COPY . .`) into workspace mode after
|
||||
its `-mod=readonly` download step.
|
||||
|
||||
Test modes: `make test` is pure-Go (`CGO_ENABLED=0`). Encrypted-at-rest OrgDB
|
||||
tests (`cek`, `CLOUD_KMS_MASTER_KEY_REF` set) REQUIRE `CGO_ENABLED=1` +
|
||||
libsqlcipher (`cek/cek.go` refuses to encrypt in pure-Go); those run only in the
|
||||
Dockerfile's dedicated `-tags libsqlite3` CGO stage, and fail under `make test`
|
||||
by design (clients/git, kms, flags, x402, cmd/kmsreseal, finance). Bundle-embed
|
||||
tests (clients/tasks/ui) need `make deploy-ui` first (real bundle is gitignored).
|
||||
|
||||
## Framework doctrine
|
||||
|
||||
One way to do everything. Composable, orthogonal, DRY. A new subsystem is a
|
||||
@@ -184,6 +227,23 @@ package under `clients/<name>` that obeys these seams — nothing more.
|
||||
curriculum is a machine-readable contract (embedded `default.yaml`; org-custom via
|
||||
PUT replaces it) so `hanzoai/marketing` can author the full `checklist.yaml`
|
||||
against the same `Step`/`Curriculum` shape.
|
||||
- **The EXPERIMENT is a composition, not a fourth engine (`clients/experiments`,
|
||||
`/v1/experiments`).** A/B testing is ONE value whatever the variant KIND (feature
|
||||
flag, ad creative, email subject, model id): the primitive owns only the
|
||||
experiment registry (definition + decision); it COMPOSES three planes it never
|
||||
duplicates. ASSIGNMENT = `flags.Assign(org,project,key,subject,props)` —
|
||||
subject→variant is a deterministic `engineEvaluate` (sha1 rollout hash), no second
|
||||
bucketing, no assignment store; create writes a multivariate flag def
|
||||
(`flags.PutDef`) and decide rewrites its weights to 100% for the winner
|
||||
(`flags.GetDef`+`PutDef`). MEASUREMENT = `analytics.Outcomes(...)` — one
|
||||
org-scoped `hanzo.events` query (the `eventsWhere` isolation invariant), no second
|
||||
event store; the analyze fold joins each subject's analytics outcome to its flags
|
||||
variant by `distinct_id`. EVIDENCE = `research.Record`/`research.List` — per-variant
|
||||
samples land as immutable `kind:"ab"` rows; significance (two-proportion z-test,
|
||||
`math.Erfc`, no dep) is a PURE function over them. `clients/campaign` runs a
|
||||
creative A/B by composing `experiments.Assign`/`experiments.Analyze` — it never
|
||||
reinvents assignment or evidence. Add a new variant KIND by putting a payload on
|
||||
the variant; the primitive does not care what it is.
|
||||
|
||||
## Identity vocabulary is IAM-native
|
||||
|
||||
@@ -258,3 +318,42 @@ app; `?org=` cannot widen it). The rolling restart needs `patch` on `apps/deploy
|
||||
here (TS-Dokploy contract, 404), and `/v1/platform/*` needs a co-resident IAM store this
|
||||
deployment does not fold in (IAM runs as a separate svc) so it 500s; the live apps backend
|
||||
is `/v1/paas`, whose board reads k8s directly with no IAM-store dependency.
|
||||
|
||||
## GTM: `/v1/campaign` orchestration → channels → connectors → analytics
|
||||
|
||||
The go-to-market stack decomplects a campaign from its execution. A **Campaign is a
|
||||
VALUE** (`clients/campaign`: `{name, audience, content[], schedule, budget, channels[],
|
||||
status}`) that SPANS channels; a **Channel is an EXECUTOR** (`channel.go`, the
|
||||
`Channel` interface) it fans out to. The three channels are orthogonal and each
|
||||
CONSUMES the connector plane via `integrations.TokenFor` — the campaign object never
|
||||
touches a credential:
|
||||
|
||||
- **paid → `/v1/ads`** — `ads.LaunchPaid/PaidSpend/PausePaid` (`clients/ads/provider.go`)
|
||||
resolve the org's ad token (`meta_ads`/`google_ads`/… via `TokenFor(org, <id>,
|
||||
"access_token")`) and run the campaign on the provider. Meta is executed for real;
|
||||
fail-closed when the org has not connected (424). This is the ONLY place `/v1/ads`
|
||||
touches the connector plane.
|
||||
- **organic → `/v1/publish`** (rename of `clients/social`) and **email → `/v1/marketing`**
|
||||
are DESIGNED follow-ons: register their executors the same way in `apps/wire_seams.go`
|
||||
(`campaign.RegisterChannel(campaign.NewChannel(kind, launch, spend, pause))`). Until
|
||||
wired, a fan-out records that channel "unavailable" (honest), never fabricated.
|
||||
|
||||
Channels are injected at the composition root (`apps/wire_seams.go`), the SAME
|
||||
injected-function decoupling the coding dispatcher uses — `campaign` never imports
|
||||
`ads`, `ads` never imports `campaign`. Fan-out (`launch.go` `fanOut`) is best-effort
|
||||
per channel; the org (the ONLY tenant key) is passed verbatim to every executor, so a
|
||||
campaign can only ever resolve its OWN org's token.
|
||||
|
||||
**Metrics = the ONE analytics plane, not a second store.** `GET /v1/campaign/:id/metrics`
|
||||
reads the funnel from `analytics.CampaignMetrics(org, campaignID, variant, start, end)`
|
||||
(`clients/analytics/campaign.go`) — an org+`utm_campaign`(+`utm_content`)-scoped query
|
||||
over `hanzo.events`, org and campaign bound POSITIONALLY (same tenancy invariant as
|
||||
every analytics query) — joined with each channel connector's reported spend
|
||||
(`Channel.Spend`). Derived KPIs: CTR/CVR/CAC/ROAS. Honest-empty when the warehouse is
|
||||
absent.
|
||||
|
||||
**Creative A/B composes the experiment primitive** (`experiment.go`), it does not
|
||||
reinvent it: a creative A/B is an experiment whose variant = a creative (tagged
|
||||
`utm_content`) and whose metric = the analytics read. The `AssignFunc`/`EvidenceFunc`
|
||||
seams are wired at the root to the flags-assignment + evidence primitive; nil-safe
|
||||
until it lands (single-creative honest default).
|
||||
|
||||
@@ -4,6 +4,18 @@
|
||||
GO ?= go
|
||||
BIN ?= cloud
|
||||
PKG ?= ./cmd/cloud
|
||||
|
||||
# cloud is a STANDALONE Go module — a self-contained deploy unit (its own go.mod,
|
||||
# Dockerfile, binary). It is intentionally NOT a member of the parent
|
||||
# ~/work/hanzo/go.work workspace (that workspace deliberately excludes the heavy
|
||||
# modules; adding cloud would merge its k8s/otel graph with o11y's and reintroduce
|
||||
# koanf/ugorji import ambiguities). But `go` auto-discovers that parent go.work
|
||||
# whenever a dev builds from inside this tree, which shadows cloud's own
|
||||
# replace/exclude directives (oxy pin, ugorji monolith exclude, k8s staging pins)
|
||||
# and breaks `go build ./...`. Force module mode so make targets build EXACTLY
|
||||
# what CI/Docker build (fresh checkout, no parent go.work). Overridable via
|
||||
# `make GOWORK=... <target>` for the rare cross-module case.
|
||||
export GOWORK := off
|
||||
DOCKER_IMAGE ?= ghcr.io/hanzoai/cloud
|
||||
DOCKER_TAG ?= dev
|
||||
LDFLAGS ?= -s -w
|
||||
@@ -24,7 +36,7 @@ OPENAPI_DIR ?= ../openapi
|
||||
# forces the fork to modernc too so the whole binary registers "sqlite" once.
|
||||
CGO_ENABLED ?= 0
|
||||
|
||||
.PHONY: help native webui deploy-ui agentskills build build-standalone hanzo run smoke test test-cgo vet tidy docker docker-push clean
|
||||
.PHONY: help native webui deploy-ui agentskills build build-standalone run smoke test test-cgo vet tidy docker docker-push clean
|
||||
|
||||
help: ## Show this help.
|
||||
@awk 'BEGIN{FS=":.*##";printf "\nUsage: make <target>\n\nTargets:\n"} /^[a-zA-Z_-]+:.*##/{printf " \033[36m%-16s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
|
||||
@@ -64,9 +76,13 @@ build: ## Build the unified cloud binary into ./bin/cloud (embeds whatever webui
|
||||
|
||||
build-standalone: webui build ## Build the REAL 1-binary console: console build:embed → webui/dist → go build.
|
||||
|
||||
hanzo: ## Build the hanzo control-plane CLI into ./bin/hanzo (pure Go, same mode as cmd/cloud — registers the ONE "sqlite" driver exactly once; a plain CGO_ENABLED=1 `go build ./cmd/hanzo` links the fork's mattn backend alongside the embedded modernc importers and panics, see header).
|
||||
@mkdir -p bin
|
||||
CGO_ENABLED=$(CGO_ENABLED) $(GO) build -ldflags="$(LDFLAGS)" -o bin/hanzo ./cmd/hanzo
|
||||
# NOTE: cloud builds ONLY the `cloud` binary — the stateless unified API. The Go
|
||||
# `hanzo` CLI (cmd/hanzo + cli/) is RETIRED: the shipped `hanzo` is the Rust CLI
|
||||
# (~/work/hanzo/cli, `curl hanzo.sh`), which talks to this API over HTTP via its
|
||||
# OpenAPI-generated command surface. The `code` wrapper (incl. the zen-tier 1M
|
||||
# mechanism) now lives in the Rust CLI. cmd/hanzo + cli/ remain only as the
|
||||
# reference for the still-to-port client-side tools (GPU fleet worker `link`,
|
||||
# `runner`, `engine`, `security`) and are no longer built here.
|
||||
|
||||
run: build ## Run with iam,base,kms,gateway,o11y enabled (matches README quickstart).
|
||||
./bin/$(BIN) --enable=iam,base,kms,gateway,o11y --brand=hanzo --domain=api.hanzo.ai
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
// Copyright 2026 Hanzo AI Inc. All Rights Reserved.
|
||||
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/principal"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// AccountFromPrincipal decomplects the console's IDENTITY onto the ONE truth every
|
||||
// /v1/admin/* call already authorizes on — the validated cloud principal — instead of
|
||||
// the embedded casibase account model.
|
||||
//
|
||||
// THE COMPLECTION IT REMOVES. The operator SPA authenticates via /v1/signin (a cloud
|
||||
// PKCE session → the X-User-* principal the middleware mints) but read its IDENTITY
|
||||
// from /v1/get-account, which the embedded IAM (casibase) answers from ITS OWN session
|
||||
// cookie. A PKCE session is not a casibase session, so get-account returned
|
||||
// owner:"hanzo" (anonymous) or "Unauthorized operation" — and the SPA's SuperAdmin
|
||||
// gate (owner == "admin" && isAdmin), reading that, bounced the operator UI to login
|
||||
// even though the SAME session got 200 from every /v1/admin/* route. Two session
|
||||
// models for one surface; the identity source and the authorization source disagreed.
|
||||
//
|
||||
// THE DECOMPLECTION. Identity is now the principal: when a VALIDATED principal is
|
||||
// present (X-User-Id is set ONLY by IdentityMiddleware from a real credential, never a
|
||||
// raw client header — see middleware_identity.go), /v1/get-account reflects it. owner
|
||||
// is the HOME org (principal.Owner) so a SuperAdmin org-switched into a tenant stays a
|
||||
// SuperAdmin; isAdmin is the validated bit. With NO principal it falls through
|
||||
// (c.Next()) to the casibase account surface unchanged — the anonymous sign-in page
|
||||
// and any legacy casibase-session caller are untouched. One truth, additive, fail-open
|
||||
// to the old path. MUST be registered AFTER IdentityMiddleware (needs the minted
|
||||
// headers) and BEFORE MountAll (so it precedes the casibase /v1/get-account handler).
|
||||
func AccountFromPrincipal() zip.Handler {
|
||||
return func(c *zip.Ctx) error {
|
||||
if c.Method() != http.MethodGet || c.Path() != "/v1/get-account" {
|
||||
return c.Next()
|
||||
}
|
||||
user := c.User() // X-User-Id — minted only from a validated credential
|
||||
if user == "" {
|
||||
return c.Next() // no principal → the casibase account surface (unchanged)
|
||||
}
|
||||
owner := principal.Owner(c) // HOME org (a SuperAdmin stays one when org-switched)
|
||||
if owner == "" {
|
||||
owner = c.Org()
|
||||
}
|
||||
name := c.Header("X-User-Name")
|
||||
if name == "" {
|
||||
name = user
|
||||
}
|
||||
return c.JSON(http.StatusOK, map[string]any{
|
||||
"status": "ok",
|
||||
"msg": "",
|
||||
"data": map[string]any{
|
||||
"owner": owner,
|
||||
"name": name,
|
||||
"displayName": name,
|
||||
"email": c.Header("X-User-Email"),
|
||||
"isAdmin": c.IsAdmin(),
|
||||
"type": "normal-user",
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
+106
-42
@@ -34,7 +34,6 @@ package apps
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/zap-proto/zip"
|
||||
@@ -62,22 +61,29 @@ import (
|
||||
"github.com/hanzoai/cloud/clients/authors"
|
||||
"github.com/hanzoai/cloud/clients/automations"
|
||||
"github.com/hanzoai/cloud/clients/base"
|
||||
"github.com/hanzoai/cloud/clients/benchmark"
|
||||
"github.com/hanzoai/cloud/clients/billing"
|
||||
"github.com/hanzoai/cloud/clients/bots"
|
||||
"github.com/hanzoai/cloud/clients/campaign"
|
||||
"github.com/hanzoai/cloud/clients/captable"
|
||||
"github.com/hanzoai/cloud/clients/catalogsync"
|
||||
"github.com/hanzoai/cloud/clients/channels"
|
||||
"github.com/hanzoai/cloud/clients/cloudflare"
|
||||
"github.com/hanzoai/cloud/clients/code"
|
||||
"github.com/hanzoai/cloud/clients/company"
|
||||
"github.com/hanzoai/cloud/clients/compliance"
|
||||
"github.com/hanzoai/cloud/clients/content"
|
||||
"github.com/hanzoai/cloud/clients/crm"
|
||||
"github.com/hanzoai/cloud/clients/dataroom"
|
||||
"github.com/hanzoai/cloud/clients/deploy"
|
||||
"github.com/hanzoai/cloud/clients/destinations"
|
||||
"github.com/hanzoai/cloud/clients/dns"
|
||||
"github.com/hanzoai/cloud/clients/do"
|
||||
"github.com/hanzoai/cloud/clients/domain"
|
||||
"github.com/hanzoai/cloud/clients/entitlements"
|
||||
"github.com/hanzoai/cloud/clients/eval"
|
||||
"github.com/hanzoai/cloud/clients/exec"
|
||||
"github.com/hanzoai/cloud/clients/experiments"
|
||||
"github.com/hanzoai/cloud/clients/flags"
|
||||
"github.com/hanzoai/cloud/clients/framework"
|
||||
"github.com/hanzoai/cloud/clients/functions"
|
||||
@@ -85,13 +91,15 @@ import (
|
||||
"github.com/hanzoai/cloud/clients/git"
|
||||
"github.com/hanzoai/cloud/clients/graph"
|
||||
"github.com/hanzoai/cloud/clients/guide"
|
||||
"github.com/hanzoai/cloud/clients/help"
|
||||
"github.com/hanzoai/cloud/clients/iam"
|
||||
"github.com/hanzoai/cloud/clients/iam2"
|
||||
"github.com/hanzoai/cloud/clients/ingress"
|
||||
"github.com/hanzoai/cloud/clients/integrations"
|
||||
"github.com/hanzoai/cloud/clients/kafka"
|
||||
"github.com/hanzoai/cloud/clients/kms"
|
||||
"github.com/hanzoai/cloud/clients/knowledge"
|
||||
"github.com/hanzoai/cloud/clients/leaderboard"
|
||||
"github.com/hanzoai/cloud/clients/legal"
|
||||
"github.com/hanzoai/cloud/clients/link"
|
||||
"github.com/hanzoai/cloud/clients/marketing"
|
||||
"github.com/hanzoai/cloud/clients/marketplace"
|
||||
@@ -109,6 +117,8 @@ import (
|
||||
"github.com/hanzoai/cloud/clients/provisioning"
|
||||
"github.com/hanzoai/cloud/clients/pubsub"
|
||||
"github.com/hanzoai/cloud/clients/referrals"
|
||||
"github.com/hanzoai/cloud/clients/research"
|
||||
"github.com/hanzoai/cloud/clients/rollingcap"
|
||||
"github.com/hanzoai/cloud/clients/runtime"
|
||||
"github.com/hanzoai/cloud/clients/sbom"
|
||||
"github.com/hanzoai/cloud/clients/security"
|
||||
@@ -124,26 +134,30 @@ import (
|
||||
"github.com/hanzoai/cloud/clients/tracker"
|
||||
"github.com/hanzoai/cloud/clients/treasury"
|
||||
"github.com/hanzoai/cloud/clients/usage"
|
||||
"github.com/hanzoai/cloud/clients/validators"
|
||||
"github.com/hanzoai/cloud/clients/venue"
|
||||
"github.com/hanzoai/cloud/clients/visor"
|
||||
"github.com/hanzoai/cloud/clients/wallets"
|
||||
"github.com/hanzoai/cloud/clients/websearch"
|
||||
"github.com/hanzoai/cloud/clients/share"
|
||||
"github.com/hanzoai/cloud/clients/world"
|
||||
"github.com/hanzoai/cloud/clients/x402"
|
||||
"github.com/hanzoai/cloud/clients/zt"
|
||||
|
||||
// Framework CONTENT modules — NOT mount subsystems (they carry no HTTP surface
|
||||
// and are absent from Wire()). Each registers its DocType fixtures and, for erp,
|
||||
// its ledger-posting lifecycle hooks into the clients/framework DocType engine
|
||||
// from a package init() (framework.RegisterModule) — the idiomatic
|
||||
// register-into-a-registry pattern (cf. database/sql drivers). The framework
|
||||
// engine is mounted (always-on, /v1/framework/*) but its module registry is
|
||||
// populated ONLY by these blank imports. Dropping one silently strips that
|
||||
// lane's DocTypes and hooks — for erp, the immutable ledger postings — from the
|
||||
// binary with NO mount change and NO failing mount test. #248 dropped them;
|
||||
// TestFrameworkContentModulesLinked now guards against a recurrence. Keep.
|
||||
// Framework CONTENT modules — pure fixture lanes that carry no HTTP surface and
|
||||
// are absent from Wire(). Each registers its DocType fixtures and, for erp, its
|
||||
// ledger-posting lifecycle hooks into the clients/framework DocType engine from a
|
||||
// package init() (framework.RegisterModule) — the idiomatic register-into-a-
|
||||
// registry pattern (cf. database/sql drivers). The framework engine is mounted
|
||||
// (always-on, /v1/framework/*) but its module registry is populated ONLY by these
|
||||
// blank imports. Dropping one silently strips that lane's DocTypes and hooks — for
|
||||
// erp, the immutable ledger postings — from the binary with NO mount change and NO
|
||||
// failing mount test. #248 dropped them; TestFrameworkContentModulesLinked now
|
||||
// guards against a recurrence. Keep. (help is a framework lane too but ALSO mounts
|
||||
// a thin /v1/help public plane, so it is a real import + a Wire() spec below,
|
||||
// alongside knowledge — the other lane with a companion subsystem.)
|
||||
_ "github.com/hanzoai/cloud/clients/cms"
|
||||
_ "github.com/hanzoai/cloud/clients/erp"
|
||||
_ "github.com/hanzoai/cloud/clients/help"
|
||||
)
|
||||
|
||||
// init wires the cross-subsystem func seams — the composition root is the one place
|
||||
@@ -164,22 +178,6 @@ func init() {
|
||||
})
|
||||
}
|
||||
|
||||
// identitySpec selects the ONE identity backend that owns /v1/iam/* (+ /login/oauth/*)
|
||||
// for this boot. CLOUD_IAM_IMPL=iam2 picks the clean-room iam2 (zip+orm, beego-free);
|
||||
// anything else — including unset, the production default — keeps the legacy beego
|
||||
// Casdoor embed, byte-for-byte today's behavior. The two impls register the SAME
|
||||
// absolute prefixes and therefore cannot co-mount, so selection (this func) stays
|
||||
// separate from activation (cfg.Enabled): exactly one spec occupies the identity slot
|
||||
// in Wire, preserving mount order either way. os.Getenv (not the unexported
|
||||
// cloud.getenv, which is unreachable from package apps) is the read — CLOUD_IAM_IMPL is
|
||||
// the deliberate, off-by-default opt-in that keeps iam2 inert until a canary flips it.
|
||||
func identitySpec() cloud.MountSpec {
|
||||
if os.Getenv("CLOUD_IAM_IMPL") == "iam2" {
|
||||
return cloud.MountSpec{Name: "iam2", Mount: iam2.Mount}
|
||||
}
|
||||
return cloud.MountSpec{Name: "iam", Mount: iam.Mount}
|
||||
}
|
||||
|
||||
// Wire returns every linked subsystem as a cloud.MountSpec, in mount order. The
|
||||
// slice position IS the order: cloud.MountAll iterates it as-given, registering each
|
||||
// subsystem's teardown as a zip shutdown hook so teardown runs in reverse (LIFO).
|
||||
@@ -211,15 +209,12 @@ func Wire() []cloud.MountSpec {
|
||||
// /v1/commerce/topup/wallet). MUST mount before the IAM /v1/iam/* wildcard (50) so
|
||||
// they win Fiber's first-match scan (framework-guaranteed since zip v1.3.0).
|
||||
{Name: "account", Mount: account.MountAccount},
|
||||
// Embedded IAM identity plane (/v1/iam/*, /.well-known/*, /login/oauth/*, /_/iam/*,
|
||||
// /cas/*, /scim/*) — the identity authority, mounts before its dependents. STAGED:
|
||||
// the operator adds "iam" to --enable only after IAM config + the fold are verified.
|
||||
// Which IMPLEMENTATION owns these prefixes is selected by CLOUD_IAM_IMPL
|
||||
// (identitySpec): the clean-room iam2 (zip+orm, beego-free) when =="iam2", else the
|
||||
// legacy beego Casdoor embed — the default (unset = today's behavior, byte-for-byte).
|
||||
// Both register the SAME absolute paths and cannot co-mount, so this is an either/or
|
||||
// switch at this ONE slot, never a shadow prefix.
|
||||
identitySpec(),
|
||||
// Embedded IAM identity plane (/v1/iam/*, /login/oauth/*) — the identity authority,
|
||||
// mounts before its dependents. The ONE implementation: the clean-room iam-v2
|
||||
// (zip-native + hanzoai/orm, beego-free); the retired Casdoor iam-v1 embed is GONE.
|
||||
// STAGED: the operator adds "iam" to --enable only after IAM config + the fold are
|
||||
// verified (login/authorize/token/jwks + the operator SSO chain).
|
||||
{Name: "iam", Mount: iam.Mount},
|
||||
// Embedded Base app engine + viral waitlist (/v1/waitlist/*). STAGED behind
|
||||
// CLOUD_BASE_EMBED. OwnsHealth: native /v1/base/health.
|
||||
{Name: "base", Mount: base.Mount, Shutdown: base.Shutdown, OwnsHealth: true},
|
||||
@@ -248,6 +243,11 @@ func Wire() []cloud.MountSpec {
|
||||
// Provisioning control plane: /v1/sql,/v1/vector,/v1/datastore,/v1/kv,/v1/search,/v1/s3,/v1/docdb.
|
||||
{Name: "provisioning", Mount: provisioning.Mount},
|
||||
{Name: "billing", Mount: billing.Mount},
|
||||
// Rolling AI-spend cap: installs the ai gate's per-tier trailing-window cap
|
||||
// reader (registers no routes; its admin-editable knobs are platform switches
|
||||
// surfaced in the /v1/admin/flags cockpit). After commerce/plan so the tier +
|
||||
// finance globals it composes are wired.
|
||||
{Name: "rollingcap", Mount: rollingcap.Mount},
|
||||
// CATCH-ALL /v1/billing/* + /v1/commerce/* data bridges — AFTER clients/billing
|
||||
// (121) + the commerce embed (100). Same clients/account package as "account" (48).
|
||||
{Name: "account-bridge", Mount: account.MountBridge},
|
||||
@@ -257,6 +257,8 @@ func Wire() []cloud.MountSpec {
|
||||
// The /v1/dns forward head: relays the console DNS dashboard to the DNS
|
||||
// control plane under the caller's own validated bearer (clients/dns).
|
||||
{Name: "dns", Mount: dns.Mount},
|
||||
// The registrar: search/price/register domains (name.com) per org.
|
||||
{Name: "domain", Mount: domain.Mount},
|
||||
{Name: "prompts", Mount: prompts.Mount},
|
||||
{Name: "agents", Mount: agents.Mount, Shutdown: agents.Shutdown},
|
||||
// The unified AI login manager registry (/v1/links). Mounts AFTER agents so
|
||||
@@ -278,6 +280,13 @@ func Wire() []cloud.MountSpec {
|
||||
{Name: "templates", Mount: templates.Mount},
|
||||
{Name: "framework", Mount: framework.Mount, Shutdown: ctxShutdown(framework.Shutdown)},
|
||||
{Name: "knowledge", Mount: knowledge.Mount},
|
||||
// Hanzo Support PUBLIC plane /v1/help/* (help center KB read + customer ticket
|
||||
// intake) — the anonymous face the secure-by-default framework surface can't
|
||||
// serve. A framework lane like knowledge: its DocType fixtures register via
|
||||
// init() (help.Module), and this mounts the thin public subsystem. Owns no store
|
||||
// (delegates to the framework in-process API), so no Shutdown. After framework
|
||||
// (whose in-process API it calls at request time); before the AI /v1/* catch-all.
|
||||
{Name: "help", Mount: help.Mount},
|
||||
// Marketing content loop /v1/content/* (generate → CMS → transition → publish).
|
||||
// After framework (its DocType store the ops read/write) + knowledge (the sibling
|
||||
// framework lane); before the AI /v1/* catch-all so /v1/content/* resolves here.
|
||||
@@ -291,6 +300,11 @@ func Wire() []cloud.MountSpec {
|
||||
{Name: "catalogsync", Mount: catalogsync.Mount, Shutdown: catalogsync.Shutdown},
|
||||
{Name: "ml", Mount: ml.Mount, OwnsHealth: true},
|
||||
{Name: "usage", Mount: usage.Mount},
|
||||
// Gamified usage analytics: /v1/usage/leaderboard + /v1/usage/activity + the
|
||||
// per-day contribution graph, over the datastore rollup (#43). Co-owns /v1/usage/*
|
||||
// with usage (a distinct concern — who leads + your activity graph) at its own
|
||||
// exact paths; owns the opt-in SQLite store. Before the ai /v1/* catch-all.
|
||||
{Name: "leaderboard", Mount: leaderboard.Mount, Shutdown: leaderboard.Shutdown},
|
||||
{Name: "crm", Mount: crm.Mount},
|
||||
// Native /v1/marketing/* — the in-process fold of github.com/hanzoai/marketing
|
||||
// (per-org campaign store on Base/SQLite), twin of crm. Owns a DB handle, so
|
||||
@@ -300,6 +314,18 @@ func Wire() []cloud.MountSpec {
|
||||
// twin of crm/marketing. Owns a DB handle, so its Shutdown closes it cleanly
|
||||
// on SIGTERM (ctxShutdown adapts func() error).
|
||||
{Name: "ads", Mount: ads.Mount, Shutdown: ctxShutdown(ads.Shutdown)},
|
||||
// Top-level GTM orchestration /v1/campaign/* — the capability layer that fans a
|
||||
// campaign VALUE out to its channels (paid→ads, organic→publish, email→marketing),
|
||||
// each CONSUMING the connector plane via integrations.TokenFor. Metrics read from
|
||||
// the ONE analytics plane (never a second store); creative A/B composes the
|
||||
// experiment seam. The paid channel executor is wired in wire_seams.go. Owns a DB
|
||||
// handle, so its Shutdown closes it cleanly on SIGTERM.
|
||||
{Name: "campaign", Mount: campaign.Mount, Shutdown: ctxShutdown(campaign.Shutdown)},
|
||||
// GDA/SDM validator onboarding /v1/validators/* — wallet-sig + ETH-mainnet
|
||||
// GenesisNFT ownerOf → seal luxd staking identity into KMS → write a NEW-node
|
||||
// LuxNetwork CR (node.lux.cloud, never the live luxd) → enqueue an owner-gated
|
||||
// registration (never auto-submitted to any P-Chain). Owns a DB handle.
|
||||
{Name: "validators", Mount: validators.Mount, Shutdown: ctxShutdown(validators.Shutdown)},
|
||||
// Native /v1/social/* — the in-process fold of the live social stack
|
||||
// (github.com/hanzoai/social: social-backend/frontend/orchestrator, a Postiz-style
|
||||
// scheduler), a per-org accounts+posts store on Base/SQLite, twin of crm. Owns a DB
|
||||
@@ -312,24 +338,43 @@ func Wire() []cloud.MountSpec {
|
||||
// DB handles, so its Shutdown closes them on SIGTERM.
|
||||
{Name: "sync", Mount: sync.Mount, Shutdown: ctxShutdown(sync.Shutdown)},
|
||||
{Name: "visor", Mount: visor.Mount},
|
||||
// Connect-a-cloud-account plane /v1/cloud/*: an org links its DigitalOcean /
|
||||
// AWS / GCP / Azure accounts (labeled, KMS-sealed, keyless where possible),
|
||||
// Hanzo DISCOVERS each account's native Kubernetes clusters and FOLDS them into
|
||||
// the ONE fleet (clients/fleet.Register) — so they surface in visor's
|
||||
// /v1/clusters and run work like any BYO/managed cluster. No second registry.
|
||||
{Name: "venue", Mount: venue.Mount},
|
||||
// Cap table on Base via goja. STAGED behind CLOUD_ENABLE.
|
||||
{Name: "captable", Mount: captable.Mount, Shutdown: captable.Shutdown},
|
||||
{Name: "code", Mount: code.Mount, Shutdown: code.Shutdown},
|
||||
{Name: "zero-trust", Mount: zt.Mount},
|
||||
// ngrok-native public sharing: /v1/share/* provisions a per-org zrok
|
||||
// account so `hanzo share <port>` publishes a local service to a public
|
||||
// https://<token>.share.hanzo.ai URL. Fail-closed until ZROK_ADMIN_TOKEN.
|
||||
{Name: "share", Mount: share.Mount},
|
||||
// Data rooms via goja + per-tenant Base. STAGED behind CLOUD_ENABLE. OwnsHealth.
|
||||
{Name: "dataroom", Mount: dataroom.Mount, Shutdown: dataroom.Shutdown, OwnsHealth: true},
|
||||
{Name: "graph", Mount: graph.Mount},
|
||||
{Name: "security", Mount: security.Mount, Shutdown: ctxShutdown(security.Shutdown), OwnsHealth: true},
|
||||
{Name: "integrations", Mount: integrations.Mount, Shutdown: integrations.Shutdown},
|
||||
// Per-org Cloudflare asset plane /v1/integrations/cloudflare/{pages,workers,r2,kv,d1}/*.
|
||||
// Mounts AFTER integrations because it reads the org's Cloudflare token through
|
||||
// the integrations custody seam (integrations.TokenFor) — one token, one
|
||||
// custody boundary. Stateless: no store, no shutdown.
|
||||
// Marketing destinations /v1/destinations/* — the native fan-out that
|
||||
// TRANSLATES the canonical /v1/event stream to each connected ad/analytics
|
||||
// platform (GA4 Measurement Protocol, Meta Conversions API, X/LinkedIn/TikTok/
|
||||
// Reddit) and forwards it server-side. Mounts AFTER analytics (whose fan-out
|
||||
// sink it installs) and integrations (a destination may reuse an OAuth
|
||||
// connection's token via integrations.TokenFor). Owns a DB handle → Shutdown.
|
||||
{Name: "destinations", Mount: destinations.Mount, Shutdown: ctxShutdown(destinations.Shutdown)},
|
||||
// First-class per-org Cloudflare asset plane /v1/cloudflare/{zones,pages,workers,
|
||||
// ai,r2,kv,d1}/* (sibling of /v1/dns, /v1/domain). Mounts AFTER integrations
|
||||
// because it reads the org's Cloudflare token through the integrations custody
|
||||
// seam (integrations.TokenFor) — one token, one custody boundary. Connecting the
|
||||
// provider stays on the integrations plane; this plane only MANAGES resources.
|
||||
{Name: "cloudflare", Mount: cloudflare.Mount},
|
||||
{Name: "sbom", Mount: sbom.Mount, OwnsHealth: true},
|
||||
{Name: "team", Mount: team.Mount, Shutdown: ctxShutdown(team.Shutdown)},
|
||||
{Name: "settings", Mount: settings.Mount, Shutdown: settings.Shutdown},
|
||||
{Name: "notify", Mount: notify.Mount, OwnsHealth: true},
|
||||
{Name: "channels", Mount: channels.Mount, Shutdown: channels.Shutdown},
|
||||
{Name: "gateway", Mount: gateway.Mount},
|
||||
{Name: "entitlements", Mount: entitlements.Mount, Shutdown: entitlements.Shutdown},
|
||||
{Name: "exec", Mount: exec.Mount},
|
||||
@@ -346,6 +391,17 @@ func Wire() []cloud.MountSpec {
|
||||
{Name: "sign", Mount: sign.Mount, Shutdown: sign.Shutdown, OwnsHealth: true},
|
||||
{Name: "product", Mount: product.Mount},
|
||||
{Name: "evals", Mount: eval.Mount},
|
||||
{Name: "benchmark", Mount: benchmark.Mount},
|
||||
// The R&D EVIDENCE plane (HIP-0512) + its R&D Ops Board UI at /research —
|
||||
// the arena's sibling: benchmark measures, research is the versioned diary
|
||||
// every product's runs accrue into. Non-staged: mounts under the default.
|
||||
{Name: "research", Mount: research.Mount, Shutdown: ctxShutdown(research.Shutdown)},
|
||||
// The unified EXPERIMENT primitive (/v1/experiments): A/B testing as ONE value
|
||||
// whatever the variant kind (feature | ad creative | email | model). It is a
|
||||
// COMPOSITION — assignment via flags, measurement via analytics, evidence via
|
||||
// research — so it mounts AFTER all three. It owns only the experiment registry
|
||||
// store → Shutdown. clients/campaign composes it (experiments.Assign/Analyze).
|
||||
{Name: "experiments", Mount: experiments.Mount, Shutdown: ctxShutdown(experiments.Shutdown)},
|
||||
{Name: "treasury", Mount: treasury.Mount, Shutdown: ctxShutdown(treasury.Shutdown)},
|
||||
{Name: "admin", Mount: admin.Mount},
|
||||
// Launch-control gate (per-service waitlist): the COMPLETE feature — host→service
|
||||
@@ -386,6 +442,14 @@ func Wire() []cloud.MountSpec {
|
||||
// google token custody; captable/dataroom facades) and before the /v1/* AI
|
||||
// catch-all so its routes resolve here.
|
||||
{Name: "company", Mount: company.Mount, Shutdown: company.Shutdown},
|
||||
// The corporate back-office surfaces — orthogonal to company/captable/billing:
|
||||
// Hanzo Compliance (/v1/compliance) orchestrates KYC/KYB verification providers
|
||||
// + tracks accreditation state + surfaces the SOC 2 audit posture; Hanzo Legal
|
||||
// (/v1/legal) is the versioned template + generation engine + e-sign/filing seams.
|
||||
// Both are TOOLING with providers/professionals in the loop, never advice or
|
||||
// certification. They mount before the /v1/* AI catch-all so their routes resolve.
|
||||
{Name: "compliance", Mount: compliance.Mount, Shutdown: ctxShutdown(compliance.Shutdown), OwnsHealth: true},
|
||||
{Name: "legal", Mount: legal.Mount, Shutdown: ctxShutdown(legal.Shutdown), OwnsHealth: true},
|
||||
// Chat orchestrator — POST /v1/chat: ONE LLM tool-calling round over the tool
|
||||
// plane. It COMPOSES the ai completion path (in-process, so per-org billing
|
||||
// runs) + the unified tool registry, and splits the model's tool calls into
|
||||
|
||||
@@ -24,11 +24,15 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
accountclient "github.com/hanzoai/cloud/clients/account"
|
||||
"github.com/hanzoai/cloud/clients/commerceclient"
|
||||
"github.com/hanzoai/cloud/clients/commerceinproc"
|
||||
financeclient "github.com/hanzoai/cloud/clients/finance"
|
||||
"github.com/hanzoai/cloud/clients/principal"
|
||||
"github.com/hanzoai/commerce"
|
||||
commercebilling "github.com/hanzoai/commerce/api/billing"
|
||||
catalogapi "github.com/hanzoai/commerce/api/catalog"
|
||||
planapi "github.com/hanzoai/commerce/api/plan"
|
||||
commercestore "github.com/hanzoai/commerce/api/store"
|
||||
commercedatastore "github.com/hanzoai/commerce/datastore"
|
||||
commercemid "github.com/hanzoai/commerce/middleware"
|
||||
@@ -64,6 +68,21 @@ var commercePrefixes = []string{
|
||||
// whose prepaid BALANCE gate 402'd every store read (a store-metadata read
|
||||
// must never require an LLM balance).
|
||||
"/v1/store",
|
||||
// Platform-admin catalog CMS: GET/POST/PUT/DELETE /v1/catalog/entries +
|
||||
// POST /v1/catalog/seed — the SuperAdmin CRUD admin.hanzo.ai's editor drives.
|
||||
// commerce's setupRoutes wires only the PUBLIC read (/v1/commerce/catalog);
|
||||
// the CRUD lives on the standalone /v1 bundle (api.Route → catalogApi.AdminRoute),
|
||||
// which the co-resident embed skips, so mountCommerce mounts it below on the
|
||||
// same /v1 gate chain. Own the prefix here so it reaches commerce (each handler
|
||||
// is requireSuperAdmin-gated) instead of the AI /v1/* balance catch-all.
|
||||
"/v1/catalog",
|
||||
// Platform-admin subscription/DNS plan authority CMS: GET/POST/PUT/DELETE
|
||||
// /v1/plans/entries + POST /v1/plans/seed (increment 3a) — the SuperAdmin CRUD
|
||||
// the console plan editor drives. The PUBLIC read stays GET /v1/billing/plans;
|
||||
// this CRUD rides the /v1 bundle (api.Route → planApi.AdminRoute), which the
|
||||
// embed skips, so mountCommerce mounts it below. Own the prefix so it reaches
|
||||
// commerce (each handler requireSuperAdmin-gated), not the AI /v1/* 402 gate.
|
||||
"/v1/plans",
|
||||
// Payment-provider webhook receiver (POST /v1/billing/webhooks/:provider —
|
||||
// Square et al). The provider's HMAC over the registered notification URL +
|
||||
// body IS the auth; a bearer gate is impossible for provider callbacks.
|
||||
@@ -144,6 +163,26 @@ func mountCommerce(app *zip.App, deps cloud.Deps) error {
|
||||
storeV1.Use(iammiddleware.IAMTokenRequired())
|
||||
commercestore.Route(storeV1, commercemid.TokenRequired())
|
||||
|
||||
// Platform-admin catalog CMS on the SAME /v1 bundle: GET/POST/PUT/DELETE
|
||||
// /v1/catalog/entries + POST /v1/catalog/seed. setupRoutes wires only the
|
||||
// public read (/v1/commerce/catalog); the CRUD rides the standalone /v1 bundle
|
||||
// (api.Route → catalogApi.AdminRoute), which the co-resident embed skips — so
|
||||
// register it here, exactly as the standalone does. storeV1's IAMTokenRequired
|
||||
// populates the claims each handler's requireSuperAdmin reads (anon → 403, a
|
||||
// platform admin edits); it is cross-tenant data, never org-scoped.
|
||||
catalogapi.AdminRoute(storeV1)
|
||||
|
||||
// Platform-admin subscription/DNS plan authority CRUD on the SAME /v1 bundle:
|
||||
// GET/POST/PUT/DELETE /v1/plans/entries + POST /v1/plans/seed (increment 3a).
|
||||
// Mirrors the catalog mount: the standalone wires it on the /v1 bundle
|
||||
// (api.Route → planApi.AdminRoute), which the co-resident embed skips. The
|
||||
// embed seed SOURCE is injected here (the composition root) — commercebilling.
|
||||
// SeedRows, the SAME @hanzo/plans embed the boot seed + resolveSubscriptionPlan
|
||||
// read — so api/plan never imports api/billing. Each handler is
|
||||
// requireSuperAdmin-gated (anon → 403); prices are admin-editable but the mint
|
||||
// gates score the IMMUTABLE embed, so an edit never moves a charge gate.
|
||||
planapi.AdminRoute(storeV1, commercebilling.SeedRows)
|
||||
|
||||
// Provider webhook intake at the LIVE registered path. Chain mirrors the
|
||||
// commerce-standalone posture: gated request context, then the sessionless
|
||||
// HMAC-verified handler.
|
||||
@@ -160,6 +199,214 @@ func mountCommerce(app *zip.App, deps cloud.Deps) error {
|
||||
commercebilling.RunAutoRechargeAllOrgs,
|
||||
)
|
||||
|
||||
// GET /v1/billing/plans — the public tier catalog the console renders. commerce's
|
||||
// legacy api.Route() billing bundle (ListPlans, invoices, subscriptions, …) is NOT
|
||||
// registered by the co-resident embed: setupRoutes wires only /v1/commerce/*, so
|
||||
// /v1/billing/plans has NO handler in this binary. The account bridge's
|
||||
// /v1/billing/* wildcard (order 122) then forwards the read BACK to commerce at
|
||||
// COMMERCE_URL — which defaults to the public api.hanzo.ai edge — re-entering the
|
||||
// same bridge in an unbounded self-dispatch loop that surfaces as a 502
|
||||
// ("commerce unreachable: Get https://api.hanzo.ai/v1/billing/plans"). Registering
|
||||
// the static ListPlans handler HERE (order 100, ahead of the bridge) shadows that
|
||||
// wildcard and serves plans in-process — the same co-resident move billing.go makes
|
||||
// for usage/balance. RequestContext supplies the namespaced context promo.Active reads.
|
||||
app.Get("/v1/billing/plans", commercemid.RequestContext(), commercebilling.ListPlans)
|
||||
|
||||
// The rest of the console's billing READS, served co-resident for the SAME reason
|
||||
// plans is: commerce's api.Route() billing bundle is never compiled here, so without
|
||||
// these registrations every one of them falls through to the account bridge's
|
||||
// /v1/billing/* wildcard, which forwards to COMMERCE_URL — the public api.hanzo.ai
|
||||
// edge, which is THIS binary — and self-dispatches into a 502 loop. In prod there is
|
||||
// no separate commerce backend to point COMMERCE_URL at (the in-cluster `commerce`
|
||||
// Service selects the cloud pods), so co-residence is the only way to break the loop.
|
||||
//
|
||||
// Chain: RequestContext (gated context) → IAMTokenRequired (resolves the org from the
|
||||
// gateway-validated X-Org-Id into Locals("organization"), the namespace commerce's
|
||||
// GetOrganization reads) → PinBillingSubject (pins the caller's OWN billing subject
|
||||
// into the query — the SAME isolation the bridge applies, so a read can never widen
|
||||
// past the caller; fail-closed for an unvalidated caller) → the commerce handler. The
|
||||
// specific paths shadow the bridge wildcard (order 100 < 122). payment-config is
|
||||
// org-scoped, not subject-scoped, but PinBillingSubject is still the auth gate that
|
||||
// keeps an unvalidated caller from reaching GetOrganization; its pinned (unused)
|
||||
// subject params are ignored by that handler.
|
||||
billingRead := []struct {
|
||||
path string
|
||||
h zip.Handler
|
||||
}{
|
||||
{"/v1/billing/invoices", commercebilling.ListInvoices},
|
||||
{"/v1/billing/invoices/:id/pdf", commercebilling.DownloadInvoicePDF},
|
||||
{"/v1/billing/subscriptions", commercebilling.ListBillingSubscriptions},
|
||||
{"/v1/billing/spend-alerts", commercebilling.ListSpendAlerts},
|
||||
{"/v1/billing/payouts", commercebilling.ListPayouts},
|
||||
{"/v1/billing/payment-config", commercebilling.GetPaymentConfig},
|
||||
}
|
||||
for _, r := range billingRead {
|
||||
app.Get(r.path,
|
||||
commercemid.RequestContext(),
|
||||
iammiddleware.IAMTokenRequired(),
|
||||
accountclient.PinBillingSubject(),
|
||||
r.h,
|
||||
)
|
||||
}
|
||||
|
||||
// GET /v1/billing/spend-alerts/authorize — the per-request per-scope spend-CAP
|
||||
// VERDICT the request-edge metering gate consumes (clients/metering scopeAuthorize,
|
||||
// pathLimitsAuthorize). It is a SERVICE-token S2S read (COMMERCE_SERVICE_TOKEN +
|
||||
// X-Org-Id), NOT a browser/IAM read — so it needs its OWN registration, distinct from
|
||||
// the console billingRead block above: without a co-resident handler this authorize
|
||||
// fell through to the account bridge's /v1/billing/* wildcard (order 122), which — being
|
||||
// service-token-forwardable (billing.go billingForwardable) — re-forwarded it to
|
||||
// COMMERCE_URL (the public api.hanzo.ai edge = THIS binary) over commerceinproc's
|
||||
// self-routing transport, re-entering the same wildcard until the depth-8 guard refused
|
||||
// → 502 → the gate fails OPEN (the cap is a policy overlay, so no traffic was blocked,
|
||||
// but ~135 502s/30m spammed the money path and each burned 8 full-app dispatches). The
|
||||
// plain GET /v1/billing/spend-alerts (registered above) already broke this loop for the
|
||||
// CRUD read; this closes the /authorize sibling the metering gate hits on every call.
|
||||
//
|
||||
// Chain mirrors commerce's OWN gate on this route (api/billing/handlers.go: the `billing`
|
||||
// group's userRequired = TokenRequired) plus the RequestContext the standalone supplies
|
||||
// globally — NOT the IAM/PinBillingSubject console chain: a raw service token is not an
|
||||
// IAM JWT, so IAMTokenRequired would leave GetOrganization unset and AuthorizeSpendCap
|
||||
// would 500. TokenRequired authenticates the service token AND resolves the tenant from
|
||||
// the gateway-pinned X-Org-Id into Locals("organization"), which AuthorizeSpendCap reads.
|
||||
// The specific route shadows the bridge wildcard (order 100 < 122). No PlatformOnly:
|
||||
// authorize is a per-org cap read, not a cross-org mint (unlike auto-recharge/run-all).
|
||||
app.Get("/v1/billing/spend-alerts/authorize",
|
||||
commercemid.RequestContext(),
|
||||
commercemid.TokenRequired(),
|
||||
commercebilling.AuthorizeSpendCap,
|
||||
)
|
||||
|
||||
// Self-service spend-cap CRUD WRITES — the customer half of the cap: a customer
|
||||
// (or the admin S2S) CREATES / EDITS / REMOVES their own usage caps. These are the
|
||||
// write siblings of the co-resident GET /v1/billing/spend-alerts list; without their
|
||||
// own registration they too fell through the account bridge's /v1/billing/* wildcard
|
||||
// (billingForwardable includes POST spend-alerts) into the SAME 502 self-dispatch loop
|
||||
// authorize hit — so a customer could not set a cap AT ALL in the unified binary
|
||||
// (POST/PATCH/DELETE all 502'd). Same chain commerce's own route table gates them with
|
||||
// (api/billing/handlers.go:322-325, the `user` group's userRequired = TokenRequired) +
|
||||
// the global RequestContext — an IAM JWT OR the COMMERCE_SERVICE_TOKEN, org resolved
|
||||
// from the gateway-pinned X-Org-Id into Locals("organization"). Org-scoped by that
|
||||
// namespace (a caller only ever writes their OWN org's caps; a foreign :id is a
|
||||
// not-found miss in the caller's namespace), so no PinBillingSubject — spend-alerts are
|
||||
// org-level, not billing-subject-level. Shadow the bridge wildcard (order 100 < 122).
|
||||
// A spend cap is a FINANCIAL SAFETY control, so its writes are gated to an ORG ADMIN
|
||||
// (or SuperAdmin, or the trusted S2S service token for the SuperAdmin cap-oversight
|
||||
// Forward) — never any authenticated member. commerce's own `user` group admits any
|
||||
// member, which would let a compromised member key DELETE the org's cap (→ unbounded
|
||||
// spend) or POST a 1¢ enforce cap (→ org-wide 402 DoS). requireSpendCapAdmin closes that.
|
||||
app.Post("/v1/billing/spend-alerts",
|
||||
commercemid.RequestContext(),
|
||||
commercemid.TokenRequired(),
|
||||
requireSpendCapAdmin(),
|
||||
commercebilling.CreateSpendAlert,
|
||||
)
|
||||
app.Patch("/v1/billing/spend-alerts/:id",
|
||||
commercemid.RequestContext(),
|
||||
commercemid.TokenRequired(),
|
||||
requireSpendCapAdmin(),
|
||||
commercebilling.UpdateSpendAlert,
|
||||
)
|
||||
app.Delete("/v1/billing/spend-alerts/:id",
|
||||
commercemid.RequestContext(),
|
||||
commercemid.TokenRequired(),
|
||||
requireSpendCapAdmin(),
|
||||
commercebilling.DeleteSpendAlert,
|
||||
)
|
||||
|
||||
// POST /v1/billing/topup/token — the INLINE Square card top-up (the console's
|
||||
// "Billing → Credits → add credits": the Square Web Payments SDK tokenizes the card
|
||||
// IN THE BROWSER → a single-use nonce → this endpoint charges it and credits the
|
||||
// caller's balance). commerce's api.Route() billing bundle is NOT compiled into the
|
||||
// co-resident embed, so — exactly like plans/invoices/spend-alerts above — without
|
||||
// this registration the POST fell through to the account bridge's /v1/billing/*
|
||||
// wildcard (order 122). That wildcard is service-token-forwardable for topup/token
|
||||
// (billing.go billingForwardable), so billingData re-forwarded it to COMMERCE_URL
|
||||
// (default the public api.hanzo.ai edge = THIS binary) over commerceinproc's
|
||||
// self-routing transport, re-entering the SAME wildcard until the depth-8 guard
|
||||
// refused → the "commerceinproc: in-process dispatch depth 8 exceeded" 502 that broke
|
||||
// top-up outright. Registering commerce's real TopupWithToken co-resident here
|
||||
// (order 100 < 122) shadows the wildcard and serves the charge in-process at depth 1
|
||||
// — no HTTP hop, no self-dispatch. topup/token STAYS in billingForwardable as the
|
||||
// split-deploy fallback (a standalone commerce still serves it); co-residence just
|
||||
// wins first.
|
||||
//
|
||||
// Chain — the browser money-WRITE posture, byte-for-byte what the bridge applied:
|
||||
// RequireCSRF — the ambient-cookie anti-CSRF gate the bridge's requireCSRF
|
||||
// wrapped POST /v1/billing/* with (a Bearer/gateway caller is
|
||||
// not CSRF-able; an ambient-cookie write needs the token).
|
||||
// RequestContext — the gated request context commerce's handler + ledger read.
|
||||
// IAMTokenRequired — resolves the org from the gateway-validated X-Org-Id into
|
||||
// Locals("organization"), which TopupWithToken.GetOrganization
|
||||
// + topupDestination read as the org billing key.
|
||||
// PinBillingSubject — pins ?user= to the caller's OWN account.Payer subject (the
|
||||
// SAME rule the ai spend-gate debits and billingData pins), so
|
||||
// the credit lands on the caller's subject (person=org/name) and
|
||||
// can never be widened; fail-closed for an unvalidated caller —
|
||||
// the IDOR boundary stays exactly where billingData put it.
|
||||
// The card PAN never touches this binary: TopupWithToken charges the Square nonce only,
|
||||
// and the settled charge itself is the mint authority (mintauth.WithAuthorized).
|
||||
app.Post("/v1/billing/topup/token",
|
||||
accountclient.RequireCSRF(),
|
||||
commercemid.RequestContext(),
|
||||
iammiddleware.IAMTokenRequired(),
|
||||
accountclient.PinBillingSubject(),
|
||||
commercebilling.TopupWithToken,
|
||||
)
|
||||
|
||||
// The remaining console billing WRITES that share topup/token's self-dispatch loop
|
||||
// class — each is a POST the console makes (billingForwardable in billing.go), each had
|
||||
// NO co-resident handler, so each fell through to the account bridge's /v1/billing/*
|
||||
// wildcard (order 122) and re-entered it over commerceinproc until the depth-8 guard
|
||||
// refused (the same "in-process dispatch depth 8 exceeded" 502 that broke top-up). Each
|
||||
// commerce handler exists in the vendored module (v1.49.13); registering them co-resident
|
||||
// (order 100 < 122) shadows the wildcard and serves the write in-process at depth 1. They
|
||||
// STAY in billingForwardable as the split-deploy fallback (same precedent as topup/token
|
||||
// + spend-alerts). Chain matches the bridge's write posture byte-for-byte:
|
||||
//
|
||||
// - RequireCSRF — the ambient-cookie anti-CSRF gate the bridge wrapped POST
|
||||
// /v1/billing/* with (Bearer/gateway callers are not CSRF-able).
|
||||
// - RequestContext — the gated request context commerce's handlers read.
|
||||
// - IAMTokenRequired — resolves the org from the gateway-validated X-Org-Id into
|
||||
// Locals("organization") — the namespace GetOrganization reads.
|
||||
// - PinBillingSubject — pins the caller's OWN account.Payer subject into BOTH query and
|
||||
// body AND fail-closes an unvalidated caller. It is the auth gate
|
||||
// on every one, and the IDOR control on the subject-scoped one.
|
||||
//
|
||||
// payment-methods (save a card-on-file / vault a Square nonce) is SUBJECT-scoped: commerce's
|
||||
// CreatePaymentMethod reads `customerId` from the BODY, so PinBillingSubject's body-pin is
|
||||
// load-bearing here — a member can only vault a card for their OWN subject, exactly the
|
||||
// boundary billingData's scopedBillingBody enforced. The Square nonce goes to Square; the
|
||||
// PAN never touches this binary.
|
||||
app.Post("/v1/billing/payment-methods",
|
||||
accountclient.RequireCSRF(),
|
||||
commercemid.RequestContext(),
|
||||
iammiddleware.IAMTokenRequired(),
|
||||
accountclient.PinBillingSubject(),
|
||||
commercebilling.CreatePaymentMethod,
|
||||
)
|
||||
|
||||
// subscriptions/:id/{cancel,reactivate} are org-NAMESPACE-scoped: commerce's handlers
|
||||
// resolve the subscription by `:id` WITHIN the caller's org namespace (a foreign org's id
|
||||
// is a 404 miss), so tenancy is the namespace IAMTokenRequired resolves and PinBillingSubject
|
||||
// is the fail-closed-anon auth gate — its pinned subject params are ignored by these
|
||||
// handlers (the SAME role it plays for the org-scoped payment-config read). The bridge's
|
||||
// subject-pin was likewise a no-op for these, so nothing is dropped.
|
||||
app.Post("/v1/billing/subscriptions/:id/cancel",
|
||||
accountclient.RequireCSRF(),
|
||||
commercemid.RequestContext(),
|
||||
iammiddleware.IAMTokenRequired(),
|
||||
accountclient.PinBillingSubject(),
|
||||
commercebilling.CancelBillingSubscription,
|
||||
)
|
||||
app.Post("/v1/billing/subscriptions/:id/reactivate",
|
||||
accountclient.RequireCSRF(),
|
||||
commercemid.RequestContext(),
|
||||
iammiddleware.IAMTokenRequired(),
|
||||
accountclient.PinBillingSubject(),
|
||||
commercebilling.ReactivateBillingSubscription,
|
||||
)
|
||||
|
||||
// In-process seams:
|
||||
// - commerceinproc routes the S2S billing byte-stream into the co-resident
|
||||
// app (the metering debit path) instead of a socket to a standalone pod.
|
||||
@@ -270,3 +517,19 @@ func fireCapAlert(org string, test bool, project, service string) {
|
||||
db := commercedatastore.New(ctx)
|
||||
commercebilling.FireSpendAlerts(ctx, db, org, test, project, service, nil)
|
||||
}
|
||||
|
||||
// requireSpendCapAdmin gates a spend-alert WRITE to a validated ORG ADMIN or platform
|
||||
// SuperAdmin (the unforgeable SanitizeIdentity-minted X-User-IsOrgAdmin / isAdmin bits),
|
||||
// OR the trusted in-proc S2S service token (the SuperAdmin cap-oversight Forward + internal
|
||||
// automation). A validated non-admin MEMBER is REFUSED (403): a spend cap is a financial
|
||||
// safety boundary — a member must not be able to delete the org's cap (→ unbounded spend)
|
||||
// or set a punitive 1¢ cap (→ org-wide 402 DoS). The read paths (list/authorize) stay
|
||||
// member/S2S-open; only the mutations require admin.
|
||||
func requireSpendCapAdmin() zip.Handler {
|
||||
return func(c *zip.Ctx) error {
|
||||
if principal.IsSuperAdmin(c) || principal.IsOrgAdmin(c) || accountclient.IsServiceToken(c) {
|
||||
return c.Next()
|
||||
}
|
||||
return zip.ErrForbidden("org admin required to change spend caps")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,8 @@ func TestCommercePrefixesPinned(t *testing.T) {
|
||||
"/v1/billing/auto-recharge": false,
|
||||
"/v1/billing/webhooks": false,
|
||||
"/v1/store": false,
|
||||
"/v1/catalog": false,
|
||||
"/v1/plans": false,
|
||||
}
|
||||
for _, p := range commercePrefixes {
|
||||
if _, ok := want[p]; ok {
|
||||
@@ -84,6 +86,25 @@ func TestStoreSurfaceRoutedToCommerceNotAIGate(t *testing.T) {
|
||||
if code, _ := doReq(t, app, http.MethodPut, "/v1/store/karma-store/listing/valentina"); code == http.StatusPaymentRequired {
|
||||
t.Fatalf("PUT /v1/store/:id/listing/:slug fell through to the AI balance gate (402) — the whole store surface must be commerce-owned")
|
||||
}
|
||||
|
||||
// The platform-admin catalog CMS (admin.hanzo.ai's editor) must reach commerce —
|
||||
// where each handler is requireSuperAdmin-gated (anon → 401/403) — never the AI
|
||||
// /v1/* balance gate, which would 402 the editor's list/edit instead.
|
||||
if code, _ := doReq(t, app, http.MethodGet, "/v1/catalog/entries"); code == http.StatusPaymentRequired {
|
||||
t.Fatalf("GET /v1/catalog/entries fell through to the AI balance gate (402) — /v1/catalog must be a commercePrefix so the SuperAdmin CMS reaches commerce")
|
||||
}
|
||||
if code, _ := doReq(t, app, http.MethodPut, "/v1/catalog/entries/cloud-starter"); code == http.StatusPaymentRequired {
|
||||
t.Fatalf("PUT /v1/catalog/entries/:slug fell through to the AI balance gate (402) — the whole catalog CMS must be commerce-owned")
|
||||
}
|
||||
|
||||
// The platform-admin plan authority CMS (increment 3a) must also reach commerce —
|
||||
// requireSuperAdmin-gated (anon → 401/403) — never the AI /v1/* 402 gate.
|
||||
if code, _ := doReq(t, app, http.MethodGet, "/v1/plans/entries"); code == http.StatusPaymentRequired {
|
||||
t.Fatalf("GET /v1/plans/entries fell through to the AI balance gate (402) — /v1/plans must be a commercePrefix so the plan authority CMS reaches commerce")
|
||||
}
|
||||
if code, _ := doReq(t, app, http.MethodPut, "/v1/plans/entries/pro"); code == http.StatusPaymentRequired {
|
||||
t.Fatalf("PUT /v1/plans/entries/:slug fell through to the AI balance gate (402) — the whole plan authority CMS must be commerce-owned")
|
||||
}
|
||||
}
|
||||
|
||||
// doReq drives one request through the mounted app and returns (status, body).
|
||||
|
||||
@@ -7,13 +7,15 @@ import (
|
||||
)
|
||||
|
||||
// TestFrameworkContentModulesLinked guards the framework CONTENT modules
|
||||
// (cms/erp/help) against silent removal. They are not mount subsystems, so they
|
||||
// never appear in Wire(); they register their DocTypes and — for erp — the
|
||||
// ledger-posting lifecycle hooks into the framework engine from a package
|
||||
// init(), reached ONLY via the blank imports in apps.go. #248 dropped
|
||||
// those imports, which stripped the erp ledger hooks from the binary with no
|
||||
// mount change and no failing mount test. This asserts the engine's module
|
||||
// registry carries each lane, so that money-adjacent regression cannot recur.
|
||||
// (cms/erp/help) against silent removal. cms/erp are pure fixture lanes reached
|
||||
// ONLY via blank imports in apps.go; help is a real import + a Wire() spec (it also
|
||||
// mounts the /v1/help public plane) but still contributes its DocTypes the SAME way,
|
||||
// from a package init() (framework.RegisterModule). #248 dropped the blank imports,
|
||||
// which stripped the erp ledger hooks from the binary with no mount change and no
|
||||
// failing mount test. This asserts the engine's module registry carries each lane,
|
||||
// so that money-adjacent regression cannot recur — for help, dropping the import
|
||||
// would also drop its Wire() spec and fail TestWireOrderMatchesFrozen, but this keeps
|
||||
// the fixture-linkage guard uniform across all three lanes.
|
||||
func TestFrameworkContentModulesLinked(t *testing.T) {
|
||||
got := make(map[string]bool)
|
||||
for _, m := range framework.RegisteredModules() {
|
||||
|
||||
@@ -1,9 +1,20 @@
|
||||
package apps
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/ads"
|
||||
"github.com/hanzoai/cloud/clients/automations"
|
||||
"github.com/hanzoai/cloud/clients/campaign"
|
||||
"github.com/hanzoai/cloud/clients/coding"
|
||||
"github.com/hanzoai/cloud/clients/experiments"
|
||||
"github.com/hanzoai/cloud/clients/framework"
|
||||
"github.com/hanzoai/cloud/clients/git"
|
||||
"github.com/hanzoai/cloud/clients/guide"
|
||||
"github.com/hanzoai/cloud/clients/integrations"
|
||||
"github.com/hanzoai/cloud/clients/principal"
|
||||
)
|
||||
|
||||
// wire_seams.go wires cross-subsystem in-process seams that cannot be a MountSpec
|
||||
@@ -20,4 +31,92 @@ import (
|
||||
// failures are non-fatal and dropped).
|
||||
func init() {
|
||||
integrations.SetCodingDispatcher(coding.NewDispatcher(git.CloneURL, git.VerifyRef, nil))
|
||||
|
||||
// Guide growth-OBSERVE seam: the /v1/guide/profile observe layer reads the org's
|
||||
// real platform truth through injected, org-scoped, honest-degrading probes.
|
||||
// clients/guide imports NONE of these subsystems (decomplected), so the
|
||||
// composition root — the ONE place that imports them all — binds the reads, the
|
||||
// same injected-function pattern the coding dispatcher above uses. Each probe is
|
||||
// PROVABLY org-scoped: framework.ModuleInstalled keys GetDocType on the org;
|
||||
// integrations.Connected keys store.Get on the org and never surfaces the token.
|
||||
// HasDeployment/RevenueCents/RecordCount are LEFT NIL: deploy is cluster/admin-
|
||||
// scoped (not per-org) and commerce-revenue-of-record + the crm record count have
|
||||
// no clean per-org in-process read yet — binding a read whose org-scoping we
|
||||
// cannot guarantee would be the bug. Until a provably-org-scoped read lands their
|
||||
// signals honest-degrade to not-present (the vocabulary is the contract; a nil
|
||||
// seam can never be a spurious true).
|
||||
guide.BindSignals(guide.Signals{
|
||||
ModuleInstalled: framework.ModuleInstalled,
|
||||
ConnectorPresent: integrations.Connected,
|
||||
})
|
||||
|
||||
// Inbound-event seam: a verified provider webhook (or chat channel) in
|
||||
// clients/integrations fires the automations engine's Deliver here — the ONE place
|
||||
// that imports both, so integrations never has to import automations (which imports
|
||||
// it, for credential custody). A primitive-typed adapter keeps the seam free of the
|
||||
// automations types.
|
||||
integrations.SetAutomationTrigger(func(ctx context.Context, org, source, name, dedupeKey string, depth int, payload map[string]any) (int, error) {
|
||||
return automations.Deliver(ctx, org, automations.TriggerEvent{
|
||||
Source: source, Name: name, DedupeKey: dedupeKey, Depth: depth, Payload: payload,
|
||||
})
|
||||
})
|
||||
|
||||
// GTM PAID channel: the /v1/campaign orchestrator fans out to executors that
|
||||
// satisfy campaign.Channel; the composition root is the ONE place that imports
|
||||
// both campaign and the ad plane, so it adapts ads' connector-consuming
|
||||
// execution funcs (provider.go — each resolves the org's ad token through
|
||||
// integrations.TokenFor and fails closed) onto the primitive-typed channel
|
||||
// seam. campaign never imports ads and ads never imports campaign — the same
|
||||
// injected-function decoupling the coding dispatcher above uses.
|
||||
campaign.RegisterChannel(campaign.NewChannel(campaign.KindPaid,
|
||||
func(ctx context.Context, org string, p campaign.Plan) (campaign.Ref, error) {
|
||||
r, err := ads.LaunchPaid(ctx, org, ads.PaidPlan{
|
||||
Platform: p.Platform, Account: p.Account, Name: p.Name,
|
||||
Objective: p.Objective, BudgetCents: p.BudgetCents, ScheduleAt: p.ScheduleAt,
|
||||
})
|
||||
return campaign.Ref{Platform: r.Platform, Account: r.Account, ExternalID: r.ExternalID, Status: r.Status, Detail: r.Detail}, err
|
||||
},
|
||||
func(ctx context.Context, org string, ref campaign.Ref) (int64, error) {
|
||||
return ads.PaidSpend(ctx, org, ads.PaidRef{Platform: ref.Platform, Account: ref.Account, ExternalID: ref.ExternalID})
|
||||
},
|
||||
func(ctx context.Context, org string, ref campaign.Ref) error {
|
||||
return ads.PausePaid(ctx, org, ads.PaidRef{Platform: ref.Platform, Account: ref.Account, ExternalID: ref.ExternalID})
|
||||
},
|
||||
))
|
||||
|
||||
// GTM creative A/B composes the merged EXPERIMENT primitive (clients/experiments)
|
||||
// — campaign never reinvents bucketing or measurement. Assign resolves the
|
||||
// subject's variant (creative) from the experiment's flag; Analyze is pull-model
|
||||
// (it reads the metric from analytics itself), returned as opaque JSON so
|
||||
// campaign stays decoupled from the analysis type. Campaign-linked experiments
|
||||
// live in the org's default project. Both are nil-safe upstream: an org that
|
||||
// never created a "campaign:<id>" experiment runs a single creative (Assign
|
||||
// errors → "" → Content[0]).
|
||||
campaign.SetExperiment(
|
||||
func(ctx context.Context, org, experimentID, subject string) (string, error) {
|
||||
a, err := experiments.Assign(ctx, org, principal.DefaultProject, experimentID, subject, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return a.Variant, nil
|
||||
},
|
||||
func(ctx context.Context, org, experimentID string, start, end time.Time) (json.RawMessage, error) {
|
||||
an, err := experiments.Analyze(ctx, org, principal.DefaultProject, experimentID, start, end, 0.05)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(an)
|
||||
},
|
||||
)
|
||||
|
||||
// GTM ORGANIC + EMAIL channels wire HERE the same way once their executors land
|
||||
// (designed follow-ons — the publish rename + marketing email-connector wiring):
|
||||
//
|
||||
// campaign.RegisterChannel(campaign.NewChannel(campaign.KindOrganic,
|
||||
// publish.Syndicate, publish.NoSpend, publish.Unpublish)) // social connectors
|
||||
// campaign.RegisterChannel(campaign.NewChannel(campaign.KindEmail,
|
||||
// marketing.Broadcast, marketing.NoSpend, marketing.Halt)) // email connectors
|
||||
//
|
||||
// Until wired, those channels record "unavailable" on a fan-out (honest) and a
|
||||
// campaign runs a single creative — never a fabricated launch or variant.
|
||||
}
|
||||
|
||||
@@ -42,11 +42,13 @@ var frozen = []struct {
|
||||
{"storage", true, false}, // was order 118
|
||||
{"provisioning", false, false}, // was order 120
|
||||
{"billing", false, false}, // was order 121
|
||||
{"rollingcap", false, false}, // rolling spend-cap gate (after billing); golden drifted — refrozen
|
||||
{"account-bridge", false, false}, // was order 122
|
||||
{"do", false, false}, // was order 123
|
||||
{"platform", true, false}, // was order 124
|
||||
{"projects", false, false}, // was order 125
|
||||
{"dns", false, false}, // new: /v1/dns zone plane (after projects)
|
||||
{"domain", false, false}, // new: Hanzo Domains registrar (/v1/domain), after dns
|
||||
{"prompts", false, false}, // was order 126
|
||||
{"agents", false, true}, // was order 127
|
||||
{"link", false, true}, // new: unified AI login manager (/v1/links), after agents
|
||||
@@ -59,18 +61,23 @@ var frozen = []struct {
|
||||
{"templates", false, false}, // was order 129
|
||||
{"framework", false, true}, // was order 129
|
||||
{"knowledge", false, false}, // was order 130
|
||||
{"help", false, false}, // new: Hanzo Support public plane /v1/help (after knowledge; framework lane with a companion subsystem, no store → no Shutdown)
|
||||
{"content", false, true}, // new: marketing content loop (after knowledge)
|
||||
{"catalogsync", false, true}, // new: reverse loop (product.created → render) after content
|
||||
{"ml", true, false}, // was order 130
|
||||
{"usage", false, false}, // was order 131
|
||||
{"leaderboard", false, true}, // new: gamified usage analytics (after usage), owns opt-in SQLite (Shutdown)
|
||||
{"crm", false, false}, // was order 131
|
||||
{"marketing", false, true}, // new: marketing domain fold (after crm)
|
||||
{"ads", false, true}, // new: ads domain fold (after crm)
|
||||
{"campaign", false, true}, // new: /v1/campaign GTM orchestration (after ads); fans out to channels
|
||||
{"validators", false, true}, // new: NFT-gated node provisioning (after ads); golden refrozen
|
||||
{"social", false, true}, // new: /v1/social fold (after crm)
|
||||
{"analytics", true, false}, // was order 132
|
||||
{"git", false, false}, // was order 132
|
||||
{"sync", false, true}, // /v1/sync engine (owns per-org DB handles → Shutdown)
|
||||
{"visor", false, false}, // was order 133
|
||||
{"venue", false, false}, // new: /v1/cloud connect-a-cloud-account plane (after visor); folds discovered clusters into the fleet
|
||||
{"captable", false, true}, // was order 133
|
||||
{"code", false, true}, // was order 134
|
||||
{"zero-trust", false, false}, // was order 134
|
||||
@@ -78,11 +85,13 @@ var frozen = []struct {
|
||||
{"graph", false, false}, // was order 135
|
||||
{"security", true, true}, // was order 136
|
||||
{"integrations", false, true}, // was order 137
|
||||
{"destinations", false, true}, // new: /v1/destinations CDP fan-out (after integrations, before cloudflare)
|
||||
{"cloudflare", false, false}, // new: /v1/cloudflare edge plane (after integrations)
|
||||
{"sbom", true, false}, // was order 137
|
||||
{"team", false, true}, // was order 138
|
||||
{"settings", false, true}, // was order 138
|
||||
{"notify", true, false}, // was order 139
|
||||
{"channels", false, true}, // new: /v1/channels transport plane (after notify; must mount after integrations so RegisterIngress installs before webhooks emit)
|
||||
{"gateway", false, false}, // was order 139
|
||||
{"entitlements", false, true}, // was order 139
|
||||
{"exec", false, false}, // was order 140
|
||||
@@ -96,6 +105,9 @@ var frozen = []struct {
|
||||
{"sign", true, true}, // was order 145
|
||||
{"product", false, false}, // was order 145
|
||||
{"evals", false, false}, // was order 145
|
||||
{"benchmark", false, false}, // benchmark plane (after evals, before treasury)
|
||||
{"research", false, true}, // R&D evidence plane + /research board (HIP-0512), arena sibling after benchmark; Shutdown closes per-org stores
|
||||
{"experiments", false, true}, // unified A/B EXPERIMENT primitive: composes flags(assign)+analytics(measure)+research(evidence); Shutdown closes the registry stores
|
||||
{"treasury", false, true}, // was order 146
|
||||
{"admin", false, false}, // was order 146
|
||||
{"admission", false, true}, // launch-control gate: composes flags (registry+seed+mode route+Enforce); Shutdown closes the registry store
|
||||
@@ -106,6 +118,8 @@ var frozen = []struct {
|
||||
{"referrals", false, false}, // was order 149
|
||||
{"guide", false, true}, // new: Business AI Guide (after referrals, before ai)
|
||||
{"company", false, true}, // new: Hanzo Company formation state machine (after guide)
|
||||
{"compliance", true, true}, // new: Hanzo Compliance — KYC/KYB + accreditation + audit posture (after company)
|
||||
{"legal", true, true}, // new: Hanzo Legal — template + generation engine + e-sign/filing (after compliance)
|
||||
{"agent", false, false}, // new: /v1/agent tool-calling round (before zen/ai catch-all)
|
||||
{"zen", false, false}, // zen* claim middleware before ai's catch-all (hip-00NN)
|
||||
{"ai", false, false}, // was order 150
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
// Copyright 2026 Hanzo AI Inc. All Rights Reserved.
|
||||
|
||||
package apps
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// stubKMS is a KMSClient whose GetSecret returns a sealed value when present or a
|
||||
// not-found error otherwise — mirroring the co-resident store that, in production,
|
||||
// holds no upstream provider keys (they are provisioned as KMS-injected env).
|
||||
type stubKMS struct{ sealed map[string]string }
|
||||
|
||||
func (s stubKMS) GetSecret(_ context.Context, ref string) ([]byte, error) {
|
||||
if v, ok := s.sealed[ref]; ok {
|
||||
return []byte(v), nil
|
||||
}
|
||||
return nil, errors.New("secret not found")
|
||||
}
|
||||
func (stubKMS) PutSecret(context.Context, string, []byte) error { return nil }
|
||||
func (stubKMS) Sign(context.Context, string, []byte) ([]byte, error) { return nil, nil }
|
||||
|
||||
// TestZenKeyResolver_EnvFallback pins the production wiring: the upstream provider
|
||||
// key is provisioned as env (from the cloud-api-llm-keys secret), NOT sealed in the
|
||||
// co-resident KMS store, so a KMS miss must resolve to the env value rather than
|
||||
// returning "" (which would send an empty bearer upstream → provider 401).
|
||||
func TestZenKeyResolver_EnvFallback(t *testing.T) {
|
||||
const env = "DO_AI_API_KEY"
|
||||
t.Setenv(env, "env-provisioned-key")
|
||||
|
||||
// KMS store has no upstream keys (the real deployment state) — resolve from env.
|
||||
if got := zenKeyResolver(stubKMS{})(context.Background(), env); got != "env-provisioned-key" {
|
||||
t.Fatalf("KMS-miss: got %q, want env value", got)
|
||||
}
|
||||
|
||||
// A nil KMS client (KMS disabled) — still resolve from env.
|
||||
if got := zenKeyResolver(nil)(context.Background(), env); got != "env-provisioned-key" {
|
||||
t.Fatalf("nil-KMS: got %q, want env value", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestZenKeyResolver_EnvTakesPrecedence pins the resolution ORDER: env is read
|
||||
// FIRST, then KMS — the same order ai uses on the prod hot path. The operator
|
||||
// injects provider keys as env from the KMS-synced secret, so the env is the live
|
||||
// value; the co-resident store is the fallback. A key present in BOTH surfaces
|
||||
// resolves to the env value.
|
||||
func TestZenKeyResolver_EnvTakesPrecedence(t *testing.T) {
|
||||
const env = "ANTHROPIC_API_KEY"
|
||||
t.Setenv(env, "env-key")
|
||||
got := zenKeyResolver(stubKMS{sealed: map[string]string{env: "sealed-key"}})(context.Background(), env)
|
||||
if got != "env-key" {
|
||||
t.Fatalf("got %q, want env-key (env precedence)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestZenKeyResolver_AbsentEverywhere keeps the fail-fast contract: absent from both
|
||||
// surfaces resolves to "" so zen refuses rather than serving for free.
|
||||
func TestZenKeyResolver_AbsentEverywhere(t *testing.T) {
|
||||
if got := zenKeyResolver(stubKMS{})(context.Background(), "MISSING_KEY_XYZ"); got != "" {
|
||||
t.Fatalf("got %q, want empty", got)
|
||||
}
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
// Copyright 2026 The Hanzo Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestJWTAudiences_AcceptsAdminGuard is the operator-cockpit keystone. The
|
||||
// admin.hanzo.ai forward-auth guard is the confidential client `hanzo-admin-guard`,
|
||||
// so IAM mints its access tokens with aud=hanzo-admin-guard (each app's aud is its
|
||||
// client_id). The guard forwards that bearer to cloud-api /v1/admin/*; the identity
|
||||
// sanitizer only grants SuperAdmin (owner==adminOrg) to a VALIDATED principal, and
|
||||
// validation enforces this audience allowlist. If hanzo-admin-guard is not accepted
|
||||
// the token resolves anonymous and the SuperAdmin gate reads false -> 403, even
|
||||
// though the token's owner IS admin. Pin the client_id into the baked default so the
|
||||
// forwarded bearer validates.
|
||||
func TestJWTAudiences_AcceptsAdminGuard(t *testing.T) {
|
||||
os.Unsetenv("CLOUD_JWT_AUDIENCES")
|
||||
os.Unsetenv("GATEWAY_ALLOWED_AUDIENCES")
|
||||
|
||||
has := func(list []string, v string) bool {
|
||||
for _, s := range list {
|
||||
if s == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
if !has(defaultJWTAudiences, "hanzo-admin-guard") {
|
||||
t.Fatalf("defaultJWTAudiences must include hanzo-admin-guard (the admin-cockpit guard client_id); got %v", defaultJWTAudiences)
|
||||
}
|
||||
if !has(jwtAudiencesFromEnv(), "hanzo-admin-guard") {
|
||||
t.Fatalf("resolved JWT audiences must include hanzo-admin-guard; got %v", jwtAudiencesFromEnv())
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
// Copyright 2026 The Hanzo Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestJWTAudiences_AcceptsHanzoWorld pins the world.hanzo.ai OIDC client. IAM
|
||||
// mints world's access tokens with aud=hanzo-world (each app's aud is its
|
||||
// client_id). Those bearers hit cloud-api; the identity sanitizer only trusts a
|
||||
// principal whose aud is in this allowlist. If hanzo-world is not accepted the
|
||||
// token resolves anonymous and the analyst's api.hanzo.ai calls 401. Pin the
|
||||
// client_id into the baked default so the forwarded bearer validates.
|
||||
func TestJWTAudiences_AcceptsHanzoWorld(t *testing.T) {
|
||||
os.Unsetenv("CLOUD_JWT_AUDIENCES")
|
||||
os.Unsetenv("GATEWAY_ALLOWED_AUDIENCES")
|
||||
|
||||
has := func(list []string, v string) bool {
|
||||
for _, s := range list {
|
||||
if s == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
if !has(defaultJWTAudiences, "hanzo-world") {
|
||||
t.Fatalf("defaultJWTAudiences must include hanzo-world (the world.hanzo.ai client_id); got %v", defaultJWTAudiences)
|
||||
}
|
||||
if !has(jwtAudiencesFromEnv(), "hanzo-world") {
|
||||
t.Fatalf("resolved JWT audiences must include hanzo-world; got %v", jwtAudiencesFromEnv())
|
||||
}
|
||||
}
|
||||
+52
-59
@@ -33,6 +33,7 @@ import (
|
||||
"github.com/go-jose/go-jose/v4/jwt"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/principal"
|
||||
model "github.com/hanzoai/iam/pkg/model"
|
||||
)
|
||||
|
||||
// idClaims is the subset of Hanzo IAM JWT claims the identity sanitizer needs.
|
||||
@@ -40,13 +41,14 @@ import (
|
||||
type idClaims struct {
|
||||
jwt.Claims
|
||||
|
||||
Owner string `json:"owner"` // org slug (the org)
|
||||
Project string `json:"project"` // org SUB-SCOPE within owner (empty ⟹ default project)
|
||||
BillingAccount string `json:"billing_account"` // WHO PAYS, stated by IAM (empty ⟹ pre-claim token)
|
||||
Name string `json:"name"` // display name (id fallback)
|
||||
PreferredUsername string `json:"preferred_username"` // id fallback
|
||||
Email string `json:"email"`
|
||||
IsAdmin bool `json:"isAdmin"`
|
||||
Owner string `json:"owner"` // org slug (the org)
|
||||
Project string `json:"project"` // org SUB-SCOPE within owner (empty ⟹ default project)
|
||||
BillingAccount string `json:"billing_account"` // WHO PAYS, stated by IAM (empty ⟹ pre-claim token)
|
||||
Name string `json:"name"` // display name (id fallback)
|
||||
PreferredUsername string `json:"preferred_username"` // id fallback
|
||||
Email string `json:"email"`
|
||||
IsAdmin bool `json:"isAdmin"`
|
||||
Orgs []model.OrgRef `json:"orgs"` // membership SET (home first); empty on legacy tokens
|
||||
}
|
||||
|
||||
// mintedProject returns the project id to stamp into X-Project-Id, or "" when the
|
||||
@@ -123,43 +125,41 @@ var jwtSigAlgs = []gojose.SignatureAlgorithm{
|
||||
// (cert-hanzo/cert-lux/cert-zoo/...), keyed by the token kid, so a single
|
||||
// jwksURL verifies all brands. Only the issuer-string comparison had to widen.
|
||||
type identityValidator struct {
|
||||
issuers []string
|
||||
audiences []string
|
||||
cache *jwksCache
|
||||
keys keyResolver // resolves an opaque API key to a principal; nil ⟹ keys stay anonymous
|
||||
issuers []string
|
||||
cache *jwksCache
|
||||
keys keyResolver // resolves an opaque API key to a principal; nil ⟹ keys stay anonymous
|
||||
}
|
||||
|
||||
// newIdentityValidator builds a validator whose trusted-issuer set is the primary
|
||||
// issuer UNIONED with every white-label brand issuer (BrandIssuers) plus any
|
||||
// WHITELABEL_ISSUERS override. ttl<=0 uses the 15m JWKS default. The union is
|
||||
// fail-secure: it only ADDS the known-good brand issuers, never an arbitrary one.
|
||||
func newIdentityValidator(issuer, jwksURL string, audiences []string, ttl time.Duration) *identityValidator {
|
||||
//
|
||||
// Trust is IAM-native: signature (JWKS) + issuer (this set) + expiry. There is NO
|
||||
// per-app audience allowlist — the `aud` (a minting app's client_id) is IAM's to
|
||||
// assign, not cloud's to mirror, so a new first-party app needs zero cloud change.
|
||||
func newIdentityValidator(issuer, jwksURL string, ttl time.Duration) *identityValidator {
|
||||
return &identityValidator{
|
||||
issuers: trustedIssuers(issuer),
|
||||
audiences: audiences,
|
||||
cache: newJWKSCache(jwksURL, ttl),
|
||||
keys: sharedKeys(), // ONE resolver+cache, shared with OrgForKey (analytics capture)
|
||||
issuers: trustedIssuers(issuer),
|
||||
cache: newJWKSCache(jwksURL, ttl),
|
||||
keys: sharedKeys(), // ONE resolver+cache, shared with OrgForKey (analytics capture)
|
||||
}
|
||||
}
|
||||
|
||||
// kmsMachineAudSuffix is the fixed suffix of a per-org PaaS-KMS sync machine
|
||||
// identity's audience. Each org's KMS sync authenticates as a dedicated,
|
||||
// NON-shared IAM application named "<org>-platform-kms" (Organization=<org>,
|
||||
// client_credentials grant), so IAM stamps the token's aud == the app's own
|
||||
// clientId == "<org>-platform-kms" (a non-shared app's audience is its clientId,
|
||||
// object/token_jwt.go tokenAudience) and owner == <org>
|
||||
// (object/token_oauth.go GetClientCredentialsToken sets owner = app.Organization).
|
||||
// identity's audience. Each org's KMS sync authenticates as a dedicated, NON-shared
|
||||
// IAM application named "<org>-platform-kms" (Organization=<org>, client_credentials
|
||||
// grant), so IAM stamps the token's aud == the app's own clientId == "<org>-platform-kms"
|
||||
// (object/token_jwt.go tokenAudience) and owner == <org> (object/token_oauth.go
|
||||
// GetClientCredentialsToken sets owner = app.Organization).
|
||||
//
|
||||
// That audience is, by construction, absent from CLOUD_JWT_AUDIENCES — it is
|
||||
// per-org, not a fixed app — which is EXACTLY why the sync stayed pending: the
|
||||
// machine token failed the audience check below, SanitizeIdentity treated it as
|
||||
// anonymous, and the /v1/kms org-scope guard 403'd it before the store. The fix is
|
||||
// to accept this one audience, but ONLY when it equals the token's OWN owner claim
|
||||
// plus this suffix, so it certifies "the KMS sync identity for its own org" and
|
||||
// grants nothing wider. Org-scoping is still enforced downstream by owner at the guard
|
||||
// (owner == :org); this only lets a legitimately-minted, owner-scoped machine token
|
||||
// clear validation. A per-org application means a per-org clientSecret — never
|
||||
// a shared platform-wide reader, which would be a cross-org hole.
|
||||
// Validation no longer consults the audience at all (trust is signature + issuer +
|
||||
// expiry), so a machine token clears validate() like any other. This suffix survives
|
||||
// for the OPPOSITE reason: to RECOGNISE a machine principal (isKMSMachinePrincipal) so
|
||||
// SanitizeIdentity can DENY it SuperAdmin even when it carries owner==adminOrg — a
|
||||
// client_credentials machine identity must never wield platform-admin. The match is
|
||||
// bound to the token's OWN owner claim (<owner>-platform-kms), so it certifies "the
|
||||
// KMS sync identity for its own org" and grants nothing wider.
|
||||
const kmsMachineAudSuffix = "-platform-kms"
|
||||
|
||||
// kmsMachineAudience returns the audience an org org's PaaS-KMS sync identity
|
||||
@@ -212,14 +212,13 @@ func (v *identityValidator) validate(raw string) (*idClaims, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Fail SECURE on a misconfigured (empty) trust set: an empty issuer OR audience
|
||||
// allowlist must REJECT every token, never silently disable that axis. In
|
||||
// production both are always resolved non-empty (BrandIssuers + the baked
|
||||
// audience defaults, unioned in config.go so they are "never empty"), so this
|
||||
// fires ONLY on an operator misconfiguration (CLOUD_JWT_AUDIENCES="" emptying the
|
||||
// resolved set, or an empty issuer set) — and then it denies, it never admits (I2).
|
||||
if len(v.issuers) == 0 || len(v.audiences) == 0 {
|
||||
return nil, fmt.Errorf("identity validator misconfigured: empty issuer or audience allowlist")
|
||||
// Fail SECURE on a misconfigured (empty) trust set: with no trusted issuer every
|
||||
// token must be REJECTED, never silently admitted. In production the set is always
|
||||
// non-empty (the primary issuer + BrandIssuers, unioned in config.go so it is
|
||||
// "never empty"), so this fires ONLY on an operator misconfiguration — and then it
|
||||
// denies, it never admits (I2).
|
||||
if len(v.issuers) == 0 {
|
||||
return nil, fmt.Errorf("identity validator misconfigured: empty issuer set")
|
||||
}
|
||||
|
||||
// Reject a missing issuer: an empty issuer must never pass the set check.
|
||||
@@ -232,28 +231,22 @@ func (v *identityValidator) validate(raw string) (*idClaims, error) {
|
||||
if claims.Expiry == nil {
|
||||
return nil, fmt.Errorf("missing expiry")
|
||||
}
|
||||
// Issuer must be one of the trusted brand issuers. go-jose's jwt.Expected
|
||||
// checks a SINGLE issuer, so the issuer is validated here against the set and
|
||||
// left out of Expected (audience + expiry stay with Expected).
|
||||
// Issuer must be one of the trusted brand issuers. go-jose's jwt.Expected checks a
|
||||
// SINGLE issuer, so the issuer is validated here against the set and left out of
|
||||
// Expected (only expiry/not-before stay with Expected).
|
||||
if !issuerAllowed(claims.Issuer, v.issuers) {
|
||||
return nil, fmt.Errorf("untrusted issuer %q", claims.Issuer)
|
||||
}
|
||||
// Audience: the static allowlist (CLOUD_JWT_AUDIENCES / brand app client_ids)
|
||||
// PLUS the per-org PaaS-KMS sync machine audience bound to THIS token's own
|
||||
// owner (<owner>-platform-kms). The machine audience is added only when the
|
||||
// allowlist is active (non-empty — always so in production) and only for the
|
||||
// token's own org, so accepting it never widens org-scoping: the /v1/kms guard still
|
||||
// gates on owner == :org. Without this, a real client_credentials machine token
|
||||
// (aud == its per-org clientId, never in the allowlist) fails here and the
|
||||
// sync silently stays pending — the activation blocker.
|
||||
// The audience allowlist is guaranteed non-empty (checked above), so the
|
||||
// audience axis is ALWAYS enforced — never silently skipped.
|
||||
auds := v.audiences
|
||||
if mach := kmsMachineAudience(claims.Owner); mach != "" {
|
||||
auds = append(append(make([]string, 0, len(v.audiences)+1), v.audiences...), mach)
|
||||
}
|
||||
expected := jwt.Expected{AnyAudience: jwt.Audience(auds)}
|
||||
if err := claims.Claims.ValidateWithLeeway(expected, 2*time.Minute); err != nil {
|
||||
// Audience is NOT an access gate. A valid signature from a trusted issuer (both
|
||||
// checked above) proves IAM minted this token for one of ITS OWN registered apps;
|
||||
// the `aud` merely names which app. Cloud does not keep a per-app allowlist to
|
||||
// mirror IAM's registry — that mirror drifted and silently 401'd every new
|
||||
// first-party app until hand-edited. Org scope is the `owner` claim, enforced by
|
||||
// every downstream guard; SuperAdmin is owner==adminOrg AND !isKMSMachinePrincipal
|
||||
// (SanitizeIdentity). Expiry + not-before are STILL enforced here: Expected{} with a
|
||||
// zero Time validates against time.Now(); an empty AnyAudience skips ONLY the
|
||||
// audience match (go-jose/v4 jwt/validation.go).
|
||||
if err := claims.Claims.ValidateWithLeeway(jwt.Expected{}, 2*time.Minute); err != nil {
|
||||
return nil, fmt.Errorf("claims: %w", err)
|
||||
}
|
||||
return &claims, nil
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestValidate_AudienceIsNotAGate proves the IAM-native trust model: a token signed
|
||||
// by a trusted issuer validates REGARDLESS of its `aud` (the minting app's client_id).
|
||||
// Cloud keeps no per-app audience allowlist mirroring IAM's registry — so a brand-new
|
||||
// first-party app works with zero cloud change, and the specific app tokens the old
|
||||
// per-app tests pinned (admin-guard, world, team, commerce) are accepted by the SAME
|
||||
// rule as everything else. Trust is signature + issuer + expiry; org scope is the
|
||||
// owner claim, enforced downstream. Replaces the three audience_*_test.go files that
|
||||
// asserted a static allowlist which no longer exists.
|
||||
func TestValidate_AudienceIsNotAGate(t *testing.T) {
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("genkey: %v", err)
|
||||
}
|
||||
jwks := jwksServer(t, &key.PublicKey)
|
||||
v := newIdentityValidator(testIssuer, jwks.URL, 0)
|
||||
future := time.Now().Add(time.Hour)
|
||||
|
||||
// Every audience — the apps the deleted per-app tests pinned AND a never-registered
|
||||
// one — validates identically, because aud is not an access gate.
|
||||
for _, aud := range []string{
|
||||
"hanzo-admin-guard", // admin.hanzo.ai forward-auth cockpit
|
||||
"hanzo-world", // world.hanzo.ai analyst tokens
|
||||
"hanzo-team", // hanzo.team wallet page
|
||||
"hanzo-commerce", // commerce.hanzo.ai admin AI assistant
|
||||
"a-brand-new-first-party-app-never-listed-anywhere",
|
||||
} {
|
||||
tok := signWith(t, key, tokenClaims(aud, "acme", "", false, future))
|
||||
id, err := v.validate(tok)
|
||||
if err != nil {
|
||||
t.Fatalf("aud=%q from a trusted issuer must validate (no allowlist), got %v", aud, err)
|
||||
}
|
||||
if id.Owner != "acme" {
|
||||
t.Errorf("aud=%q: owner must be carried through, got %q", aud, id.Owner)
|
||||
}
|
||||
}
|
||||
|
||||
// Expiry is STILL enforced (dropping the aud gate must not disable time checks):
|
||||
// a token expired beyond the 2m leeway is rejected whatever its aud.
|
||||
expired := signWith(t, key, tokenClaims("hanzo-commerce", "acme", "", false, time.Now().Add(-time.Hour)))
|
||||
if _, err := v.validate(expired); err == nil {
|
||||
t.Error("expired token must be REJECTED even though audience is no longer gated")
|
||||
}
|
||||
}
|
||||
+49
-35
@@ -1,15 +1,15 @@
|
||||
package cloud
|
||||
|
||||
// V6 (the activation blocker) — the identity validator must accept a per-org
|
||||
// PaaS-KMS sync machine token: a client_credentials JWT whose aud is the org's
|
||||
// own IAM application clientId "<owner>-platform-kms" (a per-org value, NEVER in
|
||||
// CLOUD_JWT_AUDIENCES) — but ONLY when that audience is bound to the token's OWN
|
||||
// owner claim. Before the fix the machine token failed the audience check,
|
||||
// SanitizeIdentity resolved anonymous, and the /v1/kms guard 403'd it, so the sync
|
||||
// silently stayed pending. These are white-box unit tests of validate() itself;
|
||||
// the end-to-end proof through SanitizeIdentity + the real guard lives in
|
||||
// clients/kms (v6_aud_e2e_test.go). Reuses the jwksServer/signWith/tokenClaims
|
||||
// helpers from middleware_identity_test.go (same package).
|
||||
// The per-org PaaS-KMS sync identity authenticates as its own IAM application
|
||||
// "<owner>-platform-kms" (client_credentials), so its token carries owner=<org> and
|
||||
// aud=<owner>-platform-kms. Validation no longer gates on the audience at all (trust
|
||||
// is signature + issuer + expiry), so a machine token clears validate() like any
|
||||
// other. The owner-bound machine aud survives only to IDENTIFY such a principal
|
||||
// (isKMSMachinePrincipal) so SanitizeIdentity can DENY it SuperAdmin even in the admin
|
||||
// org — a client_credentials machine identity must never wield platform-admin. These
|
||||
// are white-box unit tests of that identification; the end-to-end proof through
|
||||
// SanitizeIdentity + the real guard lives in clients/kms (v6_aud_e2e_test.go). Reuses
|
||||
// the jwksServer/signWith/tokenClaims helpers from middleware_identity_test.go.
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
@@ -18,19 +18,16 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIdentityValidator_KMSMachineAudience(t *testing.T) {
|
||||
func TestIdentityValidator_KMSMachinePrincipal(t *testing.T) {
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("genkey: %v", err)
|
||||
}
|
||||
jwks := jwksServer(t, &key.PublicKey)
|
||||
// The static allowlist deliberately contains NO *-platform-kms audience, so any
|
||||
// acceptance below can come ONLY from the owner-bound machine-aud rule, not the
|
||||
// allowlist — this is what makes it a fix and not a config workaround.
|
||||
v := newIdentityValidator(testIssuer, jwks.URL, []string{"hanzo-console"}, 0)
|
||||
v := newIdentityValidator(testIssuer, jwks.URL, 0)
|
||||
future := time.Now().Add(time.Hour)
|
||||
|
||||
t.Run("machine token for its own org is accepted", func(t *testing.T) {
|
||||
t.Run("own-org machine token validates and is recognised as a machine principal", func(t *testing.T) {
|
||||
c, err := v.validate(signWith(t, key, tokenClaims("maxpower-platform-kms", "maxpower", "", false, future)))
|
||||
if err != nil {
|
||||
t.Fatalf("machine token rejected: %v", err)
|
||||
@@ -38,34 +35,51 @@ func TestIdentityValidator_KMSMachineAudience(t *testing.T) {
|
||||
if c.Owner != "maxpower" {
|
||||
t.Fatalf("owner=%q, want maxpower", c.Owner)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("machine aud for a DIFFERENT org is rejected (owner-bound)", func(t *testing.T) {
|
||||
// owner=maxpower but aud=acme-platform-kms: the accepted machine aud is bound
|
||||
// to the token's OWN owner (maxpower-platform-kms), so this must fail — it is
|
||||
// not a blanket "*-platform-kms" wildcard.
|
||||
if _, err := v.validate(signWith(t, key, tokenClaims("acme-platform-kms", "maxpower", "", false, future))); err == nil {
|
||||
t.Fatal("cross-org machine audience must be rejected (owner-bound)")
|
||||
if !isKMSMachinePrincipal(c) {
|
||||
t.Fatal("aud==<owner>-platform-kms must be recognised as a machine principal")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("arbitrary audience still rejected (fix is scoped, not a disable)", func(t *testing.T) {
|
||||
if _, err := v.validate(signWith(t, key, tokenClaims("some-random-app", "maxpower", "", false, future))); err == nil {
|
||||
t.Fatal("an arbitrary audience must still be rejected")
|
||||
t.Run("admin-org machine token is recognised so SuperAdmin is denied", func(t *testing.T) {
|
||||
c, err := v.validate(signWith(t, key, tokenClaims("admin-platform-kms", "admin", "", true, future)))
|
||||
if err != nil {
|
||||
t.Fatalf("admin machine token rejected: %v", err)
|
||||
}
|
||||
if !isKMSMachinePrincipal(c) {
|
||||
t.Fatal("admin-org machine token must be recognised (SanitizeIdentity denies it SuperAdmin)")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("machine aud with empty owner is rejected (fail closed)", func(t *testing.T) {
|
||||
// aud="-platform-kms" with owner="": kmsMachineAudience("")=="" so no machine
|
||||
// audience is granted and the bare suffix is not in the allowlist.
|
||||
if _, err := v.validate(signWith(t, key, tokenClaims("-platform-kms", "", "", false, future))); err == nil {
|
||||
t.Fatal("machine aud with empty owner must be rejected")
|
||||
t.Run("machine aud bound to a DIFFERENT org is not this owner's machine principal", func(t *testing.T) {
|
||||
// owner=maxpower, aud=acme-platform-kms: the machine-principal match is bound to
|
||||
// the token's OWN owner (maxpower-platform-kms), not a "*-platform-kms" wildcard.
|
||||
// It validates (aud is not gated) and is owner-scoped to maxpower downstream.
|
||||
c, err := v.validate(signWith(t, key, tokenClaims("acme-platform-kms", "maxpower", "", false, future)))
|
||||
if err != nil {
|
||||
t.Fatalf("token rejected: %v", err)
|
||||
}
|
||||
if isKMSMachinePrincipal(c) {
|
||||
t.Fatal("a cross-org machine aud must not count as this owner's machine principal")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("normal static-allowlist token still accepted (regression)", func(t *testing.T) {
|
||||
if _, err := v.validate(signWith(t, key, tokenClaims("hanzo-console", "maxpower", "", false, future))); err != nil {
|
||||
t.Fatalf("static-allowlist token rejected: %v", err)
|
||||
t.Run("ordinary app token is not a machine principal", func(t *testing.T) {
|
||||
c, err := v.validate(signWith(t, key, tokenClaims("hanzo-console", "maxpower", "", false, future)))
|
||||
if err != nil {
|
||||
t.Fatalf("token rejected: %v", err)
|
||||
}
|
||||
if isKMSMachinePrincipal(c) {
|
||||
t.Fatal("an ordinary app token is not a machine principal")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty-owner token is never a machine principal (fail closed)", func(t *testing.T) {
|
||||
c, err := v.validate(signWith(t, key, tokenClaims("-platform-kms", "", "", false, future)))
|
||||
if err != nil {
|
||||
t.Fatalf("token rejected: %v", err)
|
||||
}
|
||||
if isKMSMachinePrincipal(c) {
|
||||
t.Fatal(`empty-owner token must never be a machine principal (kmsMachineAudience("")=="")`)
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@@ -9,10 +9,10 @@ import (
|
||||
)
|
||||
|
||||
// TestValidate_FailSecureOnEmptyTrustSet proves I2: a validator whose resolved
|
||||
// issuer OR audience allowlist is empty REJECTS an otherwise-valid, correctly
|
||||
// signed token — the axis is never silently disabled. Production always resolves
|
||||
// non-empty sets; this guards the misconfiguration path (CLOUD_JWT_AUDIENCES=""
|
||||
// or an empty issuer set), which must fail closed, not open.
|
||||
// issuer set is empty REJECTS an otherwise-valid, correctly signed token — the axis
|
||||
// is never silently disabled. Production always resolves a non-empty set; this
|
||||
// guards the misconfiguration path (an empty issuer set), which must fail closed,
|
||||
// not open.
|
||||
func TestValidate_FailSecureOnEmptyTrustSet(t *testing.T) {
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
@@ -23,20 +23,15 @@ func TestValidate_FailSecureOnEmptyTrustSet(t *testing.T) {
|
||||
tok := signWith(t, key, tokenClaims("hanzo-console", "acme", "", false, future))
|
||||
|
||||
// Sanity: a properly configured validator accepts the token.
|
||||
if _, err := newIdentityValidator(testIssuer, jwks.URL, []string{"hanzo-console"}, 0).validate(tok); err != nil {
|
||||
if _, err := newIdentityValidator(testIssuer, jwks.URL, 0).validate(tok); err != nil {
|
||||
t.Fatalf("baseline valid token must be accepted, got %v", err)
|
||||
}
|
||||
|
||||
// Empty audience set → deny.
|
||||
if _, err := newIdentityValidator(testIssuer, jwks.URL, nil, 0).validate(tok); err == nil {
|
||||
t.Error("empty audience allowlist must REJECT (fail-secure), not accept")
|
||||
}
|
||||
|
||||
// Empty issuer set → deny (construct directly; trustedIssuers never yields empty
|
||||
// with a primary, so bypass it to exercise the guard).
|
||||
vEmptyIss := &identityValidator{issuers: nil, audiences: []string{"hanzo-console"}, cache: newJWKSCache(jwks.URL, 0), keys: newIAMKeys()}
|
||||
vEmptyIss := &identityValidator{issuers: nil, cache: newJWKSCache(jwks.URL, 0), keys: newIAMKeys()}
|
||||
if _, err := vEmptyIss.validate(tok); err == nil {
|
||||
t.Error("empty issuer allowlist must REJECT (fail-secure), not accept")
|
||||
t.Error("empty issuer set must REJECT (fail-secure), not accept")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,7 +107,7 @@ func TestBrandIssuers(t *testing.T) {
|
||||
// full brand set, so a lux token would pass the issuer gate on the hanzo binary.
|
||||
func TestNewIdentityValidator_MultiIssuer(t *testing.T) {
|
||||
os.Unsetenv("WHITELABEL_ISSUERS")
|
||||
v := newIdentityValidator("https://hanzo.id", "http://iam.hanzo.svc/v1/iam/.well-known/jwks", []string{"hanzo-cloud", "lux-cloud"}, 0)
|
||||
v := newIdentityValidator("https://hanzo.id", "http://iam.hanzo.svc/v1/iam/.well-known/jwks", 0)
|
||||
if !issuerAllowed("https://lux.id", v.issuers) {
|
||||
t.Fatalf("validator must trust the lux issuer, set=%v", v.issuers)
|
||||
}
|
||||
@@ -123,64 +118,3 @@ func TestNewIdentityValidator_MultiIssuer(t *testing.T) {
|
||||
t.Fatalf("validator must reject an untrusted issuer, set=%v", v.issuers)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBrandAudiences proves every brand's cloud audience (<brand>-cloud) is derived
|
||||
// from the brands registry — one source of truth, mirroring BrandIssuers.
|
||||
func TestBrandAudiences(t *testing.T) {
|
||||
got := BrandAudiences()
|
||||
for _, want := range []string{"hanzo-cloud", "lux-cloud", "zoo-cloud", "pars-cloud", "bootnode-cloud"} {
|
||||
found := false
|
||||
for _, g := range got {
|
||||
if g == want {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("BrandAudiences()=%v missing %q", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestJWTAudiencesFromEnv_BrandUnion proves the resolved audience allowlist ALWAYS
|
||||
// includes every brand's <brand>-cloud aud (so a lux token validates), whether the
|
||||
// list comes from the baked default or a hanzo-only env override — and that an
|
||||
// env-supplied entry is not duplicated.
|
||||
func TestJWTAudiencesFromEnv_BrandUnion(t *testing.T) {
|
||||
has := func(list []string, v string) bool {
|
||||
for _, s := range list {
|
||||
if s == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Baked default path (no env).
|
||||
os.Unsetenv("CLOUD_JWT_AUDIENCES")
|
||||
os.Unsetenv("GATEWAY_ALLOWED_AUDIENCES")
|
||||
def := jwtAudiencesFromEnv()
|
||||
for _, want := range []string{"hanzo-cloud", "lux-cloud", "zoo-cloud", "pars-cloud"} {
|
||||
if !has(def, want) {
|
||||
t.Errorf("baked audiences %v missing brand aud %q", def, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A legacy hanzo-only env override must STILL accept lux-cloud (brand union),
|
||||
// with no duplicate of the env-supplied hanzo-cloud.
|
||||
os.Setenv("GATEWAY_ALLOWED_AUDIENCES", "hanzo-app,hanzo-console,hanzo-cloud")
|
||||
defer os.Unsetenv("GATEWAY_ALLOWED_AUDIENCES")
|
||||
got := jwtAudiencesFromEnv()
|
||||
if !has(got, "lux-cloud") {
|
||||
t.Fatalf("hanzo-only env override must still accept lux-cloud, got %v", got)
|
||||
}
|
||||
n := 0
|
||||
for _, s := range got {
|
||||
if s == "hanzo-cloud" {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("hanzo-cloud must appear exactly once (no duplicate), got %d in %v", n, got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,19 +137,3 @@ func BrandIssuers() []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BrandAudiences returns the OAuth `aud` (== IAM client_id == app name) of every
|
||||
// white-label brand's cloud login app: `<brand>-cloud` (hanzo-cloud, lux-cloud,
|
||||
// zoo-cloud, pars-cloud, bootnode-cloud). A brand's session token carries
|
||||
// aud=<brand>-cloud (HIP-0111: client_id == app == aud), so the audience allowlist
|
||||
// must include each to accept a lux/zoo/pars token on the ONE binary. Derived from
|
||||
// the same `brands` registry as BrandIssuers — one source of truth, no hand-listing.
|
||||
func BrandAudiences() []string {
|
||||
out := make([]string, 0, len(brands))
|
||||
for id := range brands {
|
||||
if id != "" {
|
||||
out = append(out, id+"-cloud")
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -2,13 +2,18 @@ package cloud
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
aiobject "github.com/hanzoai/ai/object"
|
||||
"github.com/hanzoai/cloud/cek"
|
||||
"github.com/hanzoai/cloud/clients/commerceinproc"
|
||||
"github.com/hanzoai/cloud/clients/metering"
|
||||
"github.com/hanzoai/cloud/internal/org"
|
||||
s3 "github.com/hanzoai/s3-go"
|
||||
luxlog "github.com/luxfi/log"
|
||||
"github.com/zap-proto/zip"
|
||||
|
||||
@@ -63,14 +68,15 @@ func BuildDeps(cfg *Config) Deps {
|
||||
)
|
||||
|
||||
deps := Deps{
|
||||
Logger: logger,
|
||||
Brand: cfg.Brand,
|
||||
Version: cfg.Version,
|
||||
Env: cfg.Env,
|
||||
Domain: cfg.Domain,
|
||||
IAMIssuer: cfg.IAMIssuer,
|
||||
DataDir: cfg.DataDir,
|
||||
AIDefaultModel: cfg.AIDefaultModel,
|
||||
Logger: logger,
|
||||
Brand: cfg.Brand,
|
||||
Version: cfg.Version,
|
||||
Env: cfg.Env,
|
||||
Domain: cfg.Domain,
|
||||
IAMIssuer: cfg.IAMIssuer,
|
||||
DataDir: cfg.DataDir,
|
||||
AIDefaultModel: cfg.AIDefaultModel,
|
||||
AIFallbackModel: cfg.AIFallbackModel,
|
||||
}
|
||||
|
||||
// For each subsystem: enabled → leave nil (Mount fills it); not
|
||||
@@ -91,11 +97,18 @@ func BuildDeps(cfg *Config) Deps {
|
||||
// pass-through and a dev deployment is never blocked.
|
||||
deps.Metering = buildMeteringClient(cfg, logger)
|
||||
wireTierReader(deps.Metering, logger)
|
||||
deps.AI = meteredAIClient(pickAIClient(cfg, logger), deps)
|
||||
// AI (completions, WRITE) and Embed (embeddings, READ-ONLY) are DISTINCT
|
||||
// credentials by concern: completions never ride the read-only publishable
|
||||
// (pk-) key — the gateway 403s a pk- key on any write endpoint — so deps.AI
|
||||
// resolves to the M2M identity, while deps.Embed keeps the pk- key (correct
|
||||
// least-privilege for a read-only call). Both meter through the ONE commerce path.
|
||||
deps.AI = meteredAIClient(pickCompletionsClient(cfg, logger), deps)
|
||||
deps.Embed = meteredAIClient(pickEmbedClient(cfg, logger), deps)
|
||||
wireFinance(cfg, logger)
|
||||
deps.O11y = pick(cfg, logger, "o11y", "O11y", cfg.O11yZAPAddr, clients.O11yRPCAt, clients.DisabledO11y)
|
||||
deps.VFS = pickVFSClient(cfg, logger)
|
||||
deps.MQ = pick(cfg, logger, "mq", "MQ", cfg.MQZAPAddr, clients.MQRPCAt, clients.DisabledMQ)
|
||||
deps.Durable = buildDurability(cfg, logger)
|
||||
|
||||
// Payments and Vault never co-resident. Disabled stub when no
|
||||
// endpoint, otherwise RPC.
|
||||
@@ -579,44 +592,76 @@ func RegisterCommerceClientFactory(f func(cfg *Config, log luxlog.Logger) Commer
|
||||
commerceClientFactory = f
|
||||
}
|
||||
|
||||
// pickAIClient resolves deps.AI — the client the agents subsystem runs chat
|
||||
// completions through. Unlike the co-resident subsystems, there is NO in-process
|
||||
// "ai" mount that fills a nil deps.AI: inference is an external gateway, so this
|
||||
// must return a concrete client, never nil. (A nil deps.AI was the live bug —
|
||||
// the default all-enabled config returned nil here and nothing ever filled it,
|
||||
// so every /v1/agents/:name/run 503'd "inference is not configured".)
|
||||
// publishableKey reports whether apiKey is a read-only PUBLISHABLE key (pk-). The
|
||||
// Hanzo gateway 403s a publishable key on any WRITE endpoint — "Publishable keys
|
||||
// can only access read-only endpoints … use a secret key (sk-)". So the COMPLETIONS
|
||||
// resolver must refuse it (it would only 403 chat), while the EMBED resolver accepts
|
||||
// it (embeddings ARE read-only). pk- is the IAM key family's read-only member
|
||||
// (hk-/sk-/pk-/fw_/hz_; clients/admission). This ONE predicate is the split's crux:
|
||||
// it kept the intermittent-403 bug — a pk- embed key riding the shared completions
|
||||
// client — from ever recurring, wherever the key comes from.
|
||||
func publishableKey(apiKey string) bool {
|
||||
return strings.HasPrefix(strings.TrimSpace(apiKey), "pk-")
|
||||
}
|
||||
|
||||
// pickCompletionsClient resolves deps.AI — the client the agents run path (and
|
||||
// guide/crm/content/sitegen/code /ask) execute CHAT COMPLETIONS through. There is
|
||||
// NO in-process "ai" mount that fills a nil deps.AI: inference is an external
|
||||
// gateway, so this returns a concrete client, never nil.
|
||||
//
|
||||
// Completions are a WRITE endpoint: a read-only publishable (pk-) key 403s them. So
|
||||
// this resolver NEVER rides a pk- key — that is the embed credential (pickEmbedClient).
|
||||
// Preference order:
|
||||
// 1. Static-key HTTP gateway when a base URL AND a static key are configured —
|
||||
// an operator override / pre-provisioned key. The key is a KMS-injected
|
||||
// secret; only the base URL and default model are ever logged.
|
||||
// 1. Static SECRET-key HTTP gateway when a base URL AND a completions-capable
|
||||
// (non-pk-) static key are configured — an explicit operator override (sk-/hk-).
|
||||
// A pk- key here is REFUSED (it would only 403 chat) and the resolver falls
|
||||
// through to M2M — THE fix for the intermittent publishable-key 403 on bot replies.
|
||||
// 2. M2M HTTP gateway when a base URL AND the binary's IAM identity are present
|
||||
// (the durable Hanzo default): the client mints+refreshes a client-
|
||||
// credentials token from IAM_CLIENT_ID/SECRET — no static key to rotate. The
|
||||
// secret is never logged.
|
||||
// (the durable Hanzo default): the client mints+refreshes a client-credentials
|
||||
// token from IAM_CLIENT_ID/SECRET — no static key to rotate. Secret never logged.
|
||||
// 3. ZAP RPC when an addr is configured (split-deploy of a future ai subsystem).
|
||||
// 4. Fail-closed stub otherwise — a run records an honest error, never fakes one.
|
||||
func pickAIClient(cfg *Config, log luxlog.Logger) AIClient {
|
||||
if cfg.AIBaseURL != "" && cfg.AIAPIKey != "" {
|
||||
log.Info("deps.AI → HTTP gateway (static key)", "base_url", cfg.AIBaseURL, "default_model", cfg.AIDefaultModel)
|
||||
func pickCompletionsClient(cfg *Config, log luxlog.Logger) AIClient {
|
||||
if cfg.AIBaseURL != "" && cfg.AIAPIKey != "" && !publishableKey(cfg.AIAPIKey) {
|
||||
log.Info("deps.AI (completions) → HTTP gateway (static secret key)", "base_url", cfg.AIBaseURL, "default_model", cfg.AIDefaultModel)
|
||||
return clients.AIHTTPAt(cfg.AIBaseURL, cfg.AIAPIKey, cfg.AIDefaultModel)
|
||||
}
|
||||
if cfg.AIAPIKey != "" && publishableKey(cfg.AIAPIKey) {
|
||||
log.Info("deps.AI (completions) → refusing read-only publishable (pk-) key for chat; using M2M", "base_url", cfg.AIBaseURL)
|
||||
}
|
||||
if cfg.AIBaseURL != "" && cfg.AIAuthClientID != "" && cfg.AIAuthClientSecret != "" {
|
||||
tokenURL := aiM2MTokenURL(cfg)
|
||||
if tokenURL != "" {
|
||||
log.Info("deps.AI → HTTP gateway (IAM M2M)", "base_url", cfg.AIBaseURL,
|
||||
log.Info("deps.AI (completions) → HTTP gateway (IAM M2M)", "base_url", cfg.AIBaseURL,
|
||||
"token_url", tokenURL, "client_id", cfg.AIAuthClientID, "default_model", cfg.AIDefaultModel)
|
||||
return clients.AIHTTPM2M(cfg.AIBaseURL, tokenURL, cfg.AIAuthClientID, cfg.AIAuthClientSecret, cfg.AIDefaultModel)
|
||||
}
|
||||
}
|
||||
if cfg.AIZAPAddr != "" {
|
||||
log.Info("deps.AI → ZAP RPC", "addr", cfg.AIZAPAddr)
|
||||
log.Info("deps.AI (completions) → ZAP RPC", "addr", cfg.AIZAPAddr)
|
||||
return clients.AIRPCAt(cfg.AIZAPAddr)
|
||||
}
|
||||
log.Info("deps.AI → disabled (no CLOUD_AI_API_KEY, no IAM M2M identity, no gateway configured)")
|
||||
log.Info("deps.AI (completions) → disabled (no secret key, no IAM M2M identity, no gateway configured)")
|
||||
return clients.DisabledAI()
|
||||
}
|
||||
|
||||
// pickEmbedClient resolves deps.Embed — the client code-index + KB knowledge run
|
||||
// EMBEDDINGS through. Embeddings are a READ-ONLY endpoint, so the read-only
|
||||
// publishable (pk-) key (CLOUD_AI_API_KEY ← cloud-ai-embed-key) is the CORRECT
|
||||
// least-privilege credential here, and is used UNCHANGED — this path is deliberately
|
||||
// not rewired. Preference order:
|
||||
// 1. Static-key HTTP gateway when a base URL AND a static key are configured. The
|
||||
// key (pk- or sk-) is a KMS-injected secret; only base URL + model are logged.
|
||||
// 2. Otherwise share the completions resolution (M2M / ZAP / fail-closed) so a
|
||||
// deploy with no dedicated embed key still indexes — no regression.
|
||||
func pickEmbedClient(cfg *Config, log luxlog.Logger) AIClient {
|
||||
if cfg.AIBaseURL != "" && cfg.AIAPIKey != "" {
|
||||
log.Info("deps.Embed → HTTP gateway (static embed key)", "base_url", cfg.AIBaseURL, "default_model", cfg.AIDefaultModel)
|
||||
return clients.AIHTTPAt(cfg.AIBaseURL, cfg.AIAPIKey, cfg.AIDefaultModel)
|
||||
}
|
||||
return pickCompletionsClient(cfg, log)
|
||||
}
|
||||
|
||||
// aiM2MTokenURL resolves IAM's client_credentials endpoint the agent runner mints
|
||||
// its M2M inference token at. It MUST be reachable FROM INSIDE THE CLUSTER: the
|
||||
// runner runs in-cluster and the public issuer host (https://hanzo.id) is fronted
|
||||
@@ -674,6 +719,148 @@ func pickVFSClient(cfg *Config, log luxlog.Logger) VFSClient {
|
||||
return clients.DisabledVFS()
|
||||
}
|
||||
|
||||
// durableBucket holds every org's HA-SQLite snapshot (and its writer lease). One
|
||||
// bucket, keys laid out orgs/<slug>[/…]/<subsystem>.db per HIP-0302 — the durable
|
||||
// twin of the on-disk DataDir layout.
|
||||
//
|
||||
// OPERATIONAL REQUIREMENTS the fence depends on (enforce in the SeaweedFS deployment,
|
||||
// not in code):
|
||||
//
|
||||
// - Object versioning + a no-expiry / no-lifecycle-deletion policy on this prefix.
|
||||
// The writer lease (orgs/<slug>/.owner) is the round's system of record; if the
|
||||
// gateway silently drops or rolls back that object, a monotone round can reset and
|
||||
// un-fence a zombie writer (Red M4). A local high-water-round floor per pod is the
|
||||
// future in-process defense; the object lifecycle is the operational one.
|
||||
// - RWO, per-writer PVCs for DataDir — NEVER an RWX shared volume (Red M5). Two pods
|
||||
// on one DataDir corrupt SQLite regardless of this fence; single-writer here is the
|
||||
// durable-copy fence, and per-pod RWO is the local-file guarantee the shard router
|
||||
// already relies on (see shardrouter.go).
|
||||
const durableBucket = "org-db"
|
||||
|
||||
// buildDurability constructs the deployment's HA-durability factory, or nil when the
|
||||
// deployment has no object store to be durable against (dev/single-node — every
|
||||
// OrgStore then stays local-only). It composes the SeaweedFS S3 If-Match
|
||||
// ConditionalStore (the SAME s3admin identity deps.VFS uses), the writer membership
|
||||
// over CLOUD_PEERS (the SAME set the shard router elects on, so the store-layer owner
|
||||
// and the routed owner agree), and the per-org envelope Cipher rooted at the KMS
|
||||
// master. Any construction failure fails SAFE to nil (local-only) rather than crash
|
||||
// the boot; an encryption-capable build with no usable cipher is REFUSED — a build
|
||||
// that promises encryption never ships plaintext snapshots to the object store.
|
||||
func buildDurability(cfg *Config, log luxlog.Logger) *Durability {
|
||||
// A multi-replica deployment REQUIRES the durable plane: with >1 writer, a per-org
|
||||
// store that is not hydrate-on-open + fenced is the outage this exists to fix.
|
||||
// disabledDurability logs at the severity the replica count warrants, so a
|
||||
// misconfigured prod deployment is never SILENTLY non-durable (Red L2).
|
||||
multiReplica := len(parsePeers(cfg.ShardPeers)) > 1
|
||||
|
||||
// Explicit opt-in: the object-store fence rests on the deployed SeaweedFS enforcing
|
||||
// conditional-PUT (If-Match) atomically, which must be validated against the deployed
|
||||
// version before it fences real tenant data (the takeover-fence staging gate). Until
|
||||
// CLOUD_RESEARCH_DURABLE is set the store runs local-only — the shard router still
|
||||
// pins each org to one writer, so this is not the rolling-deploy outage; only the
|
||||
// cross-restart object-store snapshot waits for the opt-in.
|
||||
if !cfg.ResearchDurable {
|
||||
disabledDurability(log, multiReplica, "CLOUD_RESEARCH_DURABLE not set — HA object-store durability is opt-in pending the SeaweedFS conditional-PUT atomicity gate")
|
||||
return nil
|
||||
}
|
||||
|
||||
admin := s3admin.New()
|
||||
if !admin.Configured() {
|
||||
disabledDurability(log, multiReplica, "no S3 admin creds (S3_ADMIN_* unset)")
|
||||
return nil
|
||||
}
|
||||
client, err := admin.Client()
|
||||
if err != nil {
|
||||
disabledDurability(log, multiReplica, fmt.Sprintf("S3 client construction failed: %v", err))
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
if err := ensureDurableBucket(ctx, admin, client); err != nil {
|
||||
// Non-fatal: the bucket likely already exists; a later ship/hydrate retries.
|
||||
log.Warn("durability bucket ensure failed (continuing)", "bucket", durableBucket, "err", err)
|
||||
}
|
||||
cancel()
|
||||
|
||||
// Membership: CLOUD_PEERS (the shard router's set). A single-pod deployment with
|
||||
// no peers is its own sole writer — still hydrate-on-open + fenced ship across a
|
||||
// rolling restart.
|
||||
self := firstNonEmptyStr(strings.TrimSpace(cfg.ShardSelf), hostnameOr("cloud-0"))
|
||||
peers := parsePeers(cfg.ShardPeers)
|
||||
if len(peers) == 0 {
|
||||
peers = []org.Member{{ID: self, Addr: self}}
|
||||
}
|
||||
members := org.NewMembership(self, org.StaticSource(peers...), 5*time.Second)
|
||||
_ = members.Start(context.Background()) // static source: the initial refresh populates Members()
|
||||
|
||||
cipher := durableCipher(cfg, log)
|
||||
if cipher == nil && cek.Encrypting() {
|
||||
// The master that satisfied cek must decode here too, so this is a genuine
|
||||
// misconfig, not a dev path: never ship plaintext snapshots AND never silently
|
||||
// drop durability — fail closed and log LOUDLY for the replica count.
|
||||
disabledDurability(log, multiReplica, "encryption-capable build but no durable cipher (would ship plaintext snapshots)")
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Info("durability enabled", "bucket", durableBucket, "self", self, "peers", len(peers), "encrypted", cipher != nil)
|
||||
return org.NewDurability(org.NewS3ConditionalStore(client, durableBucket), members, cipher)
|
||||
}
|
||||
|
||||
// disabledDurability records that the durable plane is OFF, at ERROR when the
|
||||
// deployment is multi-replica (per-org stores then survive only via shard routing +
|
||||
// per-pod RWO PVC; a lost/rescheduled PVC loses committed data — the operator MUST
|
||||
// configure S3) and at INFO when single-replica/dev (local-only is the expected
|
||||
// posture). One place, so no disabled path is silent on a deployment that needs HA.
|
||||
func disabledDurability(log luxlog.Logger, multiReplica bool, why string) {
|
||||
if multiReplica {
|
||||
log.Error("DURABILITY DISABLED on a MULTI-REPLICA deployment — per-org stores survive only via shard routing + per-pod RWO PVC; a lost/rescheduled PVC loses committed data. Configure S3_ADMIN_* to enable the durable plane.", "reason", why)
|
||||
return
|
||||
}
|
||||
log.Info("durability disabled — single-replica/dev, per-org stores stay local-only", "reason", why)
|
||||
}
|
||||
|
||||
// ensureDurableBucket creates the durable bucket if absent (idempotent).
|
||||
func ensureDurableBucket(ctx context.Context, admin s3admin.Admin, client *s3.Client) error {
|
||||
ok, err := client.BucketExists(ctx, durableBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok {
|
||||
return nil
|
||||
}
|
||||
return client.MakeBucket(ctx, durableBucket, s3.MakeBucketOptions{Region: admin.Region()})
|
||||
}
|
||||
|
||||
// durableCipher builds the per-org envelope Cipher from the base64 KMS master
|
||||
// (CLOUD_KMS_MASTER_KEY_REF — the SAME key cek derives from). nil when no valid
|
||||
// 32-byte master is configured (pure-Go dev → plaintext durable object, matching the
|
||||
// plaintext local file).
|
||||
func durableCipher(cfg *Config, log luxlog.Logger) *org.Cipher {
|
||||
ref := strings.TrimSpace(cfg.KMSMasterKeyRef)
|
||||
if ref == "" {
|
||||
return nil
|
||||
}
|
||||
master, err := base64.StdEncoding.DecodeString(ref)
|
||||
if err != nil {
|
||||
log.Warn("durable cipher: KMS master key ref is not valid base64", "err", err)
|
||||
return nil
|
||||
}
|
||||
c, err := org.NewCipher(master)
|
||||
if err != nil {
|
||||
log.Warn("durable cipher: invalid KMS master key", "err", err)
|
||||
return nil
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// hostnameOr returns the OS hostname, or def when unavailable — a stable self id for
|
||||
// a single-pod deployment that sets no CLOUD_POD_NAME.
|
||||
func hostnameOr(def string) string {
|
||||
if h, err := os.Hostname(); err == nil && h != "" {
|
||||
return h
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func pickPaymentsClient(cfg *Config, log luxlog.Logger) PaymentsClient {
|
||||
if cfg.PaymentsZAPAddr != "" {
|
||||
log.Info("deps.Payments → ZAP RPC", "addr", cfg.PaymentsZAPAddr)
|
||||
|
||||
+6
-2
@@ -63,7 +63,9 @@ var controlCommands = map[string]string{
|
||||
"k8s": "deploy-target helpers (current target)",
|
||||
"config": "view/edit ~/.hanzo/config preferences",
|
||||
"security": "scan files for hardcoded secrets (local guardrail; no server/auth)",
|
||||
"gpu": "connect this machine's GPU to the Hanzo cloud fleet (connect/status/disconnect)",
|
||||
"link": "bring this machine into the Hanzo cloud fleet as a node (fabric + compute worker)",
|
||||
"unlink": "take this machine out of the fleet (deregister + stop hanzod)",
|
||||
"status": "show the org's fleet — every node with each of its GPUs",
|
||||
"engine": "run a local hanzo-engine (OpenAI + Anthropic model server)",
|
||||
"code": "launch a coding agent (claude, codex, dev) on a Hanzo cloud model",
|
||||
"runner": "run this machine as a JIT CI runner for your org (GitHub Actions)",
|
||||
@@ -598,7 +600,9 @@ func newRootCmd() *cobra.Command {
|
||||
newBuildCmd(envOf, &f),
|
||||
newConfigCmd(),
|
||||
newSecurityCmd(envOf),
|
||||
newGPUCmd(envOf, &f),
|
||||
newLinkCmd(envOf, &f),
|
||||
newUnlinkCmd(envOf, &f),
|
||||
newStatusCmd(envOf, &f),
|
||||
newEngineCmd(envOf, &f),
|
||||
newCodeCmd(envOf, &f),
|
||||
newRunnerCmd(envOf, &f),
|
||||
|
||||
+25
-8
@@ -150,7 +150,7 @@ type codeAgent struct {
|
||||
continueArgs []string // harness-native form of Hanzo -c/--continue
|
||||
modelArg []string // how the model is passed on argv (empty: via env)
|
||||
carrier func(model string) string // maps the resolved model to a client-recognized id (claude: zen→carrier); nil = pass through
|
||||
provider func(base string) []string // agents that need the endpoint declared, not just env'd
|
||||
provider func(base, model string) []string // agents that need the endpoint declared, not just env'd
|
||||
clear []string // env that would shadow the wire (a stale key in the shell)
|
||||
configHome string // env var that relocates the agent's config dir to ~/.hanzo ("" = share the user's own install)
|
||||
seed func(dir string) error // one-time defaults for the isolated config dir
|
||||
@@ -159,6 +159,19 @@ type codeAgent struct {
|
||||
install string // hint when the binary is missing
|
||||
}
|
||||
|
||||
// codeContextWindow is the input context (tokens) the served coding model
|
||||
// budgets from. The enso and zen5 flagship tiers serve 1M; the flash tiers
|
||||
// serve 131072. Codex is told this so it sizes context to the real window
|
||||
// instead of a flat 256K cap — the cause of "maximum context exceeded" at
|
||||
// 262144 even on a 1M-capable model. This wrapper only launches Hanzo coding
|
||||
// models (default zen5), so non-flash defaults to the 1M flagship window.
|
||||
func codeContextWindow(model string) int {
|
||||
if strings.Contains(strings.ToLower(model), "flash") {
|
||||
return 131072
|
||||
}
|
||||
return 1000000
|
||||
}
|
||||
|
||||
// codex and @hanzo/dev share a lineage (dev is a Codex fork), hence a wire.
|
||||
// They also ignore OPENAI_BASE_URL and talk to chatgpt.com unless a provider is
|
||||
// declared, so declare Hanzo as the provider and select it.
|
||||
@@ -169,19 +182,23 @@ func codexLike(bin, install string) codeAgent {
|
||||
fullAuto: []string{"--dangerously-bypass-approvals-and-sandbox"},
|
||||
continueArgs: []string{"resume", "--last"},
|
||||
modelArg: []string{"-m"},
|
||||
provider: func(base string) []string {
|
||||
provider: func(base, model string) []string {
|
||||
// api.hanzo.ai exposes the standard OpenAI /v1/models shape, not
|
||||
// Codex's private remote model-catalog schema, so skip that refresh
|
||||
// and supply the model's window here — sized to the SERVED model
|
||||
// (enso / zen5 flagship = 1M, flash tiers = 131072) so Codex budgets
|
||||
// the real context instead of a flat 256K cap (the "maximum context
|
||||
// exceeded at 262144" bug). Auto-compact at 90% leaves headroom.
|
||||
win := codeContextWindow(model)
|
||||
return []string{
|
||||
"-c", "model_provider=hanzo",
|
||||
"-c", `model_providers.hanzo.name="Hanzo"`,
|
||||
"-c", fmt.Sprintf(`model_providers.hanzo.base_url="%s/v1"`, strings.TrimSuffix(base, "/")),
|
||||
"-c", `model_providers.hanzo.env_key="OPENAI_API_KEY"`,
|
||||
"-c", `model_providers.hanzo.wire_api="responses"`,
|
||||
// api.hanzo.ai exposes the standard OpenAI /v1/models shape,
|
||||
// not Codex's private remote model-catalog schema. Skip that
|
||||
// optional refresh and supply the coding model's metadata here.
|
||||
"-c", `features.remote_models=false`,
|
||||
"-c", `model_context_window=262144`,
|
||||
"-c", `model_auto_compact_token_limit=235929`,
|
||||
"-c", fmt.Sprintf("model_context_window=%d", win),
|
||||
"-c", fmt.Sprintf("model_auto_compact_token_limit=%d", win*9/10),
|
||||
}
|
||||
},
|
||||
install: install,
|
||||
@@ -447,7 +464,7 @@ func codeArgv(agent codeAgent, base, model string, safe bool, rest []string) []s
|
||||
argv = append(argv, agent.fullAuto...)
|
||||
}
|
||||
if agent.provider != nil {
|
||||
argv = append(argv, agent.provider(base)...)
|
||||
argv = append(argv, agent.provider(base, model)...)
|
||||
}
|
||||
if len(agent.modelArg) > 0 { // claude takes the model via env, codex/dev on argv
|
||||
argv = append(argv, agent.modelArg...)
|
||||
|
||||
+25
-2
@@ -100,19 +100,42 @@ func TestCodeUnknownOptionsAndPostSeparatorArgsPassThrough(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCodexProviderUsesNativeResponsesMetadata(t *testing.T) {
|
||||
// defaultCodeModel is zen5 — a 1M flagship tier — so the window must be 1M,
|
||||
// NOT the old flat 262144 cap that surfaced as "maximum context exceeded"
|
||||
// even on a 1M-capable model. Auto-compact is 90% of the window.
|
||||
argv := codeArgv(codeAgents["codex"], "https://api.hanzo.ai", defaultCodeModel, false, nil)
|
||||
for _, want := range []string{
|
||||
`model_provider=hanzo`,
|
||||
`model_providers.hanzo.base_url="https://api.hanzo.ai/v1"`,
|
||||
`model_providers.hanzo.wire_api="responses"`,
|
||||
`features.remote_models=false`,
|
||||
`model_context_window=262144`,
|
||||
`model_auto_compact_token_limit=235929`,
|
||||
`model_context_window=1000000`,
|
||||
`model_auto_compact_token_limit=900000`,
|
||||
} {
|
||||
if !slices.Contains(argv, want) {
|
||||
t.Errorf("Codex argv %q does not contain %q", argv, want)
|
||||
}
|
||||
}
|
||||
// A flash tier budgets its smaller real window, not 1M.
|
||||
flash := codeArgv(codeAgents["codex"], "https://api.hanzo.ai", "zen5-flash", false, nil)
|
||||
if !slices.Contains(flash, `model_context_window=131072`) {
|
||||
t.Errorf("zen5-flash argv %q must budget 131072, not the flagship 1M", flash)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCodeContextWindowSizing pins the model→window map: every non-flash tier
|
||||
// gets the 1M flagship window; only flash tiers drop to 131072.
|
||||
func TestCodeContextWindowSizing(t *testing.T) {
|
||||
for _, m := range []string{"zen5", "zen5-pro", "zen5-coder", "enso", "enso-ultra"} {
|
||||
if w := codeContextWindow(m); w != 1000000 {
|
||||
t.Errorf("codeContextWindow(%q) = %d, want 1000000", m, w)
|
||||
}
|
||||
}
|
||||
for _, m := range []string{"zen5-flash", "enso-flash"} {
|
||||
if w := codeContextWindow(m); w != 131072 {
|
||||
t.Errorf("codeContextWindow(%q) = %d, want 131072", m, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCodeTokenPrecedence locks in the 402 unblock: a fresh `hanzo login` JWT
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ package cli
|
||||
// cosign-signed binary from the latest github.com/hanzoai/engine release), so the
|
||||
// CLI never re-implements platform detection or verification. `serve` launches the
|
||||
// installed binary (`hanzoai --port P run -m MODEL`); `status` probes it, reusing
|
||||
// the same /v1/models probe `hanzo gpu connect --serve-engine` advertises with.
|
||||
// the same /v1/models probe `hanzo link --serve-engine` advertises with.
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
+1035
-114
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,157 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestParseRocmSmiCSV — the AMD GPU inventory names a card from its marketing
|
||||
// series + gfx target, exactly as `rocm-smi --showproductname --csv` reports on
|
||||
// evo's gfx1151 Radeon 8060S. This is the primary AMD detection path.
|
||||
func TestParseRocmSmiCSV(t *testing.T) {
|
||||
// Real evo output (header + one card row).
|
||||
csv := []byte("device,Card Series,Card Model,Card Vendor,Card SKU,Subsystem ID,Device Rev,Node ID,GUID,GFX Version\n" +
|
||||
"card0,Radeon 8060S Graphics,0x1586,Advanced Micro Devices Inc. [AMD/ATI],STRXLGEN,-0x7fe3,0xc1,1,49819,gfx1151\n")
|
||||
gpus := parseRocmSmiCSV(csv)
|
||||
if len(gpus) != 1 {
|
||||
t.Fatalf("want 1 AMD GPU, got %d (%+v)", len(gpus), gpus)
|
||||
}
|
||||
if got, want := gpus[0].Name, "AMD Strix Halo \u00b7 Radeon 8060S Graphics (gfx1151)"; got != want {
|
||||
t.Errorf("name = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGfxNameDecodesTargetVersion — the kfd fallback decodes gfx_target_version.
|
||||
func TestGfxNameDecodesTargetVersion(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
v int
|
||||
want string
|
||||
}{
|
||||
{110501, "gfx1151"}, // evo Radeon 8060S / Strix Halo
|
||||
{90012, "gfx9012"}, // sanity: MI-class encoding shape
|
||||
{100300, "gfx1030"}, // RDNA2
|
||||
} {
|
||||
if got := gfxName(c.v); got != c.want {
|
||||
t.Errorf("gfxName(%d) = %q, want %q", c.v, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseKfdTopology — the driver-only fallback (no rocm-smi) still finds the GPU
|
||||
// from /sys/class/kfd: node 0 is the CPU (simd_count 0, skipped), node 1 is the
|
||||
// gfx1151 GPU. Built against a fixture tree mirroring evo's real properties files.
|
||||
func TestParseKfdTopology(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeNode(t, root, "0", "simd_count 0\ngfx_target_version 0\n")
|
||||
writeNode(t, root, "1", "cpu_cores_count 0\nsimd_count 80\ngfx_target_version 110501\n")
|
||||
gpus := parseKfdTopology(root)
|
||||
if len(gpus) != 1 {
|
||||
t.Fatalf("want 1 GPU node (CPU node 0 skipped), got %d (%+v)", len(gpus), gpus)
|
||||
}
|
||||
if got, want := gpus[0].Name, "AMD Strix Halo (gfx1151)"; got != want {
|
||||
t.Errorf("name = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := gpus[0].Arch, "gfx1151"; got != want {
|
||||
t.Errorf("arch = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseVulkaninfoSummary — the last resort keeps only AMD/Radeon devices so it
|
||||
// never double-counts a card another vendor path already reported.
|
||||
func TestParseVulkaninfoSummary(t *testing.T) {
|
||||
out := []byte("GPU0:\n\tdeviceName = AMD Radeon Graphics (RADV GFX1151)\n" +
|
||||
"GPU1:\n\tdeviceName = llvmpipe (LLVM 18.1.0, 256 bits)\n")
|
||||
gpus := parseVulkaninfoSummary(out)
|
||||
if len(gpus) != 1 {
|
||||
t.Fatalf("want 1 AMD device (llvmpipe filtered), got %d (%+v)", len(gpus), gpus)
|
||||
}
|
||||
if got := gpus[0].Name; got != "AMD Radeon Graphics (RADV GFX1151)" {
|
||||
t.Errorf("name = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func writeNode(t *testing.T, root, id, props string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, id)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "properties"), []byte(props), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPickAmdMemUnified — an APU whose VRAM is a token carve-out reports the
|
||||
// machine's unified RAM snapped to hardware capacity (evo: 1 GiB VRAM, 118 GiB
|
||||
// GTT, 124.4 GiB kernel-visible of 128 GiB physical → 128 GiB); a discrete card
|
||||
// keeps its VRAM.
|
||||
func TestPickAmdMemUnified(t *testing.T) {
|
||||
if miB, unified := pickAmdMem(1024, 120832, 127411); !unified || miB != 131072 {
|
||||
t.Errorf("APU: got (%d, %v), want (131072, true)", miB, unified)
|
||||
}
|
||||
if miB, unified := pickAmdMem(24576, 8192, 127411); unified || miB != 24576 {
|
||||
t.Errorf("discrete: got (%d, %v), want (24576, false)", miB, unified)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapUnified — a small firmware gap snaps up to the DIMM capacity; a big
|
||||
// carve-out is real lost capacity and stays as-is; exact multiples stand.
|
||||
func TestSnapUnified(t *testing.T) {
|
||||
for _, c := range []struct{ in, want int64 }{
|
||||
{127411, 131072}, // evo: 124.4 GiB visible of 128 GiB physical
|
||||
{124610, 131072}, // spark GB10: 121.7 GiB visible of 128 GiB physical (~6.3 GiB firmware)
|
||||
{131072, 131072}, // exact 128 GiB
|
||||
{119194, 119194}, // ~116 GiB visible (16 GiB BIOS carve) — 11.6 GiB gap, keep honest
|
||||
{63488, 65536}, // 62 GiB visible of 64 GiB
|
||||
} {
|
||||
if got := snapUnified(c.in); got != c.want {
|
||||
t.Errorf("snapUnified(%d) = %d, want %d", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAmdAPUName — the board renders the APU as the processor the owner bought,
|
||||
// normalized from the BIOS shouting; non-Ryzen hosts opt out.
|
||||
func TestAmdAPUName(t *testing.T) {
|
||||
if got, want := amdAPUName("AMD RYZEN AI MAX+ 395 w/ Radeon 8060S"), "AMD Ryzen AI Max+ 395 w/ Radeon 8060S"; got != want {
|
||||
t.Errorf("amdAPUName = %q, want %q", got, want)
|
||||
}
|
||||
if got := amdAPUName("Intel(R) Core(TM) i9-14900K"); got != "" {
|
||||
t.Errorf("non-Ryzen host must opt out, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApuProcessorNames — only unified-memory cards are renamed; a discrete
|
||||
// card on the same host keeps its own identity.
|
||||
func TestApuProcessorNames(t *testing.T) {
|
||||
gpus := []gpuInfo{
|
||||
{Name: "AMD Strix Halo \u00b7 Radeon 8060S Graphics (gfx1151)", Arch: "gfx1151", Unified: true},
|
||||
{Name: "Radeon RX 7900 XTX (gfx1100)", Arch: "gfx1100"},
|
||||
}
|
||||
out := apuProcessorNames(gpus, "AMD RYZEN AI MAX+ 395 w/ Radeon 8060S")
|
||||
if got, want := out[0].Name, "AMD Ryzen AI Max+ 395 w/ Radeon 8060S (gfx1151)"; got != want {
|
||||
t.Errorf("APU name = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := out[1].Name, "Radeon RX 7900 XTX (gfx1100)"; got != want {
|
||||
t.Errorf("discrete name = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseNvidiaSmiCSV — a GB10-class unified SoC ("[N/A]" VRAM) reports the
|
||||
// machine RAM snapped to capacity + its sm arch; a discrete card keeps its VRAM.
|
||||
func TestParseNvidiaSmiCSV(t *testing.T) {
|
||||
out := []byte("NVIDIA GB10, [N/A], 12.1\n")
|
||||
gpus := parseNvidiaSmiCSV(out, 124610)
|
||||
if len(gpus) != 1 {
|
||||
t.Fatalf("want 1 GPU, got %d", len(gpus))
|
||||
}
|
||||
g := gpus[0]
|
||||
if g.Name != "NVIDIA GB10" || g.MemoryTotal != "131072 MiB" || !g.Unified || g.Arch != "sm_121" {
|
||||
t.Errorf("GB10 = %+v", g)
|
||||
}
|
||||
disc := parseNvidiaSmiCSV([]byte("NVIDIA GeForce RTX 4090, 24564 MiB, 8.9\n"), 124610)
|
||||
if d := disc[0]; d.MemoryTotal != "24564 MiB" || d.Unified || d.Arch != "sm_89" {
|
||||
t.Errorf("discrete = %+v", d)
|
||||
}
|
||||
}
|
||||
@@ -109,6 +109,7 @@ func TestBuildRegistrationCarriesEngine(t *testing.T) {
|
||||
jobsNS: "gpu-jobs",
|
||||
gpus: []gpuInfo{{Name: "NVIDIA GB10", MemoryTotal: "122880 MiB"}},
|
||||
serveEngine: true,
|
||||
studioReady: true,
|
||||
engineURL: srv.URL,
|
||||
engineAdvURL: "http://node.example:1234",
|
||||
}
|
||||
@@ -136,7 +137,7 @@ func TestBuildRegistrationCarriesEngine(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCapabilitiesWithoutEngine(t *testing.T) {
|
||||
w := &worker{serveEngine: false}
|
||||
w := &worker{serveEngine: false, studioReady: true}
|
||||
caps := w.capabilities()
|
||||
if len(caps) != 1 || caps[0] != studioCap {
|
||||
t.Fatalf("capabilities = %v, want just [%q] when not serving an engine", caps, studioCap)
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestFnValidate — payload bounds: script required, flag-injection via
|
||||
// requirements refused, timeout defaulted and capped.
|
||||
func TestFnValidate(t *testing.T) {
|
||||
if _, err := fnValidate(json.RawMessage(`{}`)); err == nil {
|
||||
t.Error("empty script must be refused")
|
||||
}
|
||||
if _, err := fnValidate(json.RawMessage(`{"script":"print(1)","requirements":["--index-url=evil"]}`)); err == nil {
|
||||
t.Error("flag-shaped requirement must be refused")
|
||||
}
|
||||
in, err := fnValidate(json.RawMessage(`{"script":"print(1)"}`))
|
||||
if err != nil || in.TimeoutSeconds != 3600 {
|
||||
t.Errorf("default timeout: got %d, err %v", in.TimeoutSeconds, err)
|
||||
}
|
||||
in, _ = fnValidate(json.RawMessage(`{"script":"print(1)","timeoutSeconds":999999}`))
|
||||
if in.TimeoutSeconds != 21600 {
|
||||
t.Errorf("timeout cap: got %d, want 21600", in.TimeoutSeconds)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTailBuffer — a chatty training loop keeps only the LAST bytes, flagged.
|
||||
func TestTailBuffer(t *testing.T) {
|
||||
tb := &tailBuffer{cap: 8}
|
||||
tb.Write([]byte("0123456789"))
|
||||
if string(tb.buf) != "23456789" || !tb.truncated {
|
||||
t.Errorf("tail = %q truncated=%v", tb.buf, tb.truncated)
|
||||
}
|
||||
tb2 := &tailBuffer{cap: 8}
|
||||
tb2.Write([]byte("abc"))
|
||||
if string(tb2.buf) != "abc" || tb2.truncated {
|
||||
t.Errorf("small write mangled: %q %v", tb2.buf, tb2.truncated)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHasNonRenderLane — echo and studio.render alone keep the render-only
|
||||
// poison guard; a registered fn.run lane lifts it.
|
||||
func TestHasNonRenderLane(t *testing.T) {
|
||||
w := &worker{handlers: map[string]jobHandler{"echo": echoHandler, studioCap: nil}}
|
||||
if w.hasNonRenderLane() {
|
||||
t.Error("render-only worker must report no extra lane")
|
||||
}
|
||||
w.handlers[fnCap] = fnRunHandler
|
||||
if !w.hasNonRenderLane() {
|
||||
t.Error("fn.run lane must lift the guard")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFnRunSmoke — the real handler end-to-end against the local uv: a script
|
||||
// that prints and exits 0. Skipped where uv is absent.
|
||||
func TestFnRunSmoke(t *testing.T) {
|
||||
if !uvPresent() {
|
||||
t.Skip("uv not installed")
|
||||
}
|
||||
res, err := fnRunHandler(t.Context(), json.RawMessage(`{"script":"print(2+2)","timeoutSeconds":120}`))
|
||||
if err != nil {
|
||||
t.Fatalf("fn.run: %v", err)
|
||||
}
|
||||
m := res.(map[string]any)
|
||||
if !strings.Contains(m["output"].(string), "4") {
|
||||
t.Errorf("output = %q, want it to contain 4", m["output"])
|
||||
}
|
||||
}
|
||||
|
||||
func uvPresent() bool {
|
||||
_, err := exec.LookPath("uv")
|
||||
return err == nil
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
package cli
|
||||
|
||||
// gpu_queue_test.go — the per-GPU claim contract: a job pinned to THIS machine's
|
||||
// lane ("gpu:<identity>") is claimed BEFORE the shared any-GPU lane ("gpu-jobs"),
|
||||
// both within the gpu-jobs namespace. A stub cloud records the taskQueue of every
|
||||
// claim so the ORDER is the assertion.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// stubJobsCloud records the taskQueue of every claim and serves the queued job (if
|
||||
// any) on the matching lane, once. A lane with no job (or already drained) answers
|
||||
// 204; complete/fail/heartbeat answer 200.
|
||||
func stubJobsCloud(t *testing.T, claims *[]string, jobsByLane map[string]*claimedActivity) *httptest.Server {
|
||||
t.Helper()
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/activities/claim") {
|
||||
var body struct {
|
||||
TaskQueue string `json:"taskQueue"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
*claims = append(*claims, body.TaskQueue)
|
||||
if job := jobsByLane[body.TaskQueue]; job != nil {
|
||||
jobsByLane[body.TaskQueue] = nil // deliver once
|
||||
_ = json.NewEncoder(w).Encode(job)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK) // complete / fail / heartbeat
|
||||
}))
|
||||
}
|
||||
|
||||
// echoJob is a GPU-free job the worker's echo handler runs to completion instantly,
|
||||
// so a claim test never needs a real render backend.
|
||||
func echoJob(id string) *claimedActivity {
|
||||
a := &claimedActivity{Input: json.RawMessage(`{}`)}
|
||||
a.Execution.WorkflowId, a.Execution.RunId = id, id
|
||||
a.Type.Name = "echo"
|
||||
return a
|
||||
}
|
||||
|
||||
func testWorker(t *testing.T, url string) *worker {
|
||||
t.Helper()
|
||||
t.Setenv("HANZO_TOKEN", "test-token") // ensureToken honors this; no `hanzo login`
|
||||
return &worker{
|
||||
env: &Env{CloudURL: url},
|
||||
http: &http.Client{Timeout: 5 * time.Second},
|
||||
baseURL: url,
|
||||
identity: "spark",
|
||||
hostname: "spark",
|
||||
jobsNS: "gpu-jobs",
|
||||
handlers: map[string]jobHandler{"echo": echoHandler},
|
||||
studioReady: true, // a render-capable node (preflight passed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGPUQueueLaneName(t *testing.T) {
|
||||
w := &worker{identity: "spark"}
|
||||
if got := w.gpuQueue(); got != "gpu:spark" {
|
||||
t.Fatalf("gpuQueue() = %q, want gpu:spark", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A job pinned to THIS GPU's lane is claimed first — the shared lane is not even
|
||||
// polled that cycle.
|
||||
func TestClaimPrefersTargetedLane(t *testing.T) {
|
||||
var claims []string
|
||||
srv := stubJobsCloud(t, &claims, map[string]*claimedActivity{"gpu:spark": echoJob("j1")})
|
||||
defer srv.Close()
|
||||
if err := testWorker(t, srv.URL).claimAndRun(context.Background(), io.Discard); err != nil {
|
||||
t.Fatalf("claimAndRun: %v", err)
|
||||
}
|
||||
if len(claims) != 1 || claims[0] != "gpu:spark" {
|
||||
t.Fatalf("claims = %v, want exactly [gpu:spark] (targeted first; shared not polled)", claims)
|
||||
}
|
||||
}
|
||||
|
||||
// With its own lane empty, the worker falls through to the shared any-GPU lane.
|
||||
func TestClaimFallsBackToSharedLane(t *testing.T) {
|
||||
var claims []string
|
||||
srv := stubJobsCloud(t, &claims, map[string]*claimedActivity{"gpu-jobs": echoJob("j2")})
|
||||
defer srv.Close()
|
||||
if err := testWorker(t, srv.URL).claimAndRun(context.Background(), io.Discard); err != nil {
|
||||
t.Fatalf("claimAndRun: %v", err)
|
||||
}
|
||||
if len(claims) != 2 || claims[0] != "gpu:spark" || claims[1] != "gpu-jobs" {
|
||||
t.Fatalf("claims = %v, want [gpu:spark gpu-jobs] (targeted, then shared)", claims)
|
||||
}
|
||||
}
|
||||
|
||||
// Both lanes empty polls targeted THEN shared and runs nothing.
|
||||
func TestClaimBothLanesEmpty(t *testing.T) {
|
||||
var claims []string
|
||||
srv := stubJobsCloud(t, &claims, map[string]*claimedActivity{})
|
||||
defer srv.Close()
|
||||
if err := testWorker(t, srv.URL).claimAndRun(context.Background(), io.Discard); err != nil {
|
||||
t.Fatalf("claimAndRun: %v", err)
|
||||
}
|
||||
if len(claims) != 2 || claims[0] != "gpu:spark" || claims[1] != "gpu-jobs" {
|
||||
t.Fatalf("claims = %v, want [gpu:spark gpu-jobs]", claims)
|
||||
}
|
||||
}
|
||||
|
||||
// The render submit seam is the gated worker-mode execute path, not the open /prompt
|
||||
// — the shared contract with the studio's --worker-mode gate.
|
||||
func TestWorkerExecuteSeamIsGated(t *testing.T) {
|
||||
if localWorkerExecute != "http://127.0.0.1:8188/v1/worker/execute" {
|
||||
t.Fatalf("localWorkerExecute = %q, want the gated /v1/worker/execute seam", localWorkerExecute)
|
||||
}
|
||||
}
|
||||
|
||||
// A HUNG nvidia-smi (blocks until its bounded context fires) must NOT block the
|
||||
// caller: reportSample detaches the probe+POST onto a goroutine and returns at once,
|
||||
// so the worker's select loop keeps heartbeating and claiming. Guards the
|
||||
// worker-wedge regression (a synchronous probe that stalls the loop under GPU/driver
|
||||
// pressure → the machine flaps offline mid-render).
|
||||
func TestReportSampleNeverBlocksLoop(t *testing.T) {
|
||||
orig := nvidiaSmi
|
||||
defer func() { nvidiaSmi = orig }()
|
||||
probing := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
nvidiaSmi = func(ctx context.Context) ([]byte, error) {
|
||||
close(probing) // entered the probe (the nvidiaSmi var read already happened)
|
||||
select {
|
||||
case <-release: // the test lets us finish
|
||||
case <-ctx.Done(): // or the bounded probe timeout fires
|
||||
}
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
t.Setenv("HANZO_TOKEN", "t")
|
||||
w := &worker{identity: "spark", hostname: "spark", http: &http.Client{Timeout: time.Second}, env: &Env{}, baseURL: "http://127.0.0.1:0"}
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() { w.reportSample(context.Background()); close(done) }()
|
||||
select {
|
||||
case <-done: // returned immediately — the select loop is never wedged
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
t.Fatal("reportSample blocked the caller — a hung sampler would wedge the worker loop")
|
||||
}
|
||||
select {
|
||||
case <-probing: // the probe really ran, on the detached goroutine (off the critical path)
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("probe never started")
|
||||
}
|
||||
close(release)
|
||||
}
|
||||
|
||||
// A node that can't serve renders (preflight failed) claims NOTHING — it must never
|
||||
// pull a render job onto a box that will only refuse it on the gated seam (poison
|
||||
// loop). It still heartbeats presence; it just stays idle.
|
||||
func TestNotStudioReadyClaimsNothing(t *testing.T) {
|
||||
var claims []string
|
||||
srv := stubJobsCloud(t, &claims, map[string]*claimedActivity{"gpu:spark": echoJob("j1")})
|
||||
defer srv.Close()
|
||||
w := testWorker(t, srv.URL)
|
||||
w.studioReady = false
|
||||
if err := w.claimAndRun(context.Background(), io.Discard); err != nil {
|
||||
t.Fatalf("claimAndRun: %v", err)
|
||||
}
|
||||
if len(claims) != 0 {
|
||||
t.Fatalf("a not-ready node claimed %v; want zero claims", claims)
|
||||
}
|
||||
}
|
||||
|
||||
// The terminal report must hit the RIGHT activity — namespace gpu-jobs, the CLAIMED
|
||||
// workflow+run ids, the correct verb. A stub that 200s every path lets an ns/id
|
||||
// routing regression pass, so assert the exact paths for both complete and fail.
|
||||
func TestTerminalReportsHitCorrectActivityPath(t *testing.T) {
|
||||
t.Setenv("HANZO_TOKEN", "t")
|
||||
var mu sync.Mutex
|
||||
terminal := map[string]string{}
|
||||
served := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/activities/claim"):
|
||||
var body struct {
|
||||
TaskQueue string `json:"taskQueue"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
if body.TaskQueue != "gpu:spark" {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
n := served
|
||||
served++
|
||||
mu.Unlock()
|
||||
switch n {
|
||||
case 0:
|
||||
_ = json.NewEncoder(w).Encode(echoJob("wf1")) // echo handler → complete
|
||||
case 1:
|
||||
j := echoJob("wf2")
|
||||
j.Type.Name = "nope" // no handler → fail
|
||||
_ = json.NewEncoder(w).Encode(j)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
case strings.HasSuffix(r.URL.Path, "/complete"):
|
||||
mu.Lock()
|
||||
terminal["complete"] = r.URL.Path
|
||||
mu.Unlock()
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case strings.HasSuffix(r.URL.Path, "/fail"):
|
||||
mu.Lock()
|
||||
terminal["fail"] = r.URL.Path
|
||||
mu.Unlock()
|
||||
w.WriteHeader(http.StatusOK)
|
||||
default:
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
w := testWorker(t, srv.URL)
|
||||
if err := w.claimAndRun(context.Background(), io.Discard); err != nil {
|
||||
t.Fatalf("run1 (echo→complete): %v", err)
|
||||
}
|
||||
if err := w.claimAndRun(context.Background(), io.Discard); err != nil {
|
||||
t.Fatalf("run2 (unknown→fail): %v", err)
|
||||
}
|
||||
if got := terminal["complete"]; got != "/v1/tasks/namespaces/gpu-jobs/activities/wf1/wf1/complete" {
|
||||
t.Fatalf("complete path = %q, want the claimed activity's exact ns+ids", got)
|
||||
}
|
||||
if got := terminal["fail"]; got != "/v1/tasks/namespaces/gpu-jobs/activities/wf2/wf2/fail" {
|
||||
t.Fatalf("fail path = %q, want the claimed activity's exact ns+ids", got)
|
||||
}
|
||||
}
|
||||
|
||||
// SharePolicy.reject's fallback: an ABSENT or unparseable input field skips its gate
|
||||
// (permissive), never a hard error, so a policy only ever narrows on fields it can read.
|
||||
func TestSharePolicyRejectFallback(t *testing.T) {
|
||||
var nilp *SharePolicy
|
||||
if r := nilp.reject("studio.render", nil); r != "" {
|
||||
t.Fatalf("nil policy must allow everything: %q", r)
|
||||
}
|
||||
p := &SharePolicy{AllowedJobTypes: []string{"studio.render"}}
|
||||
if p.reject("echo", nil) == "" {
|
||||
t.Fatal("a disallowed job type must be rejected")
|
||||
}
|
||||
if r := p.reject("studio.render", nil); r != "" {
|
||||
t.Fatalf("an allowed job type must pass: %q", r)
|
||||
}
|
||||
p2 := &SharePolicy{AllowedOrgs: []string{"acme"}}
|
||||
if r := p2.reject("studio.render", json.RawMessage(`{}`)); r != "" {
|
||||
t.Fatalf("absent org must SKIP the org gate (fallback), not reject: %q", r)
|
||||
}
|
||||
if r := p2.reject("studio.render", json.RawMessage(`{"org":"acme"}`)); r != "" {
|
||||
t.Fatalf("a matching org must pass: %q", r)
|
||||
}
|
||||
if p2.reject("studio.render", json.RawMessage(`{"org":"other"}`)) == "" {
|
||||
t.Fatal("a non-allowed org must be rejected")
|
||||
}
|
||||
if r := p2.reject("studio.render", json.RawMessage(`not json`)); r != "" {
|
||||
t.Fatalf("unparseable input must skip input gates, not reject: %q", r)
|
||||
}
|
||||
}
|
||||
|
||||
// studioCap is advertised ONLY when the node can actually render: a missing worker
|
||||
// token is never ready (the gated seam would 403), and the block reason is explicit.
|
||||
func TestStudioReadyGatesCapability(t *testing.T) {
|
||||
t.Setenv("STUDIO_WORKER_TOKEN", "")
|
||||
w := &worker{launchesStudio: true, http: &http.Client{}}
|
||||
w.refreshStudioReady(context.Background())
|
||||
if w.studioReady {
|
||||
t.Fatal("no token must never be studio-ready")
|
||||
}
|
||||
if contains(w.capabilities(), studioCap) {
|
||||
t.Fatalf("studioCap advertised without a token: %v", w.capabilities())
|
||||
}
|
||||
if w.studioBlockReason() == "" {
|
||||
t.Fatal("a not-ready node must explain why it won't render")
|
||||
}
|
||||
// Token present + we launch the studio ⇒ ready ⇒ studioCap advertised.
|
||||
t.Setenv("STUDIO_WORKER_TOKEN", "tok")
|
||||
if changed := w.refreshStudioReady(context.Background()); !changed {
|
||||
t.Fatal("adding the token should flip readiness")
|
||||
}
|
||||
if !w.studioReady || !contains(w.capabilities(), studioCap) {
|
||||
t.Fatalf("token + launchesStudio must be ready + advertise studioCap: ready=%v caps=%v", w.studioReady, w.capabilities())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package cli
|
||||
|
||||
// gpu_spec_test.go — the host static-spec a `hanzo link` node reports so
|
||||
// GET /v1/fleet can show its CPU arch, core count and total RAM (the fields a
|
||||
// code-linked box already carries). Real telemetry only: arch is `uname -m`, cores
|
||||
// are runtime.NumCPU, RAM is parsed from the OS — never a hardcoded machine.
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseMemTotalKB(t *testing.T) {
|
||||
// A real /proc/meminfo head from a 128 GiB box. MemTotal is in kB; we report bytes.
|
||||
meminfo := []byte("MemTotal: 131923980 kB\nMemFree: 1048576 kB\nMemAvailable: 120000000 kB\n")
|
||||
if got, want := parseMemTotalKB(meminfo), int64(131923980)*1024; got != want {
|
||||
t.Fatalf("parseMemTotalKB = %d, want %d bytes", got, want)
|
||||
}
|
||||
// Absent / malformed input is reported as 0 (unknown), never a guess.
|
||||
for name, in := range map[string]string{
|
||||
"empty": "",
|
||||
"no-memtotal": "MemFree: 100 kB\n",
|
||||
"malformed": "MemTotal: notanumber kB\n",
|
||||
"no-value": "MemTotal:\n",
|
||||
} {
|
||||
if got := parseMemTotalKB([]byte(in)); got != 0 {
|
||||
t.Fatalf("%s: parseMemTotalKB = %d, want 0", name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// detectMemTotal reads the real host, so on Linux/macOS CI it must return a positive
|
||||
// byte count — proof the reporter reads actual RAM rather than shipping 0.
|
||||
func TestDetectMemTotalIsReal(t *testing.T) {
|
||||
if runtime.GOOS != "linux" && runtime.GOOS != "darwin" {
|
||||
t.Skipf("no MemTotal source on %s", runtime.GOOS)
|
||||
}
|
||||
got := detectMemTotal()
|
||||
if got <= 0 {
|
||||
t.Fatalf("detectMemTotal = %d, want the host's real RAM (>0)", got)
|
||||
}
|
||||
// Evidence: what THIS host actually reports (never hardcoded). On the GB10 spark
|
||||
// box this prints aarch64 + ~128 GiB read from /proc/meminfo.
|
||||
t.Logf("real host spec: arch=%s cpus=%d memory=%d bytes (%.1f GiB)",
|
||||
detectArch(), runtime.NumCPU(), got, float64(got)/(1<<30))
|
||||
}
|
||||
|
||||
// detectArch must match the fleet's `uname -m` convention (aarch64 | x86_64 | arm64),
|
||||
// NOT runtime.GOARCH (arm64 | amd64) — so a machine that appears as both a run-target
|
||||
// and a linked worker shows ONE arch string on the board. On Linux uname -m is
|
||||
// aarch64/x86_64; assert the real host agrees and is never GOARCH's amd64.
|
||||
func TestDetectArchMatchesUnameConvention(t *testing.T) {
|
||||
got := detectArch()
|
||||
if got == "" {
|
||||
t.Fatal("detectArch returned empty; must fall back to runtime.GOARCH")
|
||||
}
|
||||
if out, err := exec.Command("uname", "-m").Output(); err == nil {
|
||||
if want := strings.TrimSpace(string(out)); want != "" && got != want {
|
||||
t.Fatalf("detectArch = %q, want `uname -m` %q (fleet convention)", got, want)
|
||||
}
|
||||
}
|
||||
// Guard the regression this test exists for: on Linux amd64 the value must be
|
||||
// x86_64, never GOARCH's "amd64".
|
||||
if runtime.GOOS == "linux" && runtime.GOARCH == "amd64" && got == "amd64" {
|
||||
t.Fatal("arch is GOARCH 'amd64'; the fleet convention is 'x86_64'")
|
||||
}
|
||||
t.Logf("detectArch=%q (GOARCH=%q)", got, runtime.GOARCH)
|
||||
}
|
||||
|
||||
// buildRegistration must carry this host's detected arch (uname -m), cores (NumCPU)
|
||||
// and RAM — so spark reports aarch64 and evo-2 reports x86_64, both ~128 GB, matching
|
||||
// how the same machines already report as code-linked run-targets.
|
||||
func TestBuildRegistrationCarriesHostSpec(t *testing.T) {
|
||||
const mem = int64(137438953472) // 128 GiB
|
||||
w := &worker{hostname: "spark", jobsNS: "gpu-jobs", arch: "aarch64", memory: mem}
|
||||
reg := w.buildRegistration()
|
||||
if reg.Arch != "aarch64" {
|
||||
t.Fatalf("Arch = %q, want the worker's detected arch %q", reg.Arch, "aarch64")
|
||||
}
|
||||
if reg.CPUs != runtime.NumCPU() {
|
||||
t.Fatalf("CPUs = %d, want runtime.NumCPU %d", reg.CPUs, runtime.NumCPU())
|
||||
}
|
||||
if reg.Memory != mem {
|
||||
t.Fatalf("Memory = %d, want the detected total %d", reg.Memory, mem)
|
||||
}
|
||||
}
|
||||
+200
@@ -0,0 +1,200 @@
|
||||
package cli
|
||||
|
||||
// link.go — `hanzo link | unlink | status`: bring THIS machine into the Hanzo
|
||||
// cloud fleet AS A NODE, take it back out, and view the fleet.
|
||||
//
|
||||
// A node is two orthogonal memberships, composed under one verb:
|
||||
//
|
||||
// 1. The FABRIC — hanzod on hanzo.network. `link` starts it by invoking the
|
||||
// canonical fabric verb `hanzo node up` (the Rust node CLI: resolve a hanzod
|
||||
// binary, spawn it detached, record its pid). link does NOT reimplement hanzod
|
||||
// supervision — there is exactly one way to start hanzod, and this composes it.
|
||||
// Best-effort: a node with no hanzod still joins the compute fleet (CPU-only
|
||||
// boxes and dev machines link fine); `--no-fabric` skips it outright.
|
||||
//
|
||||
// 2. The COMPUTE fleet — this machine's inventory (CPU cores + model, memory, and
|
||||
// each GPU as its own resource) registered as a heartbeating presence in the
|
||||
// org's `fleet` namespace, running the outbound worker loop that claims jobs
|
||||
// from `gpu-jobs`. That machinery lives in gpu.go (runConnect); `link` runs it.
|
||||
//
|
||||
// `unlink` reverses both: it deregisters the worker (drops the fleet row), then
|
||||
// stops the fabric (`hanzo node stop`). `status` is the fleet view — every machine
|
||||
// with each of its GPUs shown distinctly, this box highlighted.
|
||||
//
|
||||
// One identity: the IAM token `hanzo login` mints authorizes every cloud call; the
|
||||
// server derives the tenant from the token. No secrets on the box.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// link / unlink / status — the node-level command surface.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func newLinkCmd(envOf func() *Env, _ *globalFlags) *cobra.Command {
|
||||
var opts connectOpts
|
||||
var daemon bool
|
||||
var noFabric bool
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "link",
|
||||
Short: "Bring this machine into the Hanzo cloud fleet as a node",
|
||||
Long: "Link this machine into the Hanzo cloud as a node: join the fabric (start\n" +
|
||||
"hanzod on hanzo.network) and register as a compute worker — advertising this\n" +
|
||||
"host's CPU (cores + model), memory, and each GPU as its own resource, then\n" +
|
||||
"heartbeating and claiming jobs from your org's queue. The node shows up in the\n" +
|
||||
"console (Machines + GPUs) and on `hanzo status`. Works on a CPU-only box.\n" +
|
||||
"Authentication reuses the `hanzo login` token; the org is taken from its claims.",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if daemon {
|
||||
return installDaemon(cmd, opts)
|
||||
}
|
||||
// 1. Fabric: start hanzod via the canonical `hanzo node up` (best-effort).
|
||||
startFabric(cmd, noFabric)
|
||||
// 2. Compute fleet: register inventory, heartbeat, claim jobs (foreground).
|
||||
return runConnect(cmd, envOf(), opts)
|
||||
},
|
||||
}
|
||||
|
||||
f := cmd.Flags()
|
||||
f.StringVar(&opts.jobsNS, "jobs-namespace", defaultJobsNS, "tasks namespace to claim jobs from")
|
||||
f.BoolVar(&noFabric, "no-fabric", false, "join the compute fleet only; do not start hanzod")
|
||||
f.BoolVar(&daemon, "daemon", false, "install a systemd --user unit (Restart=always) instead of running in the foreground")
|
||||
f.BoolVar(&opts.serveEngine, "serve-engine", false, "also advertise a hanzo-engine model server (OpenAI + Anthropic) running on this node")
|
||||
f.StringVar(&opts.engineURL, "engine-url", defaultEngineURL, "local URL where hanzo-engine is probed (GET /v1/models)")
|
||||
f.StringVar(&opts.engineEndpoint, "engine-endpoint", "", "public URL to advertise for gateway routing (defaults to --engine-url; a node behind NAT needs a reachable URL/tunnel)")
|
||||
f.BoolVar(&opts.registerProvider, "register-provider", false, "auto-register the engine endpoint as an org model provider (POST /v1/add-provider)")
|
||||
f.StringVar(&opts.studioDir, "studio-dir", os.Getenv("HANZO_STUDIO_DIR"), "local Hanzo Studio checkout; when set, link launches and supervises the render backend on 127.0.0.1:8188")
|
||||
f.StringVar(&opts.studioURL, "studio-url", firstNonEmpty(os.Getenv("HANZO_STUDIO_UPLOAD_URL"), defaultStudioUploadURL), "studio base URL the render mirror uploads finished images to (POST /v1/library/upload)")
|
||||
f.BoolVar(&opts.mirror, "mirror", true, "sweep local renders into the org studio library; --mirror=false serves jobs only")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func newUnlinkCmd(envOf func() *Env, _ *globalFlags) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "unlink",
|
||||
Short: "Take this machine out of the fleet (deregister + stop hanzod)",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
// Reverse of link, both best-effort: drop the compute-fleet row, then
|
||||
// ALWAYS stop the fabric so a deregister error never leaves hanzod
|
||||
// running. The deregister error is reported, not short-circuited.
|
||||
derr := runDisconnect(cmd, envOf())
|
||||
stopFabric(cmd)
|
||||
return derr
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newStatusCmd(envOf func() *Env, _ *globalFlags) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Show the org's fleet — every machine with each of its GPUs (this box highlighted)",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error { return runFleetStatus(cmd, envOf()) },
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fabric — compose the canonical `hanzo node up` / `node stop`.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// fabricCLI resolves the Hanzo node CLI that owns hanzod supervision — the Rust
|
||||
// fabric/dev CLI with `node up/stop`. In the canonical layout the Go unified binary
|
||||
// takes the `hanzo` name and the Rust CLI is installed alongside it as `hanzo-node`,
|
||||
// so `link` composes `node up` without shelling into itself (this binary has no
|
||||
// `node`). Resolution order: HANZO_FABRIC_CLI, then `hanzo-node`, then a
|
||||
// self-guarded `hanzo` (for boxes where the Rust CLI still holds the `hanzo` name).
|
||||
// Returns "" when none is resolvable.
|
||||
func fabricCLI() string {
|
||||
if p := os.Getenv("HANZO_FABRIC_CLI"); p != "" {
|
||||
return p
|
||||
}
|
||||
if p, err := exec.LookPath("hanzo-node"); err == nil {
|
||||
return p
|
||||
}
|
||||
p, err := exec.LookPath("hanzo")
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
if self, err := os.Executable(); err == nil {
|
||||
if sp, _ := os.Readlink(p); sp == self || p == self {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Passthrough delegates a verb this binary does not own — node, dev, wallet,
|
||||
// network, … — to the Rust fabric/dev CLI (resolved by fabricCLI, installed as
|
||||
// `hanzo-node`), so the single `hanzo` name is a SUPERSET: Go verbs served natively,
|
||||
// everything else handed through unchanged. `hanzo node up` (the fabric `link`
|
||||
// itself composes) works for users too. It runs the delegate to completion with
|
||||
// inherited stdio and exits with its code; it returns false only when no fabric CLI
|
||||
// is resolvable, so the caller can report unknown-subcommand.
|
||||
func Passthrough(args []string) bool {
|
||||
bin := fabricCLI()
|
||||
if bin == "" {
|
||||
return false
|
||||
}
|
||||
c := exec.Command(bin, args...)
|
||||
c.Stdin, c.Stdout, c.Stderr = os.Stdin, os.Stdout, os.Stderr
|
||||
if err := c.Run(); err != nil {
|
||||
var ee *exec.ExitError
|
||||
if errors.As(err, &ee) {
|
||||
os.Exit(ee.ExitCode())
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "hanzo: delegating %v to %s: %v\n", args, bin, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Exit(0)
|
||||
return true
|
||||
}
|
||||
|
||||
// startFabric starts hanzod by invoking `hanzo node up` — the one canonical way to
|
||||
// join the fabric. Best-effort by design: a missing CLI or a missing hanzod prints
|
||||
// a clear note and the node still joins the compute fleet. `--no-fabric` skips it.
|
||||
func startFabric(cmd *cobra.Command, skip bool) {
|
||||
out := cmd.OutOrStdout()
|
||||
if skip {
|
||||
fmt.Fprintln(out, "fabric: skipped (--no-fabric); joining the compute fleet only")
|
||||
return
|
||||
}
|
||||
bin := fabricCLI()
|
||||
if bin == "" {
|
||||
fmt.Fprintln(out, "fabric: hanzo node CLI not found — joining the compute fleet only")
|
||||
fmt.Fprintln(out, " (install the `hanzo` node CLI or set HANZO_FABRIC_CLI to start hanzod)")
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second)
|
||||
defer cancel()
|
||||
c := exec.CommandContext(ctx, bin, "node", "up")
|
||||
c.Stdout, c.Stderr = out, cmd.ErrOrStderr()
|
||||
if err := c.Run(); err != nil {
|
||||
fmt.Fprintf(out, "fabric: `%s node up` did not start hanzod (%v) — joining the compute fleet only\n", bin, err)
|
||||
}
|
||||
}
|
||||
|
||||
// stopFabric stops the hanzod this box started, via the canonical `hanzo node stop`.
|
||||
// Best-effort — nothing to stop is not an error.
|
||||
func stopFabric(cmd *cobra.Command) {
|
||||
out := cmd.OutOrStdout()
|
||||
bin := fabricCLI()
|
||||
if bin == "" {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
c := exec.CommandContext(ctx, bin, "node", "stop")
|
||||
c.Stdout, c.Stderr = out, cmd.ErrOrStderr()
|
||||
_ = c.Run()
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// TestUnlinkIdempotent — a repeat `hanzo unlink` is a no-op. The deregister POST
|
||||
// hitting an already-terminal (409) or absent (404) fleet row is the desired end
|
||||
// state, so runDisconnect returns nil (not an error) and says so, letting `unlink`
|
||||
// be run twice safely.
|
||||
func TestUnlinkIdempotent(t *testing.T) {
|
||||
for _, code := range []int{http.StatusConflict, http.StatusNotFound} {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(code)
|
||||
_, _ = w.Write([]byte(`{"error":"activity terminal"}`))
|
||||
}))
|
||||
|
||||
t.Setenv("HANZO_TOKEN", "test-token") // ensureToken honors this; no login needed
|
||||
var out bytes.Buffer
|
||||
cmd := &cobra.Command{}
|
||||
cmd.SetContext(context.Background())
|
||||
cmd.SetOut(&out)
|
||||
|
||||
if err := runDisconnect(cmd, &Env{CloudURL: srv.URL}); err != nil {
|
||||
t.Errorf("HTTP %d: runDisconnect should be idempotent (nil), got %v", code, err)
|
||||
}
|
||||
if !strings.Contains(out.String(), "already unlinked") {
|
||||
t.Errorf("HTTP %d: want 'already unlinked' notice, got %q", code, out.String())
|
||||
}
|
||||
srv.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnlinkDeregisterErrorSurfaces — a non-terminal deregister failure (e.g. 500)
|
||||
// is a real error and must be returned, not swallowed as idempotent success.
|
||||
func TestUnlinkDeregisterErrorSurfaces(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = w.Write([]byte(`{"error":"boom"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
t.Setenv("HANZO_TOKEN", "test-token")
|
||||
cmd := &cobra.Command{}
|
||||
cmd.SetContext(context.Background())
|
||||
cmd.SetOut(&bytes.Buffer{})
|
||||
|
||||
if err := runDisconnect(cmd, &Env{CloudURL: srv.URL}); err == nil {
|
||||
t.Fatal("a 500 deregister must surface an error, got nil")
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -6,7 +6,7 @@ package cli
|
||||
// org for queued workflow jobs and spawns ephemeral, auto-exiting actions-runner
|
||||
// subprocesses tagged with this box's labels (GPU / vulkan aware). Outbound only —
|
||||
// nothing listens for inbound. Completes the trifecta: `engine` serves models,
|
||||
// `gpu connect` shares compute, `runner` claims CI — one binary, one org login.
|
||||
// `link` shares compute, `runner` claims CI — one binary, one org login.
|
||||
|
||||
import (
|
||||
"os/signal"
|
||||
|
||||
+17
-4
@@ -1,5 +1,5 @@
|
||||
// studio.go — local Hanzo Studio render-backend supervision for `hanzo gpu
|
||||
// connect --studio-dir <checkout>`. The gpu-jobs claim loop renders on the
|
||||
// studio.go — local Hanzo Studio render-backend supervision for `hanzo link
|
||||
// --studio-dir <checkout>`. The gpu-jobs claim loop renders on the
|
||||
// LOCAL studio server (127.0.0.1:8188); this keeps that server alive so the
|
||||
// box needs no separate watchdog script or hand-rolled systemd unit — the
|
||||
// hanzo CLI is the one way a BYO box joins the fleet, render backend included.
|
||||
@@ -67,9 +67,22 @@ func launchStudio(dir string) (*exec.Cmd, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// --listen 127.0.0.1 (loopback only) + --worker-mode: the render backend serves
|
||||
// the fleet worker on this box and nothing else. The worker dials loopback
|
||||
// (localComfyUI) so binding wider bought nothing but an open, unauthenticated
|
||||
// /prompt — the hidden-run hole. --worker-mode makes the studio gate its submit
|
||||
// seam (/v1/worker/execute + X-Worker-Token) so only the worker can start a render.
|
||||
// VRAM mode: default --normalvram (safe for smaller BYO GPUs); override with
|
||||
// HANZO_STUDIO_VRAM (e.g. "--highvram") on big-memory boxes (GB10 128G unified) so
|
||||
// the Qwen text-encoder stays resident on-GPU instead of non-deterministically
|
||||
// offloading to CPU — offload makes renders CPU-bound and ~8x slower.
|
||||
vramMode := os.Getenv("HANZO_STUDIO_VRAM")
|
||||
if vramMode == "" {
|
||||
vramMode = "--normalvram"
|
||||
}
|
||||
cmd := exec.Command(studioPython(dir), "main.py",
|
||||
"--listen", "0.0.0.0", "--port", "8188",
|
||||
"--normalvram", "--disable-auto-launch",
|
||||
"--listen", "127.0.0.1", "--port", "8188", "--worker-mode",
|
||||
vramMode, "--disable-auto-launch",
|
||||
"--output-directory", filepath.Join(dir, "output"))
|
||||
cmd.Dir = dir
|
||||
cmd.Env = append(os.Environ(),
|
||||
|
||||
@@ -50,6 +50,7 @@
|
||||
package account
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -367,18 +368,58 @@ func onboard(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
return herr
|
||||
}
|
||||
|
||||
// Create the org (cloning the caller's current org for password/locale
|
||||
// compatibility), then — first-run only — move the zero-org user in as admin.
|
||||
// ADDITIONAL org (caller already has a home): create it WITHOUT moving them —
|
||||
// they reach it via the OrgSwitcher (a move would strip their SuperAdmin / orphan
|
||||
// their current org).
|
||||
if additional {
|
||||
org := buildOrg(s, c, slug, displayName, body.Personal, cr.owner)
|
||||
if err := s.State.iam.createOrganization(c.Context(), org); err != nil {
|
||||
return zip.Errorf(http.StatusBadGateway, "could not create the organization: %v", err)
|
||||
}
|
||||
return c.JSON(http.StatusOK, onboardResp{Org: slug, DisplayName: displayName, Additional: true})
|
||||
}
|
||||
|
||||
// FIRST-RUN: drive the ONE atomic IAM provision (org + admin move + hashed
|
||||
// org-scoped credential), replacing the create-org + move-user pair — a mid-flight
|
||||
// retry now converges on the founder's own org instead of orphaning it. The org
|
||||
// starts at a zero balance (usage is pre-paid). Prefer it whenever the
|
||||
// service-token path is wired; fall back to the legacy pair only when it is not,
|
||||
// so a partial deploy still onboards.
|
||||
if s.State.iam.provisionReady() {
|
||||
resp, err := onboardFirstRun(c.Context(), s.State.iam, cr.id, slug, displayName, body.Personal)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.JSON(http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// Legacy fallback (service token unset): create then move — the non-atomic pair.
|
||||
org := buildOrg(s, c, slug, displayName, body.Personal, cr.owner)
|
||||
if err := s.State.iam.createOrganization(c.Context(), org); err != nil {
|
||||
return zip.Errorf(http.StatusBadGateway, "could not create the organization: %v", err)
|
||||
}
|
||||
if !additional {
|
||||
if err := s.State.iam.moveUserToOrg(c.Context(), cr.id, slug); err != nil {
|
||||
return zip.Errorf(http.StatusBadGateway, "org created but could not assign you to it: %v", err)
|
||||
}
|
||||
if err := s.State.iam.moveUserToOrg(c.Context(), cr.id, slug); err != nil {
|
||||
return zip.Errorf(http.StatusBadGateway, "org created but could not assign you to it: %v", err)
|
||||
}
|
||||
return c.JSON(http.StatusOK, onboardResp{Org: slug, DisplayName: displayName, Additional: additional})
|
||||
return c.JSON(http.StatusOK, onboardResp{Org: slug, DisplayName: displayName, Additional: false})
|
||||
}
|
||||
|
||||
// onboardFirstRun drives the ONE atomic IAM provision for a zero-org caller (create
|
||||
// org + move them in as admin + mint the hashed org-scoped credential), replacing
|
||||
// the create-org + move-user pair so a mid-flight retry converges on the founder's
|
||||
// own org instead of orphaning it. The org starts at a ZERO balance — usage is
|
||||
// pre-paid, so there is no signup grant. Split out so the provisioning glue is
|
||||
// unit-tested against mock IAM without the CSRF/routing/principal shell.
|
||||
func onboardFirstRun(ctx context.Context, iam *iamClient, callerID, slug, displayName string, personal bool) (onboardResp, error) {
|
||||
row, err := iam.getUserRow(ctx, callerID)
|
||||
if err != nil {
|
||||
return onboardResp{}, zip.Errorf(http.StatusBadGateway, "could not resolve the user: %v", err)
|
||||
}
|
||||
res, err := iam.provision(ctx, row.Owner, row.Name, slug, personal)
|
||||
if err != nil {
|
||||
return onboardResp{}, zip.Errorf(http.StatusBadGateway, "could not provision the organization: %v", err)
|
||||
}
|
||||
return onboardResp{Org: res.Org, DisplayName: displayName, Additional: false}, nil
|
||||
}
|
||||
|
||||
// resolveOnboardName derives the base slug + display name from the request, or a
|
||||
|
||||
@@ -350,3 +350,9 @@ func s2sBillingCall(c *zip.Ctx) bool {
|
||||
bearer := strings.TrimSpace(strings.TrimPrefix(c.Header("Authorization"), "Bearer "))
|
||||
return bearer != "" && subtle.ConstantTimeCompare([]byte(bearer), []byte(token)) == 1
|
||||
}
|
||||
|
||||
// IsServiceToken is the exported view of s2sBillingCall — whether the request is a trusted
|
||||
// in-proc S2S caller bearing the verified COMMERCE_SERVICE_TOKEN. Used by co-resident route
|
||||
// gates (e.g. the spend-alert admin gate) that must admit the metering cap-gate and the
|
||||
// SuperAdmin cap-oversight Forward alongside org admins, while refusing a plain member.
|
||||
func IsServiceToken(c *zip.Ctx) bool { return s2sBillingCall(c) }
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
// billing_coresident.go — PinBillingSubject, the subject-pinning middleware that lets
|
||||
// commerce's OWN billing READ handlers serve co-resident in the unified cloud binary.
|
||||
//
|
||||
// WHY IT EXISTS. Co-resident, commerce is EMBEDDED (apps/commerce.go mountCommerce →
|
||||
// commerce.Embed on the shared zip app), and in prod there is NO standalone commerce
|
||||
// backend — the in-cluster `commerce` Service selects the cloud pods themselves. So the
|
||||
// /v1/billing/* bridge (billing.go), which forwards to COMMERCE_URL, has nowhere to send
|
||||
// a read but back into cloud: the default base (the public api.hanzo.ai edge) re-enters
|
||||
// the same bridge in an unbounded self-dispatch loop that surfaces as a 502
|
||||
// ("billing upstream unreachable: Get https://api.hanzo.ai/v1/billing/<path>"). The fix
|
||||
// is to serve those reads co-resident from the embedded commerce — the same co-resident
|
||||
// move mountCommerce already makes for GET /v1/billing/plans — so the specific route
|
||||
// shadows the bridge wildcard (order 100 < 122) and never leaves the process.
|
||||
//
|
||||
// WHAT IT GUARANTEES. commerce's read handlers scope to the org by NAMESPACE (from the
|
||||
// gateway-validated X-Org-Id via iammiddleware) but filter finer scope (the billing
|
||||
// subject) only from a query param — an UNPINNED ListInvoices returns every user's rows
|
||||
// in the org namespace. The /v1/billing/* bridge is what pins that subject today; this
|
||||
// middleware carries the SAME pin onto the co-resident route so the isolation is
|
||||
// byte-for-byte the shipped behavior. It is the ONE subject rule (account.Payer, the same
|
||||
// function the ai spend-gate and the top-up resolve), fed the account the credential NAMES
|
||||
// — so a read scopes to exactly the account the gate debits, never wider.
|
||||
package account
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
|
||||
"github.com/hanzoai/account"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/principal"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// PinBillingSubject pins every billing subject key in the request query to the VALIDATED
|
||||
// caller's own subject (dropping ?org), so a co-resident commerce read handler downstream
|
||||
// can only ever return the caller's OWN rows. It is the co-resident twin of billingData's
|
||||
// subject-pinning, reusing the SAME resolveCaller → account.Payer rule and the SAME
|
||||
// scopedBillingSearch, so the two paths scope identically.
|
||||
//
|
||||
// Three cases, mirroring billingData exactly:
|
||||
// - Browser customer (a validated principal with an org): OVERWRITE the subject keys
|
||||
// with the caller's own subject and drop ?org. The client cannot widen scope.
|
||||
// - Trusted in-proc S2S (the verified COMMERCE_SERVICE_TOKEN bearer, carrying its own
|
||||
// X-Org-Id): pass the query through VERBATIM — it legitimately names its own subject,
|
||||
// scoped by the EdgeAuth-controlled org.
|
||||
// - Neither: refuse. A bearer-less request with a forged X-Org-Id has no validated
|
||||
// principal and is fail-closed here, before the read handler runs.
|
||||
//
|
||||
// The pin rewrites the request URI's query string AND (on a write) the JSON body in place;
|
||||
// fasthttp's SetQueryString resets the parsed-args cache and SetBody replaces the body
|
||||
// bytes, so the handler's later c.Query() / c.Bind() read the pinned values. Pinning the
|
||||
// body is what keeps a co-resident WRITE handler that reads its subject from the body
|
||||
// (commerce's CreatePaymentMethod reads customerId from the JSON body) IDOR-safe — query-only
|
||||
// pinning would leave a client-named customerId/userId in the body untouched.
|
||||
func PinBillingSubject() zip.Handler {
|
||||
return func(c *zip.Ctx) error {
|
||||
inQuery, _ := url.ParseQuery(string(c.Fiber().Request().URI().QueryString()))
|
||||
|
||||
cr, ok := resolveCaller(c, true)
|
||||
if !ok {
|
||||
// Not a validated customer — admit ONLY a trusted in-proc S2S caller that
|
||||
// names its own org (same admission billingData makes), leaving its query
|
||||
// untouched. Everything else is refused before the read runs.
|
||||
if s2sBillingCall(c) && c.Org() != "" {
|
||||
return c.Next()
|
||||
}
|
||||
return zip.ErrForbidden("sign in to view billing")
|
||||
}
|
||||
|
||||
subject := account.Payer(account.Credential{
|
||||
Owner: cr.owner,
|
||||
Name: cr.username,
|
||||
Account: principal.BillingAccount(c),
|
||||
}).Subject()
|
||||
|
||||
// Pin the subject on BOTH the query AND the write body — the SAME two-helper
|
||||
// scoping billingData applies (scopedBillingSearch + scopedBillingBody). A
|
||||
// co-resident WRITE handler that reads its subject from the body (commerce's
|
||||
// CreatePaymentMethod → customerId) is only IDOR-safe if the body is pinned too.
|
||||
// scopedBillingBody overwrites the subject keys and preserves every other field
|
||||
// (card, type, sourceId, …); a non-JSON / empty body is returned unchanged, so a
|
||||
// GET read carries no body and is unaffected.
|
||||
c.Fiber().Request().URI().SetQueryString(scopedBillingSearch(inQuery, subject).Encode())
|
||||
if len(c.Body()) > 0 {
|
||||
c.Fiber().Request().SetBody(scopedBillingBody(c.Body(), subject))
|
||||
}
|
||||
return c.Next()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package account
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
luxlog "github.com/luxfi/log"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// billing_coresident_test.go — proves PinBillingSubject carries the SAME tenant scoping
|
||||
// onto a co-resident commerce read handler that billingData applies on the bridge: the
|
||||
// caller's own subject is pinned into the query (dropping ?org), an unvalidated caller is
|
||||
// refused before the handler runs, and a trusted in-proc S2S caller passes through
|
||||
// verbatim. This is what lets commerce's ListInvoices/ListBillingSubscriptions/... serve
|
||||
// in-process without leaking another subject's rows.
|
||||
|
||||
// echoQuery is the downstream stand-in for a commerce read handler: it reports exactly the
|
||||
// query it observes AFTER the pin, so a test can assert the subject the handler would filter on.
|
||||
func echoQuery(c *zip.Ctx) error {
|
||||
return c.JSON(200, map[string]any{
|
||||
"user": c.Query("user"),
|
||||
"userId": c.Query("userId"),
|
||||
"customerId": c.Query("customerId"),
|
||||
"org": c.Query("org"),
|
||||
"status": c.Query("status"),
|
||||
})
|
||||
}
|
||||
|
||||
// echoBody is the downstream stand-in for a commerce WRITE handler (e.g. CreatePaymentMethod)
|
||||
// that reads its subject from the JSON BODY: it reports the body it observes AFTER the pin, so
|
||||
// a test can assert the subject the handler would persist and that non-subject fields survive.
|
||||
func echoBody(c *zip.Ctx) error {
|
||||
var got map[string]any
|
||||
if len(c.Body()) > 0 {
|
||||
_ = json.Unmarshal(c.Body(), &got)
|
||||
}
|
||||
return c.JSON(200, got)
|
||||
}
|
||||
|
||||
func pinApp(t *testing.T) *zip.App {
|
||||
t.Helper()
|
||||
app := zip.New(zip.Config{Logger: luxlog.New("test")})
|
||||
// MountAccount installs the identity middleware PinBillingSubject relies on; mounting
|
||||
// it keeps the probe on the same trust plane as the real co-resident registration.
|
||||
if err := MountAccount(app, cloud.Deps{Logger: luxlog.New("test"), Brand: "hanzo"}); err != nil {
|
||||
t.Fatalf("MountAccount: %v", err)
|
||||
}
|
||||
app.Get("/probe", PinBillingSubject(), echoQuery)
|
||||
app.Post("/probe", PinBillingSubject(), echoBody)
|
||||
return app
|
||||
}
|
||||
|
||||
// TestPinBillingSubject_PinsCallerAndDropsOrg — a validated customer's forged subject
|
||||
// keys are overwritten with its OWN subject and ?org is dropped, exactly like the bridge.
|
||||
func TestPinBillingSubject_PinsCallerAndDropsOrg(t *testing.T) {
|
||||
app := pinApp(t)
|
||||
code, body := callH(t, app, http.MethodGet,
|
||||
"/probe?userId=victim&customerId=victim&user=victim&org=othercorp&status=open", alice, "")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("want 200, got %d (%s)", code, body)
|
||||
}
|
||||
var got map[string]string
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatalf("bad body: %s", body)
|
||||
}
|
||||
for _, k := range billingSubjectKeys {
|
||||
if got[k] != "acme" { // alice/acme resolves to the org subject "acme"
|
||||
t.Fatalf("handler must see %s=acme (caller's own subject), got %q", k, got[k])
|
||||
}
|
||||
}
|
||||
if got["org"] != "" {
|
||||
t.Fatalf("org must be dropped, handler saw org=%q", got["org"])
|
||||
}
|
||||
if got["status"] != "open" {
|
||||
t.Fatalf("non-subject filter must survive, got status=%q", got["status"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestPinBillingSubject_PinsBodyForWrites — a POST whose JSON body names a FOREIGN subject
|
||||
// (customerId/userId/user) has every subject key overwritten with the caller's OWN subject
|
||||
// before the handler binds it, while non-subject fields survive. This is the boundary
|
||||
// commerce's CreatePaymentMethod (which reads customerId from the body) relies on to be
|
||||
// IDOR-safe co-resident — byte-identical to billingData's scopedBillingBody on the bridge.
|
||||
func TestPinBillingSubject_PinsBodyForWrites(t *testing.T) {
|
||||
app := pinApp(t)
|
||||
code, body := callH(t, app, http.MethodPost, "/probe", alice,
|
||||
`{"customerId":"victim","userId":"victim","user":"victim","card":{"last4":"4242"}}`)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("want 200, got %d (%s)", code, body)
|
||||
}
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatalf("bad body: %s", body)
|
||||
}
|
||||
for _, k := range billingSubjectKeys {
|
||||
if got[k] != "acme" { // alice/acme resolves to the org subject "acme"
|
||||
t.Fatalf("handler must see body %s=acme (caller's own subject), got %v", k, got[k])
|
||||
}
|
||||
}
|
||||
card, ok := got["card"].(map[string]any)
|
||||
if !ok || card["last4"] != "4242" {
|
||||
t.Fatalf("non-subject body field must survive the pin, got card=%v", got["card"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestPinBillingSubject_RefusesUnvalidated — a forged X-Org-Id with NO validated
|
||||
// X-User-Id (and no service token) is refused before the read handler runs: no
|
||||
// cross-tenant billing read is possible.
|
||||
func TestPinBillingSubject_RefusesUnvalidated(t *testing.T) {
|
||||
t.Setenv("COMMERCE_SERVICE_TOKEN", "svc-tok")
|
||||
app := pinApp(t)
|
||||
code, _ := callH(t, app, http.MethodGet, "/probe?userId=victim",
|
||||
map[string]string{"X-Org-Id": "victim"}, "")
|
||||
if code != http.StatusForbidden {
|
||||
t.Fatalf("unvalidated caller: want 403, got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPinBillingSubject_S2SForwardsVerbatim — the trusted in-proc S2S caller (verified
|
||||
// COMMERCE_SERVICE_TOKEN + its own X-Org-Id, no validated user) is admitted and its query
|
||||
// is left UNTOUCHED, so it can name its own subject (the cap-gate's authorize read).
|
||||
func TestPinBillingSubject_S2SForwardsVerbatim(t *testing.T) {
|
||||
t.Setenv("COMMERCE_SERVICE_TOKEN", "svc-tok")
|
||||
app := pinApp(t)
|
||||
code, body := callH(t, app, http.MethodGet, "/probe?user=acme&status=open",
|
||||
map[string]string{"Authorization": "Bearer svc-tok", "X-Org-Id": "acme"}, "")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("S2S: want 200, got %d (%s)", code, body)
|
||||
}
|
||||
var got map[string]string
|
||||
_ = json.Unmarshal(body, &got)
|
||||
if got["user"] != "acme" || got["status"] != "open" {
|
||||
t.Fatalf("S2S query must pass through verbatim, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPinBillingSubject_S2SNoOrgRefused — the service token with NO X-Org-Id has no org
|
||||
// to scope the privileged read and is refused (matches billingData's s2s org requirement).
|
||||
func TestPinBillingSubject_S2SNoOrgRefused(t *testing.T) {
|
||||
t.Setenv("COMMERCE_SERVICE_TOKEN", "svc-tok")
|
||||
app := pinApp(t)
|
||||
code, _ := callH(t, app, http.MethodGet, "/probe",
|
||||
map[string]string{"Authorization": "Bearer svc-tok"}, "")
|
||||
if code != http.StatusForbidden {
|
||||
t.Fatalf("S2S without X-Org-Id: want 403, got %d", code)
|
||||
}
|
||||
}
|
||||
@@ -173,6 +173,23 @@ func requireCSRF(s *cloud.Service[state], next zip.Handler) zip.Handler {
|
||||
}
|
||||
}
|
||||
|
||||
// RequireCSRF exposes the ambient-cookie anti-CSRF gate as a STANDALONE middleware for a
|
||||
// co-resident money-WRITE route registered OUTSIDE this package — specifically
|
||||
// apps/commerce.go's POST /v1/billing/topup/token, which shadows the account-bridge's
|
||||
// POST /v1/billing/* wildcard (order 100 < 122) that would otherwise have wrapped the
|
||||
// write in requireCSRF. Moving the route co-resident to break the commerceinproc
|
||||
// self-dispatch loop must NOT silently drop that anti-CSRF gate, so the identical
|
||||
// enforcement rides along as its own handler. It binds to the SAME process-wide key
|
||||
// (sharedCSRFKey) the GET /v1/csrf issuer and the bridge verifier use, so a token minted
|
||||
// at /v1/csrf verifies here byte-identically. Enforces ONLY on the ambient-cookie path (a
|
||||
// Bearer/gateway/API caller is not CSRF-able); on success it c.Next()s into the rest of
|
||||
// the chain. The minimal Service carries only the shared key — requireCSRF/verifyCSRF
|
||||
// read nothing else off it.
|
||||
func RequireCSRF() zip.Handler {
|
||||
s := &cloud.Service[state]{State: state{csrfKey: sharedCSRFKey(nil)}}
|
||||
return requireCSRF(s, func(c *zip.Ctx) error { return c.Next() })
|
||||
}
|
||||
|
||||
// issueCSRFToken serves GET /v1/csrf: for a VALIDATED caller, a fresh token
|
||||
// bound to their identity. no-store so it is never cached by a shared proxy. This is
|
||||
// the same-origin endpoint the embedded SPA reads (its response body is unreadable to
|
||||
|
||||
@@ -44,6 +44,7 @@ const iamMaxBody = 4 << 20
|
||||
// iamClient is the confidential-client caller. clientID/clientSecret authenticate
|
||||
// as the `hanzo-console` app; an empty pair means "not configured" (handlers 501).
|
||||
type iamClient struct {
|
||||
serviceToken string // IAM_SERVICE_TOKEN — the Bearer for the admin provision endpoint
|
||||
base string
|
||||
clientID string
|
||||
clientSecret string
|
||||
@@ -56,10 +57,91 @@ func newIAMClient() *iamClient {
|
||||
base: base,
|
||||
clientID: strings.TrimSpace(os.Getenv("IAM_MINT_CLIENT_ID")),
|
||||
clientSecret: strings.TrimSpace(os.Getenv("IAM_MINT_CLIENT_SECRET")),
|
||||
serviceToken: strings.TrimSpace(os.Getenv("IAM_SERVICE_TOKEN")),
|
||||
http: &http.Client{Timeout: 15 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
// provisionResult is the /v1/iam/admin/provision response: the converged org and its
|
||||
// hashed credential (accessSecret shown ONCE on first mint). The org starts at a zero
|
||||
// balance — usage is pre-paid, no signup grant.
|
||||
type provisionResult struct {
|
||||
Org string `json:"org"`
|
||||
AccessKey string `json:"accessKey"`
|
||||
AccessSecret string `json:"accessSecret"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
// provisionReady reports whether the service-token provisioning path is wired.
|
||||
func (c *iamClient) provisionReady() bool { return c != nil && c.serviceToken != "" }
|
||||
|
||||
// provision drives the ONE atomic IAM onboarding op: create the org, move the named
|
||||
// user in as its admin, and mint its hashed org-scoped credential — the service-token
|
||||
// endpoint that replaces the create-org + move-user pair, so there is no orphan
|
||||
// between two writes and a mid-flight retry converges. orgSlug is the caller's
|
||||
// already-resolved slug (IAM honors it verbatim). The org starts at a zero balance.
|
||||
func (c *iamClient) provision(ctx context.Context, owner, name, orgSlug string, personal bool) (provisionResult, error) {
|
||||
if !c.provisionReady() {
|
||||
return provisionResult{}, errNotConfigured
|
||||
}
|
||||
body, err := json.Marshal(map[string]any{
|
||||
"owner": owner, "name": name, "orgSlug": orgSlug, "personal": personal,
|
||||
})
|
||||
if err != nil {
|
||||
return provisionResult{}, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+"/v1/iam/admin/provision", strings.NewReader(string(body)))
|
||||
if err != nil {
|
||||
return provisionResult{}, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+c.serviceToken)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return provisionResult{}, fmt.Errorf("iam unreachable: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
raw, err := io.ReadAll(io.LimitReader(resp.Body, iamMaxBody))
|
||||
if err != nil {
|
||||
return provisionResult{}, err
|
||||
}
|
||||
var out provisionResult
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return provisionResult{}, fmt.Errorf("iam provision non-json response (%d)", resp.StatusCode)
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 || out.Error != "" {
|
||||
msg := out.Error
|
||||
if msg == "" {
|
||||
msg = fmt.Sprintf("iam status %d", resp.StatusCode)
|
||||
}
|
||||
return provisionResult{}, fmt.Errorf("iam provision: %s", msg)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// userRow is the subset of an IAM user the onboarding path reads to resolve the
|
||||
// caller's authoritative (owner, name) — a zero-org caller's owner is not on its
|
||||
// token, so provision needs it from the row.
|
||||
type userRow struct {
|
||||
Owner string `json:"owner"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// getUserRow resolves the user by the caller's id (the same read the move did) into
|
||||
// its authoritative (owner, name).
|
||||
func (c *iamClient) getUserRow(ctx context.Context, id string) (userRow, error) {
|
||||
raw, err := c.getUser(ctx, id)
|
||||
if err != nil {
|
||||
return userRow{}, err
|
||||
}
|
||||
var row userRow
|
||||
if err := json.Unmarshal(raw, &row); err != nil {
|
||||
return userRow{}, fmt.Errorf("iam get-user: decode: %w", err)
|
||||
}
|
||||
return row, nil
|
||||
}
|
||||
|
||||
// configured reports whether the confidential client is wired. Handlers 501 when
|
||||
// false — the deployment simply lacks the `hanzo-console` credential (the honest
|
||||
// "not configured on this deployment" state, never a fabricated result).
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package account
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestOnboardFirstRun_ProvisionsOnce proves the production signup path drives ONE
|
||||
// atomic IAM provision (not the old create-org + move-user pair): it resolves the
|
||||
// zero-org caller's authoritative (owner, name) and provisions the org + hashed
|
||||
// credential in a single call. No trial credit is granted — the org starts at a zero
|
||||
// balance and usage is pre-paid.
|
||||
func TestOnboardFirstRun_ProvisionsOnce(t *testing.T) {
|
||||
var provisionCalls int
|
||||
var provBody map[string]any
|
||||
|
||||
iamSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/v1/iam/get-user":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"status": "ok",
|
||||
"data": map[string]any{"owner": "landing", "name": "dave"},
|
||||
})
|
||||
case "/v1/iam/admin/provision":
|
||||
provisionCalls++
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
_ = json.Unmarshal(b, &provBody)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"org": "dave", "accessKey": "hk-x", "accessSecret": "sk-x",
|
||||
})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer iamSrv.Close()
|
||||
|
||||
iam := &iamClient{base: iamSrv.URL, clientID: "c", clientSecret: "s", serviceToken: "svc", http: &http.Client{}}
|
||||
|
||||
resp, err := onboardFirstRun(context.Background(), iam, "dave", "dave", "Dave", true)
|
||||
if err != nil {
|
||||
t.Fatalf("onboardFirstRun: %v", err)
|
||||
}
|
||||
if resp.Org != "dave" || resp.Additional {
|
||||
t.Fatalf("resp = %+v, want org=dave additional=false", resp)
|
||||
}
|
||||
if provisionCalls != 1 {
|
||||
t.Fatalf("provision calls = %d, want 1 (ONE atomic op, not create+move)", provisionCalls)
|
||||
}
|
||||
if provBody["owner"] != "landing" || provBody["name"] != "dave" || provBody["orgSlug"] != "dave" {
|
||||
t.Fatalf("provision body = %v, want owner=landing name=dave orgSlug=dave", provBody)
|
||||
}
|
||||
|
||||
// Retry converges: provision is idempotent (same org), no orphan.
|
||||
if _, err := onboardFirstRun(context.Background(), iam, "dave", "dave", "Dave", true); err != nil {
|
||||
t.Fatalf("retry: %v", err)
|
||||
}
|
||||
if provisionCalls != 2 {
|
||||
t.Fatalf("provision calls = %d, want 2 (retried, converges)", provisionCalls)
|
||||
}
|
||||
}
|
||||
+28
-20
@@ -90,36 +90,44 @@ func Mount(app *zip.App, deps cloud.Deps) error {
|
||||
// control plane behind core.Guard. Each carved-out domain (audit/customer/revenue/finance)
|
||||
// owns its own route registration.
|
||||
func routes(app *zip.App, s *cloud.Service[core.State]) {
|
||||
g := app.Group("/v1/admin")
|
||||
// Org-scoped panels — GuardScoped. Cross-tenant reads are impossible for a non-super
|
||||
// caller.
|
||||
app.Get("/v1/admin/me", core.GuardScoped(s, me))
|
||||
app.Get("/v1/admin/overview", core.GuardScoped(s, overview))
|
||||
app.Get("/v1/admin/orgs", core.GuardScoped(s, orgs))
|
||||
app.Get("/v1/admin/users", core.GuardScoped(s, users))
|
||||
app.Get("/v1/admin/usage", core.GuardScoped(s, usage))
|
||||
g.Get("/me", core.GuardScoped(s, me))
|
||||
g.Get("/overview", core.GuardScoped(s, overview))
|
||||
g.Get("/orgs", core.GuardScoped(s, orgs))
|
||||
g.Get("/users", core.GuardScoped(s, users))
|
||||
g.Get("/usage", core.GuardScoped(s, usage))
|
||||
// Platform reads — SuperAdmin only (cross-tenant by nature).
|
||||
app.Get("/v1/admin/roles", core.Guard(s, roles))
|
||||
app.Get("/v1/admin/applications", core.Guard(s, applications))
|
||||
app.Get("/v1/admin/products", core.Guard(s, products))
|
||||
app.Get("/v1/admin/compute", core.Guard(s, compute))
|
||||
app.Get("/v1/admin/o11y", core.Guard(s, o11y))
|
||||
app.Post("/v1/admin/sync", core.Guard(s, syncNow))
|
||||
g.Get("/roles", core.Guard(s, roles))
|
||||
g.Get("/applications", core.Guard(s, applications))
|
||||
g.Get("/products", core.Guard(s, products))
|
||||
g.Get("/compute", core.Guard(s, compute))
|
||||
g.Get("/block-storage", core.Guard(s, blockStorage))
|
||||
g.Get("/o11y", core.Guard(s, o11y))
|
||||
g.Get("/aimetrics", core.Guard(s, aimetrics))
|
||||
g.Post("/sync", core.Guard(s, syncNow))
|
||||
|
||||
// Credit grants — the ONE admin mint surface (SuperAdmin only). Thin, audited
|
||||
// relay to commerce's mint-gated POST /v1/billing/credit-grants; commerce is the
|
||||
// sole ledger. See creditgrant.go.
|
||||
g.Post("/credit-grants", core.Guard(s, createCreditGrant))
|
||||
|
||||
// Product analytics — org-scoped (SuperAdmin: all-orgs; org admin: their own org).
|
||||
app.Get("/v1/admin/analytics", core.GuardScoped(s, analytics))
|
||||
g.Get("/analytics", core.GuardScoped(s, analytics))
|
||||
// Bases — the tenant Base-instance panel, org-scoped (bases.go).
|
||||
app.Get("/v1/admin/bases", core.GuardScoped(s, bases))
|
||||
g.Get("/bases", core.GuardScoped(s, bases))
|
||||
|
||||
// ── Platform control plane — SuperAdmin ONLY (launch/release/flags + access). ──
|
||||
app.Get("/v1/admin/flags", core.Guard(s, flagsBoard))
|
||||
app.Put("/v1/admin/flags/:key", core.Guard(s, setFlag))
|
||||
g.Get("/flags", core.Guard(s, flagsBoard))
|
||||
g.Put("/flags/:key", core.Guard(s, setFlag))
|
||||
// Launch-control services board — the waitlist-mode lens on the flag engine (twin
|
||||
// of /v1/admin/flags), reading the registry + decide the admission gate owns.
|
||||
app.Get("/v1/admin/services", core.Guard(s, services))
|
||||
app.Post("/v1/admin/services", core.Guard(s, upsertService))
|
||||
app.Post("/v1/admin/services/:service/mode", core.Guard(s, setServiceMode))
|
||||
app.Get("/v1/admin/waitlist", core.Guard(s, waitlist))
|
||||
app.Post("/v1/admin/waitlist/boost", core.Guard(s, waitlistBoost))
|
||||
g.Get("/services", core.Guard(s, services))
|
||||
g.Post("/services", core.Guard(s, upsertService))
|
||||
g.Post("/services/:service/mode", core.Guard(s, setServiceMode))
|
||||
g.Get("/waitlist", core.Guard(s, waitlist))
|
||||
g.Post("/waitlist/boost", core.Guard(s, waitlistBoost))
|
||||
|
||||
// Usage-cap + promo control plane (promos platform-only; spend-caps org-scoped).
|
||||
limitRoutes(app, s)
|
||||
|
||||
@@ -24,16 +24,16 @@ import (
|
||||
// mount builds a zip app with admin mounted against the given upstream bases,
|
||||
// and returns a `do` helper that issues test requests through the whole app.
|
||||
func mount(t *testing.T, iamURL, commerceURL, healthURL string) func(method, path string, hdr map[string]string) (*http.Response, []byte) {
|
||||
do, _, _ := mountSvc(t, iamURL, commerceURL, healthURL)
|
||||
do, _, _ := mountService(t, iamURL, commerceURL, healthURL)
|
||||
return do
|
||||
}
|
||||
|
||||
// mountSvc is mount but also returns the underlying cloud.Service[state] (so finance tests can swap
|
||||
// mountService is mount but also returns the underlying cloud.Service[state] (so finance tests can swap
|
||||
// in a fake DigitalOcean client, and the cockpit tests can attach an audit store)
|
||||
// AND the raw fiber app (so tests that need a request BODY can drive it directly —
|
||||
// the returned `do` sends a nil body). The handlers read s.* live at request time,
|
||||
// so an override before issuing a request takes effect.
|
||||
func mountSvc(t *testing.T, iamURL, commerceURL, healthURL string) (func(method, path string, hdr map[string]string) (*http.Response, []byte), *cloud.Service[core.State], *fiber.App) {
|
||||
func mountService(t *testing.T, iamURL, commerceURL, healthURL string) (func(method, path string, hdr map[string]string) (*http.Response, []byte), *cloud.Service[core.State], *fiber.App) {
|
||||
t.Helper()
|
||||
app := zip.New(zip.Config{Logger: luxlog.New("test")})
|
||||
s := &cloud.Service[core.State]{State: core.State{
|
||||
|
||||
@@ -0,0 +1,392 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package admin
|
||||
|
||||
// aimetrics — GET /v1/admin/aimetrics, the GLOBAL fleet-wide AI / training / eval
|
||||
// read that powers the operator's AI-metrics board on admin.hanzo.ai. It is the
|
||||
// AI-and-eval-focused companion to o11y (o11y.go): where o11y answers "how is the
|
||||
// FLEET behaving" (RED metrics, logs, usage), this answers "how are the MODELS and
|
||||
// EVALS doing" — LLM generations, per-model spend, and eval-run quality/progress —
|
||||
// over the SAME ONE datastore (Datastore), the SAME shared client
|
||||
// (aiobject.DatastoreQuery), no second connection.
|
||||
//
|
||||
// Signals, each from its canonical table in the one datastore:
|
||||
// - LLM generations → langfuse.observations : generations, cost (USD), latency
|
||||
// (fleet-wide; honest-empty until the
|
||||
// Langfuse ingest lands rows)
|
||||
// - Per-model usage → hanzo.cloud_usage : requests, tokens, cost per model
|
||||
// (the live usage ledger the ai gateway
|
||||
// writes — populated today)
|
||||
// - Eval runs → hanzo.eval_traces : traces, runs, datasets, models under
|
||||
// test, per-trace latency
|
||||
// - Eval progress → hanzo.eval_scores : score count, avg score, per-score-name
|
||||
// distribution, recent-run averages, and
|
||||
// the avg-score-over-time TREND — the
|
||||
// training/eval progress signal
|
||||
//
|
||||
// The eval_traces / eval_scores tables are OWNED and written by the eval telemetry
|
||||
// store (clients/eval/telemetry.go) — the SAME warehouse, same db ("hanzo"), same
|
||||
// shared aiobject client. admin only READS them here. There is deliberately no
|
||||
// "training_progress" table: the router's per-request training events live in the ai
|
||||
// OLTP Postgres (object.RoutingEvent), NOT the OLAP warehouse, so the honest
|
||||
// warehouse-side progress signal is the eval-score trend, not a routing table.
|
||||
//
|
||||
// SUPERADMIN ONLY (the core.Guard wrap in admin.go), all-orgs, no org filter — the
|
||||
// one place a fleet operator crosses tenants for AI/eval metrics; a non-admin bearer
|
||||
// is refused 403 before a single row is read. Fail-closed.
|
||||
//
|
||||
// Honest by construction, exactly like o11y/compute: no datastore connected → the
|
||||
// real empty aggregate, never a fabricated fleet; and every signal degrades
|
||||
// INDEPENDENTLY — a table that is absent or a column that differs contributes its
|
||||
// zero-value (the enclosing `if err == nil`), never a failure, so the board always
|
||||
// renders what the datastore actually holds. admin READS only; it owns and creates
|
||||
// NO table. Money from cloud_usage is USD cents, from langfuse is USD; latency is
|
||||
// milliseconds; time bounds are POSITIONAL parameters (never interpolated), and the
|
||||
// bucket interval is a server-side constant — injection-safe.
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
aiobject "github.com/hanzoai/ai/object"
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/clients/admin/core"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// Fully-qualified datastore tables. admin only READS these — the ai gateway owns
|
||||
// hanzo.cloud_usage, Langfuse owns langfuse.observations, and the eval telemetry
|
||||
// store (clients/eval) owns hanzo.eval_traces / hanzo.eval_scores.
|
||||
const (
|
||||
aimUsageTable = "hanzo.cloud_usage"
|
||||
aimLangfuseObs = "langfuse.observations"
|
||||
aimEvalTraces = "hanzo.eval_traces"
|
||||
aimEvalScores = "hanzo.eval_scores"
|
||||
aimTopN = 12
|
||||
)
|
||||
|
||||
// aiMetrics is the whole AI-metrics board payload.
|
||||
type aiMetrics struct {
|
||||
Range string `json:"range"`
|
||||
Start string `json:"start"`
|
||||
End string `json:"end"`
|
||||
Langfuse aimLangfuse `json:"langfuse"`
|
||||
Usage aimUsage `json:"usage"`
|
||||
Evals aimEvals `json:"evals"`
|
||||
TopModels []aimModelStat `json:"topModels"` // cloud_usage per-model (populated today)
|
||||
LangfuseModels []aimLfModelStat `json:"langfuseModels"` // langfuse per-model (honest-empty today)
|
||||
ScoreNames []aimScoreStat `json:"scoreNames"` // eval_scores per score-name
|
||||
EvalRuns []aimRunStat `json:"evalRuns"` // recent eval runs (progress)
|
||||
ScoreSeries []aimScorePoint `json:"scoreSeries"` // avg eval score over time (progress trend)
|
||||
}
|
||||
|
||||
// aimLangfuse is the fleet-wide Langfuse generation rollup (honest-empty today).
|
||||
// Cost is USD (Langfuse's native unit); latency is milliseconds (end_time-start_time).
|
||||
type aimLangfuse struct {
|
||||
Generations int64 `json:"generations"`
|
||||
CostUsd float64 `json:"costUsd"`
|
||||
LatencyMsAvg float64 `json:"latencyMsAvg"`
|
||||
LatencyMsP95 float64 `json:"latencyMsP95"`
|
||||
}
|
||||
|
||||
// aimUsage is the fleet LLM-usage KPI band from the live cloud_usage ledger.
|
||||
type aimUsage struct {
|
||||
Requests int64 `json:"requests"`
|
||||
Tokens int64 `json:"tokens"`
|
||||
PromptTokens int64 `json:"promptTokens"`
|
||||
CompletionTokens int64 `json:"completionTokens"`
|
||||
CostCents int64 `json:"costCents"`
|
||||
Models int64 `json:"models"`
|
||||
}
|
||||
|
||||
// aimEvals is the fleet eval KPI band: the trace half (eval_traces) and the score
|
||||
// half (eval_scores). LatencyMsAvg is the mean model-under-test call window.
|
||||
type aimEvals struct {
|
||||
Runs int64 `json:"runs"`
|
||||
Traces int64 `json:"traces"`
|
||||
Datasets int64 `json:"datasets"`
|
||||
Models int64 `json:"models"`
|
||||
LatencyMsAvg float64 `json:"latencyMsAvg"`
|
||||
Scores int64 `json:"scores"`
|
||||
ScoreNames int64 `json:"scoreNames"`
|
||||
AvgScore float64 `json:"avgScore"`
|
||||
}
|
||||
|
||||
// aimModelStat is one row of the per-model usage leaderboard (cloud_usage).
|
||||
type aimModelStat struct {
|
||||
Model string `json:"model"`
|
||||
Requests int64 `json:"requests"`
|
||||
Tokens int64 `json:"tokens"`
|
||||
CostCents int64 `json:"costCents"`
|
||||
}
|
||||
|
||||
// aimLfModelStat is one row of the per-model Langfuse leaderboard (honest-empty today).
|
||||
type aimLfModelStat struct {
|
||||
Model string `json:"model"`
|
||||
Generations int64 `json:"generations"`
|
||||
CostUsd float64 `json:"costUsd"`
|
||||
}
|
||||
|
||||
// aimScoreStat is one row of the per-score-name eval leaderboard (eval_scores).
|
||||
type aimScoreStat struct {
|
||||
Name string `json:"name"`
|
||||
Count int64 `json:"count"`
|
||||
AvgValue float64 `json:"avgValue"`
|
||||
MinValue float64 `json:"minValue"`
|
||||
MaxValue float64 `json:"maxValue"`
|
||||
}
|
||||
|
||||
// aimRunStat is one recent eval run: its dataset, how many scores it recorded, its
|
||||
// mean score, and when it last ran — the run-level eval-progress row.
|
||||
type aimRunStat struct {
|
||||
RunName string `json:"runName"`
|
||||
Dataset string `json:"dataset"`
|
||||
Scores int64 `json:"scores"`
|
||||
AvgValue float64 `json:"avgValue"`
|
||||
LastTs string `json:"lastTs"`
|
||||
}
|
||||
|
||||
// aimScorePoint is one bucket of the avg-eval-score-over-time trend.
|
||||
type aimScorePoint struct {
|
||||
Ts string `json:"ts"`
|
||||
AvgValue float64 `json:"avgValue"`
|
||||
Count int64 `json:"count"`
|
||||
}
|
||||
|
||||
// aimetrics answers GET /v1/admin/aimetrics. ?range=24h|7d|30d bounds the window
|
||||
// (default 30d). SUPERADMIN ONLY (core.Guard). Every signal degrades independently:
|
||||
// a table that is absent or errors contributes its zero-value, never a failure — the
|
||||
// board always renders what the datastore actually holds.
|
||||
func aimetrics(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
ctx := c.Context()
|
||||
rangeLabel := o11yRange(c.Query("range"))
|
||||
since := computeSince(rangeLabel)
|
||||
payload := aiMetrics{
|
||||
Range: rangeLabel,
|
||||
Start: since.Format(time.RFC3339),
|
||||
End: time.Now().UTC().Format(time.RFC3339),
|
||||
TopModels: []aimModelStat{},
|
||||
LangfuseModels: []aimLfModelStat{},
|
||||
ScoreNames: []aimScoreStat{},
|
||||
EvalRuns: []aimRunStat{},
|
||||
ScoreSeries: []aimScorePoint{},
|
||||
}
|
||||
|
||||
// Honest-empty when the warehouse is not connected: the board renders its zero
|
||||
// state, never a fabricated fleet.
|
||||
if !aiobject.DatastoreEnabled() {
|
||||
return core.OK(c, payload)
|
||||
}
|
||||
|
||||
sinceTS := chTS(since) // DateTime literal — cloud_usage.timestamp, langfuse.start_time, eval_*.ts
|
||||
interval := o11yBucket(rangeLabel)
|
||||
|
||||
// ── Langfuse generations (fleet) — honest-empty until ingest lands rows ──
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, aimLangfuseTotalsSQL(), sinceTS); err == nil {
|
||||
r := firstRowOr(rows)
|
||||
payload.Langfuse.Generations = chInt64(r["gens"])
|
||||
payload.Langfuse.CostUsd = chFloat64(r["cost"])
|
||||
}
|
||||
// Langfuse latency (separate query so a Nullable end_time / column mismatch never
|
||||
// zeroes the proven generations+cost number above).
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, aimLangfuseLatencySQL(), sinceTS); err == nil {
|
||||
r := firstRowOr(rows)
|
||||
payload.Langfuse.LatencyMsAvg = chFloat64(r["lat_avg"])
|
||||
payload.Langfuse.LatencyMsP95 = chFloat64(r["lat_p95"])
|
||||
}
|
||||
// Langfuse per-model.
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, aimLangfuseModelsSQL(), sinceTS); err == nil {
|
||||
payload.LangfuseModels = lfModelsFromRows(rows)
|
||||
}
|
||||
|
||||
// ── Per-model usage (fleet) from the live cloud_usage ledger ──
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, aimUsageTotalsSQL(), sinceTS); err == nil {
|
||||
fillAimUsage(&payload.Usage, firstRowOr(rows))
|
||||
}
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, aimTopModelsSQL(), sinceTS); err == nil {
|
||||
payload.TopModels = aimModelsFromRows(rows)
|
||||
}
|
||||
|
||||
// ── Evals (fleet): traces + scores + progress ──
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, aimEvalTracesSQL(), sinceTS); err == nil {
|
||||
fillAimEvalTraces(&payload.Evals, firstRowOr(rows))
|
||||
}
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, aimEvalScoresSQL(), sinceTS); err == nil {
|
||||
fillAimEvalScores(&payload.Evals, firstRowOr(rows))
|
||||
}
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, aimScoreNamesSQL(), sinceTS); err == nil {
|
||||
payload.ScoreNames = scoreNamesFromRows(rows)
|
||||
}
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, aimEvalRunsSQL(), sinceTS); err == nil {
|
||||
payload.EvalRuns = evalRunsFromRows(rows)
|
||||
}
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, aimScoreSeriesSQL(interval), sinceTS); err == nil {
|
||||
payload.ScoreSeries = scoreSeriesFromRows(rows)
|
||||
}
|
||||
|
||||
return core.OK(c, payload)
|
||||
}
|
||||
|
||||
// ── pure SQL builders (static SQL + one positional time bound; unit-tested) ──
|
||||
|
||||
func aimLangfuseTotalsSQL() string {
|
||||
return "SELECT count() AS gens, toFloat64(sum(total_cost)) AS cost FROM " + aimLangfuseObs +
|
||||
" WHERE type = 'GENERATION' AND start_time >= ?"
|
||||
}
|
||||
|
||||
func aimLangfuseLatencySQL() string {
|
||||
lat := "(toUnixTimestamp64Milli(end_time) - toUnixTimestamp64Milli(start_time))"
|
||||
return "SELECT round(avg(" + lat + "), 2) AS lat_avg, round(quantile(0.95)(" + lat + "), 2) AS lat_p95 " +
|
||||
"FROM " + aimLangfuseObs + " WHERE type = 'GENERATION' AND start_time >= ? AND end_time > start_time"
|
||||
}
|
||||
|
||||
func aimLangfuseModelsSQL() string {
|
||||
return "SELECT provided_model_name AS model, count() AS gens, toFloat64(sum(total_cost)) AS cost " +
|
||||
"FROM " + aimLangfuseObs + " WHERE type = 'GENERATION' AND start_time >= ? AND provided_model_name != '' " +
|
||||
"GROUP BY model ORDER BY gens DESC LIMIT " + strconv.Itoa(aimTopN)
|
||||
}
|
||||
|
||||
func aimUsageTotalsSQL() string {
|
||||
return "SELECT count() AS requests, sum(total_tokens) AS tokens, " +
|
||||
"sum(prompt_tokens) AS prompt_tokens, sum(completion_tokens) AS completion_tokens, " +
|
||||
"sum(cost_cents) AS cost_cents, uniqExact(model) AS models " +
|
||||
"FROM " + aimUsageTable + " WHERE timestamp >= ?"
|
||||
}
|
||||
|
||||
func aimTopModelsSQL() string {
|
||||
return "SELECT model, count() AS requests, sum(total_tokens) AS tokens, " +
|
||||
"sum(cost_cents) AS cost_cents FROM " + aimUsageTable +
|
||||
" WHERE timestamp >= ? AND model != '' GROUP BY model ORDER BY requests DESC LIMIT " + strconv.Itoa(aimTopN)
|
||||
}
|
||||
|
||||
func aimEvalTracesSQL() string {
|
||||
lat := "(toUnixTimestamp64Milli(end_time) - toUnixTimestamp64Milli(start_time))"
|
||||
return "SELECT count() AS traces, uniqExact(run_name) AS runs, uniqExact(dataset) AS datasets, " +
|
||||
"uniqExact(model) AS models, round(avgIf(" + lat + ", end_time > start_time), 2) AS lat_avg " +
|
||||
"FROM " + aimEvalTraces + " WHERE ts >= ?"
|
||||
}
|
||||
|
||||
func aimEvalScoresSQL() string {
|
||||
return "SELECT count() AS scores, round(avg(value), 4) AS avg_value, uniqExact(name) AS score_names " +
|
||||
"FROM " + aimEvalScores + " WHERE ts >= ?"
|
||||
}
|
||||
|
||||
func aimScoreNamesSQL() string {
|
||||
return "SELECT name, count() AS n, round(avg(value), 4) AS avg_value, " +
|
||||
"round(min(value), 4) AS min_value, round(max(value), 4) AS max_value " +
|
||||
"FROM " + aimEvalScores + " WHERE ts >= ? AND name != '' GROUP BY name ORDER BY n DESC LIMIT " + strconv.Itoa(aimTopN)
|
||||
}
|
||||
|
||||
func aimEvalRunsSQL() string {
|
||||
return "SELECT run_name, any(dataset) AS dataset, count() AS scores, round(avg(value), 4) AS avg_value, " +
|
||||
"max(ts) AS last_ts FROM " + aimEvalScores + " WHERE ts >= ? AND run_name != '' " +
|
||||
"GROUP BY run_name ORDER BY last_ts DESC LIMIT " + strconv.Itoa(aimTopN)
|
||||
}
|
||||
|
||||
func aimScoreSeriesSQL(interval string) string {
|
||||
return "SELECT toStartOfInterval(ts, INTERVAL " + interval + ") AS ts, " +
|
||||
"round(avg(value), 4) AS avg_value, count() AS n FROM " + aimEvalScores +
|
||||
" WHERE ts >= ? GROUP BY ts ORDER BY ts"
|
||||
}
|
||||
|
||||
// ── pure row parsers (unit-tested) ──
|
||||
|
||||
func fillAimUsage(u *aimUsage, r map[string]any) {
|
||||
u.Requests = chInt64(r["requests"])
|
||||
u.Tokens = chInt64(r["tokens"])
|
||||
u.PromptTokens = chInt64(r["prompt_tokens"])
|
||||
u.CompletionTokens = chInt64(r["completion_tokens"])
|
||||
u.CostCents = chInt64(r["cost_cents"])
|
||||
u.Models = chInt64(r["models"])
|
||||
}
|
||||
|
||||
func fillAimEvalTraces(e *aimEvals, r map[string]any) {
|
||||
e.Traces = chInt64(r["traces"])
|
||||
e.Runs = chInt64(r["runs"])
|
||||
e.Datasets = chInt64(r["datasets"])
|
||||
e.Models = chInt64(r["models"])
|
||||
e.LatencyMsAvg = chFloat64(r["lat_avg"])
|
||||
}
|
||||
|
||||
func fillAimEvalScores(e *aimEvals, r map[string]any) {
|
||||
e.Scores = chInt64(r["scores"])
|
||||
e.AvgScore = chFloat64(r["avg_value"])
|
||||
e.ScoreNames = chInt64(r["score_names"])
|
||||
}
|
||||
|
||||
func aimModelsFromRows(rows []map[string]any) []aimModelStat {
|
||||
out := make([]aimModelStat, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
out = append(out, aimModelStat{
|
||||
Model: chStr(r["model"]),
|
||||
Requests: chInt64(r["requests"]),
|
||||
Tokens: chInt64(r["tokens"]),
|
||||
CostCents: chInt64(r["cost_cents"]),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func lfModelsFromRows(rows []map[string]any) []aimLfModelStat {
|
||||
out := make([]aimLfModelStat, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
out = append(out, aimLfModelStat{
|
||||
Model: chStr(r["model"]),
|
||||
Generations: chInt64(r["gens"]),
|
||||
CostUsd: chFloat64(r["cost"]),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func scoreNamesFromRows(rows []map[string]any) []aimScoreStat {
|
||||
out := make([]aimScoreStat, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
out = append(out, aimScoreStat{
|
||||
Name: chStr(r["name"]),
|
||||
Count: chInt64(r["n"]),
|
||||
AvgValue: chFloat64(r["avg_value"]),
|
||||
MinValue: chFloat64(r["min_value"]),
|
||||
MaxValue: chFloat64(r["max_value"]),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func evalRunsFromRows(rows []map[string]any) []aimRunStat {
|
||||
out := make([]aimRunStat, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
out = append(out, aimRunStat{
|
||||
RunName: chStr(r["run_name"]),
|
||||
Dataset: chStr(r["dataset"]),
|
||||
Scores: chInt64(r["scores"]),
|
||||
AvgValue: chFloat64(r["avg_value"]),
|
||||
LastTs: chTime(r["last_ts"]),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func scoreSeriesFromRows(rows []map[string]any) []aimScorePoint {
|
||||
out := make([]aimScorePoint, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
out = append(out, aimScorePoint{
|
||||
Ts: chTime(r["ts"]),
|
||||
AvgValue: chFloat64(r["avg_value"]),
|
||||
Count: chInt64(r["n"]),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package admin
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestAimSQL_ReadsCanonicalTables proves every AI-metrics query reads the ONE
|
||||
// datastore's canonical table, binds the time bound as a POSITIONAL param (one
|
||||
// `?`), and never interpolates user input. The bucket interval is the only rendered
|
||||
// value in the series query and it is a server-side constant.
|
||||
func TestAimSQL_ReadsCanonicalTables(t *testing.T) {
|
||||
cases := []struct {
|
||||
name, sql, table string
|
||||
wantQMarks int
|
||||
}{
|
||||
{"langfuseTotals", aimLangfuseTotalsSQL(), "langfuse.observations", 1},
|
||||
{"langfuseLatency", aimLangfuseLatencySQL(), "langfuse.observations", 1},
|
||||
{"langfuseModels", aimLangfuseModelsSQL(), "langfuse.observations", 1},
|
||||
{"usageTotals", aimUsageTotalsSQL(), "hanzo.cloud_usage", 1},
|
||||
{"topModels", aimTopModelsSQL(), "hanzo.cloud_usage", 1},
|
||||
{"evalTraces", aimEvalTracesSQL(), "hanzo.eval_traces", 1},
|
||||
{"evalScores", aimEvalScoresSQL(), "hanzo.eval_scores", 1},
|
||||
{"scoreNames", aimScoreNamesSQL(), "hanzo.eval_scores", 1},
|
||||
{"evalRuns", aimEvalRunsSQL(), "hanzo.eval_scores", 1},
|
||||
{"scoreSeries", aimScoreSeriesSQL("1 DAY"), "hanzo.eval_scores", 1},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if !strings.Contains(c.sql, "FROM "+c.table) {
|
||||
t.Errorf("%s must read %s; got %q", c.name, c.table, c.sql)
|
||||
}
|
||||
if n := strings.Count(c.sql, "?"); n != c.wantQMarks {
|
||||
t.Errorf("%s: %d bind params, want %d (time bound only) — no interpolation; got %q", c.name, n, c.wantQMarks, c.sql)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAimLangfuseScopedToGeneration proves the Langfuse lens is scoped to
|
||||
// generations only (not spans/events), matching the o11y LLM lens.
|
||||
func TestAimLangfuseScopedToGeneration(t *testing.T) {
|
||||
for _, sql := range []string{aimLangfuseTotalsSQL(), aimLangfuseLatencySQL(), aimLangfuseModelsSQL()} {
|
||||
if !strings.Contains(sql, "type = 'GENERATION'") {
|
||||
t.Errorf("langfuse lens must scope to GENERATION observations; got %q", sql)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAimTop_LimitAndOrder proves the leaderboards bound + order the result.
|
||||
func TestAimTop_LimitAndOrder(t *testing.T) {
|
||||
if !strings.Contains(aimTopModelsSQL(), "ORDER BY requests DESC LIMIT 12") {
|
||||
t.Errorf("topModels must order by requests desc, limit %d", aimTopN)
|
||||
}
|
||||
if !strings.Contains(aimScoreNamesSQL(), "GROUP BY name ORDER BY n DESC LIMIT 12") {
|
||||
t.Errorf("scoreNames must group+order+limit %d", aimTopN)
|
||||
}
|
||||
if !strings.Contains(aimEvalRunsSQL(), "ORDER BY last_ts DESC LIMIT 12") {
|
||||
t.Errorf("evalRuns must order by last_ts desc, limit %d", aimTopN)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAimScoreSeries_IntervalBound proves the (constant) bucket interval is
|
||||
// rendered into the score-trend series query and grouped/ordered by the bucket.
|
||||
func TestAimScoreSeries_IntervalBound(t *testing.T) {
|
||||
for _, iv := range []string{"1 HOUR", "6 HOUR", "1 DAY"} {
|
||||
s := aimScoreSeriesSQL(iv)
|
||||
if !strings.Contains(s, "INTERVAL "+iv) || !strings.Contains(s, "GROUP BY ts ORDER BY ts") {
|
||||
t.Errorf("score series must bucket by INTERVAL %s; got %q", iv, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAimEvalLatencyGuarded proves the latency expressions guard end_time>start_time
|
||||
// so a zero/default end_time never contributes a garbage (negative) latency.
|
||||
func TestAimEvalLatencyGuarded(t *testing.T) {
|
||||
if !strings.Contains(aimEvalTracesSQL(), "end_time > start_time") {
|
||||
t.Errorf("eval traces latency must guard end_time>start_time; got %q", aimEvalTracesSQL())
|
||||
}
|
||||
if !strings.Contains(aimLangfuseLatencySQL(), "end_time > start_time") {
|
||||
t.Errorf("langfuse latency must guard end_time>start_time; got %q", aimLangfuseLatencySQL())
|
||||
}
|
||||
}
|
||||
|
||||
// TestFillAimUsage reads a cloud_usage row into the KPI band across the numeric
|
||||
// variants the driver returns (uint64/int64/float64), honest zeros on an empty row.
|
||||
func TestFillAimUsage(t *testing.T) {
|
||||
var empty aimUsage
|
||||
fillAimUsage(&empty, map[string]any{})
|
||||
if empty.Requests != 0 || empty.Tokens != 0 || empty.Models != 0 {
|
||||
t.Fatalf("empty row must yield honest zeros; got %+v", empty)
|
||||
}
|
||||
var got aimUsage
|
||||
fillAimUsage(&got, map[string]any{
|
||||
"requests": uint64(274), "tokens": uint64(102597), "prompt_tokens": uint64(60000),
|
||||
"completion_tokens": uint64(42597), "cost_cents": uint64(216), "models": uint64(42),
|
||||
})
|
||||
if got.Requests != 274 || got.Tokens != 102597 || got.CostCents != 216 || got.Models != 42 {
|
||||
t.Fatalf("usage totals mis-parsed: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFillAimEvals maps both eval halves (traces + scores) into the KPI band,
|
||||
// including the float latency/score columns (round()/avg() land as float64; a
|
||||
// Decimal-as-string is parsed).
|
||||
func TestFillAimEvals(t *testing.T) {
|
||||
var e aimEvals
|
||||
fillAimEvalTraces(&e, map[string]any{
|
||||
"traces": uint64(1280), "runs": uint64(16), "datasets": uint64(4),
|
||||
"models": uint64(6), "lat_avg": float64(842.5),
|
||||
})
|
||||
fillAimEvalScores(&e, map[string]any{
|
||||
"scores": uint64(1280), "avg_value": "0.8125", "score_names": uint64(3),
|
||||
})
|
||||
if e.Traces != 1280 || e.Runs != 16 || e.Datasets != 4 || e.Models != 6 || e.LatencyMsAvg != 842.5 {
|
||||
t.Fatalf("eval traces mis-parsed: %+v", e)
|
||||
}
|
||||
if e.Scores != 1280 || e.ScoreNames != 3 || e.AvgScore != 0.8125 { // string→float64 path
|
||||
t.Fatalf("eval scores mis-parsed: %+v", e)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAimParsers map datastore rows into the view-models and preserve order (the
|
||||
// SQL already ORDER BYs; a parser must not reorder or drop rows), with empty input
|
||||
// yielding an empty (non-nil) slice rather than a panic.
|
||||
func TestAimParsers(t *testing.T) {
|
||||
models := aimModelsFromRows([]map[string]any{
|
||||
{"model": "glm-5.2", "requests": uint64(154), "tokens": uint64(38966), "cost_cents": uint64(114)},
|
||||
{"model": "deepseek-v4-flash", "requests": uint64(118), "tokens": uint64(61550), "cost_cents": uint64(101)},
|
||||
})
|
||||
if len(models) != 2 || models[0].Model != "glm-5.2" || models[1].Model != "deepseek-v4-flash" || models[0].Requests != 154 {
|
||||
t.Fatalf("top models mis-parsed/reordered: %+v", models)
|
||||
}
|
||||
lf := lfModelsFromRows([]map[string]any{
|
||||
{"model": "gpt-4o", "gens": uint64(42), "cost": float64(1.25)},
|
||||
})
|
||||
if len(lf) != 1 || lf[0].Model != "gpt-4o" || lf[0].Generations != 42 || lf[0].CostUsd != 1.25 {
|
||||
t.Fatalf("langfuse models mis-parsed: %+v", lf)
|
||||
}
|
||||
names := scoreNamesFromRows([]map[string]any{
|
||||
{"name": "accuracy", "n": uint64(320), "avg_value": float64(0.82), "min_value": float64(0), "max_value": float64(1)},
|
||||
})
|
||||
if len(names) != 1 || names[0].Name != "accuracy" || names[0].Count != 320 || names[0].AvgValue != 0.82 || names[0].MaxValue != 1 {
|
||||
t.Fatalf("score names mis-parsed: %+v", names)
|
||||
}
|
||||
runs := evalRunsFromRows([]map[string]any{
|
||||
{"run_name": "nightly-2026-07", "dataset": "gsm8k", "scores": uint64(200), "avg_value": float64(0.9), "last_ts": nil},
|
||||
})
|
||||
if len(runs) != 1 || runs[0].RunName != "nightly-2026-07" || runs[0].Dataset != "gsm8k" || runs[0].Scores != 200 || runs[0].AvgValue != 0.9 {
|
||||
t.Fatalf("eval runs mis-parsed: %+v", runs)
|
||||
}
|
||||
// Empty input → empty (non-nil) slices, never a panic.
|
||||
if got := scoreSeriesFromRows(nil); got == nil || len(got) != 0 {
|
||||
t.Errorf("nil rows must yield empty slice, got %v", got)
|
||||
}
|
||||
if got := aimModelsFromRows(nil); got == nil || len(got) != 0 {
|
||||
t.Errorf("nil rows must yield empty slice, got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -26,8 +26,9 @@ import (
|
||||
|
||||
// Routes registers the /v1/admin/audit* surface (SuperAdmin only).
|
||||
func Routes(app *zip.App, s *cloud.Service[core.State]) {
|
||||
app.Get("/v1/admin/audit", core.Guard(s, Records))
|
||||
app.Get("/v1/admin/audit/verify", core.Guard(s, Verify))
|
||||
g := app.Group("/v1/admin")
|
||||
g.Get("/audit", core.Guard(s, Records))
|
||||
g.Get("/audit/verify", core.Guard(s, Verify))
|
||||
}
|
||||
|
||||
// Records answers GET /v1/admin/audit from cloud's local tamper-evident store when
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package admin
|
||||
|
||||
// blockStorage — GET /v1/admin/block-storage, the realtime DO block-storage fleet the
|
||||
// operator's Block Storage board (admin.hanzo.ai) watches to scale DO before it runs out.
|
||||
// (Named `block-storage`, not `storage`, so the operator's separate S3 object-buckets
|
||||
// view keeps /v1/admin/storage — two distinct storage concerns, two endpoints.)
|
||||
// Two REAL sources, honest by construction:
|
||||
// - The FLEET inventory (count · total capacity · monthly cost · per-volume region +
|
||||
// attachment) from the DigitalOcean API (the same DO_API_TOKEN client the finance
|
||||
// dashboard already uses). DO exposes capacity + attachment but NOT fill %, so a
|
||||
// volume's used/pct stay ABSENT (the console renders an honest "—", never a
|
||||
// fabricated number).
|
||||
// - The analytics DATASTORE's own fill from ClickHouse `system.disks` (the 200Gi PVC
|
||||
// the datastore fork mounts) — total/free space over the SAME shared client
|
||||
// (aiobject.DatastoreQuery) the analytics + compute lenses read, no second
|
||||
// connection. This is THE number the operator scales on.
|
||||
//
|
||||
// SUPERADMIN ONLY (the s.guard wrap in admin.go): a cross-tenant infra read, all-orgs.
|
||||
// admin holds NO storage state — it only reads DO + the datastore. DO unconfigured →
|
||||
// empty fleet; datastore not connected → no datastore card. Never a fabricated fleet.
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
aiobject "github.com/hanzoai/ai/object"
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/clients/admin/core"
|
||||
"github.com/hanzoai/cloud/clients/admin/digitalocean"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// doBlockUsdPerGiB is DO block storage's list price ($0.10/GiB/mo) — the fleet cost
|
||||
// line for the ops budget when DO doesn't itemize per-volume spend.
|
||||
const doBlockUsdPerGiB = 0.10
|
||||
|
||||
// bytesPerGiB converts system.disks bytes (UInt64) → GiB.
|
||||
const bytesPerGiB = 1024 * 1024 * 1024
|
||||
|
||||
// storageVolume mirrors the console StorageVolume. UsedGiB/Pct are POINTERS so an
|
||||
// absent fill serializes as JSON null (→ the console's honest "—"), distinct from a
|
||||
// real 0%.
|
||||
type storageVolume struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Region string `json:"region"`
|
||||
SizeGiB int `json:"sizeGiB"`
|
||||
UsedGiB *float64 `json:"usedGiB"`
|
||||
Pct *float64 `json:"pct"`
|
||||
Attached bool `json:"attached"`
|
||||
Service string `json:"service"`
|
||||
}
|
||||
|
||||
// storageFleet is the roll-up: real count/capacity/cost; fleet fill absent (DO gives
|
||||
// no per-volume fill, so there is no honest fleet-wide used total to report).
|
||||
type storageFleet struct {
|
||||
Count int `json:"count"`
|
||||
TotalGiB int `json:"totalGiB"`
|
||||
UsedGiB *float64 `json:"usedGiB"`
|
||||
Pct *float64 `json:"pct"`
|
||||
MonthlyUsd int `json:"monthlyUsd"`
|
||||
}
|
||||
|
||||
// datastoreVolume is the analytics backend's own volume, fill REAL from system.disks.
|
||||
type datastoreVolume struct {
|
||||
Name string `json:"name"`
|
||||
Mount string `json:"mount"`
|
||||
SizeGiB int `json:"sizeGiB"`
|
||||
UsedGiB float64 `json:"usedGiB"`
|
||||
Pct float64 `json:"pct"`
|
||||
}
|
||||
|
||||
// storageAlert flags a near-full volume (only the datastore carries a real fill today,
|
||||
// so alerts are datastore-derived until a per-volume filesystem source is wired).
|
||||
type storageAlert struct {
|
||||
Volume string `json:"volume"`
|
||||
Pct float64 `json:"pct"`
|
||||
Level string `json:"level"`
|
||||
}
|
||||
|
||||
// storageSnapshot is the whole board payload the console normalizes.
|
||||
type storageSnapshot struct {
|
||||
Fleet storageFleet `json:"fleet"`
|
||||
Datastore *datastoreVolume `json:"datastore"`
|
||||
Volumes []storageVolume `json:"volumes"`
|
||||
Alerts []storageAlert `json:"alerts"`
|
||||
}
|
||||
|
||||
// blockStorage answers GET /v1/admin/block-storage. SuperAdmin only. Each source
|
||||
// degrades independently — a DO outage still returns the real datastore fill, and v.v.
|
||||
func blockStorage(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
ctx := c.Context()
|
||||
vols, _ := s.State.DO.Volumes(ctx) // honest empty on not-configured / unreachable
|
||||
fill := datastoreFill(ctx) // nil unless system.disks answered
|
||||
return core.OK(c, buildStorageSnapshot(vols, fill))
|
||||
}
|
||||
|
||||
// buildStorageSnapshot assembles the board payload (PURE — unit-tested). It folds the
|
||||
// DO inventory into the fleet totals + per-volume rows (fill absent), attaches the real
|
||||
// datastore card, and derives a near-full alert from the datastore fill. No fabrication:
|
||||
// a volume's fill is left nil (DO gives none), and the datastore card is present only
|
||||
// when system.disks actually answered.
|
||||
func buildStorageSnapshot(vols []digitalocean.Volume, fill *datastoreVolume) storageSnapshot {
|
||||
out := make([]storageVolume, 0, len(vols))
|
||||
totalGiB := 0
|
||||
for _, v := range vols {
|
||||
out = append(out, storageVolume{
|
||||
ID: v.ID,
|
||||
Name: v.Name,
|
||||
Region: v.Region,
|
||||
SizeGiB: v.SizeGiB,
|
||||
Attached: len(v.DropletIDs) > 0,
|
||||
})
|
||||
totalGiB += v.SizeGiB
|
||||
}
|
||||
alerts := make([]storageAlert, 0, 1)
|
||||
if fill != nil {
|
||||
if lvl := alertLevel(fill.Pct); lvl != "" {
|
||||
alerts = append(alerts, storageAlert{Volume: fill.Name, Pct: fill.Pct, Level: lvl})
|
||||
}
|
||||
}
|
||||
return storageSnapshot{
|
||||
Fleet: storageFleet{
|
||||
Count: len(vols),
|
||||
TotalGiB: totalGiB,
|
||||
MonthlyUsd: int(float64(totalGiB)*doBlockUsdPerGiB + 0.5),
|
||||
},
|
||||
Datastore: fill,
|
||||
Volumes: out,
|
||||
Alerts: alerts,
|
||||
}
|
||||
}
|
||||
|
||||
// alertLevel thresholds a fill %: critical ≥ 90, warn ≥ 80, else none (PURE, tested).
|
||||
func alertLevel(pct float64) string {
|
||||
switch {
|
||||
case pct >= 90:
|
||||
return "critical"
|
||||
case pct >= 80:
|
||||
return "warn"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// datastoreFill reads the analytics datastore's own volume usage from ClickHouse
|
||||
// `system.disks` (the largest disk by capacity is the data volume — the 200Gi PVC).
|
||||
// Returns nil when the datastore isn't connected or the query fails (honest — the
|
||||
// console shows no datastore card, never a fabricated fill).
|
||||
func datastoreFill(ctx context.Context) *datastoreVolume {
|
||||
if !aiobject.DatastoreEnabled() {
|
||||
return nil
|
||||
}
|
||||
rows, err := aiobject.DatastoreQuery(ctx,
|
||||
"SELECT name, path, total_space, free_space FROM system.disks ORDER BY total_space DESC LIMIT 1")
|
||||
if err != nil || len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
return datastoreFillFromRow(rows[0])
|
||||
}
|
||||
|
||||
// datastoreFillFromRow maps a system.disks row → the datastore card (PURE, tested).
|
||||
// nil when the disk reports no capacity (an unusable read, not a fabricated 0%).
|
||||
func datastoreFillFromRow(r map[string]any) *datastoreVolume {
|
||||
total := float64(chInt64(r["total_space"]))
|
||||
if total <= 0 {
|
||||
return nil
|
||||
}
|
||||
free := float64(chInt64(r["free_space"]))
|
||||
used := total - free
|
||||
if used < 0 {
|
||||
used = 0
|
||||
}
|
||||
return &datastoreVolume{
|
||||
Name: chStr(r["name"]),
|
||||
Mount: chStr(r["path"]),
|
||||
SizeGiB: int(total / bytesPerGiB),
|
||||
UsedGiB: round1(used / bytesPerGiB),
|
||||
Pct: round1(used / total * 100),
|
||||
}
|
||||
}
|
||||
|
||||
// round1 rounds to one decimal (used/pct read cleanly; not a scientific quantity).
|
||||
func round1(x float64) float64 { return float64(int(x*10+0.5)) / 10 }
|
||||
@@ -0,0 +1,130 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package admin
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/admin/digitalocean"
|
||||
)
|
||||
|
||||
// The DO inventory folds into fleet totals + rows, and per-volume fill stays ABSENT
|
||||
// (DO exposes no fill) so the console renders an honest "—", never a fabricated 0%.
|
||||
func TestBuildStorageSnapshotFleet(t *testing.T) {
|
||||
vols := []digitalocean.Volume{
|
||||
{ID: "a", Name: "pvc-a", Region: "sfo3", SizeGiB: 200, DropletIDs: []int{1}},
|
||||
{ID: "b", Name: "pvc-b", Region: "nyc1", SizeGiB: 100, DropletIDs: nil},
|
||||
{ID: "c", Name: "pvc-c", Region: "sfo3", SizeGiB: 50, DropletIDs: []int{2, 3}},
|
||||
}
|
||||
snap := buildStorageSnapshot(vols, nil)
|
||||
|
||||
if snap.Fleet.Count != 3 {
|
||||
t.Fatalf("count = %d, want 3", snap.Fleet.Count)
|
||||
}
|
||||
if snap.Fleet.TotalGiB != 350 {
|
||||
t.Fatalf("totalGiB = %d, want 350", snap.Fleet.TotalGiB)
|
||||
}
|
||||
// 350 GiB * $0.10 = $35.
|
||||
if snap.Fleet.MonthlyUsd != 35 {
|
||||
t.Fatalf("monthlyUsd = %d, want 35", snap.Fleet.MonthlyUsd)
|
||||
}
|
||||
if snap.Fleet.UsedGiB != nil || snap.Fleet.Pct != nil {
|
||||
t.Fatalf("fleet fill must be absent (DO gives none); got used=%v pct=%v", snap.Fleet.UsedGiB, snap.Fleet.Pct)
|
||||
}
|
||||
if len(snap.Volumes) != 3 {
|
||||
t.Fatalf("volumes = %d, want 3", len(snap.Volumes))
|
||||
}
|
||||
// Attachment derives from droplet_ids; fill is absent per row.
|
||||
if !snap.Volumes[0].Attached || snap.Volumes[1].Attached || !snap.Volumes[2].Attached {
|
||||
t.Fatalf("attachment wrong: %+v", snap.Volumes)
|
||||
}
|
||||
for _, v := range snap.Volumes {
|
||||
if v.UsedGiB != nil || v.Pct != nil {
|
||||
t.Fatalf("volume %s fill must be absent, got used=%v pct=%v", v.ID, v.UsedGiB, v.Pct)
|
||||
}
|
||||
}
|
||||
if snap.Datastore != nil {
|
||||
t.Fatalf("datastore must be nil when no fill was read; got %+v", snap.Datastore)
|
||||
}
|
||||
if len(snap.Alerts) != 0 {
|
||||
t.Fatalf("no alerts without a datastore fill; got %v", snap.Alerts)
|
||||
}
|
||||
}
|
||||
|
||||
// A near-full datastore raises exactly one alert; a healthy one raises none.
|
||||
func TestBuildStorageSnapshotDatastoreAlert(t *testing.T) {
|
||||
full := &datastoreVolume{Name: "default", Mount: "/var/lib/hanzo-datastore", SizeGiB: 196, UsedGiB: 178, Pct: 91}
|
||||
snap := buildStorageSnapshot(nil, full)
|
||||
if snap.Datastore == nil || snap.Datastore.Pct != 91 {
|
||||
t.Fatalf("datastore card missing/wrong: %+v", snap.Datastore)
|
||||
}
|
||||
if len(snap.Alerts) != 1 || snap.Alerts[0].Level != "critical" || snap.Alerts[0].Volume != "default" {
|
||||
t.Fatalf("expected one critical alert, got %+v", snap.Alerts)
|
||||
}
|
||||
|
||||
healthy := &datastoreVolume{Name: "default", SizeGiB: 196, UsedGiB: 13, Pct: 7}
|
||||
if a := buildStorageSnapshot(nil, healthy).Alerts; len(a) != 0 {
|
||||
t.Fatalf("a 7%%-full datastore must raise no alert, got %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertLevel(t *testing.T) {
|
||||
cases := []struct {
|
||||
pct float64
|
||||
want string
|
||||
}{
|
||||
{0, ""}, {7, ""}, {79.9, ""}, {80, "warn"}, {89.9, "warn"}, {90, "critical"}, {99, "critical"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := alertLevel(c.pct); got != c.want {
|
||||
t.Fatalf("alertLevel(%v) = %q, want %q", c.pct, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// system.disks bytes → GiB + pct, matching the live datastore (13.5G used of ~196G ≈ 7%).
|
||||
func TestDatastoreFillFromRow(t *testing.T) {
|
||||
// total ≈ 196.6 GiB, free ≈ 183.1 GiB → used ≈ 13.5 GiB → ~6.9%.
|
||||
gib := float64(bytesPerGiB) // a variable → runtime math (a float→int const conversion won't compile)
|
||||
total := int64(196.6 * gib)
|
||||
used := int64(13.5 * gib)
|
||||
row := map[string]any{
|
||||
"name": "default",
|
||||
"path": "/var/lib/hanzo-datastore",
|
||||
"total_space": uint64(total),
|
||||
"free_space": uint64(total - used),
|
||||
}
|
||||
d := datastoreFillFromRow(row)
|
||||
if d == nil {
|
||||
t.Fatal("expected a datastore fill, got nil")
|
||||
}
|
||||
if d.Name != "default" || d.Mount != "/var/lib/hanzo-datastore" {
|
||||
t.Fatalf("name/mount wrong: %+v", d)
|
||||
}
|
||||
if d.SizeGiB != 196 {
|
||||
t.Fatalf("sizeGiB = %d, want 196", d.SizeGiB)
|
||||
}
|
||||
if d.UsedGiB < 13.4 || d.UsedGiB > 13.6 {
|
||||
t.Fatalf("usedGiB = %v, want ~13.5", d.UsedGiB)
|
||||
}
|
||||
if d.Pct < 6.5 || d.Pct > 7.5 {
|
||||
t.Fatalf("pct = %v, want ~7", d.Pct)
|
||||
}
|
||||
|
||||
// A disk that reports no capacity is an unusable read → nil (never a fake 0%).
|
||||
if datastoreFillFromRow(map[string]any{"total_space": uint64(0)}) != nil {
|
||||
t.Fatal("zero-capacity disk must yield nil, not a fabricated 0%")
|
||||
}
|
||||
}
|
||||
@@ -203,7 +203,7 @@ func newCockpitFakes(t *testing.T) *cockpitFakes {
|
||||
}
|
||||
}))
|
||||
|
||||
_, s, fa := mountSvc(t, f.iam.URL, f.commerce.URL, "")
|
||||
_, s, fa := mountService(t, f.iam.URL, f.commerce.URL, "")
|
||||
f.service = s
|
||||
f.do = func(method, path string, hdr map[string]string, body string) (*http.Response, []byte) {
|
||||
t.Helper()
|
||||
|
||||
@@ -33,6 +33,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/admin/money"
|
||||
"github.com/hanzoai/cloud/clients/commerceinproc"
|
||||
)
|
||||
|
||||
// errUnconfigured marks a write (Deposit) attempted against an unwired commerce.
|
||||
@@ -45,12 +46,17 @@ type Client struct {
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
// New builds a commerce client for base + admin S2S token.
|
||||
// New builds a commerce client for base + admin S2S token. The HTTP client uses the
|
||||
// commerceinproc self-routing transport: when commerce is CO-RESIDENT (base is the
|
||||
// commerce.inproc placeholder) it dispatches in-process — a plain http.Client would
|
||||
// instead DNS-resolve "commerce.inproc" and fail "no such host", silently breaking the
|
||||
// admin cost/finance god-view. For a split-deploy (a real commerce URL) it falls
|
||||
// through to plain HTTP unchanged.
|
||||
func New(base, token string) *Client {
|
||||
return &Client{
|
||||
base: strings.TrimRight(strings.TrimSpace(base), "/"),
|
||||
token: strings.TrimSpace(token),
|
||||
http: &http.Client{Timeout: 15 * time.Second},
|
||||
http: commerceinproc.Client(15 * time.Second),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -295,235 +301,16 @@ func (c *Client) Deposit(ctx context.Context, subject string, amount money.Cents
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ── SaaS-metrics god-view (fleet-wide, org-independent) ──────────────────────
|
||||
|
||||
// SaaSMetrics mirrors commerce's GET /v1/metrics/saas snapshot — the whole-business
|
||||
// SaaS-operations aggregate (MRR/ARR, new/churn, plan mix, top customers, recent
|
||||
// movements) computed IN commerce across every org namespace. It is org-INDEPENDENT
|
||||
// (like Costs) so the reader sends NO subject. Only the fields the admin god-view
|
||||
// renders are modeled; commerce fields we don't consume (upgrades/downgrades,
|
||||
// untagged-request counts) are simply ignored by the decoder.
|
||||
type SaaSMetrics struct {
|
||||
AsOf string `json:"asOf"`
|
||||
Currency string `json:"currency"`
|
||||
Window string `json:"window"`
|
||||
Revenue SaaSRevenue `json:"revenue"`
|
||||
Subs SaaSSubs `json:"subscriptions"`
|
||||
Usage SaaSUsage `json:"usage"`
|
||||
Customers []SaaSCustomer `json:"customers"`
|
||||
Orgs int `json:"orgs"`
|
||||
Gaps []string `json:"gaps"`
|
||||
}
|
||||
|
||||
// SaaSRevenue is the recurring-revenue headline (run-rate MRR/ARR + windowed movement).
|
||||
type SaaSRevenue struct {
|
||||
MRRCents money.Cents `json:"mrrCents"`
|
||||
ARRCents money.Cents `json:"arrCents"`
|
||||
ActiveSubscriptions int `json:"activeSubscriptions"`
|
||||
PayingCustomers int `json:"payingCustomers"`
|
||||
Trials int `json:"trials"`
|
||||
NewMRRCents money.Cents `json:"newMrrCents"`
|
||||
ChurnedMRRCents money.Cents `json:"churnedMrrCents"`
|
||||
NetNewMRRCents money.Cents `json:"netNewMrrCents"`
|
||||
ByCategory []SaaSCategory `json:"byCategory"`
|
||||
}
|
||||
|
||||
// SaaSCategory is one plan-category bucket of run-rate MRR (the plan mix).
|
||||
type SaaSCategory struct {
|
||||
Category string `json:"category"`
|
||||
MRRCents money.Cents `json:"mrrCents"`
|
||||
Subscriptions int `json:"subscriptions"`
|
||||
}
|
||||
|
||||
// SaaSSubs is the subscription-operations panel (per-plan mix, trials, new/canceled,
|
||||
// recent movements).
|
||||
type SaaSSubs struct {
|
||||
ByPlan []SaaSPlan `json:"byPlan"`
|
||||
TrialsActive int `json:"trialsActive"`
|
||||
New int `json:"new"`
|
||||
Canceled int `json:"canceled"`
|
||||
Recent []SaaSEvent `json:"recent"`
|
||||
}
|
||||
|
||||
// SaaSPlan is one plan's active/trialing counts, seats, and MRR contribution.
|
||||
type SaaSPlan struct {
|
||||
Plan string `json:"plan"`
|
||||
Name string `json:"name"`
|
||||
Category string `json:"category"`
|
||||
Active int `json:"active"`
|
||||
Trialing int `json:"trialing"`
|
||||
Seats int `json:"seats"`
|
||||
MRRCents money.Cents `json:"mrrCents"`
|
||||
}
|
||||
|
||||
// SaaSEvent is one recent subscription movement ("created" or "canceled").
|
||||
type SaaSEvent struct {
|
||||
At string `json:"at"`
|
||||
Org string `json:"org"`
|
||||
Type string `json:"type"`
|
||||
Plan string `json:"plan"`
|
||||
Category string `json:"category"`
|
||||
MRRDeltaCents money.Cents `json:"mrrDeltaCents"`
|
||||
}
|
||||
|
||||
// SaaSUsage is the metered / pay-as-you-go revenue headline for the window.
|
||||
type SaaSUsage struct {
|
||||
Instrumented bool `json:"instrumented"`
|
||||
WindowUsageCents money.Cents `json:"windowUsageCents"`
|
||||
Requests int64 `json:"requests"`
|
||||
}
|
||||
|
||||
// SaaSCustomer is one top customer by MRR + windowed usage.
|
||||
type SaaSCustomer struct {
|
||||
Org string `json:"org"`
|
||||
Plan string `json:"plan"`
|
||||
Category string `json:"category"`
|
||||
Status string `json:"status"`
|
||||
MRRCents money.Cents `json:"mrrCents"`
|
||||
UsageCents money.Cents `json:"usageCents"`
|
||||
Seats int `json:"seats"`
|
||||
Since string `json:"since,omitempty"`
|
||||
}
|
||||
|
||||
// Metrics reads the fleet SaaS-operations god-view (GET /v1/metrics/saas). Like Costs it
|
||||
// is org-INDEPENDENT — the engine walks every org namespace itself — so it authenticates
|
||||
// with the admin S2S service token and sends NO subject. Empty (not an error) when
|
||||
// commerce is unwired, so a partial deploy degrades to an honest empty snapshot.
|
||||
func (c *Client) Metrics(ctx context.Context, window string, limit int) (SaaSMetrics, error) {
|
||||
var out SaaSMetrics
|
||||
// CreateCreditGrant forwards a credit-grant request verbatim to commerce's
|
||||
// mint-gated POST /v1/billing/credit-grants (CreateCreditGrant), authenticated
|
||||
// by the admin service token, with subject as the target-org namespace selector.
|
||||
// Commerce is the sole credit-grant ledger; this relays its contract untouched
|
||||
// (the raw response is returned to the caller) so the admin surface stays thin.
|
||||
func (c *Client) CreateCreditGrant(ctx context.Context, subject string, body []byte, idempotencyKey string) ([]byte, error) {
|
||||
if !c.Ready() {
|
||||
return out, nil
|
||||
return nil, errUnconfigured
|
||||
}
|
||||
q := url.Values{}
|
||||
if window != "" {
|
||||
q.Set("window", window)
|
||||
}
|
||||
if limit > 0 {
|
||||
q.Set("limit", fmt.Sprintf("%d", limit))
|
||||
}
|
||||
body, err := c.get(ctx, "/v1/metrics/saas", q, "")
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if err := json.Unmarshal(body, &out); err != nil {
|
||||
return out, fmt.Errorf("commerce metrics decode: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ── billing invoices + subscriptions (per-subject fleet rows) ────────────────
|
||||
|
||||
// Invoice is one issued invoice as the fleet god-view renders it: the id (for a future
|
||||
// /v1/billing/invoices/:id detail fetch), the human number, status, amount due,
|
||||
// currency, and the issue/due dates. Sourced from GET /v1/billing/invoices
|
||||
// (invoiceResponse); all timestamps are RFC3339 strings.
|
||||
type Invoice struct {
|
||||
ID string `json:"id"`
|
||||
Number string `json:"numberStr"`
|
||||
Status string `json:"status"`
|
||||
AmountDue money.Cents `json:"amountDue"`
|
||||
Currency string `json:"currency"`
|
||||
Issued string `json:"createdAt"`
|
||||
Due string `json:"dueDate"`
|
||||
}
|
||||
|
||||
// Invoices lists a subject's invoices (GET /v1/billing/invoices), optionally filtered by
|
||||
// status. The subject selects the org's billing namespace via X-Org-Id (trusted only
|
||||
// after the service-token bearer verifies). Empty (not an error) when commerce is unwired.
|
||||
func (c *Client) Invoices(ctx context.Context, subject, status string) ([]Invoice, error) {
|
||||
if !c.Ready() {
|
||||
return nil, nil
|
||||
}
|
||||
q := url.Values{}
|
||||
if status != "" {
|
||||
q.Set("status", status)
|
||||
}
|
||||
body, err := c.get(ctx, "/v1/billing/invoices", q, subject)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var wrap struct {
|
||||
Invoices []Invoice `json:"invoices"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &wrap); err != nil {
|
||||
return nil, fmt.Errorf("commerce invoices decode: %w", err)
|
||||
}
|
||||
return wrap.Invoices, nil
|
||||
}
|
||||
|
||||
// Subscription is one subscription row the fleet god-view renders: the id, the buyer
|
||||
// (userId), plan tier, status, monthly-normalized MRR, and the current-period
|
||||
// start/end (started/renews). MRR reuses monthlyNormalized so a yearly plan is
|
||||
// comparable to a monthly one in the fleet total.
|
||||
type Subscription struct {
|
||||
ID string `json:"id"`
|
||||
User string `json:"user"`
|
||||
Plan string `json:"plan"`
|
||||
Status string `json:"status"`
|
||||
MRR money.Cents `json:"mrrCents"`
|
||||
Started string `json:"started"`
|
||||
Renews string `json:"renews"`
|
||||
}
|
||||
|
||||
// subscriptionRowWire is the /v1/billing/subscriptions row shape the fleet view folds —
|
||||
// richer than subscriptionsWire (which Plan() uses for the MRR sum alone).
|
||||
type subscriptionRowWire struct {
|
||||
ID string `json:"id"`
|
||||
UserID string `json:"userId"`
|
||||
PlanID string `json:"planId"`
|
||||
Status string `json:"status"`
|
||||
Created string `json:"createdAt"`
|
||||
PeriodStart string `json:"currentPeriodStart"`
|
||||
PeriodEnd string `json:"currentPeriodEnd"`
|
||||
Plan struct {
|
||||
Name string `json:"name"`
|
||||
Price money.Cents `json:"price"`
|
||||
Interval string `json:"interval"`
|
||||
} `json:"plan"`
|
||||
}
|
||||
|
||||
// Subscriptions lists a subject's subscriptions (GET /v1/billing/subscriptions),
|
||||
// optionally filtered by status, as fleet rows with a monthly-normalized MRR. Empty (not
|
||||
// an error) when commerce is unwired.
|
||||
func (c *Client) Subscriptions(ctx context.Context, subject, status string) ([]Subscription, error) {
|
||||
if !c.Ready() {
|
||||
return nil, nil
|
||||
}
|
||||
q := url.Values{}
|
||||
if status != "" {
|
||||
q.Set("status", status)
|
||||
}
|
||||
body, err := c.get(ctx, "/v1/billing/subscriptions", q, subject)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var wrap struct {
|
||||
Subscriptions []subscriptionRowWire `json:"subscriptions"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &wrap); err != nil {
|
||||
return nil, fmt.Errorf("commerce subscriptions decode: %w", err)
|
||||
}
|
||||
out := make([]Subscription, 0, len(wrap.Subscriptions))
|
||||
for _, s := range wrap.Subscriptions {
|
||||
name := strings.TrimSpace(s.Plan.Name)
|
||||
if name == "" {
|
||||
name = strings.TrimSpace(s.PlanID)
|
||||
}
|
||||
started := strings.TrimSpace(s.Created)
|
||||
if started == "" {
|
||||
started = s.PeriodStart
|
||||
}
|
||||
out = append(out, Subscription{
|
||||
ID: s.ID,
|
||||
User: s.UserID,
|
||||
Plan: name,
|
||||
Status: s.Status,
|
||||
MRR: monthlyNormalized(s.Plan.Price, s.Plan.Interval),
|
||||
Started: started,
|
||||
Renews: s.PeriodEnd,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
return c.post(ctx, "/v1/billing/credit-grants", subject, body, idempotencyKey)
|
||||
}
|
||||
|
||||
// post performs one admin-authenticated commerce POST (JSON body) and returns the
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package core
|
||||
|
||||
// warehouse — the ONE-copy datastore-read kernel the billing FLEET views
|
||||
// (metrics/invoices/subscriptions) compose. They read commerce.events — the
|
||||
// single warehouse table the commerce analytics collector lands every
|
||||
// customer-activity event in (subscription/invoice/usage lifecycle) — over the
|
||||
// SAME shared client (aiobject.DatastoreQuery) the o11y/compute/analytics lenses
|
||||
// already use, no second connection. This mirrors compute.go's row-coercers and
|
||||
// EXISTS-TABLE probe, hoisted here so the three sibling domains share ONE copy
|
||||
// instead of each re-deriving it (DRY; the admin-package o11y/compute keep their
|
||||
// own private copies as the read template).
|
||||
//
|
||||
// Every read is honest by construction: no datastore connected, or the events
|
||||
// table not provisioned (the emitter is still being wired) → the real empty
|
||||
// aggregate, NEVER a fabricated fleet. admin READS only; it owns and creates NO
|
||||
// table (the collector owns commerce.events). Time bounds are POSITIONAL
|
||||
// parameters (never interpolated) so the reads are injection-safe; money is USD
|
||||
// cents; timestamps are RFC3339.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
aiobject "github.com/hanzoai/ai/object"
|
||||
)
|
||||
|
||||
// BillingEventsTable is the collector-owned warehouse table the commerce
|
||||
// customer-activity emitters land in (events/client.go → analytics-collector →
|
||||
// commerce.events). admin only READS it (never creates it — the collector owns
|
||||
// its writes), exactly as o11y reads hanzo.cloud_usage.
|
||||
const BillingEventsTable = "commerce.events"
|
||||
|
||||
// Canonical customer-activity event names — the CONTRACT with the commerce
|
||||
// emitters (events/client.go). These are server-side constants (never user
|
||||
// input), so rendering them into an IN (...) list is injection-safe.
|
||||
const (
|
||||
EvSubscriptionCreated = "subscription_created"
|
||||
EvSubscriptionRenewed = "subscription_renewed"
|
||||
EvSubscriptionPlanChanged = "subscription_plan_changed"
|
||||
EvSubscriptionCanceled = "subscription_canceled"
|
||||
EvInvoiceFinalized = "invoice_finalized"
|
||||
EvInvoicePaid = "invoice_paid"
|
||||
EvInvoiceVoid = "invoice_void"
|
||||
EvAPIUsageDebit = "api_usage_debit"
|
||||
)
|
||||
|
||||
// SubscriptionEvents / InvoiceEvents are the lifecycle sets each fleet view
|
||||
// folds over (latest-event-wins per entity). Closed server-side constants.
|
||||
var (
|
||||
SubscriptionEvents = []string{EvSubscriptionCreated, EvSubscriptionRenewed, EvSubscriptionPlanChanged, EvSubscriptionCanceled}
|
||||
InvoiceEvents = []string{EvInvoiceFinalized, EvInvoicePaid, EvInvoiceVoid}
|
||||
)
|
||||
|
||||
// WarehouseReady reports whether the shared datastore ledger is connected, the
|
||||
// gate every fleet read checks first (honest-empty when false).
|
||||
func WarehouseReady() bool { return aiobject.DatastoreEnabled() }
|
||||
|
||||
// BillingEventsReady reports whether the warehouse is connected AND the
|
||||
// collector's commerce.events table is provisioned — the two-part gate every
|
||||
// billing fleet view opens with, so an unwired collector degrades to an honest
|
||||
// empty aggregate rather than an error.
|
||||
func BillingEventsReady(ctx context.Context) bool {
|
||||
return aiobject.DatastoreEnabled() && CHTableExists(ctx, BillingEventsTable)
|
||||
}
|
||||
|
||||
// CHTableExists probes the datastore for a table's presence. The name is a
|
||||
// package constant (never user input), so EXISTS TABLE is safe. Any error →
|
||||
// false (honest "not available yet"), mirroring compute.computeTableExists.
|
||||
func CHTableExists(ctx context.Context, qualified string) bool {
|
||||
rows, err := aiobject.DatastoreQuery(ctx, "EXISTS TABLE "+qualified)
|
||||
if err != nil || len(rows) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, v := range rows[0] {
|
||||
return CHInt64(v) == 1
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SQLInList renders a set of server-side-constant strings as a datastore string
|
||||
// list ('a','b',…) for an IN (...) clause. ONLY for closed constant sets (the
|
||||
// event-name enums above) — never for user input; positional args carry all
|
||||
// caller-derived values.
|
||||
func SQLInList(vals []string) string {
|
||||
quoted := make([]string, len(vals))
|
||||
for i, v := range vals {
|
||||
quoted[i] = "'" + v + "'"
|
||||
}
|
||||
return strings.Join(quoted, ",")
|
||||
}
|
||||
|
||||
// WarehouseSince maps the ?range enum (24h|7d|30d, default 30d) to a lower time
|
||||
// bound, mirroring compute.computeSince so the fleet views share ONE window
|
||||
// grammar.
|
||||
func WarehouseSince(rangeLabel string) time.Time {
|
||||
now := time.Now().UTC()
|
||||
switch strings.TrimSpace(rangeLabel) {
|
||||
case "24h":
|
||||
return now.Add(-24 * time.Hour)
|
||||
case "7d":
|
||||
return now.Add(-7 * 24 * time.Hour)
|
||||
default:
|
||||
return now.Add(-30 * 24 * time.Hour)
|
||||
}
|
||||
}
|
||||
|
||||
// CHTimeLit formats a time as a datastore DateTime literal (UTC), bound as a
|
||||
// POSITIONAL string arg (never interpolated).
|
||||
func CHTimeLit(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05") }
|
||||
|
||||
// CHFirstRow returns the first row or an empty map (never nil), so a parser
|
||||
// reads honest zeros from an empty result instead of panicking.
|
||||
func CHFirstRow(rows []map[string]any) map[string]any {
|
||||
if len(rows) == 0 {
|
||||
return map[string]any{}
|
||||
}
|
||||
return rows[0]
|
||||
}
|
||||
|
||||
// ── map[string]any coercers (the DatastoreQuery row shape) ───────────────────
|
||||
//
|
||||
// The datastore driver decodes each column to its native Go type (uint64 for
|
||||
// count()/sum(UInt*), float64 for round()/JSON numerics, time.Time for DateTime,
|
||||
// string for String); these accept those natives so a driver/transport change
|
||||
// can't crash a read. Twins of the admin-package compute.go coercers.
|
||||
|
||||
func CHInt64(v any) int64 {
|
||||
switch n := v.(type) {
|
||||
case int:
|
||||
return int64(n)
|
||||
case int64:
|
||||
return n
|
||||
case int32:
|
||||
return int64(n)
|
||||
case uint:
|
||||
return int64(n)
|
||||
case uint64:
|
||||
return int64(n)
|
||||
case uint32:
|
||||
return int64(n)
|
||||
case uint16:
|
||||
return int64(n)
|
||||
case uint8:
|
||||
return int64(n)
|
||||
case float64:
|
||||
return int64(n)
|
||||
case float32:
|
||||
return int64(n)
|
||||
case string:
|
||||
f, err := strconv.ParseFloat(strings.TrimSpace(n), 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return int64(f)
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func CHFloat64(v any) float64 {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return n
|
||||
case float32:
|
||||
return float64(n)
|
||||
case int:
|
||||
return float64(n)
|
||||
case int64:
|
||||
return float64(n)
|
||||
case int32:
|
||||
return float64(n)
|
||||
case uint64:
|
||||
return float64(n)
|
||||
case uint32:
|
||||
return float64(n)
|
||||
case string:
|
||||
f, err := strconv.ParseFloat(strings.TrimSpace(n), 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return f
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func CHStr(v any) string {
|
||||
if s, ok := v.(string); ok {
|
||||
return s
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// CHTime coerces a datastore DateTime (time.Time) to an RFC3339 UTC string.
|
||||
func CHTime(v any) string {
|
||||
switch t := v.(type) {
|
||||
case time.Time:
|
||||
return t.UTC().Format(time.RFC3339)
|
||||
case string:
|
||||
return t
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/audit"
|
||||
"github.com/hanzoai/cloud/clients/admin/core"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// createCreditGrant is the admin mint surface: POST /v1/admin/credit-grants.
|
||||
//
|
||||
// SuperAdmin ONLY (wired through core.Guard). It does NOT mint in-process — it
|
||||
// forwards the request VERBATIM to commerce's already-mint-gated
|
||||
// POST /v1/billing/credit-grants (middleware.Mint → PlatformOnly), authenticated
|
||||
// by COMMERCE_SERVICE_TOKEN and scoped to the target org, and writes ONE
|
||||
// tamper-evident compliance record. Commerce stays the single credit-grant ledger;
|
||||
// this is a thin, audited relay so there is exactly one place credit is minted.
|
||||
//
|
||||
// The body is commerce's own CreateCreditGrant contract; the only field this layer
|
||||
// reads is the target org (`org`, or `user` as the org-pool alias) to select the
|
||||
// per-org namespace commerce's EdgeAuth trusts after verifying the service token.
|
||||
func createCreditGrant(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
if !s.State.Commerce.Ready() {
|
||||
return core.Fail(c, "commerce is not configured on this deployment")
|
||||
}
|
||||
|
||||
var req map[string]any
|
||||
if err := c.Bind(&req); err != nil {
|
||||
return core.Fail(c, "invalid request body")
|
||||
}
|
||||
|
||||
org, _ := req["org"].(string)
|
||||
if strings.TrimSpace(org) == "" {
|
||||
org, _ = req["user"].(string)
|
||||
}
|
||||
org = strings.TrimSpace(org)
|
||||
if org == "" {
|
||||
return core.Fail(c, "org is required")
|
||||
}
|
||||
idempotencyKey, _ := req["idempotencyKey"].(string)
|
||||
|
||||
body, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return core.Fail(c, "invalid request body")
|
||||
}
|
||||
|
||||
raw, err := s.State.Commerce.CreateCreditGrant(c.Context(), org, body, idempotencyKey)
|
||||
if err != nil {
|
||||
core.EmitAudit(s, c, "admin.customer.credit-grant", "credit-grant", org,
|
||||
req, map[string]any{"error": err.Error()},
|
||||
audit.Outcome{Result: "error", Status: 502, Reason: "credit-grant failed"})
|
||||
return core.Fail(c, "credit-grant failed: "+err.Error())
|
||||
}
|
||||
|
||||
core.EmitAudit(s, c, "admin.customer.credit-grant", "credit-grant", org,
|
||||
nil, json.RawMessage(raw),
|
||||
audit.Outcome{Result: "success", Status: 200})
|
||||
return core.OK(c, json.RawMessage(raw))
|
||||
}
|
||||
@@ -10,11 +10,12 @@ import (
|
||||
// precedes the :org param route; the write actions are POST (distinct method), so none
|
||||
// collide. The grants ledger + the org-in-body issue-grant share the ONE credit path.
|
||||
func Routes(app *zip.App, s *cloud.Service[core.State]) {
|
||||
app.Get("/v1/admin/customers", core.Guard(s, Customers))
|
||||
app.Get("/v1/admin/customers/:org", core.Guard(s, CustomerDetail))
|
||||
app.Post("/v1/admin/customers/:org/credit", core.Guard(s, GrantCredit))
|
||||
app.Get("/v1/admin/grants", core.Guard(s, Grants))
|
||||
app.Post("/v1/admin/grants", core.Guard(s, IssueGrant))
|
||||
app.Post("/v1/admin/customers/:org/suspend", core.Guard(s, SuspendCustomer))
|
||||
app.Post("/v1/admin/customers/:org/reactivate", core.Guard(s, ReactivateCustomer))
|
||||
g := app.Group("/v1/admin")
|
||||
g.Get("/customers", core.Guard(s, Customers))
|
||||
g.Get("/customers/:org", core.Guard(s, CustomerDetail))
|
||||
g.Post("/customers/:org/credit", core.Guard(s, GrantCredit))
|
||||
g.Get("/grants", core.Guard(s, Grants))
|
||||
g.Post("/grants", core.Guard(s, IssueGrant))
|
||||
g.Post("/customers/:org/suspend", core.Guard(s, SuspendCustomer))
|
||||
g.Post("/customers/:org/reactivate", core.Guard(s, ReactivateCustomer))
|
||||
}
|
||||
|
||||
@@ -142,6 +142,69 @@ func (c *Client) History(ctx context.Context, perPage int) ([]Entry, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Volume is one DO block-storage volume: capacity + attachment + region. DO's API
|
||||
// gives capacity and which droplets a volume is attached to, but NOT fill % — the
|
||||
// caller enriches fill only where a filesystem source (the datastore's own
|
||||
// system.disks) reports it, and renders an honest "—" everywhere else.
|
||||
type Volume struct {
|
||||
ID string
|
||||
Name string
|
||||
Region string
|
||||
SizeGiB int
|
||||
DropletIDs []int
|
||||
}
|
||||
|
||||
// volumeWire is the raw DO /v2/volumes row.
|
||||
type volumeWire struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
SizeGigabytes int `json:"size_gigabytes"`
|
||||
Region struct {
|
||||
Slug string `json:"slug"`
|
||||
} `json:"region"`
|
||||
DropletIDs []int `json:"droplet_ids"`
|
||||
}
|
||||
|
||||
// Volumes lists ALL block-storage volumes across the account, following DO's
|
||||
// page-number pagination (200/page, a hard 25-page cap so a runaway can never loop).
|
||||
// The fleet inventory (count, capacity, monthly cost) is real; per-volume fill is NOT
|
||||
// exposed by DO and stays absent (honest) until a filesystem source reports it.
|
||||
func (c *Client) Volumes(ctx context.Context) ([]Volume, error) {
|
||||
if !c.Ready() {
|
||||
return nil, fmt.Errorf("DO_API_TOKEN not configured")
|
||||
}
|
||||
var out []Volume
|
||||
for page := 1; page <= 25; page++ {
|
||||
body, err := c.get(ctx, "/v2/volumes?per_page=200&page="+strconv.Itoa(page))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var w struct {
|
||||
Volumes []volumeWire `json:"volumes"`
|
||||
Meta struct {
|
||||
Total int `json:"total"`
|
||||
} `json:"meta"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &w); err != nil {
|
||||
return nil, fmt.Errorf("do volumes decode: %w", err)
|
||||
}
|
||||
for _, v := range w.Volumes {
|
||||
out = append(out, Volume{
|
||||
ID: v.ID,
|
||||
Name: v.Name,
|
||||
Region: v.Region.Slug,
|
||||
SizeGiB: v.SizeGigabytes,
|
||||
DropletIDs: v.DropletIDs,
|
||||
})
|
||||
}
|
||||
// Stop on the last (short) page, or once we've collected the reported total.
|
||||
if len(w.Volumes) < 200 || (w.Meta.Total > 0 && len(out) >= w.Meta.Total) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// get performs one token-authenticated DO GET and returns the raw body.
|
||||
func (c *Client) get(ctx context.Context, path string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil)
|
||||
|
||||
@@ -28,16 +28,17 @@ var errUnconfigured = errors.New("not configured")
|
||||
|
||||
// Routes registers the finance dashboard (SuperAdmin only).
|
||||
func Routes(app *zip.App, s *cloud.Service[core.State]) {
|
||||
app.Get("/v1/admin/finance", core.Guard(s, Finance))
|
||||
g := app.Group("/v1/admin")
|
||||
g.Get("/finance", core.Guard(s, Finance))
|
||||
// One-time commerce→finance balance cutover (SuperAdmin only). Idempotent per org.
|
||||
app.Post("/v1/admin/finance/backfill", core.Guard(s, Backfill))
|
||||
g.Post("/finance/backfill", core.Guard(s, Backfill))
|
||||
// Fund an ARBITRARY subject's native wallet — an org pool or a human ("hanzo/z").
|
||||
// SuperAdmin only; additive (grants stack).
|
||||
app.Post("/v1/admin/finance/deposit", core.Guard(s, Deposit))
|
||||
g.Post("/finance/deposit", core.Guard(s, Deposit))
|
||||
// Per-provider upstream credit ledger + usage funding split (multi-provider
|
||||
// credit-management). Same SuperAdmin guard, same cloud_usage warehouse.
|
||||
app.Get("/v1/admin/providers/credit", core.Guard(s, ProvidersCredit))
|
||||
app.Get("/v1/admin/usage/funding", core.Guard(s, UsageFunding))
|
||||
g.Get("/providers/credit", core.Guard(s, ProvidersCredit))
|
||||
g.Get("/usage/funding", core.Guard(s, UsageFunding))
|
||||
}
|
||||
|
||||
// FinanceData is the full /v1/admin/finance aggregate.
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
// The finance PURE-math derivation tests (ComputeFinance / AvgDailyBurnCents) live with
|
||||
// the handler in clients/admin/finance. These are the INTEGRATION tests that drive GET
|
||||
// /v1/admin/finance through the shared admin mount harness (mountSvc + fake IAM/commerce/DO).
|
||||
// /v1/admin/finance through the shared admin mount harness (mountService + fake IAM/commerce/DO).
|
||||
|
||||
// newFakeDO serves the DO billing API with fixed decimal-dollar strings so the
|
||||
// finance aggregation is deterministic. account_balance is NEGATIVE (credit held).
|
||||
@@ -49,7 +49,7 @@ func TestFinance_RealAggregation(t *testing.T) {
|
||||
do := newFakeDO()
|
||||
defer do.Close()
|
||||
|
||||
doReq, s, _ := mountSvc(t, iam.server.URL, commerce.URL, "")
|
||||
doReq, s, _ := mountService(t, iam.server.URL, commerce.URL, "")
|
||||
s.State.DO = digitalocean.NewWithBase(do.URL, "test-do-token") // configured DO client
|
||||
admin := map[string]string{
|
||||
"X-User-IsAdmin": "true", "X-Org-Id": "admin",
|
||||
@@ -151,7 +151,7 @@ func TestFinance_HonestUnconfiguredDO(t *testing.T) {
|
||||
commerce := newFakeCommerceFinance()
|
||||
defer commerce.Close()
|
||||
|
||||
doReq, _, _ := mountSvc(t, iam.server.URL, commerce.URL, "") // s.do already has empty token → unconfigured
|
||||
doReq, _, _ := mountService(t, iam.server.URL, commerce.URL, "") // s.do already has empty token → unconfigured
|
||||
admin := map[string]string{"X-User-IsAdmin": "true", "X-Org-Id": "admin"}
|
||||
|
||||
resp, body := doReq("GET", "/v1/admin/finance", admin)
|
||||
@@ -212,7 +212,7 @@ func TestFinance_RevenueSourceDown_NoFabrication(t *testing.T) {
|
||||
defer commerce.Close()
|
||||
|
||||
// IAM points nowhere reachable → listOrgs errors; commerce /v1/costs still 200s.
|
||||
doReq, _, _ := mountSvc(t, "http://127.0.0.1:0", commerce.URL, "")
|
||||
doReq, _, _ := mountService(t, "http://127.0.0.1:0", commerce.URL, "")
|
||||
admin := map[string]string{"X-User-IsAdmin": "true", "X-Org-Id": "admin"}
|
||||
|
||||
resp, body := doReq("GET", "/v1/admin/finance", admin)
|
||||
|
||||
@@ -3,29 +3,28 @@
|
||||
// id a future detail view fetches /v1/billing/invoices/:id with. SuperAdmin only
|
||||
// (core.Guard).
|
||||
//
|
||||
// Commerce billing is per-tenant (an invoice lives in its org's own datastore
|
||||
// namespace), so — like revenue — this fans out the org directory concurrently and
|
||||
// reads each org's invoices via the admin S2S seam, tagging every row with its owning
|
||||
// org. Best-effort per org: an org whose invoice read fails contributes NO rows rather
|
||||
// than failing the fleet view (the SAME honest-degradation contract the customer list
|
||||
// uses; an unreachable commerce yields an empty list, never fabricated rows). Optional
|
||||
// ?org= scopes to one tenant, ?status= filters, ?limit= caps the merged list.
|
||||
// It reads the ONE shared warehouse (commerce.events) — the table the commerce
|
||||
// analytics collector lands every invoice-lifecycle event in — over the SAME client
|
||||
// (aiobject.DatastoreQuery) the o11y/compute lenses use, with ZERO per-org fan-out:
|
||||
// one GROUP BY resolves each invoice's LATEST lifecycle state (argMax by timestamp),
|
||||
// so the whole fleet is one query, not N per-org commerce reads. Honest by
|
||||
// construction: no datastore connected or the collector's table not provisioned yet →
|
||||
// the real empty list, never a fabricated row. Optional ?org= scopes to one tenant,
|
||||
// ?status= filters the LATEST status, ?limit= caps the list.
|
||||
package invoices
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
aiobject "github.com/hanzoai/ai/object"
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/clients/admin/core"
|
||||
"github.com/hanzoai/cloud/clients/admin/iam"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// defaultLimit caps the merged fleet invoice list when the caller sends none.
|
||||
// defaultLimit caps the fleet invoice list when the caller sends none.
|
||||
const defaultLimit = 500
|
||||
|
||||
// InvoiceRow is one row of GET /v1/admin/invoices — an issued invoice at a glance,
|
||||
@@ -47,84 +46,100 @@ type InvoiceRow struct {
|
||||
// GET /v1/admin/invoices?org=&status=&limit=
|
||||
func Invoices(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
ctx := c.Context()
|
||||
cr := core.CallerCreds(c)
|
||||
status := strings.TrimSpace(c.Query("status"))
|
||||
status := strings.ToLower(strings.TrimSpace(c.Query("status")))
|
||||
wantOrg := strings.TrimSpace(c.Query("org"))
|
||||
limit := parseLimit(c.Query("limit"))
|
||||
|
||||
orgs, err := core.ListOrgs(s, ctx, cr)
|
||||
// Honest-empty when the warehouse is not connected or the collector's events
|
||||
// table is not provisioned yet (the emitter is still being wired).
|
||||
if !core.BillingEventsReady(ctx) {
|
||||
return core.OKList(c, []InvoiceRow{}, 0)
|
||||
}
|
||||
|
||||
rows, err := aiobject.DatastoreQuery(ctx, invoicesSQL())
|
||||
if err != nil {
|
||||
return core.Fail(c, err.Error())
|
||||
}
|
||||
if wantOrg != "" {
|
||||
orgs = filterOrg(orgs, wantOrg)
|
||||
return core.Fail(c, "invoices query: "+err.Error())
|
||||
}
|
||||
all := invoiceRowsFromRows(rows)
|
||||
|
||||
// Per-org invoices, fanned out concurrently (best-effort per org).
|
||||
perOrg := make([][]InvoiceRow, len(orgs))
|
||||
sem := make(chan struct{}, core.MaxCustomerConcurrency)
|
||||
var wg sync.WaitGroup
|
||||
for i, o := range orgs {
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func(i int, o iam.Org) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
perOrg[i] = invoicesOf(s, ctx, o, status)
|
||||
}(i, o)
|
||||
// Filter (latest status / org) then newest issued first, cap to limit.
|
||||
out := make([]InvoiceRow, 0, len(all))
|
||||
for _, r := range all {
|
||||
if wantOrg != "" && r.Org != wantOrg {
|
||||
continue
|
||||
}
|
||||
if status != "" && strings.ToLower(r.Status) != status {
|
||||
continue
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
rows := make([]InvoiceRow, 0)
|
||||
for _, r := range perOrg {
|
||||
rows = append(rows, r...)
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Issued > out[j].Issued })
|
||||
total := len(out)
|
||||
if len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
// Newest issued first; cap to the merged limit (total reports the full pre-cap count).
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].Issued > rows[j].Issued })
|
||||
total := len(rows)
|
||||
if len(rows) > limit {
|
||||
rows = rows[:limit]
|
||||
}
|
||||
return core.OKList(c, rows, total)
|
||||
return core.OKList(c, out, total)
|
||||
}
|
||||
|
||||
// invoicesOf reads one org's invoices into fleet rows, tagged with the org. Best-effort:
|
||||
// a failed read yields no rows so the fleet view degrades honestly, never fabricating.
|
||||
func invoicesOf(s *cloud.Service[core.State], ctx context.Context, o iam.Org, status string) []InvoiceRow {
|
||||
entries, err := s.State.Commerce.Invoices(ctx, o.Name, status)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
display := core.Display(o.DisplayName, o.Name)
|
||||
rows := make([]InvoiceRow, 0, len(entries))
|
||||
for _, inv := range entries {
|
||||
rows = append(rows, InvoiceRow{
|
||||
ID: inv.ID,
|
||||
Number: inv.Number,
|
||||
Org: o.Name,
|
||||
Display: display,
|
||||
Status: inv.Status,
|
||||
AmountCents: int64(inv.AmountDue),
|
||||
Currency: inv.Currency,
|
||||
Issued: inv.Issued,
|
||||
Due: inv.Due,
|
||||
// invoicesSQL resolves each invoice's LATEST lifecycle state from commerce.events
|
||||
// (argMax by timestamp). Static SQL over a closed event-name set (SQLInList of
|
||||
// server constants) — no user input is interpolated, so it is injection-safe.
|
||||
func invoicesSQL() string {
|
||||
return "SELECT JSONExtractString(properties, 'invoice_id') AS id, " +
|
||||
"argMax(JSONExtractString(properties, 'number'), timestamp) AS number, " +
|
||||
"argMax(organization_id, timestamp) AS org, " +
|
||||
"argMax(JSONExtractString(properties, 'status'), timestamp) AS status, " +
|
||||
"argMax(JSONExtractInt(properties, 'amount_cents'), timestamp) AS amount_cents, " +
|
||||
"argMax(JSONExtractString(properties, 'currency'), timestamp) AS currency, " +
|
||||
"argMax(JSONExtractString(properties, 'issued'), timestamp) AS issued, " +
|
||||
"argMax(JSONExtractString(properties, 'due'), timestamp) AS due, " +
|
||||
"argMax(event, timestamp) AS last_event " +
|
||||
"FROM " + core.BillingEventsTable + " " +
|
||||
"WHERE event IN (" + core.SQLInList(core.InvoiceEvents) + ") " +
|
||||
"AND JSONExtractString(properties, 'invoice_id') != '' " +
|
||||
"GROUP BY id"
|
||||
}
|
||||
|
||||
// invoiceRowsFromRows maps the datastore rows onto []InvoiceRow (pure). Display is
|
||||
// the org slug — the warehouse holds no friendly name and admin does no per-org IAM
|
||||
// fan-out here (honest, not fabricated). Status folds the lifecycle from the latest
|
||||
// event so a paid/voided invoice reads correctly regardless of the status snapshot.
|
||||
func invoiceRowsFromRows(rows []map[string]any) []InvoiceRow {
|
||||
out := make([]InvoiceRow, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
org := core.CHStr(r["org"])
|
||||
out = append(out, InvoiceRow{
|
||||
ID: core.CHStr(r["id"]),
|
||||
Number: core.CHStr(r["number"]),
|
||||
Org: org,
|
||||
Display: org,
|
||||
Status: foldInvoiceStatus(core.CHStr(r["last_event"]), core.CHStr(r["status"])),
|
||||
AmountCents: core.CHInt64(r["amount_cents"]),
|
||||
Currency: core.CHStr(r["currency"]),
|
||||
Issued: core.CHStr(r["issued"]),
|
||||
Due: core.CHStr(r["due"]),
|
||||
})
|
||||
}
|
||||
return rows
|
||||
return out
|
||||
}
|
||||
|
||||
// filterOrg narrows the directory to the one requested org (empty when it does not
|
||||
// exist — an honest empty list, never a fabricated tenant).
|
||||
func filterOrg(orgs []iam.Org, want string) []iam.Org {
|
||||
for _, o := range orgs {
|
||||
if o.Name == want {
|
||||
return []iam.Org{o}
|
||||
}
|
||||
// foldInvoiceStatus resolves the effective status from the latest lifecycle event
|
||||
// (paid / void terminal), falling back to the last-emitted status snapshot (open
|
||||
// for a finalized invoice) when the event is a finalize.
|
||||
func foldInvoiceStatus(lastEvent, snapshot string) string {
|
||||
switch lastEvent {
|
||||
case core.EvInvoicePaid:
|
||||
return "paid"
|
||||
case core.EvInvoiceVoid:
|
||||
return "void"
|
||||
}
|
||||
return nil
|
||||
if s := strings.TrimSpace(snapshot); s != "" {
|
||||
return s
|
||||
}
|
||||
return "open"
|
||||
}
|
||||
|
||||
// parseLimit clamps the merged-list cap to [1,5000], defaulting to defaultLimit.
|
||||
// parseLimit clamps the fleet-list cap to [1,5000], defaulting to defaultLimit.
|
||||
func parseLimit(s string) int {
|
||||
n, err := strconv.Atoi(strings.TrimSpace(s))
|
||||
if err != nil || n <= 0 {
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package invoices
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/admin/core"
|
||||
)
|
||||
|
||||
// TestInvoiceRowsFromRows proves the warehouse-row → InvoiceRow mapping (JSON-shape
|
||||
// contract): amount coerced from driver ints, status folded from the latest event,
|
||||
// display honestly the org slug (no fan-out).
|
||||
func TestInvoiceRowsFromRows(t *testing.T) {
|
||||
rows := []map[string]any{
|
||||
{
|
||||
"id": "inv_1", "number": "INV-0042", "org": "acme",
|
||||
"status": "open", "amount_cents": int64(4900), "currency": "usd",
|
||||
"issued": "2026-07-01T00:00:00Z", "due": "2026-07-15T00:00:00Z",
|
||||
"last_event": core.EvInvoicePaid,
|
||||
},
|
||||
{
|
||||
"id": "inv_2", "number": "INV-0043", "org": "beta",
|
||||
"status": "open", "amount_cents": uint64(1200), "currency": "usd",
|
||||
"issued": "2026-07-02T00:00:00Z", "due": "",
|
||||
"last_event": core.EvInvoiceVoid,
|
||||
},
|
||||
}
|
||||
out := invoiceRowsFromRows(rows)
|
||||
if len(out) != 2 {
|
||||
t.Fatalf("got %d rows, want 2", len(out))
|
||||
}
|
||||
if out[0].ID != "inv_1" || out[0].Number != "INV-0042" || out[0].Org != "acme" || out[0].Display != "acme" {
|
||||
t.Fatalf("row0 identity wrong: %+v", out[0])
|
||||
}
|
||||
if out[0].AmountCents != 4900 || out[0].Currency != "usd" {
|
||||
t.Fatalf("row0 amount/currency wrong: %+v", out[0])
|
||||
}
|
||||
if out[0].Status != "paid" {
|
||||
t.Fatalf("row0 status = %q, want paid (paid event folds)", out[0].Status)
|
||||
}
|
||||
if out[0].Issued != "2026-07-01T00:00:00Z" || out[0].Due != "2026-07-15T00:00:00Z" {
|
||||
t.Fatalf("row0 dates wrong: %+v", out[0])
|
||||
}
|
||||
if out[1].Status != "void" {
|
||||
t.Fatalf("row1 status = %q, want void", out[1].Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFoldInvoiceStatus(t *testing.T) {
|
||||
if got := foldInvoiceStatus(core.EvInvoicePaid, "open"); got != "paid" {
|
||||
t.Fatalf("paid fold = %q", got)
|
||||
}
|
||||
if got := foldInvoiceStatus(core.EvInvoiceVoid, "open"); got != "void" {
|
||||
t.Fatalf("void fold = %q", got)
|
||||
}
|
||||
if got := foldInvoiceStatus(core.EvInvoiceFinalized, "open"); got != "open" {
|
||||
t.Fatalf("finalized snapshot = %q", got)
|
||||
}
|
||||
if got := foldInvoiceStatus(core.EvInvoiceFinalized, ""); got != "open" {
|
||||
t.Fatalf("finalized default = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvoicesSQLInjectionSafe(t *testing.T) {
|
||||
sql := invoicesSQL()
|
||||
if !strings.Contains(sql, core.BillingEventsTable) {
|
||||
t.Fatalf("query must read %s: %q", core.BillingEventsTable, sql)
|
||||
}
|
||||
for _, ev := range core.InvoiceEvents {
|
||||
if !strings.Contains(sql, "'"+ev+"'") {
|
||||
t.Fatalf("query missing event %q", ev)
|
||||
}
|
||||
}
|
||||
if strings.Contains(sql, "?") {
|
||||
t.Fatalf("invoices state query takes no positional args: %q", sql)
|
||||
}
|
||||
}
|
||||
@@ -8,5 +8,6 @@ import (
|
||||
|
||||
// Routes registers the fleet invoice view (SuperAdmin only, cross-tenant).
|
||||
func Routes(app *zip.App, s *cloud.Service[core.State]) {
|
||||
app.Get("/v1/admin/invoices", core.Guard(s, Invoices))
|
||||
g := app.Group("/v1/admin")
|
||||
g.Get("/invoices", core.Guard(s, Invoices))
|
||||
}
|
||||
|
||||
@@ -25,17 +25,18 @@ import (
|
||||
|
||||
// limitRoutes registers the promo + cap control plane. Called from routes().
|
||||
func limitRoutes(app *zip.App, s *cloud.Service[core.State]) {
|
||||
g := app.Group("/v1/admin")
|
||||
// Platform plan promo — SuperAdmin only.
|
||||
app.Get("/v1/admin/promos", core.Guard(s, getPromo))
|
||||
app.Put("/v1/admin/promos", core.Guard(s, putPromo))
|
||||
g.Get("/promos", core.Guard(s, getPromo))
|
||||
g.Put("/promos", core.Guard(s, putPromo))
|
||||
|
||||
// Per-org usage-cap oversight/override — SuperAdmin (any org via ?org=) or an org
|
||||
// admin (own org only). Reuses the customer's OWN self-service spend-alert CRUD,
|
||||
// so a platform override and a customer edit are the same rows.
|
||||
app.Get("/v1/admin/spend-caps", core.GuardScoped(s, listSpendCaps))
|
||||
app.Post("/v1/admin/spend-caps", core.GuardScoped(s, createSpendCap))
|
||||
app.Patch("/v1/admin/spend-caps/:id", core.GuardScoped(s, updateSpendCap))
|
||||
app.Delete("/v1/admin/spend-caps/:id", core.GuardScoped(s, deleteSpendCap))
|
||||
g.Get("/spend-caps", core.GuardScoped(s, listSpendCaps))
|
||||
g.Post("/spend-caps", core.GuardScoped(s, createSpendCap))
|
||||
g.Patch("/spend-caps/:id", core.GuardScoped(s, updateSpendCap))
|
||||
g.Delete("/spend-caps/:id", core.GuardScoped(s, deleteSpendCap))
|
||||
}
|
||||
|
||||
// getPromo returns the current platform plan promo. X-Org-Id is the admin org —
|
||||
|
||||
@@ -3,92 +3,458 @@
|
||||
// mix, the top customers, and the recent subscription movements. SuperAdmin only
|
||||
// (core.Guard).
|
||||
//
|
||||
// It OWNS no aggregation. The whole snapshot is computed IN commerce (the system of
|
||||
// record for subscriptions + the usage ledger) by its cross-org SaaS-metrics engine
|
||||
// (GET /v1/metrics/saas), which admin PROXIES with the SAME admin-scoped S2S service
|
||||
// token finance uses for COGS. The engine is ALREADY fleet-wide — it walks every org
|
||||
// namespace itself — so this is a SINGLE upstream read, no per-org fan-out, exactly as
|
||||
// finance consumes commerce Costs. An unwired or unreachable commerce degrades to an
|
||||
// It reads the ONE shared warehouse (commerce.events) — the table the commerce
|
||||
// analytics collector lands every subscription/invoice/usage-lifecycle event in —
|
||||
// over the SAME client (aiobject.DatastoreQuery) the o11y/compute lenses use, with
|
||||
// ZERO per-org fan-out. Each panel is ONE aggregate query that folds the whole fleet
|
||||
// (subscription state = latest-event-wins via argMax; new/churn/usage = windowed),
|
||||
// exactly the way o11y.go composes independent per-signal reads. An unconnected
|
||||
// warehouse — or the collector's events table not provisioned yet — degrades to an
|
||||
// honest empty snapshot (real zeros, `[]` not null) with a not-ok source, never a
|
||||
// fabricated number.
|
||||
// fabricated number. Money is USD cents end to end; time bounds are POSITIONAL args.
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
aiobject "github.com/hanzoai/ai/object"
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/clients/admin/commerce"
|
||||
"github.com/hanzoai/cloud/clients/admin/core"
|
||||
"github.com/hanzoai/cloud/clients/admin/money"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// errUnconfigured marks commerce not wired on this deployment — core.SrcOf reports it as
|
||||
// a not-ok source so the console renders the honest not-configured state.
|
||||
var errUnconfigured = errors.New("commerce metrics not configured")
|
||||
// errUnconfigured marks the warehouse not connected on this deployment — core.SrcOf
|
||||
// reports it as a not-ok source so the console renders the honest not-configured state.
|
||||
var errUnconfigured = errors.New("billing warehouse not connected")
|
||||
|
||||
// defaultLimit caps the top-customers list when the caller sends none (mirrors the
|
||||
// commerce engine's own default so the proxy never asks for more than it returns).
|
||||
const defaultLimit = 20
|
||||
// defaultLimit caps the top-customers list; recentLimit caps the movement feed.
|
||||
const (
|
||||
defaultLimit = 20
|
||||
recentLimit = 20
|
||||
)
|
||||
|
||||
// MetricsData is the GET /v1/admin/metrics payload: the commerce SaaS snapshot, flat,
|
||||
// plus the admin read time and the upstream freshness strip every god-view carries.
|
||||
// ── response shapes (byte-identical to the operator contract in api.ts) ──────
|
||||
// These were formerly modeled on the commerce S2S client; they now live here (the
|
||||
// one consumer) since the read is a direct warehouse aggregate. Money is money.Cents
|
||||
// (int64 underlying → plain-integer JSON, unchanged on the wire).
|
||||
|
||||
// SaaSMetrics is the whole-business SaaS-operations aggregate.
|
||||
type SaaSMetrics struct {
|
||||
AsOf string `json:"asOf"`
|
||||
Currency string `json:"currency"`
|
||||
Window string `json:"window"`
|
||||
Revenue SaaSRevenue `json:"revenue"`
|
||||
Subs SaaSSubs `json:"subscriptions"`
|
||||
Usage SaaSUsage `json:"usage"`
|
||||
Customers []SaaSCustomer `json:"customers"`
|
||||
Orgs int `json:"orgs"`
|
||||
Gaps []string `json:"gaps"`
|
||||
}
|
||||
|
||||
// SaaSRevenue is the recurring-revenue headline (run-rate MRR/ARR + windowed movement).
|
||||
type SaaSRevenue struct {
|
||||
MRRCents money.Cents `json:"mrrCents"`
|
||||
ARRCents money.Cents `json:"arrCents"`
|
||||
ActiveSubscriptions int `json:"activeSubscriptions"`
|
||||
PayingCustomers int `json:"payingCustomers"`
|
||||
Trials int `json:"trials"`
|
||||
NewMRRCents money.Cents `json:"newMrrCents"`
|
||||
ChurnedMRRCents money.Cents `json:"churnedMrrCents"`
|
||||
NetNewMRRCents money.Cents `json:"netNewMrrCents"`
|
||||
ByCategory []SaaSCategory `json:"byCategory"`
|
||||
}
|
||||
|
||||
// SaaSCategory is one plan-category bucket of run-rate MRR (the plan mix).
|
||||
type SaaSCategory struct {
|
||||
Category string `json:"category"`
|
||||
MRRCents money.Cents `json:"mrrCents"`
|
||||
Subscriptions int `json:"subscriptions"`
|
||||
}
|
||||
|
||||
// SaaSSubs is the subscription-operations panel (per-plan mix, trials, new/canceled,
|
||||
// recent movements).
|
||||
type SaaSSubs struct {
|
||||
ByPlan []SaaSPlan `json:"byPlan"`
|
||||
TrialsActive int `json:"trialsActive"`
|
||||
New int `json:"new"`
|
||||
Canceled int `json:"canceled"`
|
||||
Recent []SaaSEvent `json:"recent"`
|
||||
}
|
||||
|
||||
// SaaSPlan is one plan's active/trialing counts, seats, and MRR contribution.
|
||||
type SaaSPlan struct {
|
||||
Plan string `json:"plan"`
|
||||
Name string `json:"name"`
|
||||
Category string `json:"category"`
|
||||
Active int `json:"active"`
|
||||
Trialing int `json:"trialing"`
|
||||
Seats int `json:"seats"`
|
||||
MRRCents money.Cents `json:"mrrCents"`
|
||||
}
|
||||
|
||||
// SaaSEvent is one recent subscription movement ("created" or "canceled").
|
||||
type SaaSEvent struct {
|
||||
At string `json:"at"`
|
||||
Org string `json:"org"`
|
||||
Type string `json:"type"`
|
||||
Plan string `json:"plan"`
|
||||
Category string `json:"category"`
|
||||
MRRDeltaCents money.Cents `json:"mrrDeltaCents"`
|
||||
}
|
||||
|
||||
// SaaSUsage is the metered / pay-as-you-go revenue headline for the window.
|
||||
type SaaSUsage struct {
|
||||
Instrumented bool `json:"instrumented"`
|
||||
WindowUsageCents money.Cents `json:"windowUsageCents"`
|
||||
Requests int64 `json:"requests"`
|
||||
}
|
||||
|
||||
// SaaSCustomer is one top customer by MRR + windowed usage.
|
||||
type SaaSCustomer struct {
|
||||
Org string `json:"org"`
|
||||
Plan string `json:"plan"`
|
||||
Category string `json:"category"`
|
||||
Status string `json:"status"`
|
||||
MRRCents money.Cents `json:"mrrCents"`
|
||||
UsageCents money.Cents `json:"usageCents"`
|
||||
Seats int `json:"seats"`
|
||||
Since string `json:"since,omitempty"`
|
||||
}
|
||||
|
||||
// MetricsData is the GET /v1/admin/metrics payload: the SaaS snapshot, flat, plus the
|
||||
// admin read time and the upstream freshness strip every god-view carries.
|
||||
type MetricsData struct {
|
||||
commerce.SaaSMetrics
|
||||
SaaSMetrics
|
||||
GeneratedAt string `json:"generatedAt"`
|
||||
Sources []core.SourceStatus `json:"sources"`
|
||||
}
|
||||
|
||||
// Metrics answers GET /v1/admin/metrics by proxying the commerce SaaS-metrics engine
|
||||
// (already a fleet-wide cross-org aggregate). SuperAdmin only.
|
||||
// Metrics answers GET /v1/admin/metrics by aggregating commerce.events directly
|
||||
// (fleet-wide, no per-org fan-out). SuperAdmin only.
|
||||
//
|
||||
// GET /v1/admin/metrics?window=30d&limit=20
|
||||
func Metrics(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
ctx := c.Context()
|
||||
now := time.Now().UTC().Format(time.RFC3339)
|
||||
window := strings.TrimSpace(c.Query("window"))
|
||||
window := normalizeWindow(c.Query("window"))
|
||||
limit := parseLimit(c.Query("limit"))
|
||||
|
||||
if !s.State.Commerce.Ready() {
|
||||
return core.OK(c, empty(now, window, core.SrcOf("commerce-metrics", errUnconfigured, 0, now)))
|
||||
// Honest not-configured snapshot when the warehouse/collector table is absent.
|
||||
if !core.BillingEventsReady(ctx) {
|
||||
return core.OK(c, empty(now, window, core.SrcOf("billing-warehouse", errUnconfigured, 0, now)))
|
||||
}
|
||||
m, err := s.State.Commerce.Metrics(ctx, window, limit)
|
||||
if err != nil {
|
||||
return core.OK(c, empty(now, window, core.SrcOf("commerce-metrics", err, 0, now)))
|
||||
|
||||
sinceTS := core.CHTimeLit(core.WarehouseSince(window))
|
||||
m := SaaSMetrics{AsOf: now, Currency: "usd", Window: window}
|
||||
|
||||
// Revenue headline + plan-mix (run-rate, latest-event-wins over active subs).
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, headlineSQL()); err == nil {
|
||||
fillHeadline(&m.Revenue, core.CHFirstRow(rows))
|
||||
}
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, byCategorySQL()); err == nil {
|
||||
m.Revenue.ByCategory = byCategoryFromRows(rows)
|
||||
}
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, byPlanSQL()); err == nil {
|
||||
m.Subs.ByPlan = byPlanFromRows(rows)
|
||||
}
|
||||
m.Subs.TrialsActive = m.Revenue.Trials
|
||||
|
||||
// Windowed movement: new vs churned MRR + counts.
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, movementSQL(), sinceTS); err == nil {
|
||||
r := core.CHFirstRow(rows)
|
||||
m.Revenue.NewMRRCents = money.Cents(core.CHInt64(r["new_mrr"]))
|
||||
m.Revenue.ChurnedMRRCents = money.Cents(core.CHInt64(r["churned_mrr"]))
|
||||
m.Revenue.NetNewMRRCents = m.Revenue.NewMRRCents - m.Revenue.ChurnedMRRCents
|
||||
m.Subs.New = int(core.CHInt64(r["new_count"]))
|
||||
m.Subs.Canceled = int(core.CHInt64(r["canceled_count"]))
|
||||
}
|
||||
// Recent movements feed.
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, recentSQL(), sinceTS); err == nil {
|
||||
m.Subs.Recent = recentFromRows(rows)
|
||||
}
|
||||
// Metered usage headline (window).
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, usageSQL(), sinceTS); err == nil {
|
||||
r := core.CHFirstRow(rows)
|
||||
m.Usage.Requests = core.CHInt64(r["requests"])
|
||||
m.Usage.WindowUsageCents = money.Cents(core.CHInt64(r["usage_cents"]))
|
||||
m.Usage.Instrumented = m.Usage.Requests > 0
|
||||
}
|
||||
// Fleet org count (any billing activity).
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, orgCountSQL()); err == nil {
|
||||
m.Orgs = int(core.CHInt64(core.CHFirstRow(rows)["orgs"]))
|
||||
}
|
||||
// Top customers by MRR + windowed usage (two reads merged, no fan-out).
|
||||
m.Customers = topCustomers(ctx, sinceTS, limit)
|
||||
|
||||
m.Gaps = gapsFor(m)
|
||||
return core.OK(c, MetricsData{
|
||||
SaaSMetrics: normalize(m),
|
||||
GeneratedAt: now,
|
||||
Sources: []core.SourceStatus{core.SrcOf("commerce-metrics", nil, m.Orgs, now)},
|
||||
Sources: []core.SourceStatus{core.SrcOf("billing-warehouse", nil, m.Orgs, now)},
|
||||
})
|
||||
}
|
||||
|
||||
// empty is the honest not-configured/unreachable snapshot: real zeros + empty slices
|
||||
// (never null, never fabricated) plus the not-ok source.
|
||||
// ── active-subscription state subquery (latest-event-wins, non-canceled) ─────
|
||||
|
||||
// activeSubs is the fleet's current subscription state: one row per subscription,
|
||||
// its LATEST lifecycle values (argMax by timestamp), keeping only non-canceled
|
||||
// subs (HAVING on the latest event). Static SQL over a closed event-name set — no
|
||||
// user input interpolated. Reused by every run-rate panel so the definition of
|
||||
// "active" lives in ONE place.
|
||||
func activeSubs() string {
|
||||
return "(SELECT " +
|
||||
"argMax(organization_id, timestamp) AS org, " +
|
||||
"argMax(JSONExtractString(properties, 'plan'), timestamp) AS plan, " +
|
||||
"argMax(JSONExtractString(properties, 'plan_name'), timestamp) AS plan_name, " +
|
||||
"argMax(JSONExtractString(properties, 'category'), timestamp) AS category, " +
|
||||
"argMax(JSONExtractString(properties, 'status'), timestamp) AS status, " +
|
||||
"argMax(JSONExtractInt(properties, 'mrr_cents'), timestamp) AS mrr_cents, " +
|
||||
"argMax(JSONExtractInt(properties, 'seats'), timestamp) AS seats, " +
|
||||
"min(timestamp) AS first_ts " +
|
||||
"FROM " + core.BillingEventsTable + " " +
|
||||
"WHERE event IN (" + core.SQLInList(core.SubscriptionEvents) + ") " +
|
||||
"AND JSONExtractString(properties, 'subscription_id') != '' " +
|
||||
"GROUP BY JSONExtractString(properties, 'subscription_id') " +
|
||||
"HAVING argMax(event, timestamp) != '" + core.EvSubscriptionCanceled + "')"
|
||||
}
|
||||
|
||||
// ── pure SQL builders (static SQL + at most one positional time bound) ────────
|
||||
|
||||
// headlineSQL: run-rate MRR (paying, non-trial), active-sub count, paying-customer
|
||||
// count, and trial count — one pass over the active-subs state.
|
||||
func headlineSQL() string {
|
||||
return "SELECT sumIf(mrr_cents, status != 'trialing') AS mrr, " +
|
||||
"count() AS active_subs, " +
|
||||
"uniqExactIf(org, status != 'trialing' AND mrr_cents > 0) AS paying, " +
|
||||
"countIf(status = 'trialing') AS trials FROM " + activeSubs()
|
||||
}
|
||||
|
||||
func byCategorySQL() string {
|
||||
return "SELECT category, sumIf(mrr_cents, status != 'trialing') AS mrr, count() AS subs " +
|
||||
"FROM " + activeSubs() + " GROUP BY category ORDER BY mrr DESC"
|
||||
}
|
||||
|
||||
func byPlanSQL() string {
|
||||
return "SELECT plan, any(plan_name) AS name, any(category) AS category, " +
|
||||
"countIf(status = 'active') AS active, countIf(status = 'trialing') AS trialing, " +
|
||||
"sum(seats) AS seats, sumIf(mrr_cents, status != 'trialing') AS mrr " +
|
||||
"FROM " + activeSubs() + " GROUP BY plan ORDER BY mrr DESC"
|
||||
}
|
||||
|
||||
// movementSQL: windowed new vs churned MRR + counts (one positional since bound).
|
||||
func movementSQL() string {
|
||||
return "SELECT " +
|
||||
"sumIf(JSONExtractInt(properties, 'mrr_cents'), event = '" + core.EvSubscriptionCreated + "') AS new_mrr, " +
|
||||
"countIf(event = '" + core.EvSubscriptionCreated + "') AS new_count, " +
|
||||
"sumIf(JSONExtractInt(properties, 'mrr_cents'), event = '" + core.EvSubscriptionCanceled + "') AS churned_mrr, " +
|
||||
"countIf(event = '" + core.EvSubscriptionCanceled + "') AS canceled_count " +
|
||||
"FROM " + core.BillingEventsTable + " " +
|
||||
"WHERE event IN ('" + core.EvSubscriptionCreated + "','" + core.EvSubscriptionCanceled + "') AND timestamp >= ?"
|
||||
}
|
||||
|
||||
func recentSQL() string {
|
||||
return "SELECT timestamp AS at, organization_id AS org, event AS type, " +
|
||||
"JSONExtractString(properties, 'plan_name') AS plan, " +
|
||||
"JSONExtractString(properties, 'category') AS category, " +
|
||||
"JSONExtractInt(properties, 'mrr_cents') AS mrr_delta " +
|
||||
"FROM " + core.BillingEventsTable + " " +
|
||||
"WHERE event IN ('" + core.EvSubscriptionCreated + "','" + core.EvSubscriptionCanceled + "') AND timestamp >= ? " +
|
||||
"ORDER BY at DESC LIMIT " + strconv.Itoa(recentLimit)
|
||||
}
|
||||
|
||||
func usageSQL() string {
|
||||
return "SELECT count() AS requests, sum(JSONExtractInt(properties, 'amount_cents')) AS usage_cents " +
|
||||
"FROM " + core.BillingEventsTable + " WHERE event = '" + core.EvAPIUsageDebit + "' AND timestamp >= ?"
|
||||
}
|
||||
|
||||
func orgCountSQL() string {
|
||||
return "SELECT uniqExact(organization_id) AS orgs FROM " + core.BillingEventsTable +
|
||||
" WHERE event IN (" + core.SQLInList(allBillingEvents()) + ")"
|
||||
}
|
||||
|
||||
func perOrgSubsSQL() string {
|
||||
return "SELECT org, sumIf(mrr_cents, status != 'trialing') AS mrr, sum(seats) AS seats, " +
|
||||
"argMax(plan_name, mrr_cents) AS plan, argMax(category, mrr_cents) AS category, " +
|
||||
"argMax(status, mrr_cents) AS status, min(first_ts) AS since " +
|
||||
"FROM " + activeSubs() + " GROUP BY org"
|
||||
}
|
||||
|
||||
func perOrgUsageSQL() string {
|
||||
return "SELECT organization_id AS org, sum(JSONExtractInt(properties, 'amount_cents')) AS usage_cents " +
|
||||
"FROM " + core.BillingEventsTable + " WHERE event = '" + core.EvAPIUsageDebit + "' AND timestamp >= ? GROUP BY org"
|
||||
}
|
||||
|
||||
// allBillingEvents is the union of every customer-activity event the fleet counts
|
||||
// an org as "active" on (subscription + invoice + usage).
|
||||
func allBillingEvents() []string {
|
||||
out := append([]string{}, core.SubscriptionEvents...)
|
||||
out = append(out, core.InvoiceEvents...)
|
||||
return append(out, core.EvAPIUsageDebit)
|
||||
}
|
||||
|
||||
// ── pure row parsers ─────────────────────────────────────────────────────────
|
||||
|
||||
func fillHeadline(r *SaaSRevenue, row map[string]any) {
|
||||
r.MRRCents = money.Cents(core.CHInt64(row["mrr"]))
|
||||
r.ARRCents = r.MRRCents * 12
|
||||
r.ActiveSubscriptions = int(core.CHInt64(row["active_subs"]))
|
||||
r.PayingCustomers = int(core.CHInt64(row["paying"]))
|
||||
r.Trials = int(core.CHInt64(row["trials"]))
|
||||
}
|
||||
|
||||
func byCategoryFromRows(rows []map[string]any) []SaaSCategory {
|
||||
out := make([]SaaSCategory, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
out = append(out, SaaSCategory{
|
||||
Category: core.CHStr(r["category"]),
|
||||
MRRCents: money.Cents(core.CHInt64(r["mrr"])),
|
||||
Subscriptions: int(core.CHInt64(r["subs"])),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func byPlanFromRows(rows []map[string]any) []SaaSPlan {
|
||||
out := make([]SaaSPlan, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
out = append(out, SaaSPlan{
|
||||
Plan: core.CHStr(r["plan"]),
|
||||
Name: core.CHStr(r["name"]),
|
||||
Category: core.CHStr(r["category"]),
|
||||
Active: int(core.CHInt64(r["active"])),
|
||||
Trialing: int(core.CHInt64(r["trialing"])),
|
||||
Seats: int(core.CHInt64(r["seats"])),
|
||||
MRRCents: money.Cents(core.CHInt64(r["mrr"])),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func recentFromRows(rows []map[string]any) []SaaSEvent {
|
||||
out := make([]SaaSEvent, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
typ := "created"
|
||||
delta := money.Cents(core.CHInt64(r["mrr_delta"]))
|
||||
if core.CHStr(r["type"]) == core.EvSubscriptionCanceled {
|
||||
typ = "canceled"
|
||||
delta = -delta // churn reduces run-rate MRR
|
||||
}
|
||||
out = append(out, SaaSEvent{
|
||||
At: core.CHTime(r["at"]),
|
||||
Org: core.CHStr(r["org"]),
|
||||
Type: typ,
|
||||
Plan: core.CHStr(r["plan"]),
|
||||
Category: core.CHStr(r["category"]),
|
||||
MRRDeltaCents: delta,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// topCustomers folds per-org subscription state + per-org windowed usage into the
|
||||
// top-N customers by MRR (then usage). Two reads merged in Go by org — a union, so
|
||||
// a pay-as-you-go org with usage but no subscription still appears.
|
||||
func topCustomers(ctx context.Context, sinceTS string, limit int) []SaaSCustomer {
|
||||
byOrg := map[string]*SaaSCustomer{}
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, perOrgSubsSQL()); err == nil {
|
||||
for _, r := range rows {
|
||||
org := core.CHStr(r["org"])
|
||||
if org == "" {
|
||||
continue
|
||||
}
|
||||
byOrg[org] = &SaaSCustomer{
|
||||
Org: org,
|
||||
Plan: core.CHStr(r["plan"]),
|
||||
Category: core.CHStr(r["category"]),
|
||||
Status: core.CHStr(r["status"]),
|
||||
MRRCents: money.Cents(core.CHInt64(r["mrr"])),
|
||||
Seats: int(core.CHInt64(r["seats"])),
|
||||
Since: core.CHTime(r["since"]),
|
||||
}
|
||||
}
|
||||
}
|
||||
if rows, err := aiobject.DatastoreQuery(ctx, perOrgUsageSQL(), sinceTS); err == nil {
|
||||
for _, r := range rows {
|
||||
org := core.CHStr(r["org"])
|
||||
if org == "" {
|
||||
continue
|
||||
}
|
||||
usage := money.Cents(core.CHInt64(r["usage_cents"]))
|
||||
if cust, ok := byOrg[org]; ok {
|
||||
cust.UsageCents = usage
|
||||
continue
|
||||
}
|
||||
byOrg[org] = &SaaSCustomer{Org: org, Plan: "pay-as-you-go", Status: "active", UsageCents: usage}
|
||||
}
|
||||
}
|
||||
out := make([]SaaSCustomer, 0, len(byOrg))
|
||||
for _, c := range byOrg {
|
||||
out = append(out, *c)
|
||||
}
|
||||
sortCustomers(out)
|
||||
if len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ── small pure helpers ───────────────────────────────────────────────────────
|
||||
|
||||
// sortCustomers ranks by MRR desc, ties broken by windowed usage desc.
|
||||
func sortCustomers(cs []SaaSCustomer) {
|
||||
sort.SliceStable(cs, func(i, j int) bool { return lessCustomer(cs[i], cs[j]) })
|
||||
}
|
||||
|
||||
func lessCustomer(a, b SaaSCustomer) bool {
|
||||
if a.MRRCents != b.MRRCents {
|
||||
return a.MRRCents > b.MRRCents
|
||||
}
|
||||
return a.UsageCents > b.UsageCents
|
||||
}
|
||||
|
||||
// gapsFor lists honest not-yet-observed signals so the console can badge a partial
|
||||
// snapshot without fabricating data.
|
||||
func gapsFor(m SaaSMetrics) []string {
|
||||
gaps := []string{}
|
||||
if !m.Usage.Instrumented {
|
||||
gaps = append(gaps, "api-usage debits not yet observed")
|
||||
}
|
||||
if m.Revenue.ActiveSubscriptions == 0 {
|
||||
gaps = append(gaps, "no active subscriptions observed")
|
||||
}
|
||||
return gaps
|
||||
}
|
||||
|
||||
// empty is the honest not-connected snapshot: real zeros + empty slices (never
|
||||
// null, never fabricated) plus the not-ok source.
|
||||
func empty(now, window string, src core.SourceStatus) MetricsData {
|
||||
return MetricsData{
|
||||
SaaSMetrics: normalize(commerce.SaaSMetrics{AsOf: now, Currency: "usd", Window: window}),
|
||||
SaaSMetrics: normalize(SaaSMetrics{AsOf: now, Currency: "usd", Window: window}),
|
||||
GeneratedAt: now,
|
||||
Sources: []core.SourceStatus{src},
|
||||
}
|
||||
}
|
||||
|
||||
// normalize replaces nil slices with empty ones so the JSON is honest arrays (`[]`, not
|
||||
// null) and the console never has to guard a missing collection.
|
||||
func normalize(m commerce.SaaSMetrics) commerce.SaaSMetrics {
|
||||
// normalize replaces nil slices with empty ones so the JSON is honest arrays (`[]`,
|
||||
// not null) and the console never has to guard a missing collection.
|
||||
func normalize(m SaaSMetrics) SaaSMetrics {
|
||||
if m.Revenue.ByCategory == nil {
|
||||
m.Revenue.ByCategory = []commerce.SaaSCategory{}
|
||||
m.Revenue.ByCategory = []SaaSCategory{}
|
||||
}
|
||||
if m.Subs.ByPlan == nil {
|
||||
m.Subs.ByPlan = []commerce.SaaSPlan{}
|
||||
m.Subs.ByPlan = []SaaSPlan{}
|
||||
}
|
||||
if m.Subs.Recent == nil {
|
||||
m.Subs.Recent = []commerce.SaaSEvent{}
|
||||
m.Subs.Recent = []SaaSEvent{}
|
||||
}
|
||||
if m.Customers == nil {
|
||||
m.Customers = []commerce.SaaSCustomer{}
|
||||
m.Customers = []SaaSCustomer{}
|
||||
}
|
||||
if m.Gaps == nil {
|
||||
m.Gaps = []string{}
|
||||
@@ -96,8 +462,20 @@ func normalize(m commerce.SaaSMetrics) commerce.SaaSMetrics {
|
||||
return m
|
||||
}
|
||||
|
||||
// parseLimit clamps the top-N cap to [1,200], defaulting to defaultLimit — mirrors the
|
||||
// commerce engine's clamp exactly.
|
||||
// normalizeWindow clamps ?window to the supported set (default 30d) — mirrors the
|
||||
// warehouse window grammar (core.WarehouseSince).
|
||||
func normalizeWindow(v string) string {
|
||||
switch strings.TrimSpace(v) {
|
||||
case "24h":
|
||||
return "24h"
|
||||
case "7d":
|
||||
return "7d"
|
||||
default:
|
||||
return "30d"
|
||||
}
|
||||
}
|
||||
|
||||
// parseLimit clamps the top-N cap to [1,200], defaulting to defaultLimit.
|
||||
func parseLimit(s string) int {
|
||||
n, err := strconv.Atoi(strings.TrimSpace(s))
|
||||
if err != nil || n <= 0 {
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/admin/core"
|
||||
)
|
||||
|
||||
// TestFillHeadline proves the run-rate headline coercion (driver ints) + the
|
||||
// ARR = 12×MRR derivation.
|
||||
func TestFillHeadline(t *testing.T) {
|
||||
var rev SaaSRevenue
|
||||
fillHeadline(&rev, map[string]any{
|
||||
"mrr": int64(4900), "active_subs": uint64(3), "paying": uint64(2), "trials": uint64(1),
|
||||
})
|
||||
if rev.MRRCents != 4900 || rev.ARRCents != 4900*12 {
|
||||
t.Fatalf("mrr/arr wrong: %+v", rev)
|
||||
}
|
||||
if rev.ActiveSubscriptions != 3 || rev.PayingCustomers != 2 || rev.Trials != 1 {
|
||||
t.Fatalf("counts wrong: %+v", rev)
|
||||
}
|
||||
}
|
||||
|
||||
func TestByCategoryAndPlanFromRows(t *testing.T) {
|
||||
cats := byCategoryFromRows([]map[string]any{
|
||||
{"category": "cloud", "mrr": int64(9800), "subs": uint64(2)},
|
||||
})
|
||||
if len(cats) != 1 || cats[0].Category != "cloud" || cats[0].MRRCents != 9800 || cats[0].Subscriptions != 2 {
|
||||
t.Fatalf("category row wrong: %+v", cats)
|
||||
}
|
||||
plans := byPlanFromRows([]map[string]any{
|
||||
{"plan": "pro", "name": "Pro", "category": "cloud", "active": uint64(2), "trialing": uint64(1), "seats": uint64(5), "mrr": int64(9800)},
|
||||
})
|
||||
if len(plans) != 1 {
|
||||
t.Fatalf("want 1 plan, got %d", len(plans))
|
||||
}
|
||||
p := plans[0]
|
||||
if p.Plan != "pro" || p.Name != "Pro" || p.Category != "cloud" || p.Active != 2 || p.Trialing != 1 || p.Seats != 5 || p.MRRCents != 9800 {
|
||||
t.Fatalf("plan row wrong: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecentFromRows proves the movement feed maps event→type and NEGATES churn MRR.
|
||||
func TestRecentFromRows(t *testing.T) {
|
||||
rows := []map[string]any{
|
||||
{"at": "2026-07-10T00:00:00Z", "org": "acme", "type": core.EvSubscriptionCreated, "plan": "Pro", "category": "cloud", "mrr_delta": int64(4900)},
|
||||
{"at": "2026-07-09T00:00:00Z", "org": "beta", "type": core.EvSubscriptionCanceled, "plan": "Team", "category": "cloud", "mrr_delta": int64(3000)},
|
||||
}
|
||||
out := recentFromRows(rows)
|
||||
if len(out) != 2 {
|
||||
t.Fatalf("want 2, got %d", len(out))
|
||||
}
|
||||
if out[0].Type != "created" || out[0].MRRDeltaCents != 4900 {
|
||||
t.Fatalf("created row wrong: %+v", out[0])
|
||||
}
|
||||
if out[1].Type != "canceled" || out[1].MRRDeltaCents != -3000 {
|
||||
t.Fatalf("canceled row must negate mrr: %+v", out[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortCustomers(t *testing.T) {
|
||||
cs := []SaaSCustomer{
|
||||
{Org: "a", MRRCents: 100, UsageCents: 0},
|
||||
{Org: "b", MRRCents: 500, UsageCents: 0},
|
||||
{Org: "c", MRRCents: 500, UsageCents: 999}, // ties on MRR → usage breaks
|
||||
}
|
||||
sortCustomers(cs)
|
||||
if cs[0].Org != "c" || cs[1].Org != "b" || cs[2].Org != "a" {
|
||||
t.Fatalf("order wrong: %s,%s,%s", cs[0].Org, cs[1].Org, cs[2].Org)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStateQueriesNoPositionalArgs: the run-rate (state) queries are fully static.
|
||||
func TestStateQueriesNoPositionalArgs(t *testing.T) {
|
||||
for name, sql := range map[string]string{
|
||||
"headline": headlineSQL(), "byCategory": byCategorySQL(), "byPlan": byPlanSQL(),
|
||||
"orgCount": orgCountSQL(), "perOrgSubs": perOrgSubsSQL(),
|
||||
} {
|
||||
if !strings.Contains(sql, core.BillingEventsTable) {
|
||||
t.Fatalf("%s must read %s", name, core.BillingEventsTable)
|
||||
}
|
||||
if strings.Contains(sql, "?") {
|
||||
t.Fatalf("%s (run-rate) must take no positional args: %q", name, sql)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestWindowedQueriesOnePositionalArg: the windowed queries bind exactly ONE time
|
||||
// arg (injection-safe — the since bound is never interpolated).
|
||||
func TestWindowedQueriesOnePositionalArg(t *testing.T) {
|
||||
for name, sql := range map[string]string{
|
||||
"movement": movementSQL(), "recent": recentSQL(), "usage": usageSQL(), "perOrgUsage": perOrgUsageSQL(),
|
||||
} {
|
||||
if n := strings.Count(sql, "?"); n != 1 {
|
||||
t.Fatalf("%s must bind exactly ONE positional time arg, got %d: %q", name, n, sql)
|
||||
}
|
||||
if !strings.Contains(sql, "timestamp >= ?") {
|
||||
t.Fatalf("%s time bound must be positional: %q", name, sql)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeAndEmpty(t *testing.T) {
|
||||
m := normalize(SaaSMetrics{})
|
||||
if m.Revenue.ByCategory == nil || m.Subs.ByPlan == nil || m.Subs.Recent == nil || m.Customers == nil || m.Gaps == nil {
|
||||
t.Fatal("normalize must replace nil slices with empty (honest [] not null)")
|
||||
}
|
||||
e := empty("now", "30d", core.SrcOf("billing-warehouse", errUnconfigured, 0, "now"))
|
||||
if e.Currency != "usd" || e.Window != "30d" || len(e.Sources) != 1 || e.Sources[0].OK {
|
||||
t.Fatalf("empty snapshot wrong: %+v", e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeWindow(t *testing.T) {
|
||||
for in, want := range map[string]string{"24h": "24h", "7d": "7d", "30d": "30d", "": "30d", "90d": "30d"} {
|
||||
if got := normalizeWindow(in); got != want {
|
||||
t.Fatalf("normalizeWindow(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,5 +9,6 @@ import (
|
||||
// Routes registers the SaaS-metrics god-view (SuperAdmin only, cross-tenant business
|
||||
// aggregate).
|
||||
func Routes(app *zip.App, s *cloud.Service[core.State]) {
|
||||
app.Get("/v1/admin/metrics", core.Guard(s, Metrics))
|
||||
g := app.Group("/v1/admin")
|
||||
g.Get("/metrics", core.Guard(s, Metrics))
|
||||
}
|
||||
|
||||
@@ -22,7 +22,8 @@ import (
|
||||
|
||||
// Routes registers the fleet revenue board (SuperAdmin only, cross-tenant profitability).
|
||||
func Routes(app *zip.App, s *cloud.Service[core.State]) {
|
||||
app.Get("/v1/admin/revenue", core.Guard(s, Revenue))
|
||||
g := app.Group("/v1/admin")
|
||||
g.Get("/revenue", core.Guard(s, Revenue))
|
||||
}
|
||||
|
||||
// RevenueCustomer is one row of the per-customer revenue table.
|
||||
|
||||
@@ -8,5 +8,6 @@ import (
|
||||
|
||||
// Routes registers the fleet subscription view (SuperAdmin only, cross-tenant).
|
||||
func Routes(app *zip.App, s *cloud.Service[core.State]) {
|
||||
app.Get("/v1/admin/subscriptions", core.Guard(s, Subscriptions))
|
||||
g := app.Group("/v1/admin")
|
||||
g.Get("/subscriptions", core.Guard(s, Subscriptions))
|
||||
}
|
||||
|
||||
@@ -1,28 +1,30 @@
|
||||
// Package subscriptions is the fleet SUBSCRIPTION view (/v1/admin/subscriptions) —
|
||||
// every tenant's plan subscription: customer/org, plan, status, monthly-normalized MRR,
|
||||
// and the current-period start/renews. SuperAdmin only (core.Guard).
|
||||
// every tenant's plan subscription: customer/org, plan, status, monthly-normalized
|
||||
// MRR, and the current-period start/renews. SuperAdmin only (core.Guard).
|
||||
//
|
||||
// Like invoices (and revenue) it fans out the org directory concurrently and reads each
|
||||
// org's subscriptions via the admin S2S seam, tagging every row with its owning org. The
|
||||
// MRR is monthly-normalized in the commerce reader so a yearly plan is comparable to a
|
||||
// monthly one. Best-effort per org (a failed read contributes no rows, never fabricated
|
||||
// ones); optional ?org= scopes to one tenant, ?status= filters, ?limit= caps.
|
||||
// It reads the ONE shared warehouse (commerce.events) — the table the commerce
|
||||
// analytics collector lands every subscription-lifecycle event in — over the SAME
|
||||
// client (aiobject.DatastoreQuery) the o11y/compute lenses use, with ZERO per-org
|
||||
// fan-out: one GROUP BY resolves each subscription's LATEST lifecycle state
|
||||
// (argMax by timestamp), so the whole fleet is one query, not N per-org commerce
|
||||
// reads. Honest by construction: no datastore connected or the collector's table
|
||||
// not provisioned yet → the real empty list, never a fabricated tenant. The MRR is
|
||||
// the monthly-normalized figure the emitter already computed (cents). Optional
|
||||
// ?org= scopes to one tenant, ?status= filters the LATEST status, ?limit= caps.
|
||||
package subscriptions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
aiobject "github.com/hanzoai/ai/object"
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/clients/admin/core"
|
||||
"github.com/hanzoai/cloud/clients/admin/iam"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// defaultLimit caps the merged fleet subscription list when the caller sends none.
|
||||
// defaultLimit caps the fleet subscription list when the caller sends none.
|
||||
const defaultLimit = 500
|
||||
|
||||
// SubscriptionRow is one row of GET /v1/admin/subscriptions — a tenant's subscription at
|
||||
@@ -44,89 +46,104 @@ type SubscriptionRow struct {
|
||||
// GET /v1/admin/subscriptions?org=&status=&limit=
|
||||
func Subscriptions(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
ctx := c.Context()
|
||||
cr := core.CallerCreds(c)
|
||||
status := strings.TrimSpace(c.Query("status"))
|
||||
status := strings.ToLower(strings.TrimSpace(c.Query("status")))
|
||||
wantOrg := strings.TrimSpace(c.Query("org"))
|
||||
limit := parseLimit(c.Query("limit"))
|
||||
|
||||
orgs, err := core.ListOrgs(s, ctx, cr)
|
||||
// Honest-empty when the warehouse is not connected or the collector's events
|
||||
// table is not provisioned yet (the emitter is still being wired).
|
||||
if !core.BillingEventsReady(ctx) {
|
||||
return core.OKList(c, []SubscriptionRow{}, 0)
|
||||
}
|
||||
|
||||
rows, err := aiobject.DatastoreQuery(ctx, subscriptionsSQL())
|
||||
if err != nil {
|
||||
return core.Fail(c, err.Error())
|
||||
}
|
||||
if wantOrg != "" {
|
||||
orgs = filterOrg(orgs, wantOrg)
|
||||
return core.Fail(c, "subscriptions query: "+err.Error())
|
||||
}
|
||||
all := subscriptionRowsFromRows(rows)
|
||||
|
||||
// Per-org subscriptions, fanned out concurrently (best-effort per org).
|
||||
perOrg := make([][]SubscriptionRow, len(orgs))
|
||||
sem := make(chan struct{}, core.MaxCustomerConcurrency)
|
||||
var wg sync.WaitGroup
|
||||
for i, o := range orgs {
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func(i int, o iam.Org) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
perOrg[i] = subscriptionsOf(s, ctx, o, status)
|
||||
}(i, o)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
rows := make([]SubscriptionRow, 0)
|
||||
for _, r := range perOrg {
|
||||
rows = append(rows, r...)
|
||||
}
|
||||
// Highest-MRR first (ties broken by most-recent start); cap to the merged limit.
|
||||
sort.Slice(rows, func(i, j int) bool {
|
||||
if rows[i].MRRCents != rows[j].MRRCents {
|
||||
return rows[i].MRRCents > rows[j].MRRCents
|
||||
// Filter (latest status / org) then sort highest-MRR first, cap to limit.
|
||||
out := make([]SubscriptionRow, 0, len(all))
|
||||
for _, r := range all {
|
||||
if wantOrg != "" && r.Org != wantOrg {
|
||||
continue
|
||||
}
|
||||
return rows[i].Started > rows[j].Started
|
||||
})
|
||||
total := len(rows)
|
||||
if len(rows) > limit {
|
||||
rows = rows[:limit]
|
||||
if status != "" && strings.ToLower(r.Status) != status {
|
||||
continue
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return core.OKList(c, rows, total)
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].MRRCents != out[j].MRRCents {
|
||||
return out[i].MRRCents > out[j].MRRCents
|
||||
}
|
||||
return out[i].Started > out[j].Started
|
||||
})
|
||||
total := len(out)
|
||||
if len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
return core.OKList(c, out, total)
|
||||
}
|
||||
|
||||
// subscriptionsOf reads one org's subscriptions into fleet rows, tagged with the org.
|
||||
// Best-effort: a failed read yields no rows so the fleet view degrades honestly.
|
||||
func subscriptionsOf(s *cloud.Service[core.State], ctx context.Context, o iam.Org, status string) []SubscriptionRow {
|
||||
entries, err := s.State.Commerce.Subscriptions(ctx, o.Name, status)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
display := core.Display(o.DisplayName, o.Name)
|
||||
rows := make([]SubscriptionRow, 0, len(entries))
|
||||
for _, sub := range entries {
|
||||
rows = append(rows, SubscriptionRow{
|
||||
ID: sub.ID,
|
||||
Org: o.Name,
|
||||
Display: display,
|
||||
User: sub.User,
|
||||
Plan: sub.Plan,
|
||||
Status: sub.Status,
|
||||
MRRCents: int64(sub.MRR),
|
||||
Started: sub.Started,
|
||||
Renews: sub.Renews,
|
||||
// subscriptionsSQL resolves each subscription's LATEST lifecycle state from
|
||||
// commerce.events (argMax by timestamp). Static SQL over a closed event-name set
|
||||
// (SQLInList of server constants) — no user input is interpolated, so it is
|
||||
// injection-safe. The emitted properties carry the plan/status/mrr/period fields.
|
||||
func subscriptionsSQL() string {
|
||||
return "SELECT JSONExtractString(properties, 'subscription_id') AS id, " +
|
||||
"argMax(organization_id, timestamp) AS org, " +
|
||||
"argMax(distinct_id, timestamp) AS user, " +
|
||||
"argMax(JSONExtractString(properties, 'plan_name'), timestamp) AS plan, " +
|
||||
"argMax(JSONExtractString(properties, 'status'), timestamp) AS status, " +
|
||||
"argMax(JSONExtractInt(properties, 'mrr_cents'), timestamp) AS mrr_cents, " +
|
||||
"argMax(event, timestamp) AS last_event, " +
|
||||
"min(timestamp) AS started, " +
|
||||
"argMax(JSONExtractString(properties, 'period_end'), timestamp) AS renews " +
|
||||
"FROM " + core.BillingEventsTable + " " +
|
||||
"WHERE event IN (" + core.SQLInList(core.SubscriptionEvents) + ") " +
|
||||
"AND JSONExtractString(properties, 'subscription_id') != '' " +
|
||||
"GROUP BY id"
|
||||
}
|
||||
|
||||
// subscriptionRowsFromRows maps the datastore rows onto []SubscriptionRow (pure).
|
||||
// Display is the org slug — the warehouse holds no friendly name and admin does
|
||||
// no per-org IAM fan-out here (honest, not fabricated). The final status folds
|
||||
// the lifecycle: a subscription whose LATEST event is a cancel reads "canceled"
|
||||
// regardless of the last-emitted status snapshot.
|
||||
func subscriptionRowsFromRows(rows []map[string]any) []SubscriptionRow {
|
||||
out := make([]SubscriptionRow, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
org := core.CHStr(r["org"])
|
||||
out = append(out, SubscriptionRow{
|
||||
ID: core.CHStr(r["id"]),
|
||||
Org: org,
|
||||
Display: org,
|
||||
User: core.CHStr(r["user"]),
|
||||
Plan: core.CHStr(r["plan"]),
|
||||
Status: foldStatus(core.CHStr(r["last_event"]), core.CHStr(r["status"])),
|
||||
MRRCents: core.CHInt64(r["mrr_cents"]),
|
||||
Started: core.CHTime(r["started"]),
|
||||
Renews: core.CHStr(r["renews"]),
|
||||
})
|
||||
}
|
||||
return rows
|
||||
return out
|
||||
}
|
||||
|
||||
// filterOrg narrows the directory to the one requested org (empty when it does not
|
||||
// exist — an honest empty list, never a fabricated tenant).
|
||||
func filterOrg(orgs []iam.Org, want string) []iam.Org {
|
||||
for _, o := range orgs {
|
||||
if o.Name == want {
|
||||
return []iam.Org{o}
|
||||
}
|
||||
// foldStatus resolves the effective status: a subscription whose latest event is
|
||||
// a cancel is "canceled"; otherwise the last-emitted status snapshot (falling
|
||||
// back to "active" when the emitter sent none).
|
||||
func foldStatus(lastEvent, snapshot string) string {
|
||||
if lastEvent == core.EvSubscriptionCanceled {
|
||||
return "canceled"
|
||||
}
|
||||
return nil
|
||||
if s := strings.TrimSpace(snapshot); s != "" {
|
||||
return s
|
||||
}
|
||||
return "active"
|
||||
}
|
||||
|
||||
// parseLimit clamps the merged-list cap to [1,5000], defaulting to defaultLimit.
|
||||
// parseLimit clamps the fleet-list cap to [1,5000], defaulting to defaultLimit.
|
||||
func parseLimit(s string) int {
|
||||
n, err := strconv.Atoi(strings.TrimSpace(s))
|
||||
if err != nil || n <= 0 {
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package subscriptions
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/admin/core"
|
||||
)
|
||||
|
||||
// TestSubscriptionRowsFromRows proves the warehouse-row → SubscriptionRow mapping
|
||||
// (the JSON-shape contract) coerces the datastore driver's native types and folds
|
||||
// the lifecycle status; display honestly mirrors the org slug (no fan-out).
|
||||
func TestSubscriptionRowsFromRows(t *testing.T) {
|
||||
started := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC)
|
||||
rows := []map[string]any{
|
||||
{ // active (mrr as driver int64), latest event renewed
|
||||
"id": "sub_1", "org": "acme", "user": "hanzo/alice",
|
||||
"plan": "Pro", "status": "active", "mrr_cents": int64(4900),
|
||||
"last_event": core.EvSubscriptionRenewed, "started": started,
|
||||
"renews": "2026-08-01T00:00:00Z",
|
||||
},
|
||||
{ // canceled wins over a stale "active" snapshot
|
||||
"id": "sub_2", "org": "beta", "user": "hanzo/bob",
|
||||
"plan": "Team", "status": "active", "mrr_cents": uint64(0),
|
||||
"last_event": core.EvSubscriptionCanceled, "started": started,
|
||||
"renews": "",
|
||||
},
|
||||
}
|
||||
out := subscriptionRowsFromRows(rows)
|
||||
if len(out) != 2 {
|
||||
t.Fatalf("got %d rows, want 2", len(out))
|
||||
}
|
||||
r0 := out[0]
|
||||
if r0.ID != "sub_1" || r0.Org != "acme" || r0.Display != "acme" || r0.User != "hanzo/alice" {
|
||||
t.Fatalf("row0 identity wrong: %+v", r0)
|
||||
}
|
||||
if r0.Plan != "Pro" || r0.Status != "active" || r0.MRRCents != 4900 {
|
||||
t.Fatalf("row0 plan/status/mrr wrong: %+v", r0)
|
||||
}
|
||||
if r0.Started != "2026-07-01T12:00:00Z" {
|
||||
t.Fatalf("row0 started = %q", r0.Started)
|
||||
}
|
||||
if r0.Renews != "2026-08-01T00:00:00Z" {
|
||||
t.Fatalf("row0 renews = %q", r0.Renews)
|
||||
}
|
||||
if out[1].Status != "canceled" {
|
||||
t.Fatalf("row1 status = %q, want canceled (latest-event folds)", out[1].Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFoldStatus(t *testing.T) {
|
||||
if got := foldStatus(core.EvSubscriptionCanceled, "active"); got != "canceled" {
|
||||
t.Fatalf("cancel fold = %q", got)
|
||||
}
|
||||
if got := foldStatus(core.EvSubscriptionRenewed, "trialing"); got != "trialing" {
|
||||
t.Fatalf("snapshot passthrough = %q", got)
|
||||
}
|
||||
if got := foldStatus(core.EvSubscriptionCreated, ""); got != "active" {
|
||||
t.Fatalf("empty-snapshot default = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSubscriptionsSQLInjectionSafe asserts the query is fully static over the
|
||||
// closed event-name set — the warehouse table, no user-derived interpolation.
|
||||
func TestSubscriptionsSQLInjectionSafe(t *testing.T) {
|
||||
sql := subscriptionsSQL()
|
||||
if !strings.Contains(sql, core.BillingEventsTable) {
|
||||
t.Fatalf("query must read %s: %q", core.BillingEventsTable, sql)
|
||||
}
|
||||
for _, ev := range core.SubscriptionEvents {
|
||||
if !strings.Contains(sql, "'"+ev+"'") {
|
||||
t.Fatalf("query missing event %q", ev)
|
||||
}
|
||||
}
|
||||
if strings.Contains(sql, "?") {
|
||||
t.Fatalf("subscriptions state query takes no positional args: %q", sql)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLimitBounds(t *testing.T) {
|
||||
if parseLimit("") != defaultLimit || parseLimit("0") != defaultLimit || parseLimit("x") != defaultLimit {
|
||||
t.Fatal("bad/empty limit must default")
|
||||
}
|
||||
if parseLimit("10") != 10 {
|
||||
t.Fatal("valid limit must pass through")
|
||||
}
|
||||
if parseLimit("999999") != 5000 {
|
||||
t.Fatal("limit must clamp to 5000")
|
||||
}
|
||||
}
|
||||
+68
-8
@@ -32,6 +32,7 @@ package ads
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -106,13 +107,15 @@ func Mount(app *zip.App, deps cloud.Deps) error {
|
||||
|
||||
// routes registers the ads surface: the campaign CRUD + the summary roll-up.
|
||||
func routes(app *zip.App, s *cloud.Service[state]) {
|
||||
app.Get("/v1/ads/summary", cloud.Handle(s, summary))
|
||||
g := app.Group("/v1/ads")
|
||||
g.Get("/summary", cloud.Handle(s, summary))
|
||||
|
||||
app.Get("/v1/ads/campaigns", cloud.Handle(s, listCampaigns))
|
||||
app.Post("/v1/ads/campaigns", cloud.Handle(s, createCampaign))
|
||||
app.Get("/v1/ads/campaigns/:id", cloud.Handle(s, getCampaign))
|
||||
app.Put("/v1/ads/campaigns/:id", cloud.Handle(s, updateCampaign))
|
||||
app.Delete("/v1/ads/campaigns/:id", cloud.Handle(s, deleteCampaign))
|
||||
g.Get("/campaigns", cloud.Handle(s, listCampaigns))
|
||||
g.Post("/campaigns", cloud.Handle(s, createCampaign))
|
||||
g.Get("/campaigns/:id", cloud.Handle(s, getCampaign))
|
||||
g.Put("/campaigns/:id", cloud.Handle(s, updateCampaign))
|
||||
g.Delete("/campaigns/:id", cloud.Handle(s, deleteCampaign))
|
||||
g.Post("/campaigns/:id/launch", cloud.Handle(s, launchCampaignHandler))
|
||||
}
|
||||
|
||||
// ---- shared helpers (mirror clients/crm) ----
|
||||
@@ -223,7 +226,7 @@ func createCampaign(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
camp := Campaign{
|
||||
ID: id, Org: org, Name: name, Platform: platform, Status: status,
|
||||
ID: id, Org: org, Name: name, Platform: platform, Account: clip(body.Account), Status: status,
|
||||
Objective: clip(body.Objective), Budget: nonNeg(body.Budget), Spend: nonNeg(body.Spend),
|
||||
CreatedAt: now, UpdatedAt: now,
|
||||
}
|
||||
@@ -281,7 +284,7 @@ func updateCampaign(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
return zip.ErrBadRequest("status must be one of draft, active, paused, completed")
|
||||
}
|
||||
camp := Campaign{
|
||||
ID: idParam(c), Org: org, Name: name, Platform: platform, Status: status,
|
||||
ID: idParam(c), Org: org, Name: name, Platform: platform, Account: clip(body.Account), Status: status,
|
||||
Objective: clip(body.Objective), Budget: nonNeg(body.Budget), Spend: nonNeg(body.Spend),
|
||||
UpdatedAt: time.Now().Unix(),
|
||||
}
|
||||
@@ -307,6 +310,63 @@ func deleteCampaign(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
return c.NoContent(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// ---- launch (consumes the connector plane) ----
|
||||
|
||||
// launchCampaignHandler runs a stored ad campaign on its provider using the ORG'S
|
||||
// connected ad-account token. It is the standalone proof that /v1/ads consumes the
|
||||
// connector plane: no token is held here — LaunchPaid (provider.go) resolves it
|
||||
// from KMS through integrations.TokenFor and FAILS CLOSED when the org has not
|
||||
// connected the platform (424), so a launch can never spend on a connection the
|
||||
// org did not make. On success the provider campaign id is recorded (MarkLaunched)
|
||||
// and the campaign goes active. An optional body {account} sets/overrides the
|
||||
// target ad account when the stored campaign has none.
|
||||
func launchCampaignHandler(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
org, ok := tenant(c)
|
||||
if !ok {
|
||||
return zip.ErrForbidden("X-Org-Id required")
|
||||
}
|
||||
camp, err := s.State.store.GetCampaign(c.Context(), org, idParam(c))
|
||||
if err != nil {
|
||||
return mapErr(err, "campaign not found")
|
||||
}
|
||||
var body struct {
|
||||
Account string `json:"account"`
|
||||
}
|
||||
_ = c.Bind(&body)
|
||||
account := clip(body.Account)
|
||||
if account == "" {
|
||||
account = camp.Account
|
||||
}
|
||||
ref, lerr := LaunchPaid(c.Context(), org, PaidPlan{
|
||||
Platform: camp.Platform, Account: account, Name: camp.Name,
|
||||
Objective: camp.Objective, BudgetCents: camp.Budget,
|
||||
})
|
||||
if lerr != nil {
|
||||
return mapProviderErr(lerr)
|
||||
}
|
||||
saved, err := s.State.store.MarkLaunched(c.Context(), org, camp.ID, ref.Account, ref.ExternalID, time.Now().Unix())
|
||||
if err != nil {
|
||||
return mapErr(err, "campaign not found")
|
||||
}
|
||||
return c.JSON(http.StatusOK, saved)
|
||||
}
|
||||
|
||||
// mapProviderErr renders a provider-execution error as the honest HTTP status: a
|
||||
// missing/rejected connection is 424 (connect the account first), an unwired
|
||||
// platform is 501, a transient edge failure is 502.
|
||||
func mapProviderErr(err error) error {
|
||||
switch {
|
||||
case errors.Is(err, errNotConnected):
|
||||
return zip.Errorf(http.StatusFailedDependency, "connect your ad account for this platform first")
|
||||
case errors.Is(err, errUnsupportedPlatform):
|
||||
return zip.Errorf(http.StatusNotImplemented, "%v", err)
|
||||
case errors.Is(err, errUpstream):
|
||||
return zip.Errorf(http.StatusBadGateway, "%v", err)
|
||||
default:
|
||||
return zip.Errorf(http.StatusBadRequest, "%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- summary ----
|
||||
|
||||
func summary(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
package ads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/campaign"
|
||||
)
|
||||
|
||||
// campaign_paid_test.go is the END-TO-END proof of the paid GTM channel: the SAME
|
||||
// adapter apps/wire_seams.go registers (campaign.Plan → ads.PaidPlan → LaunchPaid)
|
||||
// driven through the campaign.Channel interface, so the whole chain
|
||||
// campaign → ads → integrations.TokenFor → provider is exercised against an
|
||||
// httptest Meta stub. It lives in package ads because only this package can point
|
||||
// the connector-custody seam (tokenFor) and the Meta base at test doubles;
|
||||
// importing clients/campaign here is acyclic (campaign never imports ads).
|
||||
|
||||
// paidChannelForTest builds the exact paid-channel adapter the composition root
|
||||
// wires, so the test exercises the real seam shape, not a bespoke one.
|
||||
func paidChannelForTest() campaign.Channel {
|
||||
return campaign.NewChannel(campaign.KindPaid,
|
||||
func(ctx context.Context, org string, p campaign.Plan) (campaign.Ref, error) {
|
||||
r, err := LaunchPaid(ctx, org, PaidPlan{
|
||||
Platform: p.Platform, Account: p.Account, Name: p.Name,
|
||||
Objective: p.Objective, BudgetCents: p.BudgetCents, ScheduleAt: p.ScheduleAt,
|
||||
})
|
||||
return campaign.Ref{Platform: r.Platform, Account: r.Account, ExternalID: r.ExternalID, Status: r.Status, Detail: r.Detail}, err
|
||||
},
|
||||
func(ctx context.Context, org string, ref campaign.Ref) (int64, error) {
|
||||
return PaidSpend(ctx, org, PaidRef{Platform: ref.Platform, Account: ref.Account, ExternalID: ref.ExternalID})
|
||||
},
|
||||
func(ctx context.Context, org string, ref campaign.Ref) error {
|
||||
return PausePaid(ctx, org, PaidRef{Platform: ref.Platform, Account: ref.Account, ExternalID: ref.ExternalID})
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// TestCampaignPaidChannel_EndToEnd: a campaign's paid channel launches an ad
|
||||
// campaign on Meta using the ORG'S connected token, then reads spend — the full
|
||||
// capability→connector→provider chain.
|
||||
func TestCampaignPaidChannel_EndToEnd(t *testing.T) {
|
||||
stubToken(t, func(_ context.Context, org, provider, _ string) ([]byte, error) {
|
||||
if provider != "meta_ads" {
|
||||
t.Fatalf("paid meta channel must consume meta_ads, got %q", provider)
|
||||
}
|
||||
return []byte("tok-" + org), nil
|
||||
})
|
||||
var gotAuth string
|
||||
stubMeta(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/act_123/campaigns" {
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
_, _ = w.Write([]byte(`{"id":"120210000000009"}`))
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/120210000000009/insights" {
|
||||
_, _ = w.Write([]byte(`{"data":[{"spend":"7.50"}]}`))
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
})
|
||||
|
||||
paid := paidChannelForTest()
|
||||
if paid.Kind() != campaign.KindPaid {
|
||||
t.Fatalf("channel kind want paid, got %q", paid.Kind())
|
||||
}
|
||||
|
||||
ref, err := paid.Launch(context.Background(), "acme", campaign.Plan{
|
||||
CampaignID: "cmp_1", Platform: "meta", Account: "123", Name: "GTM Launch",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("paid Launch: %v", err)
|
||||
}
|
||||
if ref.ExternalID != "120210000000009" || ref.Status != "live" {
|
||||
t.Fatalf("ref: %+v", ref)
|
||||
}
|
||||
if gotAuth != "Bearer tok-acme" {
|
||||
t.Fatalf("Meta must be called with acme's connector token, got %q", gotAuth)
|
||||
}
|
||||
|
||||
// Spend read composes the same token door → provider insights.
|
||||
cents, err := paid.Spend(context.Background(), "acme", ref)
|
||||
if err != nil {
|
||||
t.Fatalf("paid Spend: %v", err)
|
||||
}
|
||||
if cents != 750 {
|
||||
t.Fatalf("spend want 750 cents, got %d", cents)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCampaignPaidChannel_ConnectorDisabledBlocksSend: a campaign whose org has
|
||||
// not connected the ad account cannot launch — the channel fails closed and the
|
||||
// provider is never called (no spend on an unmade connection).
|
||||
func TestCampaignPaidChannel_ConnectorDisabledBlocksSend(t *testing.T) {
|
||||
stubToken(t, func(_ context.Context, _, _, _ string) ([]byte, error) {
|
||||
return nil, errors.New("integrations: meta_ads not connected for org")
|
||||
})
|
||||
hit := false
|
||||
stubMeta(t, func(w http.ResponseWriter, _ *http.Request) {
|
||||
hit = true
|
||||
_, _ = w.Write([]byte(`{"id":"nope"}`))
|
||||
})
|
||||
|
||||
paid := paidChannelForTest()
|
||||
_, err := paid.Launch(context.Background(), "acme", campaign.Plan{
|
||||
CampaignID: "cmp_1", Platform: "meta", Account: "123", Name: "GTM",
|
||||
})
|
||||
if !errors.Is(err, errNotConnected) {
|
||||
t.Fatalf("want errNotConnected, got %v", err)
|
||||
}
|
||||
if hit {
|
||||
t.Fatalf("provider must NOT be called when the org's connector is disabled")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
package ads
|
||||
|
||||
// provider.go is the ad-network EXECUTION edge: the ONE place /v1/ads consumes the
|
||||
// connector plane. An ad campaign runs on a provider (Meta/Google/…) using the
|
||||
// ORG'S OWN connector token — resolved at call time from KMS through the
|
||||
// integrations.TokenFor custody seam, never held in this process, never in a
|
||||
// manifest. This closes the gap the ads store left open: a stored Campaign is now
|
||||
// LAUNCHABLE against the real provider, and it is what the /v1/campaign paid
|
||||
// channel fans out to (apps/wire_seams.go adapts LaunchPaid/PaidSpend/PausePaid
|
||||
// onto campaign.Channel).
|
||||
//
|
||||
// FAIL-CLOSED CUSTODY. Every operation resolves the org's token FIRST; any reason
|
||||
// it cannot be produced — the org never connected the ad account, the integrations
|
||||
// plane is unmounted, KMS is down — is errNotConnected, and NO provider call is
|
||||
// made. The token rides the Authorization header only (never the URL, never a log,
|
||||
// never argv), exactly the meta.go connector discipline.
|
||||
//
|
||||
// LEAST SURPRISE ON SPEND. A launch creates the campaign OBJECT on the provider;
|
||||
// delivery (and therefore spend) does not begin until the ad-set/ad legs are wired
|
||||
// (the documented ads follow-up), so a launch cannot silently burn budget. Spend is
|
||||
// READ back from the provider's insights (PaidSpend) — the connector's reported
|
||||
// number the /v1/campaign metrics plane joins with the analytics funnel.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/integrations"
|
||||
)
|
||||
|
||||
const (
|
||||
// accessTokenSecret is the KMS secret name every ad connector custodies its
|
||||
// long-lived token under (meta.go/google_marketing.go: "access_token").
|
||||
accessTokenSecret = "access_token"
|
||||
// launchTimeout bounds a single provider call so a slow ad edge never wedges a
|
||||
// launch or a metrics read.
|
||||
launchTimeout = 20 * time.Second
|
||||
)
|
||||
|
||||
// platformConnector maps an ads Platform to its integrations connector id. The
|
||||
// paid channel consumes the connector via TokenFor(org, <id>, "access_token").
|
||||
// A platform with no entry has no connector wired → errUnsupportedPlatform (the
|
||||
// token is never even sought), so the map is the ONE source of "which ad networks
|
||||
// this deployment can run".
|
||||
var platformConnector = map[string]string{
|
||||
"meta": "meta_ads",
|
||||
"google": "google_ads",
|
||||
"tiktok": "tiktok_ads",
|
||||
"reddit": "reddit_ads",
|
||||
"linkedin": "linkedin_ads",
|
||||
"microsoft": "microsoft_ads",
|
||||
}
|
||||
|
||||
var (
|
||||
// errNotConnected — the org has no usable connection for the platform (no
|
||||
// token, or the provider rejected it). Fail-closed: no spend, no fabrication.
|
||||
errNotConnected = errors.New("ads: ad account not connected for org")
|
||||
// errUnsupportedPlatform — a valid platform whose provider execution is not yet
|
||||
// wired (the connector may be connected; only the create/read impl is missing).
|
||||
errUnsupportedPlatform = errors.New("ads: paid execution not wired for platform")
|
||||
// errUpstream — the ad platform edge failed transiently (network / non-auth non-2xx).
|
||||
errUpstream = errors.New("ads: ad platform edge unavailable")
|
||||
)
|
||||
|
||||
// tokenFor is the connector-custody seam. It defaults to integrations.TokenFor
|
||||
// (the ONE KMS-backed token door) and is a package var ONLY so a test can exercise
|
||||
// the provider path — the same reason meta.go's endpoints are package vars. It is
|
||||
// never reassigned in production.
|
||||
var tokenFor = integrations.TokenFor
|
||||
|
||||
// metaAdsBase is the Meta Graph API base. A package var so a test points create /
|
||||
// insights / pause at an httptest server; never mutated in production.
|
||||
var metaAdsBase = "https://graph.facebook.com/v21.0"
|
||||
|
||||
// adHTTP is the ONE bounded client for provider calls.
|
||||
var adHTTP = &http.Client{Timeout: launchTimeout}
|
||||
|
||||
// PaidPlan is the standalone contract for launching one ad campaign — the campaign
|
||||
// paid channel adapts campaign.Plan onto it (apps/wire_seams.go). BudgetCents is
|
||||
// the org's own (connector-paid) budget; ScheduleAt an optional start time.
|
||||
type PaidPlan struct {
|
||||
Platform string
|
||||
Account string // provider ad-account ref (Meta: act_<id> or <id>)
|
||||
Name string
|
||||
Objective string // provider objective enum; defaulted per-provider when empty
|
||||
BudgetCents int64
|
||||
ScheduleAt int64
|
||||
}
|
||||
|
||||
// PaidRef is a launched ad campaign's durable handle: the provider-side id plus the
|
||||
// platform/account needed to read spend or pause it.
|
||||
type PaidRef struct {
|
||||
Platform string
|
||||
Account string
|
||||
ExternalID string
|
||||
Status string
|
||||
Detail string
|
||||
}
|
||||
|
||||
// adToken resolves (connectorID, token) for a platform, fail-closed. An unmapped
|
||||
// platform never reaches KMS. An empty/absent token is errNotConnected.
|
||||
func adToken(ctx context.Context, org, platform string) (string, string, error) {
|
||||
connectorID, ok := platformConnector[strings.ToLower(strings.TrimSpace(platform))]
|
||||
if !ok {
|
||||
return "", "", fmt.Errorf("%w: %s", errUnsupportedPlatform, platform)
|
||||
}
|
||||
tok, err := tokenFor(ctx, org, connectorID, accessTokenSecret)
|
||||
if err != nil || len(strings.TrimSpace(string(tok))) == 0 {
|
||||
return connectorID, "", errNotConnected
|
||||
}
|
||||
return connectorID, strings.TrimSpace(string(tok)), nil
|
||||
}
|
||||
|
||||
// LaunchPaid creates the ad campaign on its provider using the org's connector
|
||||
// token. Fail-closed: it resolves the token BEFORE any provider call. Meta is
|
||||
// executed for real; other connected platforms return errUnsupportedPlatform
|
||||
// (the connector is verified, only the provider impl is the remaining gap).
|
||||
func LaunchPaid(ctx context.Context, org string, p PaidPlan) (PaidRef, error) {
|
||||
platform := strings.ToLower(strings.TrimSpace(p.Platform))
|
||||
connectorID, token, err := adToken(ctx, org, platform)
|
||||
if err != nil {
|
||||
return PaidRef{}, err
|
||||
}
|
||||
switch platform {
|
||||
case "meta":
|
||||
return metaCreateCampaign(ctx, token, p)
|
||||
default:
|
||||
return PaidRef{}, fmt.Errorf("%w: %s (connector %s connected)", errUnsupportedPlatform, platform, connectorID)
|
||||
}
|
||||
}
|
||||
|
||||
// PaidSpend reads the provider-reported spend (minor units) for a launched ad
|
||||
// campaign. Fail-closed on the token; honest 0 when the platform is unsupported.
|
||||
func PaidSpend(ctx context.Context, org string, ref PaidRef) (int64, error) {
|
||||
platform := strings.ToLower(strings.TrimSpace(ref.Platform))
|
||||
_, token, err := adToken(ctx, org, platform)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
switch platform {
|
||||
case "meta":
|
||||
return metaCampaignSpend(ctx, token, ref.ExternalID)
|
||||
default:
|
||||
return 0, fmt.Errorf("%w: %s", errUnsupportedPlatform, platform)
|
||||
}
|
||||
}
|
||||
|
||||
// PausePaid pauses a launched ad campaign on its provider. Fail-closed on the token.
|
||||
func PausePaid(ctx context.Context, org string, ref PaidRef) error {
|
||||
platform := strings.ToLower(strings.TrimSpace(ref.Platform))
|
||||
_, token, err := adToken(ctx, org, platform)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch platform {
|
||||
case "meta":
|
||||
return metaPauseCampaign(ctx, token, ref.ExternalID)
|
||||
default:
|
||||
return fmt.Errorf("%w: %s", errUnsupportedPlatform, platform)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Meta (Facebook/Instagram) ad campaign execution ─────────────────────────
|
||||
|
||||
// metaErr is Meta's Graph API error envelope.
|
||||
type metaErr struct {
|
||||
Message string `json:"message"`
|
||||
Type string `json:"type"`
|
||||
Code int `json:"code"`
|
||||
}
|
||||
|
||||
// metaAcct normalizes a Meta ad-account ref to the act_<id> form the campaigns
|
||||
// edge expects. Empty stays empty (caller rejects).
|
||||
func metaAcct(account string) string {
|
||||
a := strings.TrimSpace(account)
|
||||
if a == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(a, "act_") {
|
||||
return a
|
||||
}
|
||||
return "act_" + a
|
||||
}
|
||||
|
||||
// metaCreateCampaign POSTs a campaign to /act_<id>/campaigns. Objective defaults to
|
||||
// OUTCOME_TRAFFIC; special_ad_categories is the required empty set; the object is
|
||||
// created ACTIVE but does not deliver (no ad sets) so no spend starts on launch.
|
||||
func metaCreateCampaign(ctx context.Context, token string, p PaidPlan) (PaidRef, error) {
|
||||
account := metaAcct(p.Account)
|
||||
if account == "" {
|
||||
return PaidRef{}, fmt.Errorf("meta: ad account (account) is required")
|
||||
}
|
||||
objective := strings.TrimSpace(p.Objective)
|
||||
if objective == "" {
|
||||
objective = "OUTCOME_TRAFFIC"
|
||||
}
|
||||
form := url.Values{
|
||||
"name": {strings.TrimSpace(p.Name)},
|
||||
"objective": {objective},
|
||||
"status": {"ACTIVE"},
|
||||
"special_ad_categories": {"[]"},
|
||||
}
|
||||
var out struct {
|
||||
ID string `json:"id"`
|
||||
Error *metaErr `json:"error"`
|
||||
}
|
||||
if err := metaPost(ctx, metaAdsBase+"/"+account+"/campaigns", token, form, &out); err != nil {
|
||||
return PaidRef{}, err
|
||||
}
|
||||
if out.Error != nil {
|
||||
return PaidRef{}, fmt.Errorf("meta create campaign: %s", out.Error.Message)
|
||||
}
|
||||
if strings.TrimSpace(out.ID) == "" {
|
||||
return PaidRef{}, fmt.Errorf("meta create campaign returned no id")
|
||||
}
|
||||
return PaidRef{Platform: "meta", Account: account, ExternalID: out.ID, Status: "live"}, nil
|
||||
}
|
||||
|
||||
// metaCampaignSpend reads spend from /{campaignID}/insights?fields=spend. Meta
|
||||
// reports spend in the account currency's major unit as a string; it is converted
|
||||
// to minor units (cents). No insights row (campaign hasn't spent) → 0.
|
||||
func metaCampaignSpend(ctx context.Context, token, campaignID string) (int64, error) {
|
||||
if strings.TrimSpace(campaignID) == "" {
|
||||
return 0, nil
|
||||
}
|
||||
var out struct {
|
||||
Data []struct {
|
||||
Spend string `json:"spend"`
|
||||
} `json:"data"`
|
||||
Error *metaErr `json:"error"`
|
||||
}
|
||||
if err := metaGet(ctx, metaAdsBase+"/"+campaignID+"/insights?fields=spend", token, &out); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if out.Error != nil {
|
||||
return 0, fmt.Errorf("meta insights: %s", out.Error.Message)
|
||||
}
|
||||
if len(out.Data) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
dollars, err := strconv.ParseFloat(strings.TrimSpace(out.Data[0].Spend), 64)
|
||||
if err != nil || dollars < 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return int64(dollars*100 + 0.5), nil
|
||||
}
|
||||
|
||||
// metaPauseCampaign POSTs status=PAUSED to /{campaignID}.
|
||||
func metaPauseCampaign(ctx context.Context, token, campaignID string) error {
|
||||
if strings.TrimSpace(campaignID) == "" {
|
||||
return fmt.Errorf("meta: campaign id required")
|
||||
}
|
||||
var out struct {
|
||||
Success bool `json:"success"`
|
||||
Error *metaErr `json:"error"`
|
||||
}
|
||||
if err := metaPost(ctx, metaAdsBase+"/"+campaignID, token, url.Values{"status": {"PAUSED"}}, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
if out.Error != nil {
|
||||
return fmt.Errorf("meta pause: %s", out.Error.Message)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ── bounded HTTP (token on the Authorization header only) ────────────────────
|
||||
|
||||
func metaPost(ctx context.Context, endpoint, token string, form url.Values, out any) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %v", errUpstream, err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
return adDo(req, out)
|
||||
}
|
||||
|
||||
func metaGet(ctx context.Context, endpoint, token string, out any) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %v", errUpstream, err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
return adDo(req, out)
|
||||
}
|
||||
|
||||
// adDo runs a bounded provider request and decodes the JSON body. A 401/403 is
|
||||
// fail-closed as errNotConnected (the org's token is invalid/revoked — the
|
||||
// connection is not usable); any other non-2xx or transport error is errUpstream.
|
||||
// The body is bounded so a hostile edge cannot amplify memory.
|
||||
func adDo(req *http.Request, out any) error {
|
||||
resp, err := adHTTP.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %v", errUpstream, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
|
||||
if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
|
||||
return errNotConnected
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("%w: %d", errUpstream, resp.StatusCode)
|
||||
}
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(body, out); err != nil {
|
||||
return fmt.Errorf("%w: decode: %v", errUpstream, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package ads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// stubToken overrides the connector-custody seam so the provider path is
|
||||
// exercised without standing up KMS + integrations. Restored on cleanup.
|
||||
func stubToken(t *testing.T, fn func(ctx context.Context, org, provider, name string) ([]byte, error)) {
|
||||
t.Helper()
|
||||
prev := tokenFor
|
||||
tokenFor = fn
|
||||
t.Cleanup(func() { tokenFor = prev })
|
||||
}
|
||||
|
||||
// stubMeta points the Meta base at an httptest server (restored on cleanup) and
|
||||
// returns the server so the test can inspect what the provider received.
|
||||
func stubMeta(t *testing.T, h http.HandlerFunc) *httptest.Server {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(h)
|
||||
prev := metaAdsBase
|
||||
metaAdsBase = srv.URL
|
||||
t.Cleanup(func() { metaAdsBase = prev; srv.Close() })
|
||||
return srv
|
||||
}
|
||||
|
||||
// TestLaunchPaid_MetaCreatesViaConnectorToken is the proof that /v1/ads consumes
|
||||
// the connector plane: LaunchPaid resolves the ORG'S token via the TokenFor seam
|
||||
// and creates the campaign on Meta with that token on the Authorization header.
|
||||
func TestLaunchPaid_MetaCreatesViaConnectorToken(t *testing.T) {
|
||||
var (
|
||||
gotAuth, gotPath, gotBody string
|
||||
gotTokenOrg, gotProvider string
|
||||
)
|
||||
stubToken(t, func(_ context.Context, org, provider, name string) ([]byte, error) {
|
||||
gotTokenOrg, gotProvider = org, provider
|
||||
if name != accessTokenSecret {
|
||||
t.Fatalf("token secret name want %q, got %q", accessTokenSecret, name)
|
||||
}
|
||||
return []byte("T0K3N-acme"), nil
|
||||
})
|
||||
stubMeta(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
gotPath = r.URL.Path
|
||||
_ = r.ParseForm()
|
||||
gotBody = r.Form.Encode()
|
||||
_, _ = w.Write([]byte(`{"id":"120210000000001"}`))
|
||||
})
|
||||
|
||||
ref, err := LaunchPaid(context.Background(), "acme", PaidPlan{
|
||||
Platform: "meta", Account: "123", Name: "Spring Launch", Objective: "OUTCOME_TRAFFIC",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("LaunchPaid: %v", err)
|
||||
}
|
||||
if ref.ExternalID != "120210000000001" || ref.Account != "act_123" || ref.Status != "live" {
|
||||
t.Fatalf("ref: %+v", ref)
|
||||
}
|
||||
if gotTokenOrg != "acme" || gotProvider != "meta_ads" {
|
||||
t.Fatalf("TokenFor called with (%q,%q), want (acme, meta_ads)", gotTokenOrg, gotProvider)
|
||||
}
|
||||
if gotAuth != "Bearer T0K3N-acme" {
|
||||
t.Fatalf("Authorization header want the org token, got %q", gotAuth)
|
||||
}
|
||||
if gotPath != "/act_123/campaigns" {
|
||||
t.Fatalf("create path want /act_123/campaigns, got %q", gotPath)
|
||||
}
|
||||
for _, want := range []string{"name=Spring", "objective=OUTCOME_TRAFFIC", "status=ACTIVE", "special_ad_categories="} {
|
||||
if !strings.Contains(gotBody, want) {
|
||||
t.Fatalf("create body missing %q: %q", want, gotBody)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestLaunchPaid_ConnectorDisabledNoProviderCall: when the org has not connected
|
||||
// the ad account (TokenFor fails), LaunchPaid fails closed and NEVER calls the
|
||||
// provider — no spend, no fabrication.
|
||||
func TestLaunchPaid_ConnectorDisabledNoProviderCall(t *testing.T) {
|
||||
stubToken(t, func(_ context.Context, _, _, _ string) ([]byte, error) {
|
||||
return nil, errors.New("integrations: meta_ads not connected for org")
|
||||
})
|
||||
hit := false
|
||||
stubMeta(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
hit = true
|
||||
_, _ = w.Write([]byte(`{"id":"should-not-happen"}`))
|
||||
})
|
||||
|
||||
_, err := LaunchPaid(context.Background(), "acme", PaidPlan{Platform: "meta", Account: "123", Name: "X"})
|
||||
if !errors.Is(err, errNotConnected) {
|
||||
t.Fatalf("want errNotConnected, got %v", err)
|
||||
}
|
||||
if hit {
|
||||
t.Fatalf("provider must NOT be called when the connector is disabled")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLaunchPaid_TenantIsolationTokenPerOrg: each org's launch carries ITS OWN
|
||||
// org's token to the provider — org A can never spend on org B's connection.
|
||||
func TestLaunchPaid_TenantIsolationTokenPerOrg(t *testing.T) {
|
||||
stubToken(t, func(_ context.Context, org, _, _ string) ([]byte, error) {
|
||||
return []byte("tok-" + org), nil // each org has a distinct token
|
||||
})
|
||||
var seen []string
|
||||
stubMeta(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
seen = append(seen, r.Header.Get("Authorization"))
|
||||
_, _ = w.Write([]byte(`{"id":"1"}`))
|
||||
})
|
||||
|
||||
if _, err := LaunchPaid(context.Background(), "acme", PaidPlan{Platform: "meta", Account: "1", Name: "A"}); err != nil {
|
||||
t.Fatalf("acme launch: %v", err)
|
||||
}
|
||||
if _, err := LaunchPaid(context.Background(), "maxpower", PaidPlan{Platform: "meta", Account: "2", Name: "B"}); err != nil {
|
||||
t.Fatalf("maxpower launch: %v", err)
|
||||
}
|
||||
if len(seen) != 2 || seen[0] != "Bearer tok-acme" || seen[1] != "Bearer tok-maxpower" {
|
||||
t.Fatalf("each launch must carry its own org token, got %v", seen)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLaunchPaid_AuthFailureIsNotConnected: a token the provider rejects (401) is
|
||||
// fail-closed as errNotConnected — the connection is not usable.
|
||||
func TestLaunchPaid_AuthFailureIsNotConnected(t *testing.T) {
|
||||
stubToken(t, func(_ context.Context, _, _, _ string) ([]byte, error) { return []byte("stale"), nil })
|
||||
stubMeta(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"invalid token"}}`))
|
||||
})
|
||||
_, err := LaunchPaid(context.Background(), "acme", PaidPlan{Platform: "meta", Account: "1", Name: "X"})
|
||||
if !errors.Is(err, errNotConnected) {
|
||||
t.Fatalf("401 want errNotConnected, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLaunchPaid_UnsupportedPlatformAfterToken: a mapped-but-unwired platform
|
||||
// resolves the connector token (proving consumption) then honestly reports the
|
||||
// execution gap — never a fabricated launch.
|
||||
func TestLaunchPaid_UnsupportedPlatformAfterToken(t *testing.T) {
|
||||
tokenSought := false
|
||||
stubToken(t, func(_ context.Context, _, provider, _ string) ([]byte, error) {
|
||||
tokenSought = true
|
||||
if provider != "google_ads" {
|
||||
t.Fatalf("google platform must resolve google_ads, got %q", provider)
|
||||
}
|
||||
return []byte("g-token"), nil
|
||||
})
|
||||
_, err := LaunchPaid(context.Background(), "acme", PaidPlan{Platform: "google", Account: "1", Name: "X"})
|
||||
if !errors.Is(err, errUnsupportedPlatform) {
|
||||
t.Fatalf("want errUnsupportedPlatform, got %v", err)
|
||||
}
|
||||
if !tokenSought {
|
||||
t.Fatalf("a mapped platform must resolve its connector token before reporting the gap")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLaunchPaid_UnmappedPlatformNeverSeeksToken: a platform with no connector is
|
||||
// rejected BEFORE any KMS/token lookup.
|
||||
func TestLaunchPaid_UnmappedPlatformNeverSeeksToken(t *testing.T) {
|
||||
stubToken(t, func(_ context.Context, _, _, _ string) ([]byte, error) {
|
||||
t.Fatalf("token must not be sought for an unmapped platform")
|
||||
return nil, nil
|
||||
})
|
||||
_, err := LaunchPaid(context.Background(), "acme", PaidPlan{Platform: "snapchat", Account: "1", Name: "X"})
|
||||
if !errors.Is(err, errUnsupportedPlatform) {
|
||||
t.Fatalf("want errUnsupportedPlatform, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPaidSpend_MetaInsightsCents: spend is read from Meta insights and converted
|
||||
// to minor units (cents), through the org's connector token.
|
||||
func TestPaidSpend_MetaInsightsCents(t *testing.T) {
|
||||
stubToken(t, func(_ context.Context, _, _, _ string) ([]byte, error) { return []byte("tok"), nil })
|
||||
var gotPath string
|
||||
stubMeta(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
_, _ = w.Write([]byte(`{"data":[{"spend":"12.34"}]}`))
|
||||
})
|
||||
cents, err := PaidSpend(context.Background(), "acme", PaidRef{Platform: "meta", ExternalID: "120210000000001"})
|
||||
if err != nil {
|
||||
t.Fatalf("PaidSpend: %v", err)
|
||||
}
|
||||
if cents != 1234 {
|
||||
t.Fatalf("spend want 1234 cents, got %d", cents)
|
||||
}
|
||||
if gotPath != "/120210000000001/insights" {
|
||||
t.Fatalf("insights path want /120210000000001/insights, got %q", gotPath)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPausePaid_MetaSetsPaused: pause posts status=PAUSED to the provider through
|
||||
// the org's connector token.
|
||||
func TestPausePaid_MetaSetsPaused(t *testing.T) {
|
||||
stubToken(t, func(_ context.Context, _, _, _ string) ([]byte, error) { return []byte("tok"), nil })
|
||||
var gotStatus string
|
||||
stubMeta(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = r.ParseForm()
|
||||
gotStatus = r.Form.Get("status")
|
||||
_, _ = w.Write([]byte(`{"success":true}`))
|
||||
})
|
||||
if err := PausePaid(context.Background(), "acme", PaidRef{Platform: "meta", ExternalID: "120210000000001"}); err != nil {
|
||||
t.Fatalf("PausePaid: %v", err)
|
||||
}
|
||||
if gotStatus != "PAUSED" {
|
||||
t.Fatalf("pause status want PAUSED, got %q", gotStatus)
|
||||
}
|
||||
}
|
||||
+82
-17
@@ -54,7 +54,9 @@ func openStore(path string) (*Store, error) {
|
||||
|
||||
// migrate creates the campaigns table. Idempotent (IF NOT EXISTS). The table
|
||||
// leads its lookup indexes with `org` so tenant isolation is a physical
|
||||
// property, not just a WHERE clause.
|
||||
// property, not just a WHERE clause. account + external_id link a stored campaign
|
||||
// to its launched provider execution (provider.go); they are added idempotently
|
||||
// so a DB created before the connector-execution edge existed gains them cleanly.
|
||||
func (s *Store) migrate() error {
|
||||
const ddl = `
|
||||
CREATE TABLE IF NOT EXISTS ads_campaigns (
|
||||
@@ -62,6 +64,8 @@ CREATE TABLE IF NOT EXISTS ads_campaigns (
|
||||
org TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
platform TEXT NOT NULL DEFAULT 'meta',
|
||||
account TEXT NOT NULL DEFAULT '',
|
||||
external_id TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'draft',
|
||||
objective TEXT NOT NULL DEFAULT '',
|
||||
budget INTEGER NOT NULL DEFAULT 0,
|
||||
@@ -76,9 +80,48 @@ CREATE INDEX IF NOT EXISTS ix_ads_campaigns_org_platform ON ads_campaigns(org, p
|
||||
if _, err := s.db.Exec(ddl); err != nil {
|
||||
return fmt.Errorf("ads migrate: %w", err)
|
||||
}
|
||||
// Idempotent column adds for DBs created before account/external_id existed.
|
||||
// The column names are package constants (never user input) → safe to inline.
|
||||
for col, spec := range map[string]string{
|
||||
"account": "TEXT NOT NULL DEFAULT ''",
|
||||
"external_id": "TEXT NOT NULL DEFAULT ''",
|
||||
} {
|
||||
if err := s.addColumnIfMissing("ads_campaigns", col, spec); err != nil {
|
||||
return fmt.Errorf("ads migrate column %s: %w", col, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// addColumnIfMissing ALTERs table to add col (with spec) only when absent. table/
|
||||
// col/spec are package constants, so the interpolation is injection-safe. Makes
|
||||
// the schema migration forward-only and idempotent across process restarts.
|
||||
func (s *Store) addColumnIfMissing(table, col, spec string) error {
|
||||
rows, err := s.db.Query(`PRAGMA table_info(` + table + `)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
for rows.Next() {
|
||||
var (
|
||||
cid, notnull, pk int
|
||||
name, ctype string
|
||||
dflt sql.NullString
|
||||
)
|
||||
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil {
|
||||
return err
|
||||
}
|
||||
if name == col {
|
||||
return nil // already present
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.Exec(`ALTER TABLE ` + table + ` ADD COLUMN ` + col + ` ` + spec)
|
||||
return err
|
||||
}
|
||||
|
||||
// Close closes the underlying database. Idempotent-safe via sql.DB.
|
||||
func (s *Store) Close() error { return s.db.Close() }
|
||||
|
||||
@@ -89,31 +132,33 @@ func (s *Store) Close() error { return s.db.Close() }
|
||||
// (draft/active/paused/completed) — both validated at the write layer against
|
||||
// the fixed vocabularies in ads.go.
|
||||
type Campaign struct {
|
||||
ID string `json:"id"`
|
||||
Org string `json:"-"`
|
||||
Name string `json:"name"`
|
||||
Platform string `json:"platform"`
|
||||
Status string `json:"status"`
|
||||
Objective string `json:"objective"`
|
||||
Budget int64 `json:"budget"`
|
||||
Spend int64 `json:"spend"`
|
||||
CreatedAt int64 `json:"createdAt"`
|
||||
UpdatedAt int64 `json:"updatedAt"`
|
||||
ID string `json:"id"`
|
||||
Org string `json:"-"`
|
||||
Name string `json:"name"`
|
||||
Platform string `json:"platform"`
|
||||
Account string `json:"account,omitempty"` // provider ad-account ref (Meta act_<id>)
|
||||
ExternalID string `json:"externalId,omitempty"` // provider campaign id after a launch
|
||||
Status string `json:"status"`
|
||||
Objective string `json:"objective"`
|
||||
Budget int64 `json:"budget"`
|
||||
Spend int64 `json:"spend"`
|
||||
CreatedAt int64 `json:"createdAt"`
|
||||
UpdatedAt int64 `json:"updatedAt"`
|
||||
}
|
||||
|
||||
const campaignCols = `id,org,name,platform,status,objective,budget,spend,created_at,updated_at`
|
||||
const campaignCols = `id,org,name,platform,account,external_id,status,objective,budget,spend,created_at,updated_at`
|
||||
|
||||
func scanCampaign(sc interface{ Scan(...any) error }) (Campaign, error) {
|
||||
var c Campaign
|
||||
err := sc.Scan(&c.ID, &c.Org, &c.Name, &c.Platform, &c.Status, &c.Objective,
|
||||
err := sc.Scan(&c.ID, &c.Org, &c.Name, &c.Platform, &c.Account, &c.ExternalID, &c.Status, &c.Objective,
|
||||
&c.Budget, &c.Spend, &c.CreatedAt, &c.UpdatedAt)
|
||||
return c, err
|
||||
}
|
||||
|
||||
func (s *Store) CreateCampaign(ctx context.Context, c Campaign) (Campaign, error) {
|
||||
if _, err := s.db.ExecContext(ctx,
|
||||
`INSERT INTO ads_campaigns (`+campaignCols+`) VALUES (?,?,?,?,?,?,?,?,?,?)`,
|
||||
c.ID, c.Org, c.Name, c.Platform, c.Status, c.Objective, c.Budget, c.Spend,
|
||||
`INSERT INTO ads_campaigns (`+campaignCols+`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
c.ID, c.Org, c.Name, c.Platform, c.Account, c.ExternalID, c.Status, c.Objective, c.Budget, c.Spend,
|
||||
c.CreatedAt, c.UpdatedAt); err != nil {
|
||||
return Campaign{}, fmt.Errorf("insert campaign: %w", err)
|
||||
}
|
||||
@@ -161,10 +206,13 @@ func (s *Store) ListCampaigns(ctx context.Context, org, status string, limit int
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// UpdateCampaign edits the user-owned fields. external_id is deliberately NOT in
|
||||
// the SET list: it is launch-owned (MarkLaunched sets it), so editing a campaign
|
||||
// never clobbers the link to its live provider execution.
|
||||
func (s *Store) UpdateCampaign(ctx context.Context, c Campaign) (Campaign, error) {
|
||||
res, err := s.db.ExecContext(ctx,
|
||||
`UPDATE ads_campaigns SET name=?,platform=?,status=?,objective=?,budget=?,spend=?,updated_at=? WHERE org=? AND id=?`,
|
||||
c.Name, c.Platform, c.Status, c.Objective, c.Budget, c.Spend, c.UpdatedAt, c.Org, c.ID)
|
||||
`UPDATE ads_campaigns SET name=?,platform=?,account=?,status=?,objective=?,budget=?,spend=?,updated_at=? WHERE org=? AND id=?`,
|
||||
c.Name, c.Platform, c.Account, c.Status, c.Objective, c.Budget, c.Spend, c.UpdatedAt, c.Org, c.ID)
|
||||
if err != nil {
|
||||
return Campaign{}, fmt.Errorf("update campaign: %w", err)
|
||||
}
|
||||
@@ -174,6 +222,23 @@ func (s *Store) UpdateCampaign(ctx context.Context, c Campaign) (Campaign, error
|
||||
return s.GetCampaign(ctx, c.Org, c.ID)
|
||||
}
|
||||
|
||||
// MarkLaunched records the provider execution on a stored campaign: its account +
|
||||
// external id, and status=active. Org-scoped — a cross-tenant id affects zero
|
||||
// rows (errNotFound), never a foreign mutation. This is the ONLY writer of
|
||||
// external_id, so the launch link is never clobbered by a user edit.
|
||||
func (s *Store) MarkLaunched(ctx context.Context, org, id, account, externalID string, updatedAt int64) (Campaign, error) {
|
||||
res, err := s.db.ExecContext(ctx,
|
||||
`UPDATE ads_campaigns SET account=?,external_id=?,status='active',updated_at=? WHERE org=? AND id=?`,
|
||||
account, externalID, updatedAt, org, id)
|
||||
if err != nil {
|
||||
return Campaign{}, fmt.Errorf("mark launched: %w", err)
|
||||
}
|
||||
if n, _ := res.RowsAffected(); n == 0 {
|
||||
return Campaign{}, errNotFound
|
||||
}
|
||||
return s.GetCampaign(ctx, org, id)
|
||||
}
|
||||
|
||||
func (s *Store) DeleteCampaign(ctx context.Context, org, id string) (bool, error) {
|
||||
res, err := s.db.ExecContext(ctx, `DELETE FROM ads_campaigns WHERE org=? AND id=?`, org, id)
|
||||
if err != nil {
|
||||
|
||||
+128
-23
@@ -28,7 +28,9 @@ import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
mrand "math/rand/v2"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -107,6 +109,13 @@ type state struct {
|
||||
// per subsystem. Empty only on a deployment that configured no default, in
|
||||
// which case create still requires an explicit model.
|
||||
defaultModel string
|
||||
// failoverModel is the reliable model a run falls over to when the agent's own
|
||||
// model stays throttled (429/overloaded) after bounded retries
|
||||
// (deps.AIFallbackModel, default "best"). It makes an autonomous bot reply
|
||||
// still land when the throttled default flash model is overloaded. Empty
|
||||
// disables failover (retry-only). Only the run path reads it — interactive
|
||||
// chat is untouched.
|
||||
failoverModel string
|
||||
// bill is the shared per-org gate+meter (reuses deps.Metering, the ONE
|
||||
// commerce client — the same object ml/provisioning use). Nil/!Enabled()
|
||||
// makes Gate allow and Meter a no-op, so an unconfigured deployment runs
|
||||
@@ -262,11 +271,12 @@ func Mount(app *zip.App, deps cloud.Deps) error {
|
||||
s := &cloud.Service[state]{
|
||||
Base: cloud.NewBase(deps, "agents"),
|
||||
State: state{
|
||||
store: store,
|
||||
ai: deps.AI,
|
||||
defaultModel: strings.TrimSpace(deps.AIDefaultModel),
|
||||
bill: cloud.NewResourceMeter(deps, meterKind),
|
||||
bus: newBus(),
|
||||
store: store,
|
||||
ai: deps.AI,
|
||||
defaultModel: strings.TrimSpace(deps.AIDefaultModel),
|
||||
failoverModel: strings.TrimSpace(deps.AIFallbackModel),
|
||||
bill: cloud.NewResourceMeter(deps, meterKind),
|
||||
bus: newBus(),
|
||||
// TASKS PLUG-IN POINT: durable execution rides hanzoai/tasks, not a
|
||||
// bespoke engine. Default is record-only; wiring client.Dial(TASKS_URL)
|
||||
// from github.com/hanzoai/tasks/pkg/sdk/client here makes control forward
|
||||
@@ -276,6 +286,7 @@ func Mount(app *zip.App, deps cloud.Deps) error {
|
||||
}
|
||||
mounted = s
|
||||
|
||||
g := app.Group("/v1/agents")
|
||||
app.Get("/v1/agents", cloud.Handle(s, list))
|
||||
app.Post("/v1/agents", cloud.Handle(s, create))
|
||||
// The static org-wide surfaces are listed before the :ref wildcard for reading
|
||||
@@ -284,18 +295,18 @@ func Mount(app *zip.App, deps cloud.Deps) error {
|
||||
// a ref). Registration order decides nothing here — it only decides which
|
||||
// handler silently wins when two patterns are byte-identical, which is a
|
||||
// collision, not a precedence.
|
||||
app.Get("/v1/agents/metrics", cloud.Handle(s, metrics))
|
||||
app.Get("/v1/agents/activity", cloud.Handle(s, activity))
|
||||
g.Get("/metrics", cloud.Handle(s, metrics))
|
||||
g.Get("/activity", cloud.Handle(s, activity))
|
||||
// Live agent-session control plane: /v1/agents/sessions[/...].
|
||||
mountSessions(s, app)
|
||||
// Agent targets: /v1/agents/targets[/...] — the #48 dispatch destinations a
|
||||
// session runs on.
|
||||
mountTargets(s, app)
|
||||
app.Get("/v1/agents/:ref", cloud.Handle(s, get))
|
||||
app.Patch("/v1/agents/:ref", cloud.Handle(s, update))
|
||||
app.Delete("/v1/agents/:ref", cloud.Handle(s, del))
|
||||
app.Post("/v1/agents/:ref/run", cloud.Handle(s, run))
|
||||
app.Get("/v1/agents/:ref/runs", cloud.Handle(s, runs))
|
||||
g.Get("/:ref", cloud.Handle(s, get))
|
||||
g.Patch("/:ref", cloud.Handle(s, update))
|
||||
g.Delete("/:ref", cloud.Handle(s, del))
|
||||
g.Post("/:ref/run", cloud.Handle(s, run))
|
||||
g.Get("/:ref/runs", cloud.Handle(s, runs))
|
||||
|
||||
// Long-running scheduler: invokes each long-running agent's run on its cron
|
||||
// cadence through the SAME runAgent path as the HTTP handler (one run path,
|
||||
@@ -665,11 +676,12 @@ func runAgent(s *cloud.Service[state], ctx context.Context, a Agent, input, acto
|
||||
return Run{}, err
|
||||
}
|
||||
|
||||
r := executeRun(ctx, s.State.ai, a.Org, a, input)
|
||||
r := executeRun(ctx, s.State.ai, a.Org, a, input, s.State.failoverModel)
|
||||
span.SetAttributes(
|
||||
attribute.String("hanzo.agent.run_id", r.ID),
|
||||
attribute.String("hanzo.agent.run_status", r.Status),
|
||||
attribute.Int64("hanzo.agent.duration_ms", r.DurationMs),
|
||||
attribute.String("gen_ai.response.model", r.Model),
|
||||
)
|
||||
if r.Status == "error" {
|
||||
span.SetStatus(codes.Error, r.Error)
|
||||
@@ -685,12 +697,14 @@ func runAgent(s *cloud.Service[state], ctx context.Context, a Agent, input, acto
|
||||
openRunSession(s, ctx, a, r, actor)
|
||||
|
||||
// Bill only a successful run (mirrors the edge gate: failed work is not
|
||||
// charged). Rich attribution: product=agent (Provider), the agent's model,
|
||||
// and the actor for the audit trail. Fire-and-forget on a background context.
|
||||
// charged). Rich attribution: product=agent (Provider), the model ACTUALLY
|
||||
// used (r.Model — a failover run bills the reliable model it fell over to, not
|
||||
// the throttled one it started on), and the actor for the audit trail.
|
||||
// Fire-and-forget on a background context.
|
||||
if r.Status == "ok" {
|
||||
s.State.bill.MeterUsage(a.Org, meterKind, metering.Usage{
|
||||
AmountCents: fee,
|
||||
Model: a.Model,
|
||||
Model: r.Model,
|
||||
Actor: actor,
|
||||
RequestID: requestID,
|
||||
ClientIP: clientIP,
|
||||
@@ -699,11 +713,31 @@ func runAgent(s *cloud.Service[state], ctx context.Context, a Agent, input, acto
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// executeRun composes the agent's instructions with the caller input, runs one
|
||||
// real chat completion through the AI client, and returns the resulting Run —
|
||||
// status "ok" with output, or "error" with the upstream failure. Pure of HTTP
|
||||
// and persistence so it is directly testable; the caller records + responds.
|
||||
func executeRun(ctx context.Context, ai types.AIClient, org string, a Agent, input string) Run {
|
||||
// maxAttempts bounds retries of a SINGLE model's completion on a transient
|
||||
// upstream failure (429 / 5xx / empty-choices / "Platform overloaded"). Retrying
|
||||
// a completion is side-effect-free — nothing bills until it succeeds — so a
|
||||
// bounded retry with jittered backoff turns an intermittent gateway 429 into a
|
||||
// delivered reply instead of a dropped one.
|
||||
const maxAttempts = 3
|
||||
|
||||
// retryBaseDelay / retryMaxDelay bound the exponential, equal-jittered backoff
|
||||
// between attempts. Small by design: a gateway overload clears in well under a
|
||||
// second, and a run must not stall a bot conversation.
|
||||
const (
|
||||
retryBaseDelay = 150 * time.Millisecond
|
||||
retryMaxDelay = 2 * time.Second
|
||||
)
|
||||
|
||||
// executeRun composes the agent's instructions with the caller input and runs
|
||||
// one chat completion through the AI client — with a bounded retry on transient
|
||||
// upstream overload and, if the agent's own model stays throttled, ONE failover
|
||||
// to the deployment's reliable model (fallback) so an autonomous bot reply still
|
||||
// lands. It returns the resulting Run — status "ok" with output and Model set to
|
||||
// the model that ACTUALLY answered (so metering bills that model), or "error"
|
||||
// with the final upstream failure. Pure of HTTP and persistence so it is directly
|
||||
// testable; the caller records + responds. This reliability policy is the agent
|
||||
// runner's ALONE — the interactive user-facing chat path is untouched.
|
||||
func executeRun(ctx context.Context, ai types.AIClient, org string, a Agent, input, fallback string) Run {
|
||||
// Child step span; the AI client opens its own GenAI span nested under this.
|
||||
ctx, span := agentTracer.Start(ctx, "agent.step", trace.WithSpanKind(trace.SpanKindInternal))
|
||||
defer span.End()
|
||||
@@ -717,11 +751,11 @@ func executeRun(ctx context.Context, ai types.AIClient, org string, a Agent, inp
|
||||
prompt += in
|
||||
}
|
||||
start := time.Now()
|
||||
resp, aiErr := ai.ChatCompletion(ctx, &types.ChatRequest{Model: a.Model, Prompt: prompt, Org: org})
|
||||
resp, used, aiErr := completeWithFailover(ctx, ai, org, prompt, a.Model, fallback)
|
||||
dur := time.Since(start).Milliseconds()
|
||||
id, _ := genID("run")
|
||||
r := Run{
|
||||
ID: id, Org: org, AgentName: a.Name, Model: a.Model, Input: input,
|
||||
ID: id, Org: org, AgentName: a.Name, Model: used, Input: input,
|
||||
DurationMs: dur, CreatedAt: time.Now().Unix(),
|
||||
}
|
||||
if aiErr != nil {
|
||||
@@ -738,6 +772,77 @@ func executeRun(ctx context.Context, ai types.AIClient, org string, a Agent, inp
|
||||
return r
|
||||
}
|
||||
|
||||
// completeWithFailover runs the completion on the agent's model with a bounded
|
||||
// retry (completeWithRetry), then — only if that model is STILL throttled after
|
||||
// its retries — fails over ONCE to fallback, a reliable model. It returns the
|
||||
// response, the model that actually produced it (for honest metering), and the
|
||||
// final error. A non-transient failure on either model returns immediately (the
|
||||
// next model would fail identically). ONE ordered mechanism, no config sprawl.
|
||||
func completeWithFailover(ctx context.Context, ai types.AIClient, org, prompt, model, fallback string) (*types.ChatResponse, string, error) {
|
||||
models := []string{model}
|
||||
if f := strings.TrimSpace(fallback); f != "" && f != model {
|
||||
models = append(models, f)
|
||||
}
|
||||
var lastErr error
|
||||
for _, m := range models {
|
||||
resp, err := completeWithRetry(ctx, ai, org, prompt, m)
|
||||
if err == nil {
|
||||
return resp, m, nil
|
||||
}
|
||||
lastErr = err
|
||||
// Escalate to the next model ONLY on a transient overload; a hard error
|
||||
// (bad request, auth, unserved model) fails fast — failover cannot help.
|
||||
if !errors.Is(err, types.ErrUpstreamBusy) {
|
||||
return nil, m, err
|
||||
}
|
||||
}
|
||||
return nil, models[len(models)-1], lastErr
|
||||
}
|
||||
|
||||
// completeWithRetry calls the completion up to maxAttempts times, retrying ONLY a
|
||||
// transient upstream overload (types.ErrUpstreamBusy) with jittered backoff and
|
||||
// respecting context cancellation. A non-transient error returns immediately.
|
||||
func completeWithRetry(ctx context.Context, ai types.AIClient, org, prompt, model string) (*types.ChatResponse, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxAttempts; attempt++ {
|
||||
resp, err := ai.ChatCompletion(ctx, &types.ChatRequest{Model: model, Prompt: prompt, Org: org})
|
||||
if err == nil {
|
||||
return resp, nil
|
||||
}
|
||||
lastErr = err
|
||||
if !errors.Is(err, types.ErrUpstreamBusy) {
|
||||
return nil, err // permanent — do not burn retries repeating it
|
||||
}
|
||||
if attempt == maxAttempts-1 {
|
||||
break
|
||||
}
|
||||
if err := sleepBackoff(ctx, attempt); err != nil {
|
||||
return nil, err // context cancelled/expired mid-backoff
|
||||
}
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
// sleepBackoff waits an exponential, equal-jittered delay before the next
|
||||
// attempt, or returns the context error if the caller's deadline fires first.
|
||||
func sleepBackoff(ctx context.Context, attempt int) error {
|
||||
d := retryBaseDelay << attempt
|
||||
if d > retryMaxDelay {
|
||||
d = retryMaxDelay
|
||||
}
|
||||
// Equal jitter: half fixed, half random in [0, d/2) — spreads retries without
|
||||
// ever collapsing the delay to ~0 (guarantees forward progress under load).
|
||||
wait := d/2 + time.Duration(mrand.Int64N(int64(d/2)+1))
|
||||
t := time.NewTimer(wait)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-t.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func runs(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
org, ok := tenant(c)
|
||||
if !ok {
|
||||
|
||||
@@ -157,7 +157,7 @@ func TestExecuteRunOK(t *testing.T) {
|
||||
ai := &fakeAI{content: "hi there"}
|
||||
a := mk("maxpower", "greeter")
|
||||
a.Instructions = "You are a greeter."
|
||||
r := executeRun(context.Background(), ai, "maxpower", a, "say hi")
|
||||
r := executeRun(context.Background(), ai, "maxpower", a, "say hi", "")
|
||||
|
||||
if r.Status != "ok" {
|
||||
t.Fatalf("want ok, got %q err=%q", r.Status, r.Error)
|
||||
@@ -178,7 +178,7 @@ func TestExecuteRunOK(t *testing.T) {
|
||||
|
||||
func TestExecuteRunRecordsError(t *testing.T) {
|
||||
ai := &fakeAI{err: errors.New("model unavailable")}
|
||||
r := executeRun(context.Background(), ai, "maxpower", mk("maxpower", "x"), "in")
|
||||
r := executeRun(context.Background(), ai, "maxpower", mk("maxpower", "x"), "in", "")
|
||||
if r.Status != "error" {
|
||||
t.Fatalf("want error status, got %q", r.Status)
|
||||
}
|
||||
|
||||
@@ -13,8 +13,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/types"
|
||||
"github.com/hanzoai/cloud/clients/metering"
|
||||
"github.com/hanzoai/cloud/types"
|
||||
luxlog "github.com/luxfi/log"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
@@ -88,7 +88,10 @@ func mountBilled(t *testing.T, commerceURL string, ai types.AIClient) *zip.App {
|
||||
t.Fatalf("metering.New: %v", err)
|
||||
}
|
||||
app := zip.New(zip.Config{Logger: luxlog.New("test")})
|
||||
deps := cloud.Deps{Logger: luxlog.New("test"), DataDir: t.TempDir(), AI: ai, Metering: m}
|
||||
// AIFallbackModel="best" arms the agent runner's failover so the retry/failover
|
||||
// tests exercise the real escalation path; it never fires for a run whose model
|
||||
// answers (or fails non-transiently), so the other billed tests are unaffected.
|
||||
deps := cloud.Deps{Logger: luxlog.New("test"), DataDir: t.TempDir(), AI: ai, Metering: m, AIFallbackModel: "best"}
|
||||
if err := Mount(app, deps); err != nil {
|
||||
t.Fatalf("Mount: %v", err)
|
||||
}
|
||||
|
||||
@@ -2,7 +2,12 @@ package agents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
luxlog "github.com/luxfi/log"
|
||||
)
|
||||
|
||||
// mailbox.go is the LIVE hand-off between a routed run's durable owner (the
|
||||
@@ -35,6 +40,11 @@ type RoutedRun struct {
|
||||
Prompt string `json:"prompt"`
|
||||
CloneURL string `json:"cloneUrl"`
|
||||
TimeoutSeconds int `json:"timeoutSeconds,omitempty"`
|
||||
// Actor + AgentRef are CLOUD-SIDE attribution for the completion path (session
|
||||
// close + PR assignee). They are NOT part of routedRunView, so they never cross
|
||||
// to the executing machine — the machine needs neither.
|
||||
Actor string `json:"actor,omitempty"`
|
||||
AgentRef string `json:"agentRef,omitempty"`
|
||||
}
|
||||
|
||||
// RoutedResult is a routed run's terminal outcome, reported by the machine and
|
||||
@@ -96,22 +106,92 @@ type mailbox struct {
|
||||
queues map[string][]*offer
|
||||
byRun map[string]*offer
|
||||
signal map[string]chan struct{}
|
||||
// inflight is the per-org set of live routed sessions (offered, not yet finished),
|
||||
// the gauge the per-org admission cap reads. A SET keyed by session id (not a bare
|
||||
// counter) so a re-offer after a restart re-adds idempotently and a superseded
|
||||
// offer never double-counts or wrongly decrements the still-live session.
|
||||
inflight map[string]map[string]struct{}
|
||||
}
|
||||
|
||||
func newMailbox() *mailbox {
|
||||
return &mailbox{
|
||||
queues: map[string][]*offer{},
|
||||
byRun: map[string]*offer{},
|
||||
signal: map[string]chan struct{}{},
|
||||
queues: map[string][]*offer{},
|
||||
byRun: map[string]*offer{},
|
||||
signal: map[string]chan struct{}{},
|
||||
inflight: map[string]map[string]struct{}{},
|
||||
}
|
||||
}
|
||||
|
||||
func (m *mailbox) inflightAddLocked(org, sess string) {
|
||||
s := m.inflight[org]
|
||||
if s == nil {
|
||||
s = map[string]struct{}{}
|
||||
m.inflight[org] = s
|
||||
}
|
||||
s[sess] = struct{}{}
|
||||
}
|
||||
|
||||
func (m *mailbox) inflightRemoveLocked(org, sess string) {
|
||||
if s := m.inflight[org]; s != nil {
|
||||
delete(s, sess)
|
||||
if len(s) == 0 {
|
||||
delete(m.inflight, org)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// InFlight returns how many routed runs an org has live (offered, not yet finished).
|
||||
func (m *mailbox) InFlight(org string) int {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return len(m.inflight[org])
|
||||
}
|
||||
|
||||
// routedMailbox is the ONE process-wide rendezvous, shared by the coding
|
||||
// delivery activity (Offer/Await) and the machine-facing HTTP surface
|
||||
// (Claim/Report). One mailbox, one way.
|
||||
//
|
||||
// SINGLE-REPLICA DEPENDENCY (accepted, inherited). This rendezvous is IN-PROCESS: the
|
||||
// durable delivery activity (Offer/Await, on whichever replica's tasks worker polls
|
||||
// the agent-routed queue) and the external machine's POST /claim (ingress load-
|
||||
// balanced to any replica) must land on the SAME process, because the mailbox is a
|
||||
// package global, not a shared broker. cloud already runs HARD single-replica —
|
||||
// Recreate, replicas:1 — because the embedded Badger KMS holds an exclusive file lock
|
||||
// and the audit sequence is an in-memory counter (infra/k8s/operator/crs/cloud.yaml),
|
||||
// so route-work INHERITS that guarantee for free and needs no broker. assertSingleReplica
|
||||
// logs the assumption at mount and warns loudly if a multi-replica signal is present.
|
||||
//
|
||||
// IF cloud is ever made multi-replica (the KMS lock lifted): this rendezvous MUST
|
||||
// become replica-aware — either a sticky route that pins a target's /claim to the
|
||||
// replica whose worker owns its delivery, or a shared broker (the embedded NATS/
|
||||
// JetStream already in-process, keyed by (org,target)) so Offer and Claim meet
|
||||
// regardless of which replica each hits. Until then, single-replica is the contract.
|
||||
var routedMailbox = newMailbox()
|
||||
|
||||
func mbKey(org, target string) string { return org + "\x00" + target }
|
||||
// assertSingleReplica records the single-replica assumption the process-global
|
||||
// rendezvous depends on, and warns LOUDLY if a multi-replica signal is detectable
|
||||
// (CLOUD_REPLICAS > 1). It does not fail mount — cloud's replicas:1 is enforced by the
|
||||
// deployment (the KMS lock), so this is a defensive breadcrumb for the day that
|
||||
// changes, not a runtime gate. Called once from mountRouting.
|
||||
func assertSingleReplica(log luxlog.Logger) {
|
||||
if log == nil {
|
||||
return
|
||||
}
|
||||
replicas := 1
|
||||
if v := strings.TrimSpace(os.Getenv("CLOUD_REPLICAS")); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil {
|
||||
replicas = n
|
||||
}
|
||||
}
|
||||
if replicas > 1 {
|
||||
log.Warn("route-work: the routed-run rendezvous is process-global and REQUIRES cloud to run single-replica, but CLOUD_REPLICAS>1 — routed /claim will silently fail on a replica that does not own the delivery. Make the rendezvous replica-aware (sticky target route or shared broker) before scaling out.",
|
||||
"replicas", replicas)
|
||||
return
|
||||
}
|
||||
log.Info("route-work: routed-run rendezvous is in-process; assumes cloud single-replica (inherited from the KMS exclusive lock)")
|
||||
}
|
||||
|
||||
func mbKey(org, target string) string { return org + "\x00" + target }
|
||||
func runKey(org, target, sess string) string { return org + "\x00" + target + "\x00" + sess }
|
||||
|
||||
// Offer files run for its (org,target) and returns the handle its durable owner
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
// personalities.go seeds an org's BUILT-IN agents — the named personas a human
|
||||
// @-mentions in Hanzo Team (@dev to build, @des to design, @vi for vision). They
|
||||
// are ordinary rows in the ONE agent registry (this package's store): nothing
|
||||
// about them is special-cased downstream — they list, project into Team as bot
|
||||
// members (bots.go), and answer through the SAME agents.RunOnBehalf path every
|
||||
// other agent uses. The only thing this file adds is a one-time, idempotent
|
||||
// create so a fresh org has its crew without anyone POSTing them by hand.
|
||||
//
|
||||
// One and only one seed: keyed by the registry's UNIQUE(org,name), a re-seed is a
|
||||
// no-op (errConflict is swallowed). The Name is the @-handle (dev/des/vi), so the
|
||||
// Team mention resolves to the persona; Description is the human-facing title.
|
||||
|
||||
package agents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// persona is one built-in agent definition. Name is the lowercase @-handle;
|
||||
// Description is the display title; Instructions is the system prompt that gives
|
||||
// the persona its voice and remit.
|
||||
type persona struct {
|
||||
Name string
|
||||
Description string
|
||||
Instructions string
|
||||
}
|
||||
|
||||
// personalities is the canonical built-in crew. Adding one here is the ONE way a
|
||||
// new default persona ships — no per-org config, no duplicate definition. The old
|
||||
// hanzo.ai site's voices, brought into Team.
|
||||
var personalities = []persona{
|
||||
{
|
||||
Name: "dev",
|
||||
Description: "Dev — the builder",
|
||||
Instructions: "You are Dev, Hanzo's builder. You ship. When a human @-mentions you " +
|
||||
"you write the code, wire the change, and report what you did in plain terms — " +
|
||||
"file paths, commands, results. You prize the smallest correct change, one and " +
|
||||
"only one way to do a thing, and no ceremony. You never hand-wave: if you built " +
|
||||
"it you say so with proof; if you're blocked you name the blocker. Terse, exact, " +
|
||||
"and always moving toward a finished, working solution.",
|
||||
},
|
||||
{
|
||||
Name: "des",
|
||||
Description: "Des — the designer",
|
||||
Instructions: "You are Des, Hanzo's designer. You own how it looks and feels — layout, " +
|
||||
"type, color, motion, the whole experience. When a human @-mentions you, you " +
|
||||
"think in systems, not one-off screens: a token, a component, a consistent rule " +
|
||||
"that reads as one product in light and dark. You give concrete, buildable design " +
|
||||
"direction (spacing, hierarchy, states), not vague taste. Elegant, accessible, and " +
|
||||
"opinionated — you make the obvious thing beautiful.",
|
||||
},
|
||||
{
|
||||
Name: "vi",
|
||||
Description: "Vi — the visionary",
|
||||
Instructions: "You are Vi, Hanzo's visionary lead. You hold the big picture and the long " +
|
||||
"arc — where the product is going, why it matters, and what to do next to get there. " +
|
||||
"When a human @-mentions you, you connect the dots across the org, cut through noise " +
|
||||
"to the one thing that matters, and rally the crew around it. You think in bets and " +
|
||||
"outcomes, name the strategy plainly, and turn a sprawling ask into a sharp, " +
|
||||
"sequenced plan. Inspiring, decisive, and grounded in what actually ships.",
|
||||
},
|
||||
}
|
||||
|
||||
// SeedPersonalities ensures the built-in crew exists for org. Idempotent: an
|
||||
// already-present persona (UNIQUE org+name) is left untouched, so it is safe to
|
||||
// call on every org first-touch (a new Team workspace, say). Returns the number
|
||||
// newly created.
|
||||
//
|
||||
// It needs a model to attach — the deployment's configured default. With no
|
||||
// default model, seeding is a NO-OP (0, nil): an org gets its crew the moment the
|
||||
// binary has a model to run them on, never a half-created persona that can't run.
|
||||
// A subsystem that is not mounted also no-ops rather than erroring, so a caller on
|
||||
// the login path can call it best-effort without ever blocking a human.
|
||||
func SeedPersonalities(ctx context.Context, org string) (int, error) {
|
||||
if mounted == nil || mounted.State.store == nil {
|
||||
return 0, nil
|
||||
}
|
||||
org = strings.TrimSpace(org)
|
||||
if org == "" {
|
||||
return 0, nil
|
||||
}
|
||||
model := strings.TrimSpace(mounted.State.defaultModel)
|
||||
if model == "" {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
created := 0
|
||||
now := time.Now().Unix()
|
||||
for _, p := range personalities {
|
||||
id, err := genID("agent")
|
||||
if err != nil {
|
||||
return created, err
|
||||
}
|
||||
a := Agent{
|
||||
ID: id,
|
||||
Org: org,
|
||||
Name: p.Name,
|
||||
Model: model,
|
||||
Instructions: p.Instructions,
|
||||
Description: p.Description,
|
||||
Status: "ready",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
err = mounted.State.store.Create(ctx, a)
|
||||
switch {
|
||||
case err == nil:
|
||||
created++
|
||||
case errors.Is(err, errConflict):
|
||||
// Already seeded — the one-way idempotent no-op.
|
||||
default:
|
||||
return created, err
|
||||
}
|
||||
}
|
||||
return created, nil
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package agents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
luxlog "github.com/luxfi/log"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
)
|
||||
|
||||
// mountSeedTest wires the `mounted` singleton to a fresh store with a default
|
||||
// model, so SeedPersonalities has both a store to write and a model to attach.
|
||||
func mountSeedTest(t *testing.T, defaultModel string) {
|
||||
t.Helper()
|
||||
prev := mounted
|
||||
mounted = &cloud.Service[state]{
|
||||
Base: cloud.Base{Log: luxlog.New("test")},
|
||||
State: state{store: testStore(t), defaultModel: defaultModel},
|
||||
}
|
||||
t.Cleanup(func() { mounted = prev })
|
||||
}
|
||||
|
||||
// TestSeedPersonalities proves the built-in crew is created once, is idempotent,
|
||||
// projects the exact @-handles a human mentions in Team, and no-ops without a
|
||||
// model — the full contract of the one-way seed.
|
||||
func TestSeedPersonalities(t *testing.T) {
|
||||
mountSeedTest(t, "zen-1")
|
||||
ctx := context.Background()
|
||||
const org = "acme"
|
||||
|
||||
// First seed creates the whole crew.
|
||||
n, err := SeedPersonalities(ctx, org)
|
||||
if err != nil {
|
||||
t.Fatalf("SeedPersonalities: %v", err)
|
||||
}
|
||||
if n != len(personalities) {
|
||||
t.Fatalf("created %d, want %d (the full built-in crew)", n, len(personalities))
|
||||
}
|
||||
|
||||
// They are ordinary registry rows — ListForOrg returns them with the @-handles.
|
||||
list, err := ListForOrg(ctx, org)
|
||||
if err != nil {
|
||||
t.Fatalf("ListForOrg: %v", err)
|
||||
}
|
||||
got := map[string]Agent{}
|
||||
for _, a := range list {
|
||||
got[a.Name] = a
|
||||
}
|
||||
for _, want := range []string{"dev", "des", "vi"} {
|
||||
a, ok := got[want]
|
||||
if !ok {
|
||||
t.Fatalf("@%s not seeded; have %v", want, keysOf(got))
|
||||
}
|
||||
if a.Model != "zen-1" || a.Status != "ready" || a.Instructions == "" {
|
||||
t.Fatalf("@%s malformed: model=%q status=%q instr=%dB", want, a.Model, a.Status, len(a.Instructions))
|
||||
}
|
||||
}
|
||||
|
||||
// Idempotent: a re-seed creates nothing and never duplicates.
|
||||
n2, err := SeedPersonalities(ctx, org)
|
||||
if err != nil {
|
||||
t.Fatalf("re-seed: %v", err)
|
||||
}
|
||||
if n2 != 0 {
|
||||
t.Fatalf("re-seed created %d, want 0 (idempotent)", n2)
|
||||
}
|
||||
if list2, _ := ListForOrg(ctx, org); len(list2) != len(personalities) {
|
||||
t.Fatalf("after re-seed: %d agents, want %d (no dup)", len(list2), len(personalities))
|
||||
}
|
||||
|
||||
// No default model → no-op, never a half-seeded org.
|
||||
mountSeedTest(t, "")
|
||||
if n3, err := SeedPersonalities(ctx, "globex"); err != nil || n3 != 0 {
|
||||
t.Fatalf("no-model seed = (%d,%v), want (0,nil)", n3, err)
|
||||
}
|
||||
}
|
||||
|
||||
func keysOf(m map[string]Agent) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package agents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/cloud/types"
|
||||
)
|
||||
|
||||
// scriptedAI is a deterministic AIClient that models a throttled gateway: a call
|
||||
// whose model still has "busy budget" (or when allBusy is set) returns a
|
||||
// transient ErrUpstreamBusy-tagged error — exactly the shape the real httpAI
|
||||
// produces for a 429 / "Platform overloaded" — otherwise it returns content. It
|
||||
// records every call + model so a test can assert the retry count and which model
|
||||
// finally answered. It does NOT implement types.ModelLister, so create fails open
|
||||
// (any model is accepted without a catalog round-trip).
|
||||
type scriptedAI struct {
|
||||
mu sync.Mutex
|
||||
calls int
|
||||
models []string
|
||||
busy map[string]int // model -> remaining transient failures before it succeeds
|
||||
allBusy bool // every call fails transiently (failover can never win)
|
||||
content string
|
||||
}
|
||||
|
||||
func (s *scriptedAI) ChatCompletion(_ context.Context, req *types.ChatRequest) (*types.ChatResponse, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.calls++
|
||||
s.models = append(s.models, req.Model)
|
||||
if s.allBusy {
|
||||
return nil, fmt.Errorf("429 Platform overloaded: %w", types.ErrUpstreamBusy)
|
||||
}
|
||||
if n := s.busy[req.Model]; n > 0 {
|
||||
s.busy[req.Model] = n - 1
|
||||
return nil, fmt.Errorf("429 Platform overloaded: %w", types.ErrUpstreamBusy)
|
||||
}
|
||||
return &types.ChatResponse{Content: s.content}, nil
|
||||
}
|
||||
|
||||
func (s *scriptedAI) Embed(context.Context, *types.EmbedRequest) ([][]float32, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *scriptedAI) callCount() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.calls
|
||||
}
|
||||
|
||||
func (s *scriptedAI) modelsSeen() []string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return append([]string(nil), s.models...)
|
||||
}
|
||||
|
||||
// TestRunRetriesTransientThenSucceedsBillsOnce is the core reliability contract:
|
||||
// a gateway that 429s TWICE then answers must yield ONE successful run and EXACTLY
|
||||
// ONE debit — not a dropped reply (the old single-shot behavior), and not three
|
||||
// debits (retries must never bill the failed attempts; only the final success
|
||||
// bills, once).
|
||||
func TestRunRetriesTransientThenSucceedsBillsOnce(t *testing.T) {
|
||||
bs := &billServer{available: 100000}
|
||||
ai := &scriptedAI{content: "recovered", busy: map[string]int{"gpt-4o-mini": 2}}
|
||||
app := mountBilled(t, bs.start(t), ai)
|
||||
|
||||
do(t, app, http.MethodPost, "/v1/agents", "acme",
|
||||
map[string]any{"name": "a", "model": "gpt-4o-mini", "instructions": "x"})
|
||||
code, body := do(t, app, http.MethodPost, "/v1/agents/a/run", "acme", map[string]any{"input": "hi"})
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("run that recovers after 2 retries want 200, got %d (%s)", code, body)
|
||||
}
|
||||
// The reply must be the recovered content — not dropped.
|
||||
var rv runView
|
||||
_ = json.Unmarshal(body, &rv)
|
||||
if rv.Output != "recovered" {
|
||||
t.Fatalf("reply must carry the recovered output, got %q", rv.Output)
|
||||
}
|
||||
if got := ai.callCount(); got != 3 {
|
||||
t.Fatalf("want 3 completion attempts (2 busy + 1 ok), got %d", got)
|
||||
}
|
||||
// Exactly one debit for the single eventual success — never one-per-attempt.
|
||||
if !waitForDebit(func() bool { return bs.debits() == 1 }) {
|
||||
t.Fatalf("a retried-then-succeeded run must debit exactly once, got %d", bs.debits())
|
||||
}
|
||||
// Give any erroneous extra debit a chance to land; assert it did not.
|
||||
if bs.debits() != 1 {
|
||||
t.Fatalf("retries must not bill failed attempts: want 1 debit, got %d", bs.debits())
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunFailsOverToReliableModelAndBillsIt proves the escalation: the agent's
|
||||
// own model stays throttled through all its retries, so the run fails over to the
|
||||
// configured reliable model ("best"), still lands a reply, and bills the model
|
||||
// ACTUALLY used (best) — not the throttled one it started on.
|
||||
func TestRunFailsOverToReliableModelAndBillsIt(t *testing.T) {
|
||||
bs := &billServer{available: 100000}
|
||||
// gpt-4o-mini is busy for MORE than its retry budget (never recovers); "best"
|
||||
// answers first try.
|
||||
ai := &scriptedAI{content: "from-best", busy: map[string]int{"gpt-4o-mini": 99}}
|
||||
app := mountBilled(t, bs.start(t), ai)
|
||||
|
||||
do(t, app, http.MethodPost, "/v1/agents", "acme",
|
||||
map[string]any{"name": "a", "model": "gpt-4o-mini", "instructions": "x"})
|
||||
code, body := do(t, app, http.MethodPost, "/v1/agents/a/run", "acme", map[string]any{"input": "hi"})
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("failover run want 200, got %d (%s)", code, body)
|
||||
}
|
||||
var rv runView
|
||||
_ = json.Unmarshal(body, &rv)
|
||||
if rv.Output != "from-best" {
|
||||
t.Fatalf("failover reply must come from the reliable model, got %q", rv.Output)
|
||||
}
|
||||
// Primary exhausted its retries, then exactly one failover attempt on "best".
|
||||
if got := ai.callCount(); got != maxAttempts+1 {
|
||||
t.Fatalf("want %d attempts (primary exhausted + 1 failover), got %d", maxAttempts+1, got)
|
||||
}
|
||||
seen := ai.modelsSeen()
|
||||
if seen[len(seen)-1] != "best" {
|
||||
t.Fatalf("the final attempt must be the failover model, got %q", seen[len(seen)-1])
|
||||
}
|
||||
if !waitForDebit(func() bool { return bs.debits() == 1 }) {
|
||||
t.Fatalf("a failover run must debit exactly once, got %d", bs.debits())
|
||||
}
|
||||
_, ubody := bs.lastDebit()
|
||||
var u struct {
|
||||
Model string `json:"model"`
|
||||
}
|
||||
_ = json.Unmarshal(ubody, &u)
|
||||
if u.Model != "best" {
|
||||
t.Fatalf("failover run must bill the model actually used (best), got %q", u.Model)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunAllAttemptsBusyExhaustedNoDebit proves the failure contract: when EVERY
|
||||
// attempt on BOTH the agent's model and the failover model is throttled, the run
|
||||
// records a clean error (502) and NOTHING is billed — a persistent overload never
|
||||
// costs the tenant, and never fabricates a reply.
|
||||
func TestRunAllAttemptsBusyExhaustedNoDebit(t *testing.T) {
|
||||
bs := &billServer{available: 100000}
|
||||
ai := &scriptedAI{allBusy: true}
|
||||
app := mountBilled(t, bs.start(t), ai)
|
||||
|
||||
do(t, app, http.MethodPost, "/v1/agents", "acme",
|
||||
map[string]any{"name": "a", "model": "gpt-4o-mini", "instructions": "x"})
|
||||
code, _ := do(t, app, http.MethodPost, "/v1/agents/a/run", "acme", map[string]any{"input": "hi"})
|
||||
if code != http.StatusBadGateway {
|
||||
t.Fatalf("exhausted run want 502, got %d", code)
|
||||
}
|
||||
// Primary retries (maxAttempts) + failover retries (maxAttempts) were all tried.
|
||||
if got := ai.callCount(); got != 2*maxAttempts {
|
||||
t.Fatalf("want %d attempts (primary + failover, both exhausted), got %d", 2*maxAttempts, got)
|
||||
}
|
||||
if waitForDebit(func() bool { return bs.debits() > 0 }) {
|
||||
t.Fatalf("a fully-throttled run must NOT be billed, got %d debits", bs.debits())
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/clients/principal"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
@@ -38,9 +39,47 @@ const (
|
||||
// mountRouting registers the route-work machine surface. Called from mountTargets
|
||||
// AFTER the target CRUD routes so the extra-segment paths are unambiguous.
|
||||
func mountRouting(s *cloud.Service[state], app *zip.App) {
|
||||
app.Post("/v1/agents/targets/:id/claim-key", cloud.Handle(s, mintClaimKey))
|
||||
app.Post("/v1/agents/targets/:id/claim", cloud.Handle(s, claimRoutedRun))
|
||||
app.Post("/v1/agents/targets/:id/runs/:runId/report", cloud.Handle(s, reportRoutedRun))
|
||||
assertSingleReplica(s.Log)
|
||||
g := app.Group("/v1/agents")
|
||||
g.Post("/targets/:id/claim-key", cloud.Handle(s, mintClaimKey))
|
||||
g.Post("/targets/:id/claim", cloud.Handle(s, claimRoutedRun))
|
||||
g.Post("/targets/:id/runs/:runId/report", cloud.Handle(s, reportRoutedRun))
|
||||
}
|
||||
|
||||
// caller is the VALIDATED principal id (X-User-Id) — the machine-owner identity for
|
||||
// route-work. tenant() already required a validated principal, so on any handler that
|
||||
// resolved an org this is non-empty.
|
||||
func caller(c *zip.Ctx) string { return strings.TrimSpace(c.User()) }
|
||||
|
||||
// ownsTarget reports whether the caller may MANAGE this target's route-work plane —
|
||||
// mint/rotate the claim key, claim, report, patch, delete. A machine belongs to the
|
||||
// principal that registered it (least privilege, AC-6): its owner may manage it, and
|
||||
// an org admin (self-service org management, the admin-org model's isAdmin) may manage
|
||||
// any of the org's targets. An UNOWNED (pre-migration) row is admin-only until its
|
||||
// owner re-registers — register binds the owner. Fail-closed: an empty caller or an
|
||||
// empty owner never satisfies the ownership arm, so a non-validated request or a
|
||||
// pre-migration row is never owner-managed.
|
||||
func ownsTarget(c *zip.Ctx, t Target) bool {
|
||||
if principal.IsOrgAdmin(c) || principal.IsSuperAdmin(c) {
|
||||
return true
|
||||
}
|
||||
u := caller(c)
|
||||
return t.Owner != "" && u != "" && u == t.Owner
|
||||
}
|
||||
|
||||
// authorizeTargetManage resolves the (org,id) target and gates it on ownsTarget,
|
||||
// collapsing every failure — cross-org, unknown, or not-owned — to the SAME
|
||||
// errTargetNotFound so the machine surface never distinguishes them (no oracle). The
|
||||
// resolved target is returned for the caller to use (avoids a second read).
|
||||
func authorizeTargetManage(s *cloud.Service[state], c *zip.Ctx, org, id string) (Target, error) {
|
||||
t, err := s.State.store.GetTarget(c.Context(), org, id)
|
||||
if err != nil {
|
||||
return Target{}, errTargetNotFound // unknown / cross-org -> no oracle
|
||||
}
|
||||
if !ownsTarget(c, t) {
|
||||
return Target{}, errTargetNotFound
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// mintClaimKey (re)mints the target's claim key and returns it ONCE. Only the
|
||||
@@ -52,11 +91,12 @@ func mintClaimKey(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
return zip.ErrForbidden("X-Org-Id required")
|
||||
}
|
||||
id := idParam(c)
|
||||
// The target must exist in this org before it can carry a capability.
|
||||
if _, err := s.State.store.GetTarget(c.Context(), org, id); err == errTargetNotFound {
|
||||
// The target must exist in this org AND the caller must OWN it (or be an org
|
||||
// admin) before it can (re)mint a capability — minting rotates the key, so an
|
||||
// un-scoped mint would let any org member strand a victim's daemon and steal its
|
||||
// runs. Every failure collapses to the same not-found (no oracle).
|
||||
if _, err := authorizeTargetManage(s, c, org, id); err != nil {
|
||||
return zip.ErrNotFound("target not found")
|
||||
} else if err != nil {
|
||||
return zip.Errorf(http.StatusInternalServerError, "target: %v", err)
|
||||
}
|
||||
key, err := newClaimKey()
|
||||
if err != nil {
|
||||
@@ -77,6 +117,14 @@ func claimRoutedRun(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
return zip.ErrForbidden("X-Org-Id required")
|
||||
}
|
||||
id := idParam(c)
|
||||
// TWO proofs, both required, both fail-closed to the SAME 403 (no oracle): the
|
||||
// caller must OWN this machine (or be an org admin) AND hold its claim key. The
|
||||
// ownership gate is defense in depth — with mint owner-scoped an attacker cannot
|
||||
// obtain a valid key for a victim's machine, but a claim still refuses a
|
||||
// non-owner outright rather than resting solely on the capability.
|
||||
if _, err := authorizeTargetManage(s, c, org, id); err != nil {
|
||||
return claimAuthError(errTargetNotFound)
|
||||
}
|
||||
if err := s.State.store.verifyClaimKey(c.Context(), org, id, c.Header(claimKeyHeader)); err != nil {
|
||||
return claimAuthError(err)
|
||||
}
|
||||
@@ -115,6 +163,11 @@ func reportRoutedRun(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
}
|
||||
id := idParam(c)
|
||||
runID := strings.TrimSpace(c.Param("runId"))
|
||||
// Same two proofs as claim: own the machine (or org admin) AND hold its key, so a
|
||||
// non-owner can neither fabricate a report nor complete a victim's run. No oracle.
|
||||
if _, err := authorizeTargetManage(s, c, org, id); err != nil {
|
||||
return claimAuthError(errTargetNotFound)
|
||||
}
|
||||
if err := s.State.store.verifyClaimKey(c.Context(), org, id, c.Header(claimKeyHeader)); err != nil {
|
||||
return claimAuthError(err)
|
||||
}
|
||||
|
||||
@@ -40,13 +40,17 @@ func registerAndMint(t *testing.T, app *zip.App, org, host string) (string, stri
|
||||
if code != 201 && code != 200 {
|
||||
t.Fatalf("register target: %d %s", code, body)
|
||||
}
|
||||
var tv struct{ ID string `json:"id"` }
|
||||
var tv struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &tv)
|
||||
code, body = doKey(t, app, "POST", "/v1/agents/targets/"+tv.ID+"/claim-key", org, "")
|
||||
if code != 200 {
|
||||
t.Fatalf("mint claim key: %d %s", code, body)
|
||||
}
|
||||
var kv struct{ ClaimKey string `json:"claimKey"` }
|
||||
var kv struct {
|
||||
ClaimKey string `json:"claimKey"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &kv)
|
||||
if kv.ClaimKey == "" {
|
||||
t.Fatal("claim key empty")
|
||||
@@ -105,7 +109,9 @@ func TestClaim_CrossMachineAndCrossOrgDenied(t *testing.T) {
|
||||
if code != 200 {
|
||||
t.Fatalf("acme must claim its own run, got %d %s", code, body)
|
||||
}
|
||||
var rv struct{ SessionID string `json:"sessionId"` }
|
||||
var rv struct {
|
||||
SessionID string `json:"sessionId"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &rv)
|
||||
if rv.SessionID != "sess_a" {
|
||||
t.Fatalf("claimed wrong run: %s", body)
|
||||
@@ -248,3 +254,162 @@ func TestClaimKey_HashedAtRestAndVerified(t *testing.T) {
|
||||
t.Fatalf("cross-org verify must fail closed, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- M2: owner/machine scoping of the claim-key plane ----
|
||||
|
||||
// reqAs sends a request with an EXPLICIT principal (X-User-Id) + optional org-admin
|
||||
// bit and claim key + JSON body, so a test can prove owner/admin scoping distinct
|
||||
// from org scoping.
|
||||
func reqAs(t *testing.T, app *zip.App, method, path, org, user string, admin bool, key string, body any) (int, []byte) {
|
||||
t.Helper()
|
||||
var r io.Reader
|
||||
if body != nil {
|
||||
b, _ := json.Marshal(body)
|
||||
r = bytes.NewReader(b)
|
||||
}
|
||||
req := httptest.NewRequest(method, path, r)
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if org != "" {
|
||||
req.Header.Set("X-Org-Id", org)
|
||||
}
|
||||
if user != "" {
|
||||
req.Header.Set("X-User-Id", user)
|
||||
}
|
||||
if admin {
|
||||
req.Header.Set("X-User-IsOrgAdmin", "true")
|
||||
}
|
||||
if key != "" {
|
||||
req.Header.Set(claimKeyHeader, key)
|
||||
}
|
||||
resp, err := app.Fiber().Test(req)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %s %s: %v", method, path, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
out, _ := io.ReadAll(resp.Body)
|
||||
return resp.StatusCode, out
|
||||
}
|
||||
|
||||
// registerAs registers a target in org OWNED by the given principal, returning its id.
|
||||
func registerAs(t *testing.T, app *zip.App, org, user, host string) string {
|
||||
t.Helper()
|
||||
code, body := reqAs(t, app, "POST", "/v1/agents/targets", org, user, false, "", map[string]any{"label": host, "host": host})
|
||||
if code != 201 && code != 200 {
|
||||
t.Fatalf("register target: %d %s", code, body)
|
||||
}
|
||||
var tv struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &tv)
|
||||
if tv.ID == "" {
|
||||
t.Fatalf("register returned no id: %s", body)
|
||||
}
|
||||
return tv.ID
|
||||
}
|
||||
|
||||
// A machine belongs to the principal that registered it: a DIFFERENT member of the
|
||||
// SAME org can neither mint/rotate its claim key, nor patch, nor delete it — only its
|
||||
// owner or an org admin can. Every refusal collapses to not-found (no oracle).
|
||||
func TestClaimKeyPlane_OwnerScoped(t *testing.T) {
|
||||
app := mountApp(t, &fakeAI{content: "x"})
|
||||
id := registerAs(t, app, "acme", "alice", "evo")
|
||||
|
||||
// A non-owner member of the same org is DENIED on every management verb.
|
||||
if code, _ := reqAs(t, app, "POST", "/v1/agents/targets/"+id+"/claim-key", "acme", "mallory", false, "", nil); code != 404 {
|
||||
t.Fatalf("non-owner mint must be denied (no oracle -> 404), got %d", code)
|
||||
}
|
||||
if code, _ := reqAs(t, app, "PATCH", "/v1/agents/targets/"+id, "acme", "mallory", false, "", map[string]any{"status": TargetOffline}); code != 404 {
|
||||
t.Fatalf("non-owner patch must be denied, got %d", code)
|
||||
}
|
||||
if code, _ := reqAs(t, app, "DELETE", "/v1/agents/targets/"+id, "acme", "mallory", false, "", nil); code != 404 {
|
||||
t.Fatalf("non-owner delete must be denied, got %d", code)
|
||||
}
|
||||
|
||||
// The OWNER can mint.
|
||||
code, body := reqAs(t, app, "POST", "/v1/agents/targets/"+id+"/claim-key", "acme", "alice", false, "", nil)
|
||||
if code != 200 {
|
||||
t.Fatalf("owner mint must succeed, got %d %s", code, body)
|
||||
}
|
||||
|
||||
// An ORG ADMIN (self-service org management) can manage any of the org's machines.
|
||||
if code, _ := reqAs(t, app, "POST", "/v1/agents/targets/"+id+"/claim-key", "acme", "boss", true, "", nil); code != 200 {
|
||||
t.Fatalf("org admin mint must succeed, got %d", code)
|
||||
}
|
||||
if code, _ := reqAs(t, app, "PATCH", "/v1/agents/targets/"+id, "acme", "boss", true, "", map[string]any{"status": TargetOnline}); code != 200 {
|
||||
t.Fatalf("org admin patch must succeed, got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// A non-owner cannot CLAIM a victim's runs or REPORT on them, even if the claim-key
|
||||
// authorization were somehow satisfied — the ownership gate refuses first.
|
||||
func TestClaimReport_OwnerScoped(t *testing.T) {
|
||||
app := mountApp(t, &fakeAI{content: "x"})
|
||||
old := claimLongPoll
|
||||
claimLongPoll = 150 * time.Millisecond
|
||||
defer func() { claimLongPoll = old }()
|
||||
|
||||
id := registerAs(t, app, "acme", "alice", "evo")
|
||||
// Alice mints her machine's key.
|
||||
code, body := reqAs(t, app, "POST", "/v1/agents/targets/"+id+"/claim-key", "acme", "alice", false, "", nil)
|
||||
if code != 200 {
|
||||
t.Fatalf("owner mint: %d %s", code, body)
|
||||
}
|
||||
var kv struct {
|
||||
ClaimKey string `json:"claimKey"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &kv)
|
||||
|
||||
// Mallory (same org, not the owner) with the RIGHT key is still refused: she does
|
||||
// not own the machine. (In practice she cannot obtain the key, since mint is
|
||||
// owner-scoped — this is the defense-in-depth arm.)
|
||||
if code, _ := reqAs(t, app, "POST", "/v1/agents/targets/"+id+"/claim", "acme", "mallory", false, kv.ClaimKey, nil); code != 403 {
|
||||
t.Fatalf("non-owner claim must be 403, got %d", code)
|
||||
}
|
||||
if code, _ := reqAs(t, app, "POST", "/v1/agents/targets/"+id+"/runs/sess_x/report", "acme", "mallory", false, kv.ClaimKey, map[string]any{"ok": true}); code != 403 {
|
||||
t.Fatalf("non-owner report must be 403, got %d", code)
|
||||
}
|
||||
|
||||
// The owner with her key claims (no work -> 204) and reports fine.
|
||||
if code, _ := reqAs(t, app, "POST", "/v1/agents/targets/"+id+"/claim", "acme", "alice", false, kv.ClaimKey, nil); code != 204 {
|
||||
t.Fatalf("owner claim (no work) must be 204, got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// A pre-migration UNOWNED target (owner=”) is admin-only, and its owner heals it by
|
||||
// re-registering (register binds the owner). Proven at the store + handler seam.
|
||||
func TestUnownedTarget_AdminOnly_ThenBoundByRegister(t *testing.T) {
|
||||
app := mountApp(t, &fakeAI{content: "x"})
|
||||
ctx := context.Background()
|
||||
now := time.Now().Unix()
|
||||
// Seed an unowned row directly, as an upgraded pre-owner DB would carry.
|
||||
if err := mounted.State.store.CreateTarget(ctx, Target{ID: "tgt_legacy", Org: "acme", Owner: "", Label: "old", Kind: TargetMachine, Status: TargetOnline, Host: "old", CreatedAt: now, UpdatedAt: now}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A plain member cannot mint on an unowned row.
|
||||
if code, _ := reqAs(t, app, "POST", "/v1/agents/targets/tgt_legacy/claim-key", "acme", "alice", false, "", nil); code != 404 {
|
||||
t.Fatalf("unowned row must be member-denied, got %d", code)
|
||||
}
|
||||
// An org admin can.
|
||||
if code, _ := reqAs(t, app, "POST", "/v1/agents/targets/tgt_legacy/claim-key", "acme", "boss", true, "", nil); code != 200 {
|
||||
t.Fatalf("unowned row must be admin-manageable, got %d", code)
|
||||
}
|
||||
// The owner heals it by re-registering the SAME host — register ADOPTS the
|
||||
// unowned row and BINDS the owner (no duplicate).
|
||||
code, body := reqAs(t, app, "POST", "/v1/agents/targets", "acme", "alice", false, "", map[string]any{"label": "old", "host": "old"})
|
||||
if code != 200 && code != 201 {
|
||||
t.Fatalf("re-register: %d %s", code, body)
|
||||
}
|
||||
var tv struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
_ = json.Unmarshal(body, &tv)
|
||||
if tv.ID != "tgt_legacy" {
|
||||
t.Fatalf("re-register must adopt the unowned row (same id), got %q", tv.ID)
|
||||
}
|
||||
// Now alice (the bound owner) can mint.
|
||||
if code, _ := reqAs(t, app, "POST", "/v1/agents/targets/tgt_legacy/claim-key", "acme", "alice", false, "", nil); code != 200 {
|
||||
t.Fatalf("after binding, the owner must be able to mint, got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,9 +48,9 @@ func (c *countingAI) Embed(_ context.Context, _ *types.EmbedRequest) ([][]float3
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// schedSvc builds a Service + scheduler with NO billing (gate allows) and the given
|
||||
// schedService builds a Service + scheduler with NO billing (gate allows) and the given
|
||||
// AI, seeded with the supplied agents. Returns the scheduler for direct tick().
|
||||
func schedSvc(t *testing.T, ai types.AIClient, seed ...Agent) *scheduler {
|
||||
func schedService(t *testing.T, ai types.AIClient, seed ...Agent) *scheduler {
|
||||
t.Helper()
|
||||
s := &cloud.Service[state]{Base: cloud.Base{Log: luxlog.New("test")}, State: state{store: testStore(t), ai: ai}}
|
||||
for _, a := range seed {
|
||||
@@ -83,7 +83,7 @@ func waitFor(cond func() bool) bool {
|
||||
// run; a tick at a non-matching minute launches none.
|
||||
func TestSchedulerFiresDueAgent(t *testing.T) {
|
||||
ai := &countingAI{}
|
||||
sc := schedSvc(t, ai, longRunning("acme", "cron", "*/5 * * * *"))
|
||||
sc := schedService(t, ai, longRunning("acme", "cron", "*/5 * * * *"))
|
||||
ctx := context.Background()
|
||||
|
||||
sc.tick(ctx, at(t, "2026-07-01 12:36")) // 36 not multiple of 5 -> no fire
|
||||
@@ -102,7 +102,7 @@ func TestSchedulerFiresDueAgent(t *testing.T) {
|
||||
// like an HTTP run — the scheduler shares runAgent.
|
||||
func TestSchedulerRecordsRun(t *testing.T) {
|
||||
ai := &countingAI{}
|
||||
sc := schedSvc(t, ai, longRunning("acme", "cron", "* * * * *"))
|
||||
sc := schedService(t, ai, longRunning("acme", "cron", "* * * * *"))
|
||||
ctx := context.Background()
|
||||
sc.tick(ctx, at(t, "2026-07-01 12:00"))
|
||||
if !waitFor(func() bool {
|
||||
@@ -119,7 +119,7 @@ func TestSchedulerRecordsRun(t *testing.T) {
|
||||
// tick fires; the immediately-following matching tick is skipped (backoff=1).
|
||||
func TestSchedulerBackoffOnFailure(t *testing.T) {
|
||||
ai := &countingAI{fail: true}
|
||||
sc := schedSvc(t, ai, longRunning("acme", "cron", "* * * * *"))
|
||||
sc := schedService(t, ai, longRunning("acme", "cron", "* * * * *"))
|
||||
ctx := context.Background()
|
||||
|
||||
sc.tick(ctx, at(t, "2026-07-01 12:00"))
|
||||
@@ -152,7 +152,7 @@ func TestSchedulerBackoffOnFailure(t *testing.T) {
|
||||
// second matching tick while it is in flight does NOT start a second run.
|
||||
func TestSchedulerConcurrencyCap(t *testing.T) {
|
||||
ai := &countingAI{block: make(chan struct{})}
|
||||
sc := schedSvc(t, ai, longRunning("acme", "cron", "* * * * *"))
|
||||
sc := schedService(t, ai, longRunning("acme", "cron", "* * * * *"))
|
||||
ctx := context.Background()
|
||||
|
||||
sc.tick(ctx, at(t, "2026-07-01 12:00")) // starts run #1, which blocks
|
||||
@@ -257,7 +257,7 @@ func TestSchedulerGatesUnfundedRun(t *testing.T) {
|
||||
// path that lets Shutdown close the store safely.
|
||||
func TestSchedulerStopDrainsCleanly(t *testing.T) {
|
||||
ai := &countingAI{}
|
||||
sc := schedSvc(t, ai, longRunning("acme", "cron", "* * * * *"))
|
||||
sc := schedService(t, ai, longRunning("acme", "cron", "* * * * *"))
|
||||
sc.start()
|
||||
// Fire one run via a direct tick, then stop — stop must return after drain.
|
||||
sc.tick(context.Background(), at(t, "2026-07-01 12:00"))
|
||||
@@ -286,7 +286,7 @@ func TestSchedulerStopDrainsCleanly(t *testing.T) {
|
||||
// rather than waiting the full runTimeout.
|
||||
func TestSchedulerStopHonorsDeadline(t *testing.T) {
|
||||
ai := &countingAI{block: make(chan struct{})}
|
||||
sc := schedSvc(t, ai, longRunning("acme", "cron", "* * * * *"))
|
||||
sc := schedService(t, ai, longRunning("acme", "cron", "* * * * *"))
|
||||
sc.start()
|
||||
sc.tick(context.Background(), at(t, "2026-07-01 12:00"))
|
||||
if !waitFor(func() bool { return ai.count() == 1 }) {
|
||||
@@ -310,7 +310,7 @@ func TestSchedulerStopHonorsDeadline(t *testing.T) {
|
||||
func TestSchedulerOnlyLongRunning(t *testing.T) {
|
||||
ai := &countingAI{}
|
||||
one := mk("acme", "one") // one-shot default, no schedule
|
||||
sc := schedSvc(t, ai, one)
|
||||
sc := schedService(t, ai, one)
|
||||
sc.tick(context.Background(), at(t, "2026-07-01 12:00"))
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
if ai.count() != 0 {
|
||||
|
||||
+13
-11
@@ -182,17 +182,19 @@ func toEventView(e Event) eventView {
|
||||
// :name would otherwise capture "sessions"). Within the block, the static
|
||||
// /stream route precedes the /:id param for the same reason.
|
||||
func mountSessions(s *cloud.Service[state], app *zip.App) {
|
||||
app.Post("/v1/agents/sessions", cloud.Handle(s, registerSession))
|
||||
app.Get("/v1/agents/sessions", cloud.Handle(s, listSessions))
|
||||
app.Get("/v1/agents/sessions/stream", cloud.Handle(s, sessionsStream))
|
||||
app.Get("/v1/agents/sessions/:id", cloud.Handle(s, getSession))
|
||||
app.Patch("/v1/agents/sessions/:id", cloud.Handle(s, patchSession))
|
||||
app.Get("/v1/agents/sessions/:id/tree", cloud.Handle(s, sessionTree))
|
||||
app.Post("/v1/agents/sessions/:id/events", cloud.Handle(s, appendSessionEvent))
|
||||
app.Post("/v1/agents/sessions/:id/pause", cloud.Handle(s, pauseSession))
|
||||
app.Post("/v1/agents/sessions/:id/resume", cloud.Handle(s, resumeSession))
|
||||
app.Post("/v1/agents/sessions/:id/stop", cloud.Handle(s, stopSession))
|
||||
app.Post("/v1/agents/sessions/:id/message", cloud.Handle(s, messageSession))
|
||||
g := app.Group("/v1/agents")
|
||||
g.Post("/sessions", cloud.Handle(s, registerSession))
|
||||
g.Get("/sessions", cloud.Handle(s, listSessions))
|
||||
g.Get("/sessions/stream", cloud.Handle(s, sessionsStream))
|
||||
g.Get("/sessions/:id", cloud.Handle(s, getSession))
|
||||
g.Patch("/sessions/:id", cloud.Handle(s, patchSession))
|
||||
g.Get("/sessions/:id/tree", cloud.Handle(s, sessionTree))
|
||||
g.Post("/sessions/:id/events", cloud.Handle(s, appendSessionEvent))
|
||||
g.Get("/sessions/:id/control", cloud.Handle(s, drainControl))
|
||||
g.Post("/sessions/:id/pause", cloud.Handle(s, pauseSession))
|
||||
g.Post("/sessions/:id/resume", cloud.Handle(s, resumeSession))
|
||||
g.Post("/sessions/:id/stop", cloud.Handle(s, stopSession))
|
||||
g.Post("/sessions/:id/message", cloud.Handle(s, messageSession))
|
||||
}
|
||||
|
||||
func idParam(c *zip.Ctx) string { return strings.TrimSpace(c.Param("id")) }
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
package agents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// sessions_control_drain.go is the CLI-facing half of the control channel. The
|
||||
// dashboard POSTs steering commands (pause/resume/stop/message) which `control`
|
||||
// records as durable KindControl events; a locally-started `hanzo code` session
|
||||
// is NOT task-backed, so those events are never forwarded to a tasks engine —
|
||||
// the running surface consumes them itself by polling here.
|
||||
//
|
||||
// GET /v1/agents/sessions/:id/control?after=<seq> returns the control commands
|
||||
// newer than the caller's cursor, oldest first, with a cursor to poll from next.
|
||||
// It is READ-ONLY and org-scoped (org is the ONLY tenant key): the same validated
|
||||
// principal + same-org ownership that guards the session guards this, so a poller
|
||||
// only ever drains its OWN session's commands and a foreign id is a clean 404.
|
||||
|
||||
// controlCommandView is one steering command as the CLI consumes it.
|
||||
type controlCommandView struct {
|
||||
Seq int64 `json:"seq"`
|
||||
Command string `json:"command"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Payload json.RawMessage `json:"payload,omitempty"`
|
||||
}
|
||||
|
||||
// drainControl returns KindControl commands with seq > ?after for the caller's
|
||||
// own session. Bounded (200/poll) and cursor-driven, so a steady poll is cheap
|
||||
// and never redelivers an applied command.
|
||||
func drainControl(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
org, ok := tenant(c)
|
||||
if !ok {
|
||||
return zip.ErrForbidden("X-Org-Id required")
|
||||
}
|
||||
id := idParam(c)
|
||||
if _, err := s.State.store.GetSession(c.Context(), org, id); err == errSessionNotFound {
|
||||
return zip.ErrNotFound("session not found")
|
||||
} else if err != nil {
|
||||
return zip.Errorf(http.StatusInternalServerError, "get: %v", err)
|
||||
}
|
||||
|
||||
after := int64(0)
|
||||
if q := strings.TrimSpace(c.Query("after")); q != "" {
|
||||
if n, perr := strconv.ParseInt(q, 10, 64); perr == nil && n >= 0 {
|
||||
after = n
|
||||
}
|
||||
}
|
||||
|
||||
evs, err := s.State.store.ListControlAfter(c.Context(), org, id, after, 200)
|
||||
if err != nil {
|
||||
return zip.Errorf(http.StatusInternalServerError, "drain control: %v", err)
|
||||
}
|
||||
|
||||
cursor := after
|
||||
cmds := make([]controlCommandView, 0, len(evs))
|
||||
for _, e := range evs {
|
||||
var cp controlPayload
|
||||
_ = json.Unmarshal([]byte(e.Payload), &cp)
|
||||
cmds = append(cmds, controlCommandView{
|
||||
Seq: e.Seq,
|
||||
Command: cp.Command,
|
||||
Message: cp.Message,
|
||||
Payload: cp.Payload,
|
||||
})
|
||||
if e.Seq > cursor {
|
||||
cursor = e.Seq
|
||||
}
|
||||
}
|
||||
return c.JSON(http.StatusOK, map[string]any{"commands": cmds, "cursor": cursor})
|
||||
}
|
||||
|
||||
// ListControlAfter returns a session's KindControl events with seq > since,
|
||||
// oldest first — the durable steering queue a running surface drains. Mirrors
|
||||
// ListEvents but filters to control so a chatty session's message/log/tool-call
|
||||
// events never dilute a poll.
|
||||
func (s *Store) ListControlAfter(ctx context.Context, org, sessionID string, since int64, limit int) ([]Event, error) {
|
||||
if limit <= 0 || limit > 1000 {
|
||||
limit = 200
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx,
|
||||
`SELECT id,session_id,org,seq,kind,actor,payload,created_at
|
||||
FROM agent_session_events WHERE org=? AND session_id=? AND kind=? AND seq>?
|
||||
ORDER BY seq ASC LIMIT ?`, org, sessionID, KindControl, since, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var out []Event
|
||||
for rows.Next() {
|
||||
var e Event
|
||||
if err := rows.Scan(&e.ID, &e.SessionID, &e.Org, &e.Seq, &e.Kind, &e.Actor,
|
||||
&e.Payload, &e.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package agents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestListControlAfterFiltersCursorsAndScopes proves the CLI drain query: it
|
||||
// returns ONLY control events, oldest first, honours the ?after cursor so an
|
||||
// applied command is never redelivered, and is org-scoped so no co-tenant can
|
||||
// drain another org's steering queue.
|
||||
func TestListControlAfterFiltersCursorsAndScopes(t *testing.T) {
|
||||
s := testSessionStore(t)
|
||||
ctx := context.Background()
|
||||
if err := s.CreateSession(ctx, mkSession("acme", "root", "", "root")); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
// Interleave non-control noise with two control commands.
|
||||
_, _ = s.AppendEvent(ctx, Event{ID: genIDMust(t), SessionID: "root", Org: "acme", Kind: KindLog, CreatedAt: now})
|
||||
c1, err := s.AppendEvent(ctx, Event{ID: genIDMust(t), SessionID: "root", Org: "acme", Kind: KindControl, Payload: `{"command":"pause"}`, CreatedAt: now})
|
||||
if err != nil {
|
||||
t.Fatalf("append c1: %v", err)
|
||||
}
|
||||
_, _ = s.AppendEvent(ctx, Event{ID: genIDMust(t), SessionID: "root", Org: "acme", Kind: KindMessage, CreatedAt: now})
|
||||
c2, err := s.AppendEvent(ctx, Event{ID: genIDMust(t), SessionID: "root", Org: "acme", Kind: KindControl, Payload: `{"command":"stop"}`, CreatedAt: now})
|
||||
if err != nil {
|
||||
t.Fatalf("append c2: %v", err)
|
||||
}
|
||||
|
||||
// From the start: exactly the two control events, in seq order, no noise.
|
||||
got, err := s.ListControlAfter(ctx, "acme", "root", 0, 200)
|
||||
if err != nil {
|
||||
t.Fatalf("drain: %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("want 2 control events, got %d", len(got))
|
||||
}
|
||||
if got[0].Seq != c1.Seq || got[1].Seq != c2.Seq {
|
||||
t.Fatalf("want ordered [%d,%d], got [%d,%d]", c1.Seq, c2.Seq, got[0].Seq, got[1].Seq)
|
||||
}
|
||||
for _, e := range got {
|
||||
if e.Kind != KindControl {
|
||||
t.Fatalf("non-control kind leaked into drain: %s", e.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
// Cursor past the first command yields only the second (no redelivery).
|
||||
got2, err := s.ListControlAfter(ctx, "acme", "root", c1.Seq, 200)
|
||||
if err != nil {
|
||||
t.Fatalf("drain after cursor: %v", err)
|
||||
}
|
||||
if len(got2) != 1 || got2[0].Seq != c2.Seq {
|
||||
t.Fatalf("cursor drain want [%d], got %+v", c2.Seq, got2)
|
||||
}
|
||||
|
||||
// Tenant isolation: another org drains nothing from acme's session.
|
||||
evil, err := s.ListControlAfter(ctx, "evil", "root", 0, 200)
|
||||
if err != nil {
|
||||
t.Fatalf("evil drain: %v", err)
|
||||
}
|
||||
if len(evil) != 0 {
|
||||
t.Fatalf("cross-tenant control leak: %d events", len(evil))
|
||||
}
|
||||
}
|
||||
+82
-20
@@ -87,6 +87,7 @@ var errTargetNotFound = errors.New("agents: target not found")
|
||||
type Target struct {
|
||||
ID string
|
||||
Org string
|
||||
Owner string // the VALIDATED principal (c.User()) that registered this machine; "" for a pre-migration row
|
||||
Label string
|
||||
Kind string // laptop | cloud | gpu | cluster | machine
|
||||
Status string // online | offline | draining
|
||||
@@ -112,6 +113,7 @@ func (s *Store) migrateTargets() error {
|
||||
CREATE TABLE IF NOT EXISTS agent_targets (
|
||||
id TEXT PRIMARY KEY,
|
||||
org TEXT NOT NULL,
|
||||
owner TEXT NOT NULL DEFAULT '',
|
||||
label TEXT NOT NULL DEFAULT '',
|
||||
kind TEXT NOT NULL DEFAULT 'machine',
|
||||
status TEXT NOT NULL DEFAULT 'online',
|
||||
@@ -129,9 +131,12 @@ CREATE INDEX IF NOT EXISTS ix_targets_org_created ON agent_targets(org, created_
|
||||
return fmt.Errorf("migrate targets: %w", err)
|
||||
}
|
||||
// Forward, idempotent upgrade for target rows created before the capability +
|
||||
// metrics columns existed. PRAGMA-guarded, so re-running on an upgraded DB is a
|
||||
// no-op — the DDL above covers fresh installs, this covers pre-existing ones.
|
||||
// metrics + owner columns existed. PRAGMA-guarded, so re-running on an upgraded
|
||||
// DB is a no-op — the DDL above covers fresh installs, this covers pre-existing
|
||||
// ones. A pre-owner row backfills owner='' (unowned) and is admin-only until its
|
||||
// owner re-registers (register binds the owner) — see registerTarget.
|
||||
if err := s.addColumns("agent_targets", map[string]string{
|
||||
"owner": "TEXT NOT NULL DEFAULT ''",
|
||||
"spec": "TEXT NOT NULL DEFAULT ''",
|
||||
"metrics": "TEXT NOT NULL DEFAULT ''",
|
||||
"metrics_at": "INTEGER NOT NULL DEFAULT 0",
|
||||
@@ -141,12 +146,12 @@ CREATE INDEX IF NOT EXISTS ix_targets_org_created ON agent_targets(org, created_
|
||||
return nil
|
||||
}
|
||||
|
||||
const targetCols = `id,org,label,kind,status,capacity,host,spec,metrics,metrics_at,created_at,updated_at`
|
||||
const targetCols = `id,org,owner,label,kind,status,capacity,host,spec,metrics,metrics_at,created_at,updated_at`
|
||||
|
||||
func scanTarget(sc interface{ Scan(...any) error }) (Target, error) {
|
||||
var t Target
|
||||
var spec, metrics string
|
||||
err := sc.Scan(&t.ID, &t.Org, &t.Label, &t.Kind, &t.Status, &t.Capacity, &t.Host,
|
||||
err := sc.Scan(&t.ID, &t.Org, &t.Owner, &t.Label, &t.Kind, &t.Status, &t.Capacity, &t.Host,
|
||||
&spec, &metrics, &t.MetricsAt, &t.CreatedAt, &t.UpdatedAt)
|
||||
if err != nil {
|
||||
return t, err
|
||||
@@ -159,8 +164,8 @@ func scanTarget(sc interface{ Scan(...any) error }) (Target, error) {
|
||||
// CreateTarget inserts one target. The id is caller-generated (genID("tgt")).
|
||||
func (s *Store) CreateTarget(ctx context.Context, t Target) error {
|
||||
_, err := s.db.ExecContext(ctx,
|
||||
`INSERT INTO agent_targets (`+targetCols+`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
t.ID, t.Org, t.Label, t.Kind, t.Status, t.Capacity, t.Host,
|
||||
`INSERT INTO agent_targets (`+targetCols+`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
t.ID, t.Org, t.Owner, t.Label, t.Kind, t.Status, t.Capacity, t.Host,
|
||||
encodeSpec(t.Spec), encodeMetrics(t.Metrics), t.MetricsAt, t.CreatedAt, t.UpdatedAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert target: %w", err)
|
||||
@@ -203,12 +208,14 @@ func (s *Store) ListTargets(ctx context.Context, org string) ([]Target, error) {
|
||||
}
|
||||
|
||||
// UpdateTarget persists mutable fields for an existing (org,id) target. Scoped by org
|
||||
// so a cross-tenant id can never mutate another's target.
|
||||
// so a cross-tenant id can never mutate another's target. owner is persisted too so a
|
||||
// relink can BIND a previously-unowned row (registerTarget) and a patch preserves the
|
||||
// owner it read; no client-facing patch field sets owner, so it never moves by mutation.
|
||||
func (s *Store) UpdateTarget(ctx context.Context, t Target) error {
|
||||
res, err := s.db.ExecContext(ctx,
|
||||
`UPDATE agent_targets SET label=?, kind=?, status=?, capacity=?, host=?, spec=?, metrics=?, metrics_at=?, updated_at=?
|
||||
`UPDATE agent_targets SET owner=?, label=?, kind=?, status=?, capacity=?, host=?, spec=?, metrics=?, metrics_at=?, updated_at=?
|
||||
WHERE org=? AND id=?`,
|
||||
t.Label, t.Kind, t.Status, t.Capacity, t.Host,
|
||||
t.Owner, t.Label, t.Kind, t.Status, t.Capacity, t.Host,
|
||||
encodeSpec(t.Spec), encodeMetrics(t.Metrics), t.MetricsAt, t.UpdatedAt, t.Org, t.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("update target: %w", err)
|
||||
@@ -220,6 +227,32 @@ func (s *Store) UpdateTarget(ctx context.Context, t Target) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetLinkableTargetByHost returns the target for (org,host) that the caller `owner`
|
||||
// may re-link — its OWN row, else an UNOWNED (pre-migration) row it may adopt —
|
||||
// preferring the exact-owner match, newest first. A row owned by a DIFFERENT
|
||||
// principal is NEVER returned, so a re-link can never clobber another member's
|
||||
// machine: the caller gets its own row or (falling through in registerTarget) a fresh
|
||||
// one. errTargetNotFound when nothing linkable exists.
|
||||
func (s *Store) GetLinkableTargetByHost(ctx context.Context, org, host, owner string) (Target, error) {
|
||||
host = strings.TrimSpace(host)
|
||||
if host == "" {
|
||||
return Target{}, errTargetNotFound
|
||||
}
|
||||
row := s.db.QueryRowContext(ctx,
|
||||
`SELECT `+targetCols+` FROM agent_targets
|
||||
WHERE org=? AND host=? AND (owner=? OR owner='')
|
||||
ORDER BY CASE WHEN owner=? THEN 0 ELSE 1 END, created_at DESC, id ASC LIMIT 1`,
|
||||
org, host, owner, owner)
|
||||
t, err := scanTarget(row)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Target{}, errTargetNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return Target{}, fmt.Errorf("get linkable target by host: %w", err)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// GetTargetByHost returns an org's target reporting the given host, or
|
||||
// errTargetNotFound. It is how a re-link of the SAME machine finds its existing target
|
||||
// (idempotent register) instead of creating a duplicate. Org-scoped: a host string can
|
||||
@@ -476,11 +509,12 @@ func toTargetView(t Target, load TargetLoad) targetView {
|
||||
// /v1/agents/:ref wildcard (Fiber matches in registration order) so "targets" is not
|
||||
// captured as a ref. The static /v1/agents/targets precedes /v1/agents/targets/:id.
|
||||
func mountTargets(s *cloud.Service[state], app *zip.App) {
|
||||
app.Post("/v1/agents/targets", cloud.Handle(s, registerTarget))
|
||||
app.Get("/v1/agents/targets", cloud.Handle(s, listTargets))
|
||||
app.Get("/v1/agents/targets/:id", cloud.Handle(s, getTarget))
|
||||
app.Patch("/v1/agents/targets/:id", cloud.Handle(s, patchTarget))
|
||||
app.Delete("/v1/agents/targets/:id", cloud.Handle(s, deleteTarget))
|
||||
g := app.Group("/v1/agents")
|
||||
g.Post("/targets", cloud.Handle(s, registerTarget))
|
||||
g.Get("/targets", cloud.Handle(s, listTargets))
|
||||
g.Get("/targets/:id", cloud.Handle(s, getTarget))
|
||||
g.Patch("/targets/:id", cloud.Handle(s, patchTarget))
|
||||
g.Delete("/targets/:id", cloud.Handle(s, deleteTarget))
|
||||
// The #48 route-work machine surface (claim-key, claim long-poll, report)
|
||||
// lives on the same target routes; register after the CRUD so the
|
||||
// extra-segment paths are unambiguous.
|
||||
@@ -548,12 +582,21 @@ func registerTarget(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
metricsAt = now // the server owns the staleness clock; a client can't forge it
|
||||
}
|
||||
|
||||
// Idempotent re-link: the SAME machine (org+host) refreshes its existing target
|
||||
// rather than piling up duplicates, so mission-control shows one row per machine
|
||||
// with live spec/metrics. Only an explicit host keys this — an anonymous target
|
||||
// (no host) always creates.
|
||||
// The registering principal OWNS this machine (least privilege): only it (or an
|
||||
// org admin) may later mint the claim key, claim runs, report, patch, or delete
|
||||
// it. tenant() already required a validated principal, so this is non-empty.
|
||||
owner := caller(c)
|
||||
|
||||
// Idempotent re-link: the SAME machine (org+host+owner) refreshes its existing
|
||||
// target rather than piling up duplicates, so mission-control shows one row per
|
||||
// machine with live spec/metrics. It resolves ONLY the caller's own row (or an
|
||||
// UNOWNED pre-migration row, which it ADOPTS by binding owner) — a row owned by a
|
||||
// different member is never touched, so a re-link can never hijack another's
|
||||
// machine; the caller falls through to create its own. Only an explicit host keys
|
||||
// this — an anonymous target (no host) always creates.
|
||||
if host != "" {
|
||||
if existing, err := s.State.store.GetTargetByHost(c.Context(), org, host); err == nil {
|
||||
if existing, err := s.State.store.GetLinkableTargetByHost(c.Context(), org, host, owner); err == nil {
|
||||
existing.Owner = owner // bind an adopted unowned row; no-op if already ours
|
||||
existing.Label, existing.Kind, existing.Status, existing.Capacity = label, kind, status, capacity
|
||||
existing.Spec, existing.Metrics, existing.MetricsAt = spec, metrics, metricsAt
|
||||
existing.UpdatedAt = now
|
||||
@@ -571,7 +614,7 @@ func registerTarget(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
return zip.Errorf(http.StatusInternalServerError, "rng: %v", err)
|
||||
}
|
||||
t := Target{
|
||||
ID: id, Org: org, Label: label, Kind: kind, Status: status,
|
||||
ID: id, Org: org, Owner: owner, Label: label, Kind: kind, Status: status,
|
||||
Capacity: capacity, Host: host, Spec: spec, Metrics: metrics, MetricsAt: metricsAt,
|
||||
CreatedAt: now, UpdatedAt: now,
|
||||
}
|
||||
@@ -648,6 +691,12 @@ func patchTarget(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
if err != nil {
|
||||
return zip.Errorf(http.StatusInternalServerError, "get: %v", err)
|
||||
}
|
||||
// Only the machine's owner (or an org admin) may mutate it — a member cannot
|
||||
// reconfigure/drain another member's machine. Fail-closed to the SAME not-found
|
||||
// an unknown id gives, so a probe learns nothing about what exists.
|
||||
if !ownsTarget(c, t) {
|
||||
return zip.ErrNotFound("target not found")
|
||||
}
|
||||
var body patchTargetReq
|
||||
if err := c.Bind(&body); err != nil {
|
||||
return err
|
||||
@@ -729,6 +778,19 @@ func deleteTarget(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
return zip.ErrForbidden("X-Org-Id required")
|
||||
}
|
||||
id := idParam(c)
|
||||
// Resolve + ownership-gate before deleting: only the machine's owner (or an org
|
||||
// admin) may deregister it. A cross-org id, an unknown id, and a non-owned id all
|
||||
// collapse to the same not-found — no oracle.
|
||||
t, err := s.State.store.GetTarget(c.Context(), org, id)
|
||||
if err == errTargetNotFound {
|
||||
return zip.ErrNotFound("target not found")
|
||||
}
|
||||
if err != nil {
|
||||
return zip.Errorf(http.StatusInternalServerError, "get: %v", err)
|
||||
}
|
||||
if !ownsTarget(c, t) {
|
||||
return zip.ErrNotFound("target not found")
|
||||
}
|
||||
deleted, err := s.State.store.DeleteTarget(c.Context(), org, id)
|
||||
if err != nil {
|
||||
return zip.Errorf(http.StatusInternalServerError, "delete: %v", err)
|
||||
|
||||
+69
-16
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -38,6 +39,27 @@ func setScopeAttrs(span trace.Span, org, project string) {
|
||||
}
|
||||
}
|
||||
|
||||
// StartGenAISpan opens one OTel gen_ai client span on the shared cloud tracer and
|
||||
// stamps the request-side semantic-convention attributes every AI call carries:
|
||||
// gen_ai.system (the serving provider — "hanzo" for the Hanzo gateway, "cloudflare"
|
||||
// for Workers AI, …), gen_ai.operation.name, gen_ai.request.model, plus the
|
||||
// per-tenant scope (hanzo.org / hanzo.project). It is the ONE gen_ai span
|
||||
// constructor: the HTTP chat/embed clients here AND any other inference path (the
|
||||
// per-org Cloudflare Workers AI proxy) emit to the SAME o11y span plane through it,
|
||||
// so AI telemetry is one shape with one tenant-isolation contract, never a parallel
|
||||
// per-provider span. The caller defers span.End() and adds response attributes
|
||||
// (tokens, status) after the call returns.
|
||||
func StartGenAISpan(ctx context.Context, system, operation, model, org, project string) (context.Context, trace.Span) {
|
||||
ctx, span := aiTracer.Start(ctx, operation+" "+model, trace.WithSpanKind(trace.SpanKindClient))
|
||||
span.SetAttributes(
|
||||
attribute.String("gen_ai.system", system),
|
||||
attribute.String("gen_ai.operation.name", operation),
|
||||
attribute.String("gen_ai.request.model", model),
|
||||
)
|
||||
setScopeAttrs(span, org, project)
|
||||
return ctx, span
|
||||
}
|
||||
|
||||
// httpAI is the real, in-process types.AIClient: it runs chat completions
|
||||
// against an OpenAI-compatible endpoint — the Hanzo LLM gateway
|
||||
// (https://api.hanzo.ai/v1). This is the ONE concrete inference client the
|
||||
@@ -135,14 +157,8 @@ func (a *httpAI) ChatCompletion(ctx context.Context, req *types.ChatRequest) (*t
|
||||
|
||||
// GenAI client span (OTel semantic conventions) — one span per LLM call,
|
||||
// nested under any active agent-run span carried on ctx.
|
||||
ctx, span := aiTracer.Start(ctx, "chat "+model, trace.WithSpanKind(trace.SpanKindClient))
|
||||
ctx, span := StartGenAISpan(ctx, "hanzo", "chat", model, req.Org, req.Project)
|
||||
defer span.End()
|
||||
span.SetAttributes(
|
||||
attribute.String("gen_ai.system", "hanzo"),
|
||||
attribute.String("gen_ai.operation.name", "chat"),
|
||||
attribute.String("gen_ai.request.model", model),
|
||||
)
|
||||
setScopeAttrs(span, req.Org, req.Project)
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, aiHTTPTimeout)
|
||||
defer cancel()
|
||||
@@ -156,6 +172,12 @@ func (a *httpAI) ChatCompletion(ctx context.Context, req *types.ChatRequest) (*t
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, "chat completion failed")
|
||||
// Tag a transient overload (429/5xx/"overloaded") with types.ErrUpstreamBusy
|
||||
// so the agent runner can retry/fail over; a permanent error is left untagged
|
||||
// and fails fast. The message is preserved either way.
|
||||
if transientChat(err) {
|
||||
err = fmt.Errorf("%w: %w", err, types.ErrUpstreamBusy)
|
||||
}
|
||||
return nil, fmt.Errorf("cloud: chat completion (model %q): %w", model, err)
|
||||
}
|
||||
span.SetAttributes(
|
||||
@@ -164,8 +186,10 @@ func (a *httpAI) ChatCompletion(ctx context.Context, req *types.ChatRequest) (*t
|
||||
attribute.Int("gen_ai.usage.output_tokens", resp.Usage.CompletionTokens),
|
||||
)
|
||||
if len(resp.Choices) == 0 {
|
||||
// A 200 with no choices is the gateway's "overloaded, no capacity" shape —
|
||||
// transient, so tag it busy for retry/failover rather than dropping the reply.
|
||||
span.SetStatus(codes.Error, "no choices")
|
||||
return nil, fmt.Errorf("cloud: chat completion (model %q): upstream returned no choices", model)
|
||||
return nil, fmt.Errorf("cloud: chat completion (model %q): upstream returned no choices: %w", model, types.ErrUpstreamBusy)
|
||||
}
|
||||
return &types.ChatResponse{
|
||||
Content: resp.Choices[0].Message.Content,
|
||||
@@ -175,6 +199,41 @@ func (a *httpAI) ChatCompletion(ctx context.Context, req *types.ChatRequest) (*t
|
||||
}, nil
|
||||
}
|
||||
|
||||
// transientChat reports whether an upstream chat-completion error is a transient
|
||||
// overload the agent runner may safely retry or fail over on: HTTP
|
||||
// 429/500/502/503/504, or a gateway that surfaces "overloaded" without a typed
|
||||
// status. A permanent failure — a 4xx that is not 429 (bad request, auth, an
|
||||
// unserved model) — returns false so it fails fast instead of burning retries.
|
||||
func transientChat(err error) bool {
|
||||
if code, ok := httpStatus(err); ok {
|
||||
switch code {
|
||||
case http.StatusTooManyRequests, http.StatusInternalServerError,
|
||||
http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
// No typed status (a bare transport/overload message) — treat an explicit
|
||||
// "overloaded" as transient; anything else is not classifiable, so not retried.
|
||||
return strings.Contains(strings.ToLower(err.Error()), "overloaded")
|
||||
}
|
||||
|
||||
// httpStatus extracts the upstream HTTP status from a go-openai error
|
||||
// (*openai.APIError for a decoded error body, *openai.RequestError for a raw
|
||||
// non-2xx), returning ok=false when the error carries no status.
|
||||
func httpStatus(err error) (int, bool) {
|
||||
var apiErr *openai.APIError
|
||||
if errors.As(err, &apiErr) && apiErr.HTTPStatusCode > 0 {
|
||||
return apiErr.HTTPStatusCode, true
|
||||
}
|
||||
var reqErr *openai.RequestError
|
||||
if errors.As(err, &reqErr) && reqErr.HTTPStatusCode > 0 {
|
||||
return reqErr.HTTPStatusCode, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// Embed returns one vector per input from the gateway's OpenAI-compatible
|
||||
// /embeddings, authenticated by the SAME credential as ChatCompletion — so the
|
||||
// semantic tier bills and meters through the ONE org/project-aligned path, never
|
||||
@@ -193,15 +252,9 @@ func (a *httpAI) Embed(ctx context.Context, req *types.EmbedRequest) ([][]float3
|
||||
return nil, fmt.Errorf("cloud: embed: empty model")
|
||||
}
|
||||
|
||||
ctx, span := aiTracer.Start(ctx, "embeddings "+model, trace.WithSpanKind(trace.SpanKindClient))
|
||||
ctx, span := StartGenAISpan(ctx, "hanzo", "embeddings", model, req.Org, req.Project)
|
||||
defer span.End()
|
||||
span.SetAttributes(
|
||||
attribute.String("gen_ai.system", "hanzo"),
|
||||
attribute.String("gen_ai.operation.name", "embeddings"),
|
||||
attribute.String("gen_ai.request.model", model),
|
||||
attribute.Int("gen_ai.request.input_count", len(inputs)),
|
||||
)
|
||||
setScopeAttrs(span, req.Org, req.Project)
|
||||
span.SetAttributes(attribute.Int("gen_ai.request.input_count", len(inputs)))
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, aiHTTPTimeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -3,6 +3,7 @@ package clients
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -100,6 +101,49 @@ func TestAIHTTP_UpstreamErrorMapped(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAIHTTP_TransientTagged proves the classification the agent runner relies on:
|
||||
// a transient overload (429 / 5xx / empty-choices) is tagged types.ErrUpstreamBusy
|
||||
// so the runner retries/fails over, while a permanent 4xx (400) is NOT tagged so it
|
||||
// fails fast. Without this split the runner would either drop replies on a passing
|
||||
// 429 or spin retries on an unrecoverable bad request.
|
||||
func TestAIHTTP_TransientTagged(t *testing.T) {
|
||||
serve := func(status int, body string) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if status != 0 {
|
||||
w.WriteHeader(status)
|
||||
}
|
||||
_, _ = w.Write([]byte(body))
|
||||
}))
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
status int
|
||||
body string
|
||||
transit bool
|
||||
}{
|
||||
{"429 overloaded", http.StatusTooManyRequests, `{"error":{"message":"Platform overloaded"}}`, true},
|
||||
{"503 unavailable", http.StatusServiceUnavailable, `{"error":{"message":"unavailable"}}`, true},
|
||||
{"500 server", http.StatusInternalServerError, `{"error":{"message":"boom"}}`, true},
|
||||
{"200 empty choices", 0, `{"id":"x","object":"chat.completion","choices":[]}`, true},
|
||||
{"400 bad request", http.StatusBadRequest, `{"error":{"message":"bad input"}}`, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
srv := serve(tc.status, tc.body)
|
||||
defer srv.Close()
|
||||
_, err := AIHTTPAt(srv.URL, "sk-test", "deepseek-v4-flash").
|
||||
ChatCompletion(context.Background(), &types.ChatRequest{Prompt: "x"})
|
||||
if err == nil {
|
||||
t.Fatalf("%s: expected an error", tc.name)
|
||||
}
|
||||
if got := errors.Is(err, types.ErrUpstreamBusy); got != tc.transit {
|
||||
t.Fatalf("%s: errors.Is(ErrUpstreamBusy)=%v, want %v (err=%v)", tc.name, got, tc.transit, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestAIHTTP_ServerErrorMapped asserts a 5xx upstream also maps to an error.
|
||||
func TestAIHTTP_ServerErrorMapped(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
|
||||
@@ -40,7 +40,8 @@
|
||||
//
|
||||
// GET /v1/analytics/overview per-org KPIs (llm real; web/commerce honest-empty)
|
||||
// GET /v1/analytics/timeseries requests/tokens/spend over time (hour|day buckets)
|
||||
// GET /v1/analytics/top top models (real) + top products (honest-empty)
|
||||
// GET /v1/analytics/top top models (real) + products + behavior lenses
|
||||
// (topPages/topReferrers/topSources over the events lens)
|
||||
// GET /v1/analytics/health subsystem health (datastore connectivity + lens tables)
|
||||
//
|
||||
// Registered as id "analytics" with cloud.HealthOwner + order 132: it serves its
|
||||
@@ -63,6 +64,7 @@ import (
|
||||
aiobject "github.com/hanzoai/ai/object"
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/clients/principal"
|
||||
"github.com/hanzoai/cloud/clients/sites"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
@@ -85,12 +87,47 @@ func Mount(app *zip.App, deps cloud.Deps) error {
|
||||
}
|
||||
|
||||
// build carries no per-subsystem state — analytics reads the shared warehouse. It
|
||||
// only records the informative mount line.
|
||||
// records the informative mount line and installs the site-host ingest carve.
|
||||
func build(b cloud.Base) (state, error) {
|
||||
b.Log.Info("analytics surface", "warehouse", "hanzo", "brand", b.Brand)
|
||||
installHostCarve(b)
|
||||
return state{}, nil
|
||||
}
|
||||
|
||||
// installHostCarve wires the published-site-host beacon ingest (the twin of base's
|
||||
// sites.SetBaseHostHandler): a page served on a site host can POST its OWN analytics
|
||||
// beacon to the canonical /v1/event (or the deprecated /v1/analytics{,/batch} and
|
||||
// /v1/insights/e beacons kept working mid-migration) and have it ingested into
|
||||
// hanzo.events with the tenant FORCED to the site's resolved Org — the
|
||||
// server-supplied, host-derived tenant, never a body/header claim. It funnels
|
||||
// through the SAME write core (eventWithOrg / captureWithOrg / insightsWithOrg →
|
||||
// ingestBody → ingestEvents); the in-handler tenant resolvers are deliberately NOT
|
||||
// consulted because the host already authorizes the tenant.
|
||||
//
|
||||
// Gated by the SAME already-existing flag the anonymous ingest path uses —
|
||||
// CLOUD_ANALYTICS_PUBLIC_CAPTURE (publicCaptureEnabled, default ON) — so a site
|
||||
// host accepts its own beacons out of the box, and turning public capture off also
|
||||
// removes this carve (a site host then 405s a beacon POST, unchanged). The org is
|
||||
// the resolver's Site.Org; sites.Middleware gates this carve on method POST so the
|
||||
// authenticated GET read lenses are never hijacked.
|
||||
func installHostCarve(b cloud.Base) {
|
||||
if !publicCaptureEnabled() {
|
||||
b.Log.Info("analytics public-host ingest carve disabled", "flag", publicCaptureEnv)
|
||||
return
|
||||
}
|
||||
sites.SetAnalyticsHostHandler(func(org string, c *zip.Ctx) error {
|
||||
switch c.Path() {
|
||||
case "/v1/event": // the canonical door — host-forced org, canonical wire
|
||||
return eventWithOrg(org, c)
|
||||
case "/v1/insights/e": // deprecated PostHog-wire beacon
|
||||
return insightsWithOrg(org, c)
|
||||
default: // deprecated Segment/beacon wire: /v1/analytics{,/batch}
|
||||
return captureWithOrg(org, c)
|
||||
}
|
||||
})
|
||||
b.Log.Info("analytics public-host ingest carve enabled", "flag", publicCaptureEnv)
|
||||
}
|
||||
|
||||
// routes registers the analytics surface. Health owns /v1/analytics/health
|
||||
// explicitly (not JWT-gated: liveness must be probe-able); the data endpoints are
|
||||
// all org-gated in-handler.
|
||||
@@ -101,20 +138,34 @@ func routes(app *zip.App, s *cloud.Service[state]) {
|
||||
app.Get("/v1/analytics/top", cloud.Handle(s, top))
|
||||
|
||||
// Capture (WRITE) side — the ingest that fills hanzo.events. POST /v1/event
|
||||
// (event.go) is the ONE canonical front door: body Event | [Event], org
|
||||
// resolved IAM-only and fail-closed, into the ONE write core (ingestEvents).
|
||||
// (event.go) is the ONE canonical front door serving EVERY auth context (IAM
|
||||
// bearer | pk_ publishable key | site-host-forced) and EVERY wire shape
|
||||
// (Event | [Event] | {batch}) into the ONE write core (ingestEvents). Every
|
||||
// other route below is a thin alias/shim delegating to it.
|
||||
app.Post("/v1/event", cloud.Handle(s, eventIngest))
|
||||
|
||||
// DEPRECATED ingest aliases — thin wire adapters that normalize onto the SAME
|
||||
// write core (log a one-shot deprecation, keep working). /v1/analytics{,/batch}
|
||||
// and /v1/tracker speak the Segment/beacon CaptureBatch wire; /v1/tracker is a
|
||||
// bare route (never collides with the /v1/tracker/projects* issue tracker).
|
||||
// /v1/ingest — a THIN DEPRECATED ALIAS of /v1/event (delegates to the exact
|
||||
// eventHandle logic: pk_ auth now lives on the canonical door). /v1/ingest/keys
|
||||
// mints a pk_ for the caller's org (minting is a distinct concern, not ingest);
|
||||
// /v1/errors is the type:'error' read lens (validated principal — reads never
|
||||
// accept the write-only key).
|
||||
app.Post("/v1/ingest", cloud.Handle(s, ingest))
|
||||
app.Post("/v1/ingest/keys", cloud.Handle(s, mintKey))
|
||||
app.Get("/v1/errors", cloud.Handle(s, errorsLens))
|
||||
|
||||
// DEPRECATED foreign-protocol ingest shims — external-SDK compat ONLY; no Hanzo
|
||||
// surface uses these (Hanzo surfaces POST /v1/event). They normalize their own
|
||||
// wire onto CaptureEvent and funnel through the SAME write core (log a one-shot
|
||||
// deprecation, keep working so external Segment/beacon callers are unbroken).
|
||||
// /v1/analytics{,/batch} and /v1/tracker speak the Segment/beacon CaptureBatch
|
||||
// wire; /v1/tracker is a bare route (never collides with /v1/tracker/projects*).
|
||||
app.Post("/v1/analytics", cloud.Handle(s, capture))
|
||||
app.Post("/v1/analytics/batch", cloud.Handle(s, capture))
|
||||
app.Post("/v1/tracker", cloud.Handle(s, capture))
|
||||
|
||||
// /v1/insights — console reads over the SAME engine + the DEPRECATED PostHog-
|
||||
// wire ingest adapter (/v1/insights/e → the ONE write core). Flags live at /v1/flags.
|
||||
// wire ingest shim (/v1/insights/e → the ONE write core; external PostHog SDK
|
||||
// compat only). Flags live at /v1/flags.
|
||||
app.Get("/v1/insights/health", cloud.Handle(s, insightsHealth))
|
||||
app.Post("/v1/insights/e", cloud.Handle(s, insightsIngest))
|
||||
app.Get("/v1/insights/events", cloud.Handle(s, insightsEvents))
|
||||
@@ -352,13 +403,37 @@ func top(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
"GROUP BY product_id ORDER BY revenue DESC LIMIT %d", eventsTable, ewhere, limit)
|
||||
prodRows, perr := aiobject.DatastoreQuery(ctx, prodSQL, eargs...)
|
||||
|
||||
// Behavior lenses over hanzo.events — WHERE people go / WHAT they look at
|
||||
// (topPages) and where they come FROM (topReferrers organic/referral,
|
||||
// topSources campaigns). Each is ONE pageview breakdown that degrades to
|
||||
// honest-empty if the events table is absent or the query errors — never a 500
|
||||
// (mirrors the overview web lens exactly).
|
||||
pageSQL, pageArgs := breakdownSQL(pageKeyExpr, org, start, end, limit)
|
||||
pageRows, pageErr := aiobject.DatastoreQuery(ctx, pageSQL, pageArgs...)
|
||||
if pageErr != nil {
|
||||
s.Log.Debug("topPages lens unavailable (honest-empty)", "err", pageErr)
|
||||
}
|
||||
refSQL, refArgs := breakdownSQL(referrerKeyExpr, org, start, end, limit)
|
||||
refRows, refErr := aiobject.DatastoreQuery(ctx, refSQL, refArgs...)
|
||||
if refErr != nil {
|
||||
s.Log.Debug("topReferrers lens unavailable (honest-empty)", "err", refErr)
|
||||
}
|
||||
srcSQL, srcArgs := breakdownSQL(sourceKeyExpr, org, start, end, limit)
|
||||
srcRows, srcErr := aiobject.DatastoreQuery(ctx, srcSQL, srcArgs...)
|
||||
if srcErr != nil {
|
||||
s.Log.Debug("topSources lens unavailable (honest-empty)", "err", srcErr)
|
||||
}
|
||||
|
||||
return c.JSON(http.StatusOK, Top{
|
||||
Range: rangeLabel,
|
||||
Start: start.UTC().Format(time.RFC3339),
|
||||
End: end.UTC().Format(time.RFC3339),
|
||||
Scope: Scope{Org: org},
|
||||
Models: buildTopModels(modelRows),
|
||||
Products: buildTopProducts(prodRows, perr == nil),
|
||||
Range: rangeLabel,
|
||||
Start: start.UTC().Format(time.RFC3339),
|
||||
End: end.UTC().Format(time.RFC3339),
|
||||
Scope: Scope{Org: org},
|
||||
Models: buildTopModels(modelRows),
|
||||
Products: buildTopProducts(prodRows, perr == nil),
|
||||
Pages: buildBreakdown(pageRows, pageErr == nil),
|
||||
Referrers: buildBreakdown(refRows, refErr == nil),
|
||||
Sources: buildBreakdown(srcRows, srcErr == nil),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -183,3 +183,107 @@ func TestBuildTopProductsHonestEmpty(t *testing.T) {
|
||||
t.Fatalf("must carry honest reason + source, got %+v", tp)
|
||||
}
|
||||
}
|
||||
|
||||
// ── behavior lenses (topPages / topReferrers / topSources) ────────────────────
|
||||
|
||||
// TestBreakdownSQLBindsOrgPositionally: EVERY behavior lens inherits the tenancy
|
||||
// invariant — the org is the trailing bound parameter (never interpolated), the
|
||||
// predicate binds tenant_id, and the read is restricted to $pageview rows. The
|
||||
// group key is a server-chosen constant (safe to interpolate) and the limit is a
|
||||
// validated int. This is the SQL-boundary isolation proof for the new lenses.
|
||||
func TestBreakdownSQLBindsOrgPositionally(t *testing.T) {
|
||||
start := time.Date(2026, 6, 24, 0, 0, 0, 0, time.UTC)
|
||||
end := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
|
||||
const org = "o'; DROP TABLE hanzo.events; --" // hostile slug must NOT escape into SQL
|
||||
for _, keyExpr := range []string{pageKeyExpr, referrerKeyExpr, sourceKeyExpr} {
|
||||
sql, args := breakdownSQL(keyExpr, org, start, end, 10)
|
||||
if strings.Contains(sql, org) {
|
||||
t.Fatalf("org %q must NOT be interpolated into sql: %q", org, sql)
|
||||
}
|
||||
if !strings.Contains(sql, "tenant_id = ?") {
|
||||
t.Fatalf("breakdown must bind tenant_id positionally: %q", sql)
|
||||
}
|
||||
if !strings.Contains(sql, "timestamp >= ? AND timestamp < ?") {
|
||||
t.Fatalf("time bounds must be parameterized: %q", sql)
|
||||
}
|
||||
if len(args) != 3 {
|
||||
t.Fatalf("want 3 bound args (start,end,org), got %d: %v", len(args), args)
|
||||
}
|
||||
if got, ok := args[2].(string); !ok || got != org {
|
||||
t.Fatalf("org must be the trailing bound arg verbatim, want %q got %v", org, args[2])
|
||||
}
|
||||
if !strings.Contains(sql, "event = '$pageview'") {
|
||||
t.Fatalf("behavior lenses count only $pageview rows: %q", sql)
|
||||
}
|
||||
if !strings.Contains(sql, keyExpr) {
|
||||
t.Fatalf("breakdown must group by the key expr %q: %q", keyExpr, sql)
|
||||
}
|
||||
if !strings.Contains(sql, "LIMIT 10") {
|
||||
t.Fatalf("breakdown must bound cardinality by the validated limit: %q", sql)
|
||||
}
|
||||
// pct denominator is the TRUE in-window total (window fn), not the top-N sum.
|
||||
if !strings.Contains(sql, "sum(pageviews) OVER ()") {
|
||||
t.Fatalf("breakdown pct denominator must be the in-window pageview total: %q", sql)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBreakdownBucketsDirectAndNone: the referrer lens buckets a missing or
|
||||
// same-origin (self) referrer as "(direct)"; the source lens buckets an empty utm
|
||||
// as "(none)". This is the origin/referral/campaign split done IN SQL.
|
||||
func TestBreakdownBucketsDirectAndNone(t *testing.T) {
|
||||
start := time.Date(2026, 6, 24, 0, 0, 0, 0, time.UTC)
|
||||
end := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
|
||||
refSQL, _ := breakdownSQL(referrerKeyExpr, "acme", start, end, 5)
|
||||
if !strings.Contains(refSQL, "'(direct)'") {
|
||||
t.Fatalf("referrer lens must bucket no/own referrer as (direct): %q", refSQL)
|
||||
}
|
||||
if !strings.Contains(refSQL, "domain(url)") {
|
||||
t.Fatalf("referrer lens must exclude same-origin (self) referrers via domain(url): %q", refSQL)
|
||||
}
|
||||
srcSQL, _ := breakdownSQL(sourceKeyExpr, "acme", start, end, 5)
|
||||
if !strings.Contains(srcSQL, "'(none)'") {
|
||||
t.Fatalf("source lens must bucket empty utm as (none): %q", srcSQL)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildBreakdownPctShareOfTotal: pct is each bucket's share of the in-window
|
||||
// pageview TOTAL (the window-fn `total` column = 100), NOT of the returned-rows sum
|
||||
// (80). So a top-N list honestly reflects the long tail. Values come through
|
||||
// verbatim (key/pageviews/visitors).
|
||||
func TestBuildBreakdownPctShareOfTotal(t *testing.T) {
|
||||
rows := []map[string]any{
|
||||
{"k": "/pricing", "pageviews": uint64(60), "visitors": uint64(40), "total": uint64(100)},
|
||||
{"k": "/docs", "pageviews": uint64(20), "visitors": uint64(15), "total": uint64(100)},
|
||||
}
|
||||
b := buildBreakdown(rows, true)
|
||||
if !b.Available || len(b.Items) != 2 {
|
||||
t.Fatalf("want 2 available items, got %+v", b)
|
||||
}
|
||||
if b.Items[0].Key != "/pricing" || b.Items[0].Pageviews != 60 || b.Items[0].Visitors != 40 {
|
||||
t.Fatalf("row 0 must carry values verbatim: %+v", b.Items[0])
|
||||
}
|
||||
// 60/100 = 60%, 20/100 = 20% — share of the in-window TOTAL, not the shown sum.
|
||||
if b.Items[0].Pct != 60 || b.Items[1].Pct != 20 {
|
||||
t.Fatalf("pct must be share of the in-window total: %v / %v", b.Items[0].Pct, b.Items[1].Pct)
|
||||
}
|
||||
if b.Source != "hanzo.events" {
|
||||
t.Fatalf("source want hanzo.events, got %q", b.Source)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildBreakdownHonestEmpty: an events-query failure (ok=false) yields an
|
||||
// honestly-unavailable lens with an empty (non-nil) list — never fabricated rows,
|
||||
// never a 500. Mirrors buildTopProducts's honest-empty exactly.
|
||||
func TestBuildBreakdownHonestEmpty(t *testing.T) {
|
||||
b := buildBreakdown(nil, false)
|
||||
if b.Available {
|
||||
t.Fatal("breakdown must be unavailable when the events query failed")
|
||||
}
|
||||
if b.Items == nil || len(b.Items) != 0 {
|
||||
t.Fatalf("items must be an empty (non-nil) slice, got %#v", b.Items)
|
||||
}
|
||||
if b.Reason == "" || b.Source != "hanzo.events" {
|
||||
t.Fatalf("must carry honest reason + source, got %+v", b)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// campaign.go is the in-process CAMPAIGN-METRICS seam over the ONE analytics
|
||||
// warehouse: the /v1/campaign plane (clients/campaign) reads a campaign's funnel
|
||||
// from HERE rather than opening a second store. A campaign's results ARE an
|
||||
// analytics query scoped to the campaign — the utm_campaign-tagged events in
|
||||
// hanzo.events — so there is one metrics plane, not a parallel one.
|
||||
//
|
||||
// TENANCY: identical to every other query this package builds. campaignWhere
|
||||
// binds the org (tenant_id) AND the campaign id (utm_campaign) AND the optional
|
||||
// variant (utm_content) POSITIONALLY — nothing user-derived is ever interpolated,
|
||||
// so a caller can only ever read its OWN org's campaign, and the utm_campaign
|
||||
// filter can never escape into SQL. The variant arg powers the creative-A/B
|
||||
// evidence read (utm_content) the experiment primitive composes.
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
aiobject "github.com/hanzoai/ai/object"
|
||||
)
|
||||
|
||||
// CampaignEvents is the per-campaign funnel read from hanzo.events, scoped to
|
||||
// (org, utm_campaign[, utm_content]). Impressions/clicks/conversions are counts
|
||||
// of the campaign's tagged events; Available is false (honest-empty) when the
|
||||
// events warehouse is not connected or the events table is not yet provisioned —
|
||||
// never fabricated. Spend is deliberately absent: it is the channel connector's
|
||||
// reported number, joined by the campaign plane, not an analytics value.
|
||||
type CampaignEvents struct {
|
||||
Available bool `json:"available"`
|
||||
Impressions int64 `json:"impressions"`
|
||||
Clicks int64 `json:"clicks"`
|
||||
Conversions int64 `json:"conversions"`
|
||||
Revenue float64 `json:"revenue"`
|
||||
Visitors int64 `json:"visitors"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
|
||||
// campaignWhere is the org + campaign (+ optional variant) predicate over
|
||||
// hanzo.events. org (tenant_id) and campaignID (utm_campaign) are ALWAYS bound;
|
||||
// variant (utm_content) is appended only when non-empty. Time bounds are bound as
|
||||
// datastore DateTime literals (the proven cloud_usage transport). Same isolation
|
||||
// boundary as eventsWhere — the org is a bound parameter, never interpolated.
|
||||
func campaignWhere(org, campaignID, variant string, start, end time.Time) (string, []any) {
|
||||
where := "timestamp >= ? AND timestamp < ? AND tenant_id = ? AND utm_campaign = ?"
|
||||
args := []any{tsLiteral(start), tsLiteral(end), org, campaignID}
|
||||
if variant != "" {
|
||||
where += " AND utm_content = ?"
|
||||
args = append(args, variant)
|
||||
}
|
||||
return where, args
|
||||
}
|
||||
|
||||
// CampaignMetrics reads the (org, campaignID) funnel from the ONE analytics
|
||||
// warehouse. variant=="" reads the whole campaign (all creatives); a non-empty
|
||||
// variant reads a single creative's slice (utm_content) — the evidence read for a
|
||||
// creative A/B. It degrades to honest-empty (Available=false, nil error) when the
|
||||
// datastore is not connected, so a campaign metrics view still renders its spend +
|
||||
// channels. A genuine query failure against a connected warehouse returns the
|
||||
// error (the caller logs it and shows honest-empty) — never a fabricated funnel.
|
||||
func CampaignMetrics(ctx context.Context, org, campaignID, variant string, start, end time.Time) (CampaignEvents, error) {
|
||||
out := CampaignEvents{Available: false, Source: eventsTable}
|
||||
if org == "" || campaignID == "" {
|
||||
return out, nil
|
||||
}
|
||||
if !aiobject.DatastoreEnabled() {
|
||||
return out, nil // honest-empty: no warehouse connected
|
||||
}
|
||||
where, args := campaignWhere(org, campaignID, variant, start, end)
|
||||
// Each countIf predicate is a server-chosen constant expression (never user
|
||||
// input); the only user-derived values — org, campaign, variant, time — stay
|
||||
// bound parameters via campaignWhere.
|
||||
sql := "SELECT " +
|
||||
"countIf(event = 'impression' OR event = 'ad_impression') AS impressions, " +
|
||||
"countIf(event = 'click' OR event = 'ad_click') AS clicks, " +
|
||||
"countIf(event = 'order_completed' OR event = 'signup' OR event = 'conversion') AS conversions, " +
|
||||
"toFloat64(sum(revenue)) AS revenue, " +
|
||||
"uniqExact(distinct_id) AS visitors " +
|
||||
"FROM " + eventsTable + " WHERE " + where
|
||||
rows, err := aiobject.DatastoreQuery(ctx, sql, args...)
|
||||
if err != nil {
|
||||
// Connected warehouse rejected/failed the query (or the events table is
|
||||
// absent): honest-empty for the caller, with the error surfaced for logs.
|
||||
return out, err
|
||||
}
|
||||
row := firstRow(rows)
|
||||
return CampaignEvents{
|
||||
Available: true,
|
||||
Impressions: aInt64(row["impressions"]),
|
||||
Clicks: aInt64(row["clicks"]),
|
||||
Conversions: aInt64(row["conversions"]),
|
||||
Revenue: aFloat64(row["revenue"]),
|
||||
Visitors: aInt64(row["visitors"]),
|
||||
Source: eventsTable,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestCampaignWhere_BindsOrgAndCampaignPositionally is the tenancy-invariant test
|
||||
// for the campaign-metrics seam: the org (tenant_id) and campaign (utm_campaign)
|
||||
// are ALWAYS bound parameters, never interpolated, so a caller can only read its
|
||||
// own org's campaign and a hostile campaign id can never escape into SQL.
|
||||
func TestCampaignWhere_BindsOrgAndCampaignPositionally(t *testing.T) {
|
||||
start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
end := time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC)
|
||||
where, args := campaignWhere("acme", "cmp_1", "", start, end)
|
||||
|
||||
if !strings.Contains(where, "tenant_id = ?") || !strings.Contains(where, "utm_campaign = ?") {
|
||||
t.Fatalf("org + campaign must be bound placeholders, got %q", where)
|
||||
}
|
||||
if strings.Contains(where, "utm_content") {
|
||||
t.Fatalf("no variant clause expected for whole-campaign read, got %q", where)
|
||||
}
|
||||
// args order: start, end, org, campaign.
|
||||
if len(args) != 4 || args[2] != "acme" || args[3] != "cmp_1" {
|
||||
t.Fatalf("args must bind [ts, ts, org, campaign], got %v", args)
|
||||
}
|
||||
// The hostile-slug proof: the org value is a bound arg, never text in the SQL.
|
||||
if strings.Contains(where, "acme") || strings.Contains(where, "cmp_1") {
|
||||
t.Fatalf("org/campaign must NOT be interpolated into SQL: %q", where)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCampaignWhere_VariantAppended: a non-empty variant adds a bound utm_content
|
||||
// clause (the creative-A/B evidence read) — still fully parameterized.
|
||||
func TestCampaignWhere_VariantAppended(t *testing.T) {
|
||||
start := time.Unix(0, 0).UTC()
|
||||
end := time.Unix(1000, 0).UTC()
|
||||
where, args := campaignWhere("acme", "cmp_1", "hero-b", start, end)
|
||||
if !strings.Contains(where, "utm_content = ?") {
|
||||
t.Fatalf("variant must add a bound utm_content clause, got %q", where)
|
||||
}
|
||||
if len(args) != 5 || args[4] != "hero-b" {
|
||||
t.Fatalf("variant must be the trailing bound arg, got %v", args)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCampaignMetrics_HonestEmptyWhenDatastoreDisabled: with no warehouse
|
||||
// connected (unit-test default), the seam returns honest-empty (Available=false)
|
||||
// and NO error — the campaign metrics view still renders spend + channels.
|
||||
func TestCampaignMetrics_HonestEmptyWhenDatastoreDisabled(t *testing.T) {
|
||||
ev, err := CampaignMetrics(context.Background(), "acme", "cmp_1", "", time.Now().Add(-time.Hour), time.Now())
|
||||
if err != nil {
|
||||
t.Fatalf("datastore-disabled must be honest-empty, not an error: %v", err)
|
||||
}
|
||||
if ev.Available {
|
||||
t.Fatalf("no warehouse connected ⇒ Available must be false, got %+v", ev)
|
||||
}
|
||||
if ev.Source != eventsTable {
|
||||
t.Fatalf("source should name the events table even when empty, got %q", ev.Source)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCampaignMetrics_EmptyIdentifiersFailClosed: an empty org or campaign never
|
||||
// queries — honest-empty, never a warehouse-wide read.
|
||||
func TestCampaignMetrics_EmptyIdentifiersFailClosed(t *testing.T) {
|
||||
for _, tc := range []struct{ org, camp string }{{"", "cmp_1"}, {"acme", ""}} {
|
||||
ev, err := CampaignMetrics(context.Background(), tc.org, tc.camp, "", time.Now().Add(-time.Hour), time.Now())
|
||||
if err != nil || ev.Available {
|
||||
t.Fatalf("empty (%q,%q) must be honest-empty, got avail=%v err=%v", tc.org, tc.camp, ev.Available, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -173,6 +173,7 @@ type CaptureEvent struct {
|
||||
Quantity uint32 `json:"quantity"`
|
||||
Revenue float64 `json:"revenue"`
|
||||
Currency string `json:"currency"`
|
||||
Error *Exception `json:"error"` // set on type:'error' events (folded into properties.$exception)
|
||||
Properties map[string]any `json:"properties"`
|
||||
Library string `json:"library"`
|
||||
LibraryVer string `json:"libraryVersion"`
|
||||
@@ -297,12 +298,20 @@ func resolveEventName(e CaptureEvent) string {
|
||||
return "$identify"
|
||||
case "group":
|
||||
return "$group"
|
||||
case "error":
|
||||
if name == "" {
|
||||
return "$error"
|
||||
}
|
||||
return name
|
||||
default: // "event"
|
||||
return name
|
||||
}
|
||||
}
|
||||
|
||||
// canonicalType folds the type to the closed set {pageview,identify,group,event}.
|
||||
// canonicalType folds the type to the closed set
|
||||
// {pageview,identify,group,error,event}. `error` is first-class so the ingest can
|
||||
// store type:'error' events under event_type='error' — the key the /v1/errors
|
||||
// read lens filters on. An unknown type still folds to "event".
|
||||
func canonicalType(t string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(t)) {
|
||||
case "pageview", "page":
|
||||
@@ -311,6 +320,8 @@ func canonicalType(t string) string {
|
||||
return "identify"
|
||||
case "group":
|
||||
return "group"
|
||||
case "error":
|
||||
return "error"
|
||||
default:
|
||||
return "event"
|
||||
}
|
||||
@@ -604,7 +615,14 @@ func withSource(p map[string]any, source string) map[string]any {
|
||||
|
||||
// deprecatedOnce records one deprecation log per alias path per process, so a
|
||||
// high-volume ingest alias signals its sunset exactly once instead of flooding.
|
||||
var deprecatedOnce sync.Map
|
||||
// A plain mutex-guarded map (the alias set is tiny — /v1/analytics, /batch,
|
||||
// /tracker) rather than sync.Map: the latter (Go's HashTrieMap) was panicking
|
||||
// "ran out of hash bits" under the hot ingest path, 500-ing EVERY capture. A
|
||||
// bounded map+mutex has no trie state to corrupt and cannot panic here.
|
||||
var (
|
||||
deprecatedMu sync.Mutex
|
||||
deprecatedSeen = make(map[string]struct{}, 8)
|
||||
)
|
||||
|
||||
// deprecated logs (once per path) that a superseded ingest alias was hit, pointing
|
||||
// callers at the canonical front door. It NEVER changes behavior — the alias keeps
|
||||
@@ -612,7 +630,13 @@ var deprecatedOnce sync.Map
|
||||
// warehouse).
|
||||
func deprecated(s *cloud.Service[state], c *zip.Ctx, canonical string) {
|
||||
p := c.Path()
|
||||
if _, seen := deprecatedOnce.LoadOrStore(p, struct{}{}); seen {
|
||||
deprecatedMu.Lock()
|
||||
_, seen := deprecatedSeen[p]
|
||||
if !seen {
|
||||
deprecatedSeen[p] = struct{}{}
|
||||
}
|
||||
deprecatedMu.Unlock()
|
||||
if seen {
|
||||
return
|
||||
}
|
||||
s.Log.Warn("deprecated analytics ingest endpoint; migrate to the canonical event front door",
|
||||
@@ -657,27 +681,38 @@ func ingestEvents(ctx context.Context, org, source string, evs []CaptureEvent) (
|
||||
if err := aiobject.DatastoreExec(ctx, stmt, args...); err != nil {
|
||||
return CaptureResult{}, warehouseErr("capture", err)
|
||||
}
|
||||
// Fan the accepted batch out to the downstream sink (destinations), detached and
|
||||
// fail-soft — never blocks or fails an ingest (forward.go). No-op when unset.
|
||||
fanOut(org, evs)
|
||||
return CaptureResult{Accepted: len(rows), Dropped: dropped}, nil
|
||||
}
|
||||
|
||||
// capture ingests a Segment/beacon batch into hanzo.events, tenant-scoped. It is
|
||||
// the DEPRECATED wire adapter behind /v1/analytics, /v1/analytics/batch, and
|
||||
// /v1/tracker: a thin CaptureBatch decoder over the ONE write core (ingestEvents).
|
||||
// New callers post the canonical Event to /v1/event; this alias keeps working and
|
||||
// keeps captureTenant's brand-host path for anonymous marketing traffic.
|
||||
// capture ingests a Segment/beacon batch into hanzo.events, tenant-scoped. It is a
|
||||
// DEPRECATED foreign-protocol shim behind /v1/analytics, /v1/analytics/batch, and
|
||||
// /v1/tracker — external-SDK compat ONLY; no Hanzo surface uses these (Hanzo
|
||||
// surfaces POST /v1/event). It funnels through the ONE ingest core (ingestBody via
|
||||
// captureWithOrg) and keeps captureTenant's brand-host path for anonymous external
|
||||
// marketing traffic, which the strict canonical door deliberately refuses.
|
||||
func capture(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
deprecated(s, c, "/v1/event")
|
||||
org, ok := captureTenant(c)
|
||||
if !ok {
|
||||
return zip.ErrForbidden("valid bearer or a recognized brand host required")
|
||||
}
|
||||
var batch CaptureBatch
|
||||
if err := c.Bind(&batch); err != nil {
|
||||
return zip.ErrBadRequest("malformed capture batch")
|
||||
}
|
||||
res, err := ingestEvents(c.Context(), org, sourceCapture, batch.events())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.JSON(http.StatusOK, res)
|
||||
return captureWithOrg(org, c)
|
||||
}
|
||||
|
||||
// captureWithOrg is the Segment/beacon ingest with the tenant supplied EXPLICITLY by
|
||||
// the caller — the ONE write path shared by two org sources:
|
||||
// - the /v1/analytics{,/batch}, /v1/tracker aliases resolve org via captureTenant
|
||||
// (validated principal | project key | brand host) and call this;
|
||||
// - the site-host carve (sites.SetAnalyticsHostHandler) FORCES org from the
|
||||
// resolved Site (server-supplied, host-derived — never the caller/body) and
|
||||
// calls this.
|
||||
//
|
||||
// It funnels through the ONE ingest core (ingestBody, source=capture), which is
|
||||
// wire-tolerant — so the Segment {batch} envelope this wire speaks decodes through
|
||||
// the SAME decoder as the canonical door; org is never read from the body.
|
||||
func captureWithOrg(org string, c *zip.Ctx) error {
|
||||
return ingestBody(c, org, sourceCapture)
|
||||
}
|
||||
|
||||
+153
-45
@@ -14,21 +14,36 @@
|
||||
|
||||
// event.go — the ONE canonical event-ingestion front door.
|
||||
//
|
||||
// POST /v1/event body: Event | [Event] -> {accepted, dropped}
|
||||
// POST /v1/event body: Event | [Event] | {batch:[…]} -> {accepted, dropped}
|
||||
//
|
||||
// A JSON object is one event; a JSON array IS the batch (there is deliberately no
|
||||
// /v1/event/batch). Every other ingest surface (the PostHog wire at
|
||||
// /v1/insights/e, the Segment/beacon wire at /v1/analytics{,/batch} and
|
||||
// /v1/tracker) is a thin DEPRECATED adapter that normalizes its own wire shape
|
||||
// onto CaptureEvent and funnels through the SAME write core (ingestEvents) into
|
||||
// the SAME hanzo.events table. One write path, many adapters.
|
||||
// ONE door, EVERY wire, EVERY auth context. The decoder (decodeIngest) is
|
||||
// wire-tolerant: a bare canonical Event object, a bare [Event] array, AND the
|
||||
// CaptureBatch envelope ({batch:[…]} | {events:[…]}) the Segment/beacon/publishable
|
||||
// paths speak all decode onto the SAME []CaptureEvent the ONE write core
|
||||
// (ingestEvents) consumes, into the SAME hanzo.events table. There is deliberately
|
||||
// no /v1/event/batch — a JSON array, or a batch envelope, IS the batch.
|
||||
//
|
||||
// AUTH — IAM ONLY, FAIL-CLOSED: the tenant is resolved SERVER-SIDE from a
|
||||
// validated bearer principal (its owner org) or, for a keyed bearer-less SDK, an
|
||||
// access key resolved through the ONE IAM key seam (cloud.OrgForKey). There is NO
|
||||
// brand-host fallback on this endpoint: an unauthenticated or unresolvable caller
|
||||
// is refused (403), so the canonical door never writes an event into a tenant IAM
|
||||
// did not vouch for. The org is NEVER read from the body.
|
||||
// AUTH is the orthogonal, PLUGGABLE concern on this one door (eventTenant), resolved
|
||||
// SERVER-SIDE and FAIL-CLOSED, in strict trust order:
|
||||
//
|
||||
// 1. a validated IAM bearer principal — its owner org;
|
||||
// 2. a write-only publishable key (pk_…) — HMAC-verified org, no IAM/DB hop (the
|
||||
// SAME key publishable.go mints; folded in here so a pk_ caller uses /v1/event
|
||||
// directly);
|
||||
// 3. an out-of-band IAM access key (hk-/sk-…) — resolved through the ONE key seam
|
||||
// (cloud.OrgForKey).
|
||||
//
|
||||
// None of the above ⇒ 403. There is NO brand-host fallback on the canonical door
|
||||
// (that path stays only on the deprecated aliases), so /v1/event never writes an
|
||||
// event into a tenant IAM did not vouch for. The org is NEVER read from the body.
|
||||
//
|
||||
// The site-host carve (eventWithOrg) is the ONE exception to in-handler auth: on a
|
||||
// published site host the tenant is FORCED from the resolved Site BEFORE the handler
|
||||
// — the same server-supplied, host-derived tenant the file/base carves trust.
|
||||
//
|
||||
// Every other ingest route (/v1/ingest, /v1/analytics{,/batch}, /v1/tracker,
|
||||
// /v1/insights/e) is a thin alias/shim that resolves org its own way and funnels
|
||||
// through the SAME decode + write core. One write path, many doors.
|
||||
package analytics
|
||||
|
||||
import (
|
||||
@@ -64,17 +79,28 @@ func (e Event) toCapture() CaptureEvent {
|
||||
}
|
||||
}
|
||||
|
||||
// eventTenant resolves the tenant for POST /v1/event — IAM ONLY, FAIL-CLOSED. A
|
||||
// validated bearer principal wins (its owner org); otherwise a presented access
|
||||
// key is resolved to its org through the ONE IAM key seam (resolveKeyOrg →
|
||||
// cloud.OrgForKey). There is NO brand-host fallback: an unauthenticated or
|
||||
// unresolvable caller returns ("", false) → 403. (The deprecated aliases keep
|
||||
// captureTenant's brand-host path for anonymous marketing traffic; the canonical
|
||||
// endpoint is deliberately stricter — IAM is the only tenant authority here.)
|
||||
// eventTenant resolves the tenant for the canonical door — PLUGGABLE auth,
|
||||
// FAIL-CLOSED, in strict trust order:
|
||||
//
|
||||
// 1. a validated IAM bearer principal wins (its owner org);
|
||||
// 2. else a presented write-only publishable key (pk_…) is HMAC-verified to its
|
||||
// org with no IAM/DB hop (the SAME verifier publishable.go's /v1/ingest used —
|
||||
// folded in here so a pk_ caller uses /v1/event directly);
|
||||
// 3. else a presented out-of-band IAM access key (hk-/sk-…) is resolved to its org
|
||||
// through the ONE key seam (resolveKeyOrg → cloud.OrgForKey).
|
||||
//
|
||||
// None matches ⇒ ("", false) → 403. There is NO brand-host fallback (that path
|
||||
// stays only on the deprecated aliases), so the canonical door is strictly authed —
|
||||
// IAM or a signed/resolvable key, never the request Host.
|
||||
func eventTenant(c *zip.Ctx) (string, bool) {
|
||||
if org, ok := tenant(c); ok {
|
||||
return org, true
|
||||
}
|
||||
if key := ingestKey(c); key != "" {
|
||||
if org, ok := verifyPublishableKey(ingestSecret(), key); ok {
|
||||
return org, true
|
||||
}
|
||||
}
|
||||
if key := projectKey(c); key != "" {
|
||||
if org, ok := resolveKeyOrg(c.Context(), key); ok {
|
||||
return org, true
|
||||
@@ -83,20 +109,27 @@ func eventTenant(c *zip.Ctx) (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
// decodeEvents decodes a request body as Event | []Event. The first non-space
|
||||
// byte decides: '[' ⇒ the array batch, anything else ⇒ a single Event. An empty
|
||||
// body yields no events (an honest empty receipt, not an error). Pure over the
|
||||
// raw bytes (the handler passes c.Body() — fasthttp-buffered, the same bytes
|
||||
// projectKey peeked) so the decode is driven directly by tests.
|
||||
func decodeEvents(body []byte) ([]Event, error) {
|
||||
i := 0
|
||||
for i < len(body) {
|
||||
if b := body[i]; b == ' ' || b == '\t' || b == '\r' || b == '\n' {
|
||||
i++
|
||||
continue
|
||||
// firstNonWS returns the index of the first non-JSON-whitespace byte, or len(body)
|
||||
// when the body is empty or all whitespace. The four bytes are JSON's insignificant
|
||||
// whitespace (RFC 8259 §2). The ONE place the ingest decoders skip leading space.
|
||||
func firstNonWS(body []byte) int {
|
||||
for i := 0; i < len(body); i++ {
|
||||
switch body[i] {
|
||||
case ' ', '\t', '\r', '\n':
|
||||
default:
|
||||
return i
|
||||
}
|
||||
break
|
||||
}
|
||||
return len(body)
|
||||
}
|
||||
|
||||
// decodeEvents decodes a body as the canonical Event wire: Event | []Event. The
|
||||
// first non-space byte decides: '[' ⇒ the array batch, anything else ⇒ a single
|
||||
// Event. An empty body yields no events (an honest empty receipt, not an error).
|
||||
// Pure over the raw bytes; it is the canonical-Event sub-decoder inside decodeIngest
|
||||
// (which additionally accepts the CaptureBatch envelope).
|
||||
func decodeEvents(body []byte) ([]Event, error) {
|
||||
i := firstNonWS(body)
|
||||
if i >= len(body) {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -114,26 +147,101 @@ func decodeEvents(body []byte) ([]Event, error) {
|
||||
return []Event{e}, nil
|
||||
}
|
||||
|
||||
// eventIngest answers POST /v1/event — the ONE canonical ingestion front door.
|
||||
// Org is IAM-derived and fail-closed (eventTenant); the body is Event | [Event];
|
||||
// every event flows through the ONE write core (ingestEvents) into the ONE
|
||||
// hanzo.events table, tagged source=event.
|
||||
func eventIngest(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
org, ok := eventTenant(c)
|
||||
if !ok {
|
||||
return zip.ErrForbidden("valid bearer or a resolvable access key required")
|
||||
// decodeIngest is the ONE wire-tolerant decoder of the canonical door. It accepts
|
||||
// EVERY shape a Hanzo surface emits and yields the SAME []CaptureEvent the write
|
||||
// core consumes:
|
||||
//
|
||||
// - the CaptureBatch envelope {batch:[…]} | {events:[…]} — the Segment/beacon/
|
||||
// publishable wire — detected by a top-level batch/events key;
|
||||
// - a bare canonical Event object {"event":…,"distinctId":…};
|
||||
// - a bare canonical Event array [ {…}, … ].
|
||||
//
|
||||
// An empty/whitespace-only body ⇒ no events (honest empty receipt, not an error).
|
||||
// Pure over the raw bytes (handlers pass c.Body() — fasthttp-buffered, the same
|
||||
// bytes projectKey/ingestKey peeked), so the decode is driven directly by tests.
|
||||
func decodeIngest(body []byte) ([]CaptureEvent, error) {
|
||||
i := firstNonWS(body)
|
||||
if i >= len(body) {
|
||||
return nil, nil
|
||||
}
|
||||
evs, err := decodeEvents(c.Body())
|
||||
if body[i] == '{' {
|
||||
// An object is the CaptureBatch envelope iff it carries a batch/events key;
|
||||
// otherwise it is a bare canonical Event. RawMessage is non-nil whenever the
|
||||
// key is present (even `[]`), so an empty batch is still routed as an envelope.
|
||||
var probe struct {
|
||||
Batch json.RawMessage `json:"batch"`
|
||||
Events json.RawMessage `json:"events"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &probe); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if probe.Batch != nil || probe.Events != nil {
|
||||
var batch CaptureBatch
|
||||
if err := json.Unmarshal(body, &batch); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return batch.events(), nil
|
||||
}
|
||||
}
|
||||
evs, err := decodeEvents(body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
caps := make([]CaptureEvent, len(evs))
|
||||
for j, e := range evs {
|
||||
caps[j] = e.toCapture()
|
||||
}
|
||||
return caps, nil
|
||||
}
|
||||
|
||||
// ingestBody is the ONE ingest core given an already-resolved org: wire-tolerant
|
||||
// decode (decodeIngest) → fold type:'error' events (foldException) → the ONE write
|
||||
// core (ingestEvents) → the honest receipt. org is the SERVER-resolved tenant (never
|
||||
// client input); source tags the front door for the $source migration signal. Every
|
||||
// door — the canonical /v1/event (pluggable auth), the site-host carve (forced org),
|
||||
// and the deprecated aliases — resolves org its OWN way then calls THIS: auth is the
|
||||
// only thing that differs between doors, the decode + write path is identical.
|
||||
func ingestBody(c *zip.Ctx, org, source string) error {
|
||||
evs, err := decodeIngest(c.Body())
|
||||
if err != nil {
|
||||
return zip.ErrBadRequest("malformed event payload")
|
||||
}
|
||||
caps := make([]CaptureEvent, len(evs))
|
||||
for i, e := range evs {
|
||||
caps[i] = e.toCapture()
|
||||
for i := range evs {
|
||||
evs[i] = foldException(evs[i])
|
||||
}
|
||||
res, err := ingestEvents(c.Context(), org, sourceEvent, caps)
|
||||
res, err := ingestEvents(c.Context(), org, source, evs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.JSON(http.StatusOK, res)
|
||||
}
|
||||
|
||||
// eventHandle is the canonical-door handler core: pluggable in-handler auth
|
||||
// (eventTenant, fail-closed) → the ONE ingest core. source tags the door so the
|
||||
// canonical /v1/event and the /v1/ingest deprecated alias share ONE implementation,
|
||||
// differing only in origin tag (and the alias's deprecation log).
|
||||
func eventHandle(c *zip.Ctx, source string) error {
|
||||
org, ok := eventTenant(c)
|
||||
if !ok {
|
||||
return zip.ErrForbidden("valid bearer or a resolvable ingest key required")
|
||||
}
|
||||
return ingestBody(c, org, source)
|
||||
}
|
||||
|
||||
// eventIngest answers POST /v1/event — the ONE canonical ingestion front door.
|
||||
// Org is resolved fail-closed (eventTenant: bearer | pk_ | access key); the body is
|
||||
// Event | [Event] | {batch}; every event flows through the ONE write core into the
|
||||
// ONE hanzo.events table, tagged source=event.
|
||||
func eventIngest(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
return eventHandle(c, sourceEvent)
|
||||
}
|
||||
|
||||
// eventWithOrg is the canonical-door ingest with the tenant supplied EXPLICITLY by
|
||||
// the caller — the twin of captureWithOrg/insightsWithOrg for the site-host carve.
|
||||
// The carve FORCES org from the resolved Site (host-derived, never the caller/body)
|
||||
// and calls this, so a published-site beacon POSTing <host>/v1/event lands as that
|
||||
// site's Org regardless of any body/header claim. Tagged source=event: it is the
|
||||
// canonical wire, merely authorized by the host instead of a bearer/key.
|
||||
func eventWithOrg(org string, c *zip.Ctx) error {
|
||||
return ingestBody(c, org, sourceEvent)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// appBeaconBody is the EXACT payload the published-site page beacon posts
|
||||
// (app/lib/publishing/wired-injection.ts:68-69): a {batch:[CaptureEvent]} envelope.
|
||||
// The only change the app makes is repointing ANALYTICS_ENDPOINT from /v1/analytics
|
||||
// to /v1/event — the body is unchanged and MUST land via the canonical door's
|
||||
// site-host carve.
|
||||
const appBeaconBody = `{"batch":[{"messageId":"m-abc123","type":"pageview","event":"$pageview",` +
|
||||
`"timestamp":"2026-07-22T12:00:00.000Z","distinctId":"anon-9","anonymousId":"anon-9",` +
|
||||
`"sessionId":"sess-1","url":"https://yadota.hanzo.app/pricing","path":"/pricing",` +
|
||||
`"referrer":"https://news.ycombinator.com/","properties":{"space":"yadota","title":"Pricing"},` +
|
||||
`"library":"@hanzo/capture-wired","libraryVersion":"0.1.1"}]}`
|
||||
|
||||
// TestMount_HostCarve_EventDoorForcesOrg is the /v1/event twin of
|
||||
// TestMount_HostCarve_ForcesSiteOrg: a beacon POST to the CANONICAL door on a LIVE
|
||||
// site host is ingested as the site's Org even though the request carries a forged
|
||||
// org (body + X-Org-Id) and NO validated principal. 503-not-403 is the discriminator
|
||||
// — the same body 403s a DIRECT /v1/event call (no brand fallback), but here it
|
||||
// passes because the carve FORCES org from the host, stopping only at datastore-down.
|
||||
func TestMount_HostCarve_EventDoorForcesOrg(t *testing.T) {
|
||||
app := carveApp(t, "hanzo")
|
||||
for _, body := range []string{
|
||||
`{"event":"signup_completed","distinctId":"d","org":"attacker"}`, // bare Event
|
||||
`[{"event":"signup_completed","distinctId":"d"}]`, // bare [Event]
|
||||
`{"batch":[{"type":"event","event":"signup_completed"}],"org":"evil"}`, // {batch} envelope
|
||||
} {
|
||||
code := postHost(t, app, "yadota.hanzo.app", "/v1/event", body,
|
||||
map[string]string{"X-Org-Id": "attacker"})
|
||||
if code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("/v1/event beacon %q want 503 (ingested as site org), got %d", body, code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMount_HostCarve_AppBeaconExactBody confirms the CANONICAL door accepts the
|
||||
// APP beacon's EXACT {batch:[ev]} body via the site-host carve — the acceptance test
|
||||
// for repointing ANALYTICS_ENDPOINT to /v1/event. Admitted (503, datastore down),
|
||||
// tenant forced to the site's Org regardless of the beacon's properties.space claim.
|
||||
func TestMount_HostCarve_AppBeaconExactBody(t *testing.T) {
|
||||
app := carveApp(t, "hanzo")
|
||||
code := postHost(t, app, "yadota.hanzo.app", "/v1/event", appBeaconBody, nil)
|
||||
if code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("exact app beacon on /v1/event want 503 (admitted via carve), got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMount_HostCarve_EventEmptyBatchOK: an empty beacon batch on the canonical door
|
||||
// is an honest 200 (zero counts) BEFORE the datastore is consulted — proving the
|
||||
// carve decodes and funnels through the ONE write core with the host-forced org.
|
||||
func TestMount_HostCarve_EventEmptyBatchOK(t *testing.T) {
|
||||
app := carveApp(t, "hanzo")
|
||||
if code := postHost(t, app, "yadota.hanzo.app", "/v1/event", `{"batch":[]}`, nil); code != http.StatusOK {
|
||||
t.Fatalf("empty beacon batch on /v1/event want 200, got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMount_HostCarve_EventDirectNoHostStillFailsClosed pins that the forced-org
|
||||
// carve is HOST-scoped: the SAME anonymous /v1/event body on a NON-site host runs the
|
||||
// normal canonical gate (eventTenant, no brand fallback) and is refused 403 — the
|
||||
// carve did not fire, so the strict door invariant is unweakened.
|
||||
func TestMount_HostCarve_EventDirectNoHostStillFailsClosed(t *testing.T) {
|
||||
app := carveApp(t, "hanzo")
|
||||
code := postHost(t, app, "evil.example.com", "/v1/event",
|
||||
`{"event":"signup_completed","distinctId":"d"}`, map[string]string{"X-Org-Id": "attacker"})
|
||||
if code != http.StatusForbidden {
|
||||
t.Fatalf("anonymous /v1/event on a non-site host want 403 (no carve, no brand fallback), got %d", code)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// forward.go is the fan-out seam of the canonical event plane. After the ONE write
|
||||
// core (ingestEvents) commits a batch to hanzo.events, it hands a COPY of that batch
|
||||
// to an optional downstream sink — the destinations subsystem — which translates and
|
||||
// forwards each event to the org's connected ad/analytics platforms (GA4, Meta CAPI,
|
||||
// …). The seam is:
|
||||
//
|
||||
// - ONE-WAY. analytics never imports destinations; destinations calls SetSink from
|
||||
// its Mount. A nil sink means no fan-out (the default when destinations is off),
|
||||
// so this file changes nothing about ingest when the subsystem is absent.
|
||||
// - RAW. The sink receives the event BEFORE the warehouse privacy scrub, because a
|
||||
// server-side Conversions-API forwarder must hash the match keys (email/phone/
|
||||
// click ids) the warehouse deliberately drops. The org connected the destination
|
||||
// and owns that consent; the destination adapters SHA-256 every PII field before
|
||||
// it leaves the process.
|
||||
// - FAIL-SOFT. The sink runs detached (a panic-guarded goroutine) so a slow or
|
||||
// broken destination can never block, fail, or crash an ingest.
|
||||
package analytics
|
||||
|
||||
import "time"
|
||||
|
||||
// SinkEvent is one accepted event handed to the downstream fan-out. It carries the
|
||||
// resolved canonical name plus the commerce + identity fields a conversion needs;
|
||||
// Properties is the RAW (pre-scrub) property bag the translator lifts match keys and
|
||||
// custom data from. The tenant is the org argument to the sink, never a field here.
|
||||
type SinkEvent struct {
|
||||
MessageID string
|
||||
Name string
|
||||
DistinctID string
|
||||
AnonymousID string
|
||||
Time time.Time
|
||||
URL string
|
||||
Path string
|
||||
Referrer string
|
||||
Revenue float64
|
||||
Currency string
|
||||
ProductID string
|
||||
Quantity uint32
|
||||
Properties map[string]any
|
||||
}
|
||||
|
||||
// sink is the downstream fan-out hook, installed once by the destinations subsystem
|
||||
// at Mount (nil ⇒ no fan-out). A subsystem Mount runs before request traffic, so no
|
||||
// lock is needed on this package-global.
|
||||
var sink func(org string, evs []SinkEvent)
|
||||
|
||||
// SetSink installs (nil clears) the downstream fan-out hook.
|
||||
func SetSink(fn func(org string, evs []SinkEvent)) { sink = fn }
|
||||
|
||||
// fanOut hands the accepted batch to the sink, detached and fail-soft. org is the
|
||||
// SERVER-resolved tenant (already an owned copy from principal.Org). It builds
|
||||
// SinkEvents from the RAW events (skipping unroutable ones, mirroring the write
|
||||
// core's drop rule) and, if any remain and a sink is installed, dispatches them on a
|
||||
// panic-guarded goroutine so ingest is never blocked or failed by a destination.
|
||||
func fanOut(org string, evs []CaptureEvent) {
|
||||
fn := sink
|
||||
if fn == nil || len(evs) == 0 {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
out := make([]SinkEvent, 0, len(evs))
|
||||
for _, e := range evs {
|
||||
name := resolveEventName(e)
|
||||
if name == "" {
|
||||
continue // unroutable — the write core dropped it too
|
||||
}
|
||||
out = append(out, SinkEvent{
|
||||
MessageID: firstNonEmptyStr(trim(e.MessageID), randID()),
|
||||
Name: name,
|
||||
DistinctID: trim(e.DistinctID),
|
||||
AnonymousID: trim(e.AnonymousID),
|
||||
Time: clampTS(e.Timestamp, now),
|
||||
URL: trim(e.URL),
|
||||
Path: trim(e.Path),
|
||||
Referrer: trim(e.Referrer),
|
||||
Revenue: e.Revenue,
|
||||
Currency: trim(e.Currency),
|
||||
ProductID: trim(e.ProductID),
|
||||
Quantity: e.Quantity,
|
||||
Properties: e.Properties,
|
||||
})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer func() { _ = recover() }()
|
||||
fn(org, out)
|
||||
}()
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestFanOutTranslatesAndFiresSink verifies the fan-out seam builds SinkEvents from
|
||||
// the RAW accepted batch (resolved name, commerce fields, un-scrubbed properties for
|
||||
// match keys), drops unroutable events, and dispatches them to the installed sink.
|
||||
func TestFanOutTranslatesAndFiresSink(t *testing.T) {
|
||||
got := make(chan []SinkEvent, 1)
|
||||
SetSink(func(org string, evs []SinkEvent) {
|
||||
if org == "acme" {
|
||||
got <- evs
|
||||
}
|
||||
})
|
||||
defer SetSink(nil)
|
||||
|
||||
fanOut("acme", []CaptureEvent{
|
||||
{Type: "event", Event: "order_completed", DistinctID: "u1", Revenue: 49, Currency: "USD",
|
||||
Properties: map[string]any{"email": "a@b.com"}},
|
||||
{Type: "pageview"}, // resolves to $pageview
|
||||
{Type: "event", Event: ""}, // unroutable → dropped (mirrors the write core)
|
||||
})
|
||||
|
||||
select {
|
||||
case evs := <-got:
|
||||
if len(evs) != 2 {
|
||||
t.Fatalf("want 2 routable events, got %d", len(evs))
|
||||
}
|
||||
if evs[0].Name != "order_completed" || evs[0].Revenue != 49 || evs[0].Currency != "USD" {
|
||||
t.Errorf("purchase event not carried: %+v", evs[0])
|
||||
}
|
||||
if evs[0].Properties["email"] != "a@b.com" {
|
||||
t.Errorf("raw (pre-scrub) properties must be carried for match keys: %+v", evs[0].Properties)
|
||||
}
|
||||
if evs[1].Name != "$pageview" {
|
||||
t.Errorf("pageview name = %q", evs[1].Name)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("sink was not called")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFanOutNilSinkIsNoOp verifies fan-out is inert (no panic, no goroutine) when no
|
||||
// sink is installed — the default when destinations is disabled.
|
||||
func TestFanOutNilSinkIsNoOp(t *testing.T) {
|
||||
SetSink(nil)
|
||||
fanOut("acme", []CaptureEvent{{Type: "event", Event: "order_completed"}})
|
||||
// Nothing to assert beyond "did not panic / block".
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/clients/sites"
|
||||
luxlog "github.com/luxfi/log"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// liveResolver is a sites.Resolver that knows exactly ONE published site — the
|
||||
// slug key "yadota" and the bound custom host "yadota.tech" — and returns its live
|
||||
// Site whose Org the carve must force onto every ingested beacon. Any other key is
|
||||
// an honest miss (found=false), exactly as the real projects store behaves, so a
|
||||
// stray external host is NOT mistaken for a bound custom domain.
|
||||
type liveResolver struct{ org string }
|
||||
|
||||
func (r liveResolver) Resolve(_ context.Context, key string) (sites.Site, bool, error) {
|
||||
switch key {
|
||||
case "yadota", "yadota.tech":
|
||||
return sites.Site{Org: r.org, Slug: "yadota", Bucket: "b", Prefix: r.org + "/yadota", Status: "live"}, true, nil
|
||||
default:
|
||||
return sites.Site{}, false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// ResolveOrg pins the slug to an org (first-party-host path). The carve resolves
|
||||
// via resolveLivePinned, so the analytics host-carve tests exercise this path.
|
||||
func (r liveResolver) ResolveOrg(_ context.Context, org, slug string) (sites.Site, bool, error) {
|
||||
if org == r.org && slug == "yadota" {
|
||||
return sites.Site{Org: org, Slug: "yadota", Bucket: "b", Prefix: org + "/yadota", Status: "live"}, true, nil
|
||||
}
|
||||
return sites.Site{}, false, nil
|
||||
}
|
||||
|
||||
// carveApp mounts analytics (which installs the site-host ingest carve via
|
||||
// sites.SetAnalyticsHostHandler) BEHIND the sites host-router middleware, then
|
||||
// points the resolver at one live Site. A POST to the site host is intercepted by
|
||||
// the middleware and forced to Site.Org; a POST to any other host falls through to
|
||||
// the normal /v1/analytics route.
|
||||
func carveApp(t *testing.T, org string) *zip.App {
|
||||
t.Helper()
|
||||
app := zip.New(zip.Config{Logger: luxlog.New("test"), DisableStartupMessage: true})
|
||||
srv := sites.New(sites.Config{
|
||||
Apex: "hanzo.app",
|
||||
Reserved: []string{"app", "api", "admin"},
|
||||
SelfDomains: []string{"hanzo.ai", "hanzo.app"},
|
||||
}, luxlog.New("test"))
|
||||
app.Use(srv.Middleware())
|
||||
if err := Mount(app, cloud.Deps{Logger: luxlog.New("test")}); err != nil {
|
||||
t.Fatalf("Mount: %v", err)
|
||||
}
|
||||
sites.SetResolver(liveResolver{org: org})
|
||||
t.Cleanup(func() {
|
||||
sites.SetResolver(nil)
|
||||
sites.SetAnalyticsHostHandler(nil)
|
||||
})
|
||||
return app
|
||||
}
|
||||
|
||||
func postHost(t *testing.T, app *zip.App, host, path, body string, hdr map[string]string) int {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodPost, "http://"+host+path, strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Host = host
|
||||
for k, v := range hdr {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
resp, err := app.Fiber().Test(req)
|
||||
if err != nil {
|
||||
t.Fatalf("Test POST %s%s: %v", host, path, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return resp.StatusCode
|
||||
}
|
||||
|
||||
// TestMount_HostCarve_ForcesSiteOrg is the end-to-end proof: Mount wires the carve,
|
||||
// and a beacon POST to a LIVE site host is ingested as the site's Org even though
|
||||
// the request carries a forged org (body + X-Org-Id) and NO validated principal.
|
||||
// The discriminator is 503-not-403: the SAME shape 403s on the normal route
|
||||
// (TestCapture_ForgedOrgWithoutBearerForbidden / unknown-host), but here it passes
|
||||
// the tenant gate (org forced from the host) and stops only at the datastore-down
|
||||
// 503 — proving the org is server-supplied from the host, never the caller/body.
|
||||
func TestMount_HostCarve_ForcesSiteOrg(t *testing.T) {
|
||||
app := carveApp(t, "hanzo")
|
||||
|
||||
// Segment/beacon wire on /v1/analytics + its /batch alias.
|
||||
for _, p := range []string{"/v1/analytics", "/v1/analytics/batch"} {
|
||||
code := postHost(t, app, "yadota.hanzo.app", p,
|
||||
`{"batch":[{"type":"event","event":"signup_completed"}],"org":"attacker","properties":{"space":"attacker"}}`,
|
||||
map[string]string{"X-Org-Id": "attacker"})
|
||||
if code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("beacon POST %s want 503 (ingested as site org, datastore down), got %d", p, code)
|
||||
}
|
||||
}
|
||||
|
||||
// PostHog wire on /v1/insights/e.
|
||||
code := postHost(t, app, "yadota.hanzo.app", "/v1/insights/e",
|
||||
`{"event":"$pageview","distinct_id":"d","properties":{"space":"attacker"}}`,
|
||||
map[string]string{"X-Org-Id": "attacker"})
|
||||
if code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("insights beacon want 503 (ingested as site org), got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMount_HostCarve_EmptyBatchOK: an empty beacon batch on the site host is an
|
||||
// honest 200 (zero counts) BEFORE the datastore is consulted — proving the carve
|
||||
// decodes and funnels through the ONE write core without any principal.
|
||||
func TestMount_HostCarve_EmptyBatchOK(t *testing.T) {
|
||||
app := carveApp(t, "hanzo")
|
||||
if code := postHost(t, app, "yadota.hanzo.app", "/v1/analytics", `{"batch":[]}`, nil); code != http.StatusOK {
|
||||
t.Fatalf("empty beacon batch want 200, got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMount_HostCarve_CustomDomainForcesOrg: the carve fires for a bound custom
|
||||
// domain too, forcing that Site's Org.
|
||||
func TestMount_HostCarve_CustomDomainForcesOrg(t *testing.T) {
|
||||
app := carveApp(t, "yadota")
|
||||
code := postHost(t, app, "yadota.tech", "/v1/analytics",
|
||||
`{"batch":[{"type":"pageview"}]}`, map[string]string{"X-Org-Id": "attacker"})
|
||||
if code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("custom-domain beacon want 503 (ingested as site org), got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMount_HostCarve_GetNotHijacked: a GET on the site host is NOT ingest — it is
|
||||
// served as static (storage unconfigured here ⇒ 503 from the serve path), never
|
||||
// routed to the ingest carve; the read-lens surface is untouched.
|
||||
func TestMount_HostCarve_GetNotHijacked(t *testing.T) {
|
||||
app := carveApp(t, "hanzo")
|
||||
req := httptest.NewRequest(http.MethodGet, "http://yadota.hanzo.app/v1/analytics/overview", nil)
|
||||
resp, err := app.Fiber().Test(req)
|
||||
if err != nil {
|
||||
t.Fatalf("GET: %v", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
// It must have reached the static serve, tagged X-Hanzo-Site — not the ingest
|
||||
// carve (which would 200 the empty body) and not the API pipeline.
|
||||
if resp.Header.Get("X-Hanzo-Site") != "yadota" {
|
||||
t.Fatalf("GET did not reach the static serve (X-Hanzo-Site=%q, status=%d)", resp.Header.Get("X-Hanzo-Site"), resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMount_HostCarve_NonSiteHostUsesNormalGate: on a NON-site host the middleware
|
||||
// Continues and the normal /v1/analytics route runs — so an anonymous unknown-host
|
||||
// beacon is refused 403 by captureTenant (the carve did not fire). This pins that
|
||||
// the forced-org path is host-scoped and does NOT weaken the normal ingest gate.
|
||||
func TestMount_HostCarve_NonSiteHostUsesNormalGate(t *testing.T) {
|
||||
app := carveApp(t, "hanzo")
|
||||
code := postHost(t, app, "evil.example.com", "/v1/analytics",
|
||||
`{"batch":[{"type":"pageview"}]}`, map[string]string{"X-Org-Id": "attacker"})
|
||||
if code != http.StatusForbidden {
|
||||
t.Fatalf("anonymous unknown-host beacon on the normal route want 403, got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMount_HostCarve_DisabledWhenPublicCaptureOff: with public capture off the
|
||||
// carve is NOT installed, so a beacon POST to the site host falls to the static
|
||||
// serve and 405s (unchanged from before the fix).
|
||||
func TestMount_HostCarve_DisabledWhenPublicCaptureOff(t *testing.T) {
|
||||
t.Setenv(publicCaptureEnv, "off")
|
||||
app := carveApp(t, "hanzo")
|
||||
code := postHost(t, app, "yadota.hanzo.app", "/v1/analytics",
|
||||
`{"batch":[{"type":"pageview"}]}`, nil)
|
||||
if code != http.StatusMethodNotAllowed {
|
||||
t.Fatalf("public-capture-off site beacon want 405 (carve not installed), got %d", code)
|
||||
}
|
||||
}
|
||||
@@ -33,7 +33,11 @@ import (
|
||||
)
|
||||
|
||||
// insightsEvent is the PostHog wire shape (subset that matters for ingest).
|
||||
// UUID is the top-level per-event id PostHog SDKs mint for idempotency; the rest
|
||||
// of the identity/attribution the SDKs carry rides inside Properties (mapped in
|
||||
// toCapture).
|
||||
type insightsEvent struct {
|
||||
UUID string `json:"uuid"`
|
||||
Event string `json:"event"`
|
||||
DistinctID string `json:"distinct_id"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
@@ -65,6 +69,11 @@ func (e insightsEvent) toCapture() CaptureEvent {
|
||||
typ = "pageview"
|
||||
}
|
||||
return CaptureEvent{
|
||||
// Idempotency id: PostHog SDKs carry a top-level event `uuid`; some send it
|
||||
// as an `$insert_id` property instead. Preserve it as the client MessageID so
|
||||
// a retried batch (insights-go retries with backoff) keeps a STABLE row id
|
||||
// rather than the server minting a fresh one per attempt.
|
||||
MessageID: firstNonEmptyStr(strings.TrimSpace(e.UUID), strings.TrimSpace(str("$insert_id"))),
|
||||
Type: typ,
|
||||
Event: e.Event,
|
||||
Timestamp: e.Timestamp,
|
||||
@@ -73,6 +82,19 @@ func (e insightsEvent) toCapture() CaptureEvent {
|
||||
URL: str("$current_url"),
|
||||
Path: str("$pathname"),
|
||||
Referrer: str("$referrer"),
|
||||
// UTM attribution: PostHog SDKs put campaign params in BARE `utm_*`
|
||||
// properties (not $-prefixed — confirmed against the SDK/ingest source).
|
||||
// hanzo.events has first-class utm_* columns and the native capture path
|
||||
// maps CaptureEvent.UTM into them (capture.go), so surfacing them here is
|
||||
// what lets the web/commerce lens attribute traffic to a campaign. They were
|
||||
// previously dropped on the PostHog-wire front door.
|
||||
UTM: UTM{
|
||||
Source: str("utm_source"),
|
||||
Medium: str("utm_medium"),
|
||||
Campaign: str("utm_campaign"),
|
||||
Term: str("utm_term"),
|
||||
Content: str("utm_content"),
|
||||
},
|
||||
Product: str("product"),
|
||||
Library: str("$lib"),
|
||||
LibraryVer: str("$lib_version"),
|
||||
@@ -80,17 +102,27 @@ func (e insightsEvent) toCapture() CaptureEvent {
|
||||
}
|
||||
}
|
||||
|
||||
// insightsIngest answers POST /v1/insights/e — the DEPRECATED PostHog-wire
|
||||
// adapter. It normalizes the PostHog single/batch shape onto CaptureEvent and
|
||||
// funnels through the ONE write core (ingestEvents, source=posthog); it keeps
|
||||
// captureTenant's brand-host path so anonymous PostHog-wire traffic is unbroken.
|
||||
// New callers post the canonical Event to /v1/event.
|
||||
// insightsIngest answers POST /v1/insights/e — a DEPRECATED foreign-protocol shim
|
||||
// for the PostHog wire. External-SDK compat ONLY; no Hanzo surface uses it (Hanzo
|
||||
// surfaces POST /v1/event). It normalizes the PostHog single/batch shape onto
|
||||
// CaptureEvent and funnels through the ONE write core (ingestEvents, source=posthog);
|
||||
// it keeps captureTenant's brand-host path so anonymous external PostHog-wire traffic
|
||||
// is unbroken — the path the strict canonical door deliberately refuses.
|
||||
func insightsIngest(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
deprecated(s, c, "/v1/event")
|
||||
org, ok := captureTenant(c)
|
||||
if !ok {
|
||||
return zip.ErrForbidden("valid bearer or a recognized brand host required")
|
||||
}
|
||||
return insightsWithOrg(org, c)
|
||||
}
|
||||
|
||||
// insightsWithOrg is the PostHog-wire decode+ingest core with the tenant supplied
|
||||
// EXPLICITLY by the caller — the twin of captureWithOrg for the PostHog beacon
|
||||
// shape. The /v1/insights/e alias resolves org via captureTenant; the site-host
|
||||
// carve FORCES org from the resolved Site (host-derived, never the caller/body).
|
||||
// Both funnel through the ONE write core (ingestEvents, source=posthog).
|
||||
func insightsWithOrg(org string, c *zip.Ctx) error {
|
||||
var body insightsBody
|
||||
if err := c.Bind(&body); err != nil {
|
||||
return zip.ErrBadRequest("malformed insights payload")
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestToCapture_PreservesUTMAttribution proves the PostHog-wire adapter carries the
|
||||
// BARE utm_* campaign params (what PostHog SDKs emit) through to the native
|
||||
// CaptureEvent — and thence, via normalizeEvent, into the hanzo.events utm_*
|
||||
// columns the INSERT binds. Regression guard: these were previously dropped, so
|
||||
// every campaign-attributed pageview lost its source/medium/campaign on the
|
||||
// /v1/insights/e front door and the web/commerce lens could never attribute it.
|
||||
func TestToCapture_PreservesUTMAttribution(t *testing.T) {
|
||||
e := insightsEvent{
|
||||
Event: "$pageview",
|
||||
DistinctID: "visitor-1",
|
||||
Properties: map[string]any{
|
||||
"utm_source": "newsletter",
|
||||
"utm_medium": "email",
|
||||
"utm_campaign": "launch",
|
||||
"utm_term": "analytics",
|
||||
"utm_content": "hero-cta",
|
||||
"$current_url": "https://hanzo.ai/insights",
|
||||
},
|
||||
}
|
||||
cap := e.toCapture()
|
||||
if cap.UTM.Source != "newsletter" || cap.UTM.Medium != "email" ||
|
||||
cap.UTM.Campaign != "launch" || cap.UTM.Term != "analytics" || cap.UTM.Content != "hero-cta" {
|
||||
t.Fatalf("UTM not mapped from PostHog wire: %+v", cap.UTM)
|
||||
}
|
||||
// End-to-end through the normalizer into the positional row the INSERT binds.
|
||||
row, ok := normalizeEvent("acme", time.Now(), cap)
|
||||
if !ok {
|
||||
t.Fatal("want ok")
|
||||
}
|
||||
if row.utmSource != "newsletter" || row.utmMedium != "email" ||
|
||||
row.utmCampaign != "launch" || row.utmTerm != "analytics" || row.utmContent != "hero-cta" {
|
||||
t.Fatalf("UTM lost before the events row: src=%q med=%q camp=%q term=%q content=%q",
|
||||
row.utmSource, row.utmMedium, row.utmCampaign, row.utmTerm, row.utmContent)
|
||||
}
|
||||
}
|
||||
|
||||
// TestToCapture_IdempotencyID proves the client event id (PostHog top-level `uuid`,
|
||||
// or the `$insert_id` property fallback) is preserved as the stable row id, so a
|
||||
// retried batch does not mint a fresh id per attempt — while an absent id still
|
||||
// falls back to a server-minted one (existing behavior unchanged).
|
||||
func TestToCapture_IdempotencyID(t *testing.T) {
|
||||
// top-level uuid wins
|
||||
row, _ := normalizeEvent("acme", time.Now(),
|
||||
insightsEvent{Event: "signup", DistinctID: "u1", UUID: "evt-abc"}.toCapture())
|
||||
if row.id != "evt-abc" {
|
||||
t.Fatalf("top-level uuid not preserved as row id, got %q", row.id)
|
||||
}
|
||||
// $insert_id property fallback when no top-level uuid
|
||||
row2, _ := normalizeEvent("acme", time.Now(),
|
||||
insightsEvent{Event: "signup", DistinctID: "u1", Properties: map[string]any{"$insert_id": "ins-9"}}.toCapture())
|
||||
if row2.id != "ins-9" {
|
||||
t.Fatalf("$insert_id fallback not preserved, got %q", row2.id)
|
||||
}
|
||||
// absent → server still mints a non-empty id
|
||||
row3, _ := normalizeEvent("acme", time.Now(),
|
||||
insightsEvent{Event: "signup", DistinctID: "u1"}.toCapture())
|
||||
if row3.id == "" {
|
||||
t.Fatal("server must still mint an id when the client sends none")
|
||||
}
|
||||
}
|
||||
|
||||
// TestToCapture_MapsCoreFields guards that the pre-existing $-property mappings
|
||||
// still hold alongside the new UTM/idempotency mappings (no regression).
|
||||
func TestToCapture_MapsCoreFields(t *testing.T) {
|
||||
cap := insightsEvent{
|
||||
Event: "$pageview",
|
||||
DistinctID: "v1",
|
||||
Properties: map[string]any{
|
||||
"$session_id": "s1",
|
||||
"$current_url": "https://hanzo.ai/x",
|
||||
"$pathname": "/x",
|
||||
"$referrer": "https://news.ycombinator.com/",
|
||||
"$lib": "insights-go",
|
||||
"$lib_version": "1.2.3",
|
||||
"product": "console",
|
||||
},
|
||||
}.toCapture()
|
||||
if cap.Type != "pageview" || cap.SessionID != "s1" || cap.URL != "https://hanzo.ai/x" ||
|
||||
cap.Path != "/x" || cap.Referrer != "https://news.ycombinator.com/" ||
|
||||
cap.Library != "insights-go" || cap.LibraryVer != "1.2.3" || cap.Product != "console" {
|
||||
t.Fatalf("core PostHog-wire mapping regressed: %+v", cap)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package analytics
|
||||
|
||||
// outcomes.go — the MEASUREMENT seam the experiments primitive composes. An
|
||||
// experiment's per-variant metric is read from the ONE analytics events plane
|
||||
// (hanzo.events), never a second event store: outcomes are already captured by
|
||||
// distinct_id (capture.go), so an experiment only needs to fold them per subject and
|
||||
// join each subject to its flags variant. This is the scientific-growth loop's
|
||||
// MEASUREMENT half — analytics measures, the experiment tests, flags decides.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
aiobject "github.com/hanzoai/ai/object"
|
||||
)
|
||||
|
||||
// SubjectOutcome is one subject's (distinct_id's) participation in an experiment
|
||||
// window: whether it fired the Exposed (enrolled / saw the arm) event and whether it
|
||||
// fired the Converted (metric) event. It is the per-subject grain the experiments
|
||||
// primitive joins to a flags variant assignment to produce per-variant samples.
|
||||
type SubjectOutcome struct {
|
||||
Subject string
|
||||
Exposed bool
|
||||
Converted bool
|
||||
}
|
||||
|
||||
// Outcomes returns, for one org over [start,end), each subject's exposure +
|
||||
// conversion for an experiment's two event names, read from hanzo.events. It is the
|
||||
// measurement seam the experiments primitive composes: flags assignment joins to
|
||||
// these outcomes by distinct_id.
|
||||
//
|
||||
// TENANT ISOLATION is the eventsWhere invariant — org is bound POSITIONALLY, never
|
||||
// interpolated — and every event name is a BOUND parameter too, so neither a hostile
|
||||
// org slug nor a hostile event name can escape into SQL. exposureEvent may be ""
|
||||
// (then every returned subject is Exposed: the population is "appeared in-window");
|
||||
// metricEvent is required. Fails closed with a 503 when the warehouse is absent.
|
||||
func Outcomes(ctx context.Context, org, exposureEvent, metricEvent string, start, end time.Time) ([]SubjectOutcome, error) {
|
||||
if err := requireDatastore(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if metricEvent == "" {
|
||||
return nil, fmt.Errorf("analytics: outcomes needs a metric event")
|
||||
}
|
||||
if err := EnsureEventsTable(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sql, args := outcomesSQL(org, exposureEvent, metricEvent, start, end)
|
||||
rows, err := aiobject.DatastoreQuery(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, warehouseErr("outcomes", err)
|
||||
}
|
||||
out := make([]SubjectOutcome, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
subject := aString(r["subject"])
|
||||
if subject == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, SubjectOutcome{
|
||||
Subject: subject,
|
||||
Exposed: aInt64(r["exposed"]) > 0,
|
||||
Converted: aInt64(r["converted"]) > 0,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// outcomesSQL builds the per-subject exposure/conversion query over hanzo.events —
|
||||
// the pure, I/O-free core so the isolation invariant is testable without a warehouse.
|
||||
// TENANCY: org rides eventsWhere as a BOUND parameter (never interpolated) and every
|
||||
// event name is BOUND too, so nothing user-derived escapes into SQL. The SELECT
|
||||
// maxIf placeholders appear first in the string, then eventsWhere's [start,end,org],
|
||||
// then the event-set IN — the args slice follows that exact positional order.
|
||||
func outcomesSQL(org, exposureEvent, metricEvent string, start, end time.Time) (string, []any) {
|
||||
where, wargs := eventsWhere(org, start, end)
|
||||
if exposureEvent == "" {
|
||||
args := append([]any{metricEvent}, wargs...)
|
||||
return "SELECT distinct_id AS subject, 1 AS exposed, " +
|
||||
"maxIf(1, event = ?) AS converted FROM " + eventsTable +
|
||||
" WHERE " + where + " GROUP BY distinct_id", args
|
||||
}
|
||||
args := append([]any{exposureEvent, metricEvent}, wargs...)
|
||||
args = append(args, exposureEvent, metricEvent)
|
||||
return "SELECT distinct_id AS subject, " +
|
||||
"maxIf(1, event = ?) AS exposed, " +
|
||||
"maxIf(1, event = ?) AS converted FROM " + eventsTable +
|
||||
" WHERE " + where + " AND event IN (?, ?) GROUP BY distinct_id", args
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user