fix(zt): message-hygiene — 503 body names no internal env

The fail-closed 503 body no longer names ZT_CLIENT_ID/ZT_CLIENT_SECRET;
it now reads 'networking is not configured on this deployment' — a
customer-facing string the console renders as a clean 'not available yet'
state. Ops still see the env names in the Warn log at Mount. No behavior
change; the gate() fail-closed contract is identical.

.dockerignore

@@ -0,0 +1,41 @@
+# VCS
+.git/
+.gitignore
+.gitattributes
+
+# CI / repo metadata not needed inside the build
+.github/
+
+# Docs (image runs the binary; readers visit GitHub)
+*.md
+LICENSE
+SECURITY.md
+
+# Already-built binary at repo root (matches .gitignore)
+/cloud
+
+# Environment files (never bake secrets into images)
+.env
+.env.*
+
+# IDE / editor
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS metadata
+.DS_Store
+Thumbs.db
+
+# Tests stay out of the runtime image
+*_test.go
+
+# Local build outputs
+/dist/
+/build/
+/bin/
+
+# The Dockerfile itself doesn't need to be in the context it builds
+Dockerfile
+.dockerignore

.github/hero.svg

@@ -0,0 +1,9 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="1280" height="640" viewBox="0 0 1280 640" role="img" aria-label="cloud">
+<rect width="1280" height="640" fill="#0A0A0A"/>
+<svg x="96" y="215" width="210" height="210" viewBox="0 0 67 67"><path d="M22.21 67V44.6369H0V67H22.21Z" fill="#fff"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z" fill="#fff"/><path d="M22.21 0H0V22.3184H22.21V0Z" fill="#fff"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z" fill="#fff"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z" fill="#fff"/></svg>
+<text x="378" y="276" font-family="Inter,system-ui,-apple-system,sans-serif" font-size="78" font-weight="800" letter-spacing="-2" fill="#ffffff">cloud</text>
+<text x="378" y="322" font-family="Inter,system-ui,sans-serif" font-size="30" fill="#ffffff" opacity=".66">Hanzo Cloud — unified Go binary that imports every Hanzo-native…</text>
+<rect x="378" y="338" width="806" height="3" rx="1.5" fill="#ffffff" opacity=".9"/>
+<text x="378" y="390" font-family="Inter,system-ui,sans-serif" font-size="24" font-weight="600" fill="#ffffff" opacity=".5">github.com/hanzoai</text>
+<text x="1184" y="390" text-anchor="end" font-family="Inter,system-ui,sans-serif" font-size="24" font-weight="600" fill="#ffffff" opacity=".5">hanzo.ai</text>
+</svg>

.github/workflows/release.yml

@@ -0,0 +1,286 @@
+name: release
+
+# Cuts a release of ghcr.io/hanzoai/cloud. The invariant this workflow exists to
+# enforce:
+#
+#     a git tag v<X.Y.Z> exists  ⇔  an image ghcr.io/hanzoai/cloud:v<X.Y.Z>
+#     was pushed AND booted to "listening" in the smoke test.
+#
+# The tag is a RECEIPT for a proven image, minted only AFTER a successful push —
+# never a trigger for a build that might fail. The prior design triggered builds
+# FROM pushed tags, so a tag could exist with no image behind it (a failed or
+# never-run build) — universe would then try to roll that tag and the pods went
+# ImagePullBackOff (phantom v1.786.42/43). Here the order is inverted:
+#
+#     main push → compute next version → build → SMOKE → push image → tag → notify
+#
+# so a push/smoke/build failure fails the run BEFORE the tag step and leaves no
+# tag; universe is only ever notified of a version whose image is proven present.
+#
+# DO NOT push v* tags by hand anymore. This workflow OWNS them. A hand-cut tag has
+# no image behind it (exactly the phantom this prevents) and won't build (there is
+# no `tags:` trigger). Every merge to main IS the release; skip one with the usual
+# `[skip ci]` in the commit/merge message (a docs-only change need not ship).
+#
+# concurrency: a single serialized lane (cancel-in-progress:false — a
+# mid-flight push must finish, never be killed between "image pushed" and "tag
+# created"). Two main pushes can therefore never compute the same next number:
+# the queued run starts only after the running one tags, re-reads the tags, and
+# lands on the next patch. Monotonic by construction.
+#
+# The next version is max(highest git tag, highest pushed container tag) + 1 (patch
+# bump only — never a major/minor jump). Folding in the container tags means we
+# never reuse a number that already has a pushed image, even if some earlier run
+# pushed an image but died before tagging.
+#
+# ── Infra notes (unchanged, still true) ─────────────────────────────────────────
+# Self-hosted arcd amd64 scale set — NEVER GitHub-hosted runners (this org's
+# GitHub-hosted Actions are billing-frozen). GHCR login uses GH_PAT, not the repo
+# GITHUB_TOKEN: the ghcr.io/hanzoai/cloud package is linked to a DIFFERENT repo
+# (hanzoai/ai, from the cloud→ai module rename), so this repo's GITHUB_TOKEN is
+# denied write to it (permission_denied: write_package). GH_PAT (admin:org +
+# write:packages) writes any hanzoai package regardless of package-repo linkage,
+# and is the BuildKit gh_token the Dockerfile uses to fetch private cross-org Go
+# modules. amd64-only: the cluster is amd64; one platform completes on the live
+# scale set without waiting on the arm64 pool.
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+# Cut tags on the cloud repo (git tag push) → contents: write. packages: write
+# to push the image; id-token for provenance.
+permissions:
+  contents: write
+  packages: write
+  id-token: write
+
+# One serialized release lane. Never cancel in-flight: a run killed between
+# "image pushed" and "tag created" is exactly the drift we are preventing.
+concurrency:
+  group: release-cloud
+  cancel-in-progress: false
+
+jobs:
+  build-amd64:
+    # ARC ephemeral runners match jobs targeting the scale-set NAME as a label.
+    runs-on: [hanzo-build-linux-amd64]
+    outputs:
+      version: ${{ steps.ver.outputs.version }}
+      version_v: ${{ steps.ver.outputs.version_v }}
+    steps:
+      - name: Checkout (full history + all tags — the version floor is read from tags)
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Compute next version (monotonic patch bump over git + container tags)
+        id: ver
+        env:
+          GH_PAT: ${{ secrets.GH_PAT }}
+        run: |
+          set -euo pipefail
+          git fetch --tags --force --quiet
+
+          # Highest semver git tag (vX.Y.Z), normalised without the leading v.
+          git_max="$(git tag -l 'v[0-9]*.[0-9]*.[0-9]*' \
+            | sed 's/^v//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1 || true)"
+
+          # Best-effort: highest ALREADY-PUSHED container tag, so a number that has
+          # an image (even from a run that died before tagging) is never reused.
+          cont_max=""
+          if command -v gh >/dev/null 2>&1; then
+            cont_max="$(GH_TOKEN="$GH_PAT" gh api --paginate \
+              '/orgs/hanzoai/packages/container/cloud/versions' \
+              --jq '.[].metadata.container.tags[]' 2>/dev/null \
+              | sed 's/^v//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1 || true)"
+          fi
+
+          # Floor = highest of the two; fall back to 1.786.0 only if the repo has
+          # no tags at all (first release ever).
+          max="$(printf '%s\n%s\n%s\n' "1.786.0" "$git_max" "$cont_max" \
+            | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1)"
+
+          major="${max%%.*}"; rest="${max#*.}"; minor="${rest%%.*}"; patch="${rest##*.}"
+          version="${major}.${minor}.$((patch + 1))"
+
+          # Refuse to proceed if the number we intend to mint already exists as a
+          # git tag (a concurrent run beat us — the serialized lane makes this a
+          # can't-happen, but fail loud rather than clobber).
+          if git rev-parse -q --verify "refs/tags/v${version}" >/dev/null; then
+            echo "::error::computed v${version} already exists as a git tag — aborting to avoid collision"
+            exit 1
+          fi
+
+          echo "version=${version}"     >> "$GITHUB_OUTPUT"
+          echo "version_v=v${version}"  >> "$GITHUB_OUTPUT"
+          echo "major_minor=${major}.${minor}" >> "$GITHUB_OUTPUT"
+          echo "sha_short=$(git rev-parse --short "$GITHUB_SHA")" >> "$GITHUB_OUTPUT"
+          echo "Next release: v${version} (git_max='${git_max:-none}' container_max='${cont_max:-none}')"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+          driver-opts: network=host
+
+      - name: Log in to ghcr.io (GH_PAT — writes the cloud package despite its ai-repo linkage)
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: hanzo-dev
+          password: ${{ secrets.GH_PAT }}
+
+      - name: OCI labels
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/hanzoai/cloud
+          tags: type=raw,value=${{ steps.ver.outputs.version_v }}
+
+      # ── Build → SMOKE → push → tag ───────────────────────────────────────────
+      # 1. Build once to a LOCAL tag (load into the daemon, do NOT push). Warms
+      #    the BuildKit cache — the expensive console/npm + Go layers land here.
+      # 2. Boot that exact image and assert it reaches "listening" with no
+      #    startup-crash signature (the gate).
+      # 3. Re-run build with push:true and the real tags: identical context /
+      #    platform / secrets, so every layer is a cache hit from step 1 and the
+      #    step only exports + pushes the already-tested image. Nothing that failed
+      #    the smoke test can reach the registry.
+      # 4. Only after the push succeeds, mint + push the git tag (the receipt).
+      - name: Build (load locally for the smoke test)
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64
+          push: false
+          load: true
+          tags: cloud:smoke
+          labels: ${{ steps.meta.outputs.labels }}
+          # gh_token: BuildKit secret the Dockerfile consumes to fetch private
+          # cross-org Go modules (hanzoai/*, luxfi/*) over authenticated git.
+          secrets: |
+            gh_token=${{ secrets.GH_PAT }}
+
+      - name: Smoke test — the binary MUST boot to "listening" with no crash signature
+        run: |
+          set -euo pipefail
+          IMAGE=cloud:smoke
+          CID=""
+          cleanup() { [ -n "$CID" ] && docker rm -f "$CID" >/dev/null 2>&1 || true; }
+          trap cleanup EXIT
+
+          # Minimal, production-representative boot env:
+          #  • a writable ephemeral /data root — the audit store, the embedded
+          #    KMS secrets plane and every per-tenant SQLite open files under
+          #    CLOUD_DATA_DIR; an unwritable dir would fail EVERY image before
+          #    MountAll and the gate would stop discriminating good from bad; and
+          #  • a throwaway 32-byte KMS master key so the KMS plane mounts on its
+          #    normal ready path exactly as prod does (no real secret is used).
+          # The subsystem that crashed the incident (metrics, mount order 40)
+          # mounts AFTER kms (order 10), so the boot must get past kms for the
+          # gate to observe the panic — this env does exactly that.
+          KEY="$(head -c 32 /dev/urandom | base64 | tr -d '\n')"
+          CID="$(docker run -d \
+            --tmpfs /data:rw,size=64m \
+            -e CLOUD_DATA_DIR=/data \
+            -e CLOUD_ENV=smoke \
+            -e CLOUD_KMS_MASTER_KEY_REF="$KEY" \
+            "$IMAGE")"
+
+          # Poll up to 60s for boot to either finish ("listening" is logged once
+          # every subsystem has mounted and both transports are about to bind) or
+          # die (a Mount panic exits the process). A healthy boot is ~1-2s; the
+          # ceiling only guards a cold daemon.
+          listening=0
+          for _ in $(seq 1 60); do
+            logs="$(docker logs "$CID" 2>&1 || true)"
+            if printf '%s' "$logs" | grep -q '"message":"listening"'; then listening=1; break; fi
+            if [ "$(docker inspect -f '{{.State.Running}}' "$CID" 2>/dev/null || echo false)" != "true" ]; then break; fi
+            sleep 1
+          done
+
+          logs="$(docker logs "$CID" 2>&1 || true)"
+          echo "::group::cloud:smoke boot logs"
+          printf '%s\n' "$logs"
+          echo "::endgroup::"
+
+          # (1) No startup-crash signature. Catches the incident's Mount
+          #     type-assert panic AND any generic Go panic — case-insensitive so a
+          #     re-worded variant can't slip through — BEFORE a byte is pushed.
+          if printf '%s' "$logs" | grep -Eiq 'metrics\.Mount|mount metrics|panic|want \*zip\.App'; then
+            echo "SMOKE FAIL: startup-crash signature in boot logs (see above)"
+            exit 1
+          fi
+
+          # (2) Reached "listening" — proof that MountAll returned for every
+          #     enabled subsystem (a failed Mount returns before this line).
+          if [ "$listening" -ne 1 ]; then
+            echo "SMOKE FAIL: binary never reached \"listening\" (a subsystem did not mount)"
+            exit 1
+          fi
+
+          # (3) Still alive — a server that logged "listening" then exited (e.g. a
+          #     listener bind failure) is not a healthy image.
+          if [ "$(docker inspect -f '{{.State.Running}}' "$CID" 2>/dev/null || echo false)" != "true" ]; then
+            echo "SMOKE FAIL: process exited after \"listening\""
+            exit 1
+          fi
+
+          echo "SMOKE PASS: cloud:smoke booted to \"listening\" with no crash signature"
+
+      - name: Push (cache hit from the smoke build — publishes the tested image)
+        id: push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64
+          push: true
+          tags: |
+            ghcr.io/hanzoai/cloud:${{ steps.ver.outputs.version_v }}
+            ghcr.io/hanzoai/cloud:${{ steps.ver.outputs.version }}
+            ghcr.io/hanzoai/cloud:${{ steps.ver.outputs.major_minor }}
+            ghcr.io/hanzoai/cloud:sha-${{ steps.ver.outputs.sha_short }}
+            ghcr.io/hanzoai/cloud:latest
+          labels: ${{ steps.meta.outputs.labels }}
+          secrets: |
+            gh_token=${{ secrets.GH_PAT }}
+
+      # THE RECEIPT: reached only because build + smoke + push all succeeded. If
+      # any of them failed the job already stopped and no tag was minted.
+      - name: Tag the proven image (git tag = receipt for a pushed, smoke-passed image)
+        env:
+          GH_PAT: ${{ secrets.GH_PAT }}
+        run: |
+          set -euo pipefail
+          V="${{ steps.ver.outputs.version_v }}"
+          git config user.name  "hanzo-dev"
+          git config user.email "dev@hanzo.ai"
+          git tag -a "$V" -m "release $V — image ghcr.io/hanzoai/cloud:$V pushed and smoke-passed (${GITHUB_SHA})"
+          git push "https://x-access-token:${GH_PAT}@github.com/${GITHUB_REPOSITORY}.git" "$V"
+          echo "Tagged $V → ghcr.io/hanzoai/cloud:$V"
+
+  # Notify universe so the GitOps pipeline rolls the new image to prod — same
+  # image-update contract every service uses (gateway, iam, …). Runs ONLY after
+  # build-amd64 succeeds, i.e. only for a version whose image is proven pushed and
+  # tagged. A failed release never reaches here, so universe is never asked to
+  # deploy a phantom tag.
+  notify-universe:
+    needs: build-amd64
+    runs-on: [hanzo-build-linux-amd64]
+    steps:
+      - name: Repository dispatch (image-update)
+        uses: peter-evans/repository-dispatch@v3
+        with:
+          token: ${{ secrets.UNIVERSE_DISPATCH_TOKEN }}
+          repository: hanzoai/universe
+          event-type: image-update
+          client-payload: |
+            {
+              "service": "cloud",
+              "image":   "ghcr.io/hanzoai/cloud:${{ needs.build-amd64.outputs.version_v }}",
+              "sha":     "${{ github.sha }}",
+              "env":     "all"
+            }

.gitignore

@@ -0,0 +1,36 @@
+# Built binaries (Dockerfile output + `go build ./cmd/hanzo`)
+/cloud
+/hanzo
+
+# Local build directories
+/dist/
+/build/
+/bin/
+
+# Go test + coverage artifacts
+*.test
+*.out
+coverage.txt
+coverage.html
+
+# Environment files
+.env
+.env.*
+!.env.example
+
+# IDE / editor
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS metadata
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log
+.shots/
+
+.claude/

Dockerfile

@@ -1,10 +1,107 @@
-FROM golang:1.26-alpine AS build
-RUN apk add --no-cache ca-certificates tzdata
+# hanzoai/cloud — the ONE unified Hanzo Cloud binary (HIP-0106).
+#
+# This image is a SINGLE artifact that serves BOTH the /v1 API AND the console
+# UI from one process: the console is compiled into the Go binary via
+# //go:embed (see webui.go). The pipeline is:
+#
+#   1. console  stage → build the hanzoai/console2 static bundle
+#   2. (copied) → into  webui/dist/  of the Go build context
+#   3. build    stage → `go build` bakes webui/dist into the binary (go:embed)
+#
+# so the final `/cloud` binary already carries the UI. No separate console
+# Service, no second origin — the embedded console calls /v1 on its own host.
+#
+# ── console UI stage ─────────────────────────────────────────────────────────
+# Builds the console2 SPA and emits a STATIC bundle at /out. console2 is fetched
+# at a pinned ref (CONSOLE2_REF) using the same gh_token BuildKit secret the Go
+# build uses for private modules.
+#
+# console2 exposes `npm run build:embed` (scripts/build-embed.mjs): it prunes the
+# Next server route handlers (BFF proxies — they collapse to the cloud /v1/* the
+# SPA calls same-origin), wraps the client catch-all pages for output:'export',
+# and neutralizes the root layout's request-time headers() read (the per-host
+# <title>, resolved client-side in the embed) so the STATIC export prerenders
+# clean — emitting out/. This stage runs it and copies out/ into /out, which the
+# Go build drops into webui/dist so //go:embed bakes the FULL @hanzo/gui console
+# into the ONE binary. The stage still degrades to the committed fallback shell,
+# non-fatally, if build:embed is ever absent or fails (a console2 export
+# regression must never take down the cloud backend image) — but the intended,
+# working path is the real bundle.
+FROM public.ecr.aws/docker/library/node:24-alpine AS console
+ARG CONSOLE2_REPO=https://github.com/hanzoai/console2.git
+ARG CONSOLE2_REF=main
+RUN apk add --no-cache git
+WORKDIR /console
+# The static export prerenders every page (webpack compile + export prerender);
+# give the heap headroom so a large @hanzo/gui build never OOMs into the stub.
+ENV NEXT_TELEMETRY_DISABLED=1 NODE_OPTIONS=--max-old-space-size=8192
+RUN --mount=type=secret,id=gh_token \
+    if [ -s /run/secrets/gh_token ]; then \
+      git config --global url."https://x-access-token:$(cat /run/secrets/gh_token)@github.com/".insteadOf "https://github.com/"; \
+    fi && \
+    git clone --depth 1 --branch "${CONSOLE2_REF}" "${CONSOLE2_REPO}" . && \
+    npm install --no-audit --no-fund --fetch-retries=5 --fetch-retry-mintimeout=20000 --fetch-timeout=120000
+# Always emit /out. When console2 exposes a static-embed target AND it builds, /out
+# holds the real bundle; otherwise /out stays EMPTY so the Go build keeps the
+# committed fallback shell. Never fail the image — a static target that is missing
+# OR that fails to build is a degrade, not an error (the standalone console2
+# Deployment is the primary console; this embed is a same-origin convenience). A
+# console2 prerender/export crash (e.g. /signin Server-Components error) must NOT
+# take down the cloud backend image.
+RUN mkdir -p /out && \
+    if npm run 2>/dev/null | grep -q ' build:embed'; then \
+      echo ">> console2 build:embed → static bundle"; \
+      if npm run build:embed && [ -d out ]; then \
+        cp -r out/. /out/; \
+        echo ">> embedded console2 static bundle"; \
+      else \
+        echo ">> console2 build:embed FAILED — degrading to committed fallback shell (non-fatal)"; \
+      fi; \
+    else \
+      echo ">> console2 has no static-embed target yet; cloud embeds the fallback shell"; \
+    fi
+
+# ── Go build stage ───────────────────────────────────────────────────────────
+# ECR Public mirror of the Docker library image — Docker Hub's unauthenticated
+# pull rate-limit (429 toomanyrequests) fails the build on shared CI runners.
+FROM public.ecr.aws/docker/library/golang:1.26-alpine AS build
+RUN apk add --no-cache ca-certificates tzdata git
 RUN addgroup -g 65532 -S nonroot && adduser -u 65532 -S nonroot -G nonroot
 WORKDIR /src
+# hanzoai/* and luxfi/* are PUBLIC and resolve via the IMMUTABLE public proxy +
+# sumdb — go.sum pins those canonical hashes, so a force-re-pointed tag can never
+# break the build. Routing them DIRECT (the old GOPRIVATE approach) re-fetches a
+# re-tagged tree (e.g. luxfi/age@v1.5.0) whose hash differs from go.sum's proxy
+# hash → "checksum mismatch / SECURITY ERROR". This matches the drop-GOPRIVATE
+# fix already shipped in hanzoai/iam + luxfi/kms. Only zap-proto/* stays first-
+# party-direct (kept in GOPRIVATE) — authenticated git via gh_token. GOPROXY
+# still routes nested-path monorepo tags (e.g. tencentcloud-sdk-go) through the
+# proxy. The committed go.sum is the single source of truth.
+ENV GOPRIVATE=github.com/zap-proto/* \
+    GONOSUMDB=github.com/zap-proto/* \
+    GOSUMDB=off \
+    GOPROXY=https://proxy.golang.org,direct \
+    GOFLAGS=-mod=mod
+COPY go.mod go.sum ./
+# With go.sum recorded against live tag content and our orgs routed direct, this
+# verifies cleanly — no runtime go.sum regeneration. (The old `rm -f go.sum`
+# self-heal masked a stale go.sum and silently re-recorded unverified hashes on
+# ANY transient error; removed in favor of a correct, committed go.sum.)
+RUN --mount=type=secret,id=gh_token \
+    if [ -s /run/secrets/gh_token ]; then \
+      git config --global url."https://x-access-token:$(cat /run/secrets/gh_token)@github.com/".insteadOf "https://github.com/"; \
+    fi && \
+    go mod download
 COPY . .
+# Drop the console static bundle into the embed path BEFORE `go build`, so
+# //go:embed all:webui/dist bakes it into the binary. /out from the console stage
+# is either the real static build (then it overlays the committed fallback shell)
+# or empty (then webui/dist keeps the shell that `COPY . .` already brought). The
+# committed assets/.gitkeep keeps the embed's assets/ dir present either way.
+COPY --from=console /out/ /src/webui/dist/
 RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o /cloud ./cmd/cloud
 
+# ── final image ──────────────────────────────────────────────────────────────
 FROM scratch
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY --from=build /usr/share/zoneinfo /usr/share/zoneinfo

LICENSE

@@ -0,0 +1,203 @@
+Copyright (c) 2026 Hanzo AI Inc.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright (c) 2026 Hanzo AI Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -0,0 +1,73 @@
+# hanzoai/cloud — developer ergonomics for the unified Hanzo Cloud binary (HIP-0106).
+# Targets are intentionally minimal; deploy artifacts (compose, helm) live in deploy/ and helm/.
+
+GO              ?= go
+BIN             ?= cloud
+PKG             ?= ./cmd/cloud
+DOCKER_IMAGE    ?= ghcr.io/hanzoai/cloud
+DOCKER_TAG      ?= dev
+LDFLAGS         ?= -s -w
+# Path to a hanzoai/console2 checkout used to build the embedded console bundle.
+CONSOLE2_DIR    ?= ../console2
+
+# The shipped binary is pure Go (Dockerfile: CGO_ENABLED=0 → scratch). Default all
+# build/test targets to that mode so `make build`/`make test` exercise exactly
+# what prod runs — and, critically, register the ONE "sqlite" driver exactly once:
+# cloud's stores use github.com/hanzoai/sqlite (its !cgo backend IS modernc), and
+# the embedded deps (ai/base/commerce/o11y/orm/tasks) that import modernc directly
+# then resolve to the SAME package → a single registration. A plain CGO_ENABLED=1
+# build instead links the fork's mattn backend ALONGSIDE those modernc importers
+# and panics at init ("sql: Register called twice for driver sqlite"); `make
+# test-cgo` proves the cgo path via the fork's `sqlite_purego` opt-out tag, which
+# forces the fork to modernc too so the whole binary registers "sqlite" once.
+CGO_ENABLED     ?= 0
+
+.PHONY: help webui build build-standalone run smoke test test-cgo vet tidy docker docker-push clean
+
+help: ## Show this help.
+	@awk 'BEGIN{FS=":.*##";printf "\nUsage: make <target>\n\nTargets:\n"} /^[a-zA-Z_-]+:.*##/{printf "  \033[36m%-16s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+webui: ## Build the real console2 static bundle into webui/dist (go:embed source). CONSOLE2_DIR=<path to console2>.
+	@command -v npm >/dev/null 2>&1 || { echo "npm is required to build the console bundle"; exit 1; }
+	@test -f "$(CONSOLE2_DIR)/package.json" || { echo "console2 checkout not found at $(CONSOLE2_DIR) — set CONSOLE2_DIR=<path>"; exit 1; }
+	@test -d "$(CONSOLE2_DIR)/node_modules" || (cd "$(CONSOLE2_DIR)" && npm install --no-audit --no-fund)
+	cd "$(CONSOLE2_DIR)" && NEXT_TELEMETRY_DISABLED=1 NODE_OPTIONS=--max-old-space-size=8192 npm run build:embed
+	# Overlay the fresh static export onto webui/dist, keeping only the tracked
+	# fallbacks (.gitignore + assets/.gitkeep); the real bundle is build-time-only.
+	find webui/dist -mindepth 1 -maxdepth 1 ! -name .gitignore ! -name assets -exec rm -rf {} +
+	cp -r "$(CONSOLE2_DIR)/out/." webui/dist/
+	@echo ">> embedded real console2 bundle into webui/dist (index.html $$(wc -c < webui/dist/index.html) bytes)"
+
+build: ## Build the unified cloud binary into ./bin/cloud (embeds whatever webui/dist holds — run `webui` first for the real console).
+	@mkdir -p bin
+	CGO_ENABLED=$(CGO_ENABLED) $(GO) build -ldflags="$(LDFLAGS)" -o bin/$(BIN) $(PKG)
+
+build-standalone: webui build ## Build the REAL 1-binary console: console2 build:embed → webui/dist → go build.
+
+run: build ## Run with iam,base,kms,gateway,o11y enabled (matches README quickstart).
+	./bin/$(BIN) --enable=iam,base,kms,gateway,o11y --brand=hanzo --domain=api.hanzo.ai
+
+smoke: ## Build and run cmd/cloud-smoke (mount-time integration check).
+	$(GO) run ./cmd/cloud-smoke
+
+test: ## Run unit + integration tests (pure-Go, exactly as prod ships).
+	CGO_ENABLED=$(CGO_ENABLED) $(GO) test ./...
+
+test-cgo: ## Prove the cgo build works too — forces the fork's pure-Go backend via -tags sqlite_purego so the embedded modernc importers don't double-register "sqlite".
+	CGO_ENABLED=1 $(GO) test -tags sqlite_purego ./...
+
+vet: ## go vet across the module.
+	CGO_ENABLED=$(CGO_ENABLED) $(GO) vet ./...
+
+tidy: ## go mod tidy + verify go.sum.
+	$(GO) mod tidy
+	$(GO) mod verify
+
+docker: ## Build the Docker image (uses repo Dockerfile, scratch final stage).
+	docker build -t $(DOCKER_IMAGE):$(DOCKER_TAG) .
+
+docker-push: docker ## Push the Docker image to ghcr.io. Requires docker login.
+	docker push $(DOCKER_IMAGE):$(DOCKER_TAG)
+
+clean: ## Remove built artifacts.
+	rm -rf bin

README.md

@@ -1,3 +1,5 @@
+<p align="center"><img src=".github/hero.svg" alt="cloud" width="880"></p>
+
 # cloud
 
 Unified Go control plane and binary for the Hanzo platform (HIP-0106).
@@ -15,6 +17,36 @@ docker run -p 8080:8080 ghcr.io/hanzoai/cloud:latest
 
 `hanzoai/cloud` is one Go binary that mounts every Hanzo subsystem (iam, kms, base, gateway, ai, commerce, vfs, mq, dns, amqp, mcp, o11y, ...) into a single multi-tenant process. Same artifact serves `api.hanzo.ai`, `api.osage.cloud`, `api.lux.cloud`, `api.zoo.cloud`, and every white-label reseller. Brand, enabled subsystems, and tenant scope are deployment configuration.
 
+## `hanzo` — cloud control CLI
+
+The same binary is also a gcloud/doctl-class CLI. The first token selects the mode:
+
+- `hanzo <subsystem>` — **server mode**: serve a subsystem (`hanzo iam`, `hanzo cloud`, …).
+- `hanzo <verb>` — **client mode**: control the live estate. A thin client over
+  Hanzo IAM (`hanzo.id`), the platform control plane (`platform.hanzo.ai/v1`),
+  and the cloud `/v1` API — it invents no parallel API.
+
+```bash
+hanzo login                       # IAM password grant against hanzo.id → token in ~/.hanzo (0600)
+hanzo whoami                      # identity from the stored token (--verify hits IAM userinfo)
+hanzo apps list                   # platform apps board: declared/running/latest tag + drift + health
+hanzo apps get <org>/<app>/<env>  # one app row
+hanzo deploy <container> --project <p> --env <e>   # rolling, zero-downtime redeploy
+hanzo clusters list|get|create|select|target       # dedicated DOKS cluster lifecycle
+hanzo build <repo> --sha <sha> --image <img>       # platform-native (arcd/Kaniko) build, no GitHub builders
+hanzo k8s target                  # the org's resolved deploy target (kubeconfig never returned)
+hanzo config set <k> <v>          # ~/.hanzo/config preferences
+```
+
+Global flags: `--org`, `-o/--output table|json`, `--platform-url`, `--iam-issuer`,
+`--platform-token`. Tokens resolve from flag → env → `~/.hanzo` (never hardcoded):
+the IAM user token is the identity; the platform control plane is service-token
+authed (it cannot validate user tokens), so `apps`/`deploy`/`clusters` use
+`--platform-token` / `HANZO_PLATFORM_TOKEN` / `PLATFORM_SERVICE_TOKEN`, and
+`build` uses `HANZO_BUILD_TOKEN` / `PLATFORM_BUILD_CALLBACK_TOKEN`.
+
+Install: `go install github.com/hanzoai/cloud/cmd/hanzo@latest`, or `brew install hanzoai/tap/hanzo`.
+
 ## Specs
 
 Implements:
@@ -88,6 +120,48 @@ deployment configuration.
 [hanzoai/zip](https://github.com/hanzoai/zip) — Sinatra-style Go web framework
 built on Fiber v3. The ONE Go web framework. No `.Fast` escape hatch.
 
+## Console UI — embedded in the ONE binary
+
+The same `hanzoai/cloud` binary serves the [console](https://github.com/hanzoai/console2)
+(`@hanzo/gui`) UI at the web root AND the `/v1` API from one process — one
+artifact, one origin, no separate console Service. The UI is compiled in via
+`//go:embed` (see `webui.go`).
+
+Pipeline (in the `Dockerfile`, before `go build`):
+
+```
+console stage  →  build console2 static bundle  →  /out
+      COPY --from=console /out/ → src/webui/dist/     (overlays the fallback shell)
+build stage    →  go build   →  //go:embed all:webui/dist bakes it into /cloud
+```
+
+Serving (`webui.go`, registered LAST in `Serve` so it never shadows the API):
+
+- `GET /` and any client-side route (`/orgs`, `/models`, …) → the SPA shell
+  (`index.html`) with `Cache-Control: no-cache`; fingerprinted assets under
+  `assets/`/`_next/` are served `immutable` for a year, with brotli/gzip
+  precompressed negotiation when the build emits `.br`/`.gz` siblings.
+- `GET /v1/*` (and `/zap`, `/healthz`, …) → the API. Real subsystem routes are
+  registered before the console catch-all, so they always win; an **unmatched**
+  path under an API prefix returns a real 404 (JSON namespace), never HTML.
+- Same-origin: the embedded console calls `/v1` on its own host, so the session
+  cookie is first-party — no second origin, no CORS.
+
+`webui/dist/index.html` is a committed **fallback shell** (a real same-origin
+`/v1` bootstrap) so `go build` always compiles and the binary always serves a UI
+even without the Node toolchain. The image build overwrites `webui/dist` with the
+real console bundle. See `webui_test.go` for the boot-and-assert tests
+(`/` → shell, deep link → shell 200, `/v1/*` → API, unmatched `/v1` → 404).
+
+> Honest current state: console2 ships 15 Next server route handlers
+> (`app/**/route.ts`) that hold KMS-sourced service tokens and mint short-lived
+> user tokens, so it emits a Node server bundle, not a static export
+> (`output: export` would fail). Until console2 exposes a `build:embed` static
+> target — or those handlers land here as native `/v1` endpoints — the image
+> embeds the fallback shell, and the separate console2 Service stays up. The Go
+> embed/serve plumbing is complete and needs no further change to light up the
+> full console the moment the static bundle exists.
+
 ## Status
 
 Scaffold. The Mount(app, deps) integration for each subsystem lands per

audit/audit_test.go

@@ -0,0 +1,623 @@
+package audit
+
+// Tests for the tamper-evident audit chain. They exercise the REAL SQLite store
+// (an on-disk temp db, not a mock) so the append-only write path, the hash-chain
+// math, the verifier, redaction, and the filtered query are all proven
+// end-to-end. The headline test is tamper-detection: a record edited directly in
+// the database is DETECTED as breaking the chain.
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// openTemp opens a Recorder backed by a fresh on-disk SQLite file (not :memory:,
+// because tamper tests re-open the same file via a second connection to edit it
+// out-of-band — exactly what an attacker with DB access would do).
+func openTemp(t *testing.T) (*Recorder, string) {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "audit.db")
+	rec, err := Open(path, nil)
+	if err != nil {
+		t.Fatalf("Open: %v", err)
+	}
+	t.Cleanup(func() { _ = rec.Close() })
+	return rec, path
+}
+
+// sampleRecord is a representative security event (a global-admin org deletion).
+func sampleRecord(action string) Record {
+	return Record{
+		Time:      time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC),
+		Actor:     Actor{Org: "admin", Sub: "z@hanzo.ai", Email: "z@hanzo.ai"},
+		Action:    action,
+		Resource:  Resource{Type: "org", ID: "acme"},
+		Auth:      AuthContext{Method: "jwt", IsAdmin: true},
+		Outcome:   Outcome{Result: "success", Status: 200},
+		SourceIP:  "203.0.113.7",
+		UserAgent: "console2",
+		RequestID: "req-123",
+		Method:    "DELETE",
+		Path:      "/v1/admin/orgs/acme",
+	}
+}
+
+// TestChain_AppendSealsAndLinks proves each appended record gets a monotonic seq,
+// links its PrevHash to the previous record's Hash, and starts from the genesis
+// anchor.
+func TestChain_AppendSealsAndLinks(t *testing.T) {
+	rec, _ := openTemp(t)
+	ctx := context.Background()
+
+	r0, err := rec.Append(ctx, sampleRecord("DELETE /v1/admin/orgs"))
+	if err != nil {
+		t.Fatalf("append 0: %v", err)
+	}
+	if r0.Seq != 0 {
+		t.Fatalf("first seq = %d, want 0", r0.Seq)
+	}
+	if r0.PrevHash != genesisPrevHash {
+		t.Fatalf("genesis prev = %q, want %q", r0.PrevHash, genesisPrevHash)
+	}
+	if r0.Hash == "" || r0.Hash == genesisPrevHash {
+		t.Fatalf("hash not computed: %q", r0.Hash)
+	}
+
+	r1, err := rec.Append(ctx, sampleRecord("POST /v1/admin/roles"))
+	if err != nil {
+		t.Fatalf("append 1: %v", err)
+	}
+	if r1.Seq != 1 {
+		t.Fatalf("second seq = %d, want 1", r1.Seq)
+	}
+	if r1.PrevHash != r0.Hash {
+		t.Fatalf("link broken: r1.prev=%q, r0.hash=%q", r1.PrevHash, r0.Hash)
+	}
+	if r1.Hash == r0.Hash {
+		t.Fatal("distinct records must have distinct hashes")
+	}
+}
+
+// TestVerify_PassesOnUntamperedChain proves a well-formed chain verifies OK.
+func TestVerify_PassesOnUntamperedChain(t *testing.T) {
+	rec, _ := openTemp(t)
+	ctx := context.Background()
+	for i := 0; i < 25; i++ {
+		if _, err := rec.Append(ctx, sampleRecord("POST /v1/admin/sync")); err != nil {
+			t.Fatalf("append %d: %v", i, err)
+		}
+	}
+	integrity, err := rec.Verify(ctx)
+	if err != nil {
+		t.Fatalf("Verify: %v", err)
+	}
+	if !integrity.OK {
+		t.Fatalf("chain not OK: broken at %d (%s)", integrity.BrokenAt, integrity.Reason)
+	}
+	if integrity.Count != 25 {
+		t.Fatalf("count = %d, want 25", integrity.Count)
+	}
+	if integrity.BrokenAt != -1 {
+		t.Fatalf("brokenAt = %d, want -1 on a good chain", integrity.BrokenAt)
+	}
+	// Head must equal the last record's hash.
+	count, head := rec.Head()
+	if count != 25 || head != integrity.HeadHash {
+		t.Fatalf("head mismatch: (%d,%q) vs verify (%d,%q)", count, head, integrity.Count, integrity.HeadHash)
+	}
+}
+
+// TestVerify_DetectsFieldTamper is the headline: an attacker with direct DB
+// access edits a record's content (flips a denied outcome to success, or changes
+// the actor). The stored hash no longer matches the recomputed hash, so Verify
+// reports the exact seq where the chain breaks. THIS is the tamper-evidence
+// property — an audit trail that can be silently forged is worse than none.
+func TestVerify_DetectsFieldTamper(t *testing.T) {
+	rec, path := openTemp(t)
+	ctx := context.Background()
+	for i := 0; i < 10; i++ {
+		if _, err := rec.Append(ctx, sampleRecord("DELETE /v1/admin/orgs")); err != nil {
+			t.Fatalf("append %d: %v", i, err)
+		}
+	}
+	// Sanity: clean chain verifies.
+	if iv, _ := rec.Verify(ctx); !iv.OK {
+		t.Fatalf("precondition: clean chain should verify, broke at %d", iv.BrokenAt)
+	}
+
+	// Tamper OUT OF BAND — a second connection issues an UPDATE the application
+	// never would. This models an attacker who owns the file / a rogue DBA.
+	tamperOutOfBand(t, path, `UPDATE audit_log SET actor_sub='attacker', result='success' WHERE seq=4`)
+
+	iv, err := rec.Verify(ctx)
+	if err != nil {
+		t.Fatalf("Verify after tamper: %v", err)
+	}
+	if iv.OK {
+		t.Fatal("TAMPER NOT DETECTED — a modified record verified as OK; the chain is forgeable")
+	}
+	if iv.BrokenAt != 4 {
+		t.Fatalf("brokenAt = %d, want 4 (the edited record)", iv.BrokenAt)
+	}
+	if !strings.Contains(iv.Reason, "hash mismatch") {
+		t.Fatalf("reason = %q, want a hash-mismatch explanation", iv.Reason)
+	}
+}
+
+// TestVerify_DetectsDeletion proves deleting a record (or a contiguous run) breaks
+// the chain: the record after the hole has a PrevHash that no longer matches the
+// now-preceding record, and the seq sequence gaps. Either way Verify flags it.
+func TestVerify_DetectsDeletion(t *testing.T) {
+	rec, path := openTemp(t)
+	ctx := context.Background()
+	for i := 0; i < 10; i++ {
+		if _, err := rec.Append(ctx, sampleRecord("POST /v1/admin/roles")); err != nil {
+			t.Fatalf("append %d: %v", i, err)
+		}
+	}
+	// Delete a MIDDLE record — the classic "cover your tracks" edit.
+	tamperOutOfBand(t, path, `DELETE FROM audit_log WHERE seq=5`)
+
+	iv, err := rec.Verify(ctx)
+	if err != nil {
+		t.Fatalf("Verify after delete: %v", err)
+	}
+	if iv.OK {
+		t.Fatal("DELETION NOT DETECTED — a removed record left the chain verifying OK")
+	}
+	// The break is observed at seq 6 (the record whose predecessor vanished): its
+	// seq no longer follows the running counter (5 is missing), so the gap check
+	// fires first at 6.
+	if iv.BrokenAt != 6 {
+		t.Fatalf("brokenAt = %d, want 6 (record after the hole)", iv.BrokenAt)
+	}
+}
+
+// TestVerify_DetectsReorder proves swapping two records' positions (an attacker
+// trying to reorder events) breaks the prev-hash linkage.
+func TestVerify_DetectsReorder(t *testing.T) {
+	rec, path := openTemp(t)
+	ctx := context.Background()
+	for i := 0; i < 6; i++ {
+		if _, err := rec.Append(ctx, sampleRecord("POST /v1/kms/secrets")); err != nil {
+			t.Fatalf("append %d: %v", i, err)
+		}
+	}
+	// Swap the hashes of seq 2 and seq 3 (content stays, linkage corrupts). Any
+	// out-of-band shuffle that doesn't recompute the WHOLE suffix is detectable.
+	tamperOutOfBand(t, path, `
+UPDATE audit_log SET hash = (SELECT hash FROM audit_log WHERE seq=3) WHERE seq=2;`)
+
+	iv, _ := rec.Verify(ctx)
+	if iv.OK {
+		t.Fatal("REORDER/HASH-SWAP NOT DETECTED")
+	}
+	if iv.BrokenAt < 0 {
+		t.Fatalf("expected a break, got brokenAt=%d", iv.BrokenAt)
+	}
+}
+
+// TestChain_RestartContinues proves a re-opened store continues the SAME chain
+// (recovers seq + head) rather than forking — so a pod restart cannot silently
+// reset the trail.
+func TestChain_RestartContinues(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "audit.db")
+	ctx := context.Background()
+
+	rec1, err := Open(path, nil)
+	if err != nil {
+		t.Fatalf("open 1: %v", err)
+	}
+	var lastHash string
+	for i := 0; i < 5; i++ {
+		r, err := rec1.Append(ctx, sampleRecord("POST /v1/admin/sync"))
+		if err != nil {
+			t.Fatalf("append %d: %v", i, err)
+		}
+		lastHash = r.Hash
+	}
+	_ = rec1.Close()
+
+	rec2, err := Open(path, nil)
+	if err != nil {
+		t.Fatalf("open 2: %v", err)
+	}
+	defer func() { _ = rec2.Close() }()
+
+	count, head := rec2.Head()
+	if count != 5 {
+		t.Fatalf("recovered count = %d, want 5", count)
+	}
+	if head != lastHash {
+		t.Fatalf("recovered head = %q, want %q", head, lastHash)
+	}
+	// The next append must chain onto the recovered head at seq 5.
+	r5, err := rec2.Append(ctx, sampleRecord("DELETE /v1/admin/orgs"))
+	if err != nil {
+		t.Fatalf("append after restart: %v", err)
+	}
+	if r5.Seq != 5 || r5.PrevHash != lastHash {
+		t.Fatalf("chain did not continue: seq=%d prev=%q (want seq 5 prev %q)", r5.Seq, r5.PrevHash, lastHash)
+	}
+	// And the whole continued chain still verifies.
+	if iv, _ := rec2.Verify(ctx); !iv.OK {
+		t.Fatalf("continued chain broke at %d (%s)", iv.BrokenAt, iv.Reason)
+	}
+}
+
+// TestRedact_StripsSecrets proves the redactor removes credential-bearing fields
+// (by key name, recursively) while keeping non-secret structure — so an explicit
+// emit point's before/after can never carry a password/token/key.
+func TestRedact_StripsSecrets(t *testing.T) {
+	in := json.RawMessage(`{
+      "name": "acme",
+      "password": "hunter2",
+      "apiKey": "sk-live-abc123",
+      "passphrase": "correct horse",
+      "wgPrivKey": "PRIVKEYBYTES",
+      "recoveryPhrase": "twelve words here",
+      "socialSecurityNumber": "078-05-1120",
+      "config": {
+        "clientSecret": "shh",
+        "endpoint": "https://api.example.com",
+        "nested": {"private_key": "-----BEGIN-----", "region": "sfo3"}
+      },
+      "tokens": ["t1", "t2"],
+      "roles": ["admin", "viewer"]
+    }`)
+	out := Redact(in)
+
+	s := string(out)
+	// Secrets gone (incl. the edge-case key names: passphrase, privkey, phrase, ssn).
+	for _, leak := range []string{"hunter2", "sk-live-abc123", "shh", "BEGIN",
+		"correct horse", "PRIVKEYBYTES", "twelve words here", "078-05-1120"} {
+		if strings.Contains(s, leak) {
+			t.Fatalf("secret leaked through redaction: %q still present in %s", leak, s)
+		}
+	}
+	// Non-secret structure preserved.
+	for _, keep := range []string{"acme", "https://api.example.com", "sfo3", "viewer"} {
+		if !strings.Contains(s, keep) {
+			t.Fatalf("redaction dropped a non-secret value %q: %s", keep, s)
+		}
+	}
+	// The redaction marker appears where secrets were.
+	if !strings.Contains(s, redactedMarker) {
+		t.Fatalf("no redaction marker in output: %s", s)
+	}
+	// "tokens" is a secret key → the whole array is redacted (not its elements).
+	var decoded map[string]any
+	if err := json.Unmarshal(out, &decoded); err != nil {
+		t.Fatalf("redacted output is not valid JSON: %v", err)
+	}
+	if decoded["tokens"] != redactedMarker {
+		t.Fatalf("secret-keyed array not redacted whole: %v", decoded["tokens"])
+	}
+}
+
+// TestRedact_FailsClosedOnBadJSON proves unparseable input is never echoed back.
+func TestRedact_FailsClosedOnBadJSON(t *testing.T) {
+	out := Redact(json.RawMessage(`{not valid json, password=hunter2`))
+	if strings.Contains(string(out), "hunter2") {
+		t.Fatalf("bad JSON echoed a secret: %s", out)
+	}
+	if !strings.Contains(string(out), redactedMarker) {
+		t.Fatalf("bad JSON should redact to a marker, got %s", out)
+	}
+}
+
+// TestQuery_Filters proves the filtered read returns the right subset by actor,
+// action, resource, and result, newest-first, with an accurate total.
+func TestQuery_Filters(t *testing.T) {
+	rec, _ := openTemp(t)
+	ctx := context.Background()
+
+	mk := func(org, action, res, result string) Record {
+		r := sampleRecord(action)
+		r.Actor.Org = org
+		r.Resource.Type = res
+		r.Outcome.Result = result
+		return r
+	}
+	// A mixed set.
+	seed := []Record{
+		mk("admin", "DELETE /v1/admin/orgs", "org", "success"),
+		mk("acme", "POST /v1/base/records", "records", "success"),
+		mk("admin", "POST /v1/admin/roles", "roles", "deny"),
+		mk("admin", "DELETE /v1/admin/orgs", "org", "success"),
+		mk("acme", "POST /v1/kms/secrets", "secrets", "error"),
+	}
+	for i, r := range seed {
+		if _, err := rec.Append(ctx, r); err != nil {
+			t.Fatalf("seed %d: %v", i, err)
+		}
+	}
+
+	// Filter by org=admin → 3 rows.
+	rows, total, err := rec.Query(ctx, Filter{Org: "admin"})
+	if err != nil {
+		t.Fatalf("query org: %v", err)
+	}
+	if total != 3 || len(rows) != 3 {
+		t.Fatalf("org=admin: got %d rows, total %d, want 3/3", len(rows), total)
+	}
+	// Newest first: the last-appended admin row (seq 3) comes before seq 2, 0.
+	if rows[0].Seq < rows[len(rows)-1].Seq {
+		t.Fatalf("not newest-first: %d..%d", rows[0].Seq, rows[len(rows)-1].Seq)
+	}
+
+	// Filter by result=deny → 1 row (the 403-style role change).
+	denies, dtotal, err := rec.Query(ctx, Filter{Result: "deny"})
+	if err != nil {
+		t.Fatalf("query deny: %v", err)
+	}
+	if dtotal != 1 || len(denies) != 1 || denies[0].Action != "POST /v1/admin/roles" {
+		t.Fatalf("result=deny: got %d (%+v), want 1 role-change", dtotal, denies)
+	}
+
+	// Filter by resource=secrets → 1 row.
+	secs, stotal, err := rec.Query(ctx, Filter{Resource: "secrets"})
+	if err != nil {
+		t.Fatalf("query resource: %v", err)
+	}
+	if stotal != 1 || len(secs) != 1 {
+		t.Fatalf("resource=secrets: got %d, want 1", stotal)
+	}
+}
+
+// TestQuery_SQLInjectionInFilterIsInert proves a malicious filter value is a
+// parameter, never SQL: it simply matches nothing and cannot drop the table.
+func TestQuery_SQLInjectionInFilterIsInert(t *testing.T) {
+	rec, _ := openTemp(t)
+	ctx := context.Background()
+	for i := 0; i < 3; i++ {
+		if _, err := rec.Append(ctx, sampleRecord("POST /v1/admin/sync")); err != nil {
+			t.Fatalf("seed %d: %v", i, err)
+		}
+	}
+	inject := Filter{Org: "admin'; DROP TABLE audit_log;--"}
+	rows, total, err := rec.Query(ctx, inject)
+	if err != nil {
+		t.Fatalf("query should not error on injection attempt: %v", err)
+	}
+	if total != 0 || len(rows) != 0 {
+		t.Fatalf("injection value matched %d rows, want 0", total)
+	}
+	// The table survived — a normal query still returns the seeded rows.
+	if _, all, err := rec.Query(ctx, Filter{}); err != nil || all != 3 {
+		t.Fatalf("table damaged by injection attempt: all=%d err=%v", all, err)
+	}
+}
+
+// TestChain_ConcurrentAppendsStayGapless proves the serialized writer keeps the
+// chain a true, gapless total order under CONCURRENT appends: many goroutines
+// append at once, and the resulting chain must have every seq 0..N-1 exactly once
+// AND verify. A race in the head/seq handoff would surface as a duplicate seq (a
+// PRIMARY KEY error), a gap, or a broken link — all of which this catches.
+func TestChain_ConcurrentAppendsStayGapless(t *testing.T) {
+	rec, _ := openTemp(t)
+	ctx := context.Background()
+
+	const goroutines, per = 16, 20
+	total := goroutines * per
+	errCh := make(chan error, total)
+	var wg sync.WaitGroup
+	for g := 0; g < goroutines; g++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < per; i++ {
+				if _, err := rec.Append(ctx, sampleRecord("POST /v1/admin/sync")); err != nil {
+					errCh <- err
+				}
+			}
+		}()
+	}
+	wg.Wait()
+	close(errCh)
+	for err := range errCh {
+		t.Fatalf("concurrent append failed (race in seq/head handoff?): %v", err)
+	}
+
+	// The chain must verify and contain exactly `total` gapless records.
+	iv, err := rec.Verify(ctx)
+	if err != nil {
+		t.Fatalf("verify: %v", err)
+	}
+	if !iv.OK {
+		t.Fatalf("concurrent chain broke at %d (%s)", iv.BrokenAt, iv.Reason)
+	}
+	if iv.Count != uint64(total) {
+		t.Fatalf("recorded %d records, want %d (a lost/duplicated append)", iv.Count, total)
+	}
+}
+
+// checkpointMirror is a Mirror that also captures checkpoints (implements
+// CheckpointSink) so the test can assert the head digest reaches an independent
+// sink.
+type checkpointMirror struct {
+	mu  sync.Mutex
+	cps []Checkpoint
+}
+
+func (m *checkpointMirror) Append(context.Context, Record) error { return nil }
+func (m *checkpointMirror) Checkpoint(_ context.Context, cp Checkpoint) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.cps = append(m.cps, cp)
+	return nil
+}
+func (m *checkpointMirror) last() (Checkpoint, bool) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if len(m.cps) == 0 {
+		return Checkpoint{}, false
+	}
+	return m.cps[len(m.cps)-1], true
+}
+
+// TestCheckpoint_EmitsHeadDigest proves the AU-9 anchor: the periodic checkpoint
+// emits the current (count, head) to the log function AND, when the mirror is a
+// CheckpointSink, to the independent digest store — and a final checkpoint fires
+// on Close. This is what an external monitor compares to detect tail-truncation.
+func TestCheckpoint_EmitsHeadDigest(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "audit.db")
+	mirror := &checkpointMirror{}
+	rec, err := Open(path, mirror)
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	ctx := context.Background()
+
+	var logged []Checkpoint
+	var lmu sync.Mutex
+	// every=0 → no ticker; we drive checkpoints via Close (final) + a manual tick.
+	rec.StartCheckpoints(0, func(cp Checkpoint) {
+		lmu.Lock()
+		logged = append(logged, cp)
+		lmu.Unlock()
+	})
+
+	for i := 0; i < 7; i++ {
+		if _, err := rec.Append(ctx, sampleRecord("POST /v1/admin/sync")); err != nil {
+			t.Fatalf("append %d: %v", i, err)
+		}
+	}
+	// Close emits the FINAL checkpoint (count=7, head=chain head).
+	if err := rec.Close(); err != nil {
+		t.Fatalf("close: %v", err)
+	}
+
+	lmu.Lock()
+	n := len(logged)
+	var lastLogged Checkpoint
+	if n > 0 {
+		lastLogged = logged[n-1]
+	}
+	lmu.Unlock()
+	if n == 0 {
+		t.Fatal("no checkpoint logged (Close should emit a final head digest)")
+	}
+	if lastLogged.Count != 7 {
+		t.Errorf("final checkpoint count = %d, want 7", lastLogged.Count)
+	}
+	if lastLogged.Head == "" || lastLogged.Head == genesisPrevHash {
+		t.Errorf("final checkpoint head not set: %q", lastLogged.Head)
+	}
+	// The independent sink also received the final digest.
+	if cp, ok := mirror.last(); !ok || cp.Count != 7 {
+		t.Errorf("checkpoint sink final = %+v (ok=%v), want count 7", cp, ok)
+	}
+}
+
+// TestCheckpoint_DoubleStartIsSafe proves a second StartCheckpoints call is
+// ignored (no re-arm, no field/WaitGroup race) — the Red-review robustness fix.
+// Run under -race to catch a regression.
+func TestCheckpoint_DoubleStartIsSafe(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "audit.db")
+	rec, err := Open(path, nil)
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	rec.StartCheckpoints(time.Hour, func(Checkpoint) {})
+	rec.StartCheckpoints(time.Hour, func(Checkpoint) {}) // second call must be a no-op.
+	// Append + close must not race or hang.
+	if _, err := rec.Append(context.Background(), sampleRecord("POST /v1/admin/sync")); err != nil {
+		t.Fatalf("append: %v", err)
+	}
+	if err := rec.Close(); err != nil {
+		t.Fatalf("close: %v", err)
+	}
+}
+
+// TestCheckpoint_CloseSyncsToSink proves the FINAL checkpoint on Close reaches the
+// independent sink SYNCHRONOUSLY (the Red-review durability fix) — the sink has
+// the final count before Close returns, not on a detached goroutine that might
+// not run before process exit.
+func TestCheckpoint_CloseSyncsToSink(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "audit.db")
+	mirror := &checkpointMirror{}
+	rec, err := Open(path, mirror)
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	rec.StartCheckpoints(0, func(Checkpoint) {}) // no ticker; only the on-close checkpoint.
+	for i := 0; i < 4; i++ {
+		if _, err := rec.Append(context.Background(), sampleRecord("POST /v1/admin/sync")); err != nil {
+			t.Fatalf("append %d: %v", i, err)
+		}
+	}
+	if err := rec.Close(); err != nil {
+		t.Fatalf("close: %v", err)
+	}
+	// Immediately after Close returns (no sleep), the sink MUST already have the
+	// final digest — proving the Close-path write was synchronous.
+	cp, ok := mirror.last()
+	if !ok || cp.Count != 4 {
+		t.Fatalf("sink final checkpoint = %+v (ok=%v), want count 4 synchronously on Close", cp, ok)
+	}
+}
+
+// TestCheckpoint_CountMonotonicDetectsTruncation demonstrates the DETECTION an
+// external monitor performs: consecutive checkpoints have non-decreasing Count;
+// after a tail truncation the head reported by Head() drops below a prior
+// checkpoint — the signal the o11y alert fires on.
+func TestCheckpoint_CountMonotonicDetectsTruncation(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "audit.db")
+	rec, err := Open(path, nil)
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	ctx := context.Background()
+	for i := 0; i < 10; i++ {
+		if _, err := rec.Append(ctx, sampleRecord("POST /v1/admin/sync")); err != nil {
+			t.Fatalf("append %d: %v", i, err)
+		}
+	}
+	before, _ := rec.Head() // the monitor's last pinned checkpoint count.
+	if before != 10 {
+		t.Fatalf("pre-truncation count = %d, want 10", before)
+	}
+	_ = rec.Close()
+
+	// Attacker truncates the tail (deletes the last 4 records) out of band.
+	tamperOutOfBand(t, path, `DELETE FROM audit_log WHERE seq >= 6`)
+
+	rec2, err := Open(path, nil)
+	if err != nil {
+		t.Fatalf("reopen: %v", err)
+	}
+	defer func() { _ = rec2.Close() }()
+	after, _ := rec2.Head()
+	// The internal chain still verifies (a truncated prefix is self-consistent)…
+	if iv, _ := rec2.Verify(ctx); !iv.OK {
+		t.Fatalf("truncated prefix should self-verify, broke at %d", iv.BrokenAt)
+	}
+	// …but the count REGRESSED vs the pinned checkpoint — the truncation signal.
+	if after >= before {
+		t.Fatalf("count did not regress after truncation: before=%d after=%d", before, after)
+	}
+	t.Logf("truncation detected by count regression: %d → %d (chain-internal verify is OK; external anchor catches it)", before, after)
+}
+
+// tamperOutOfBand opens the SAME sqlite file on a SEPARATE connection and runs a
+// mutating statement the audit application itself never issues — modeling an
+// attacker with direct database/file access. The Recorder's own connection is
+// unaffected; Verify then re-reads and must catch the damage.
+func tamperOutOfBand(t *testing.T, path, stmt string) {
+	t.Helper()
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		t.Fatalf("tamper open: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+	if _, err := db.Exec(stmt); err != nil {
+		t.Fatalf("tamper exec %q: %v", stmt, err)
+	}
+}

audit/query.go

@@ -0,0 +1,231 @@
+package audit
+
+// The read paths: filtered Query (for /v1/admin/audit) and Verify (the
+// tamper-evidence walk for /v1/admin/audit/verify). Both are read-only — they
+// issue SELECT only, never mutate — so exposing them can never weaken the
+// append-only property.
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// Filter narrows a Query. Zero-value fields are ignored (no constraint), so an
+// empty Filter returns the most-recent Limit records. Time bounds are inclusive
+// and compared against the RFC3339Nano ts column lexicographically (RFC3339 is
+// order-preserving as text, so a string range is a correct time range).
+type Filter struct {
+	Org      string    // actor_org exact match (tenant scope)
+	Sub      string    // actor_sub exact match (a specific user)
+	Action   string    // action exact match
+	Resource string    // res_type exact match
+	Result   string    // outcome result: success|deny|error
+	Since    time.Time // ts >= Since (UTC)
+	Until    time.Time // ts <= Until (UTC)
+	Limit    int       // max rows (default 100, cap 1000)
+	Offset   int       // pagination offset
+}
+
+// Query returns records matching f, newest first, and the total count matching
+// the same predicate (ignoring Limit/Offset) for pagination. All predicates are
+// parameterized — never string-interpolated — so a filter value can never inject
+// SQL. Column names in the WHERE come from a fixed allowlist below, not caller
+// input.
+func (r *Recorder) Query(ctx context.Context, f Filter) (rows []Record, total int, err error) {
+	where, args := f.build()
+	limit := f.Limit
+	if limit <= 0 {
+		limit = 100
+	}
+	if limit > 1000 {
+		limit = 1000
+	}
+	offset := f.Offset
+	if offset < 0 {
+		offset = 0
+	}
+
+	countQ := `SELECT COUNT(*) FROM audit_log` + where
+	if err = r.db.QueryRowContext(ctx, countQ, args...).Scan(&total); err != nil {
+		return nil, 0, fmt.Errorf("audit: count: %w", err)
+	}
+
+	listQ := `SELECT ` + selectCols + ` FROM audit_log` + where +
+		` ORDER BY seq DESC LIMIT ? OFFSET ?`
+	listArgs := append(append([]any{}, args...), limit, offset)
+	rs, err := r.db.QueryContext(ctx, listQ, listArgs...)
+	if err != nil {
+		return nil, 0, fmt.Errorf("audit: query: %w", err)
+	}
+	defer func() { _ = rs.Close() }()
+	for rs.Next() {
+		rec, scanErr := scanRecord(rs)
+		if scanErr != nil {
+			return nil, 0, fmt.Errorf("audit: scan: %w", scanErr)
+		}
+		rows = append(rows, rec)
+	}
+	return rows, total, rs.Err()
+}
+
+// build assembles the parameterized WHERE clause from the non-zero filter
+// fields. Each fragment uses a fixed column name and a ? placeholder, so no
+// caller value ever reaches the SQL text.
+func (f Filter) build() (string, []any) {
+	var conds []string
+	var args []any
+	add := func(frag string, val any) {
+		conds = append(conds, frag)
+		args = append(args, val)
+	}
+	if f.Org != "" {
+		add("actor_org = ?", f.Org)
+	}
+	if f.Sub != "" {
+		add("actor_sub = ?", f.Sub)
+	}
+	if f.Action != "" {
+		add("action = ?", f.Action)
+	}
+	if f.Resource != "" {
+		add("res_type = ?", f.Resource)
+	}
+	if f.Result != "" {
+		add("result = ?", f.Result)
+	}
+	if !f.Since.IsZero() {
+		add("ts >= ?", f.Since.UTC().Format(time.RFC3339Nano))
+	}
+	if !f.Until.IsZero() {
+		add("ts <= ?", f.Until.UTC().Format(time.RFC3339Nano))
+	}
+	if len(conds) == 0 {
+		return "", nil
+	}
+	return " WHERE " + strings.Join(conds, " AND "), args
+}
+
+const selectCols = `seq, ts, actor_org, actor_sub, actor_email, action, res_type, res_id,
+  auth_method, is_admin, result, status, reason, source_ip, user_agent,
+  request_id, method, path, before, after, prev_hash, hash`
+
+// scanRecord reconstructs a Record from a row of selectCols.
+func scanRecord(sc interface{ Scan(...any) error }) (Record, error) {
+	var (
+		rec           Record
+		ts            string
+		isAdmin       int
+		before, after string
+	)
+	if err := sc.Scan(
+		&rec.Seq, &ts, &rec.Actor.Org, &rec.Actor.Sub, &rec.Actor.Email,
+		&rec.Action, &rec.Resource.Type, &rec.Resource.ID,
+		&rec.Auth.Method, &isAdmin, &rec.Outcome.Result, &rec.Outcome.Status, &rec.Outcome.Reason,
+		&rec.SourceIP, &rec.UserAgent, &rec.RequestID, &rec.Method, &rec.Path,
+		&before, &after, &rec.PrevHash, &rec.Hash,
+	); err != nil {
+		return Record{}, err
+	}
+	if t, err := time.Parse(time.RFC3339Nano, ts); err == nil {
+		rec.Time = t
+	}
+	rec.Auth.IsAdmin = isAdmin != 0
+	if before != "" {
+		rec.Before = json.RawMessage(before)
+	}
+	if after != "" {
+		rec.After = json.RawMessage(after)
+	}
+	return rec, nil
+}
+
+// Integrity is the result of a Verify walk — the AU-9 evidence that the trail has
+// not been tampered with.
+type Integrity struct {
+	// OK is true iff every record's stored hash equals the recomputed hash AND the
+	// chain links are continuous (each PrevHash == the prior record's Hash, seqs
+	// gapless from 0).
+	OK bool `json:"ok"`
+	// Count is the number of records walked.
+	Count uint64 `json:"count"`
+	// HeadHash is the hash of the last record (or the genesis anchor for an empty
+	// chain). Pin this externally over time to detect tail-truncation.
+	HeadHash string `json:"headHash"`
+	// BrokenAt is the seq of the FIRST record that failed verification, or -1 when
+	// OK. Reason describes the break (recomputed-hash mismatch, prev-hash
+	// discontinuity, or a seq gap).
+	BrokenAt int64  `json:"brokenAt"`
+	Reason   string `json:"reason,omitempty"`
+}
+
+// Verify walks the entire chain in seq order, recomputing each record's hash from
+// its content + the running prev-hash and checking continuity. It is the
+// tamper-detector: any modification (a changed field re-hashes differently), any
+// deletion or reordering (a seq gap or a broken prev-hash link), or a forged row
+// (its recomputed hash won't match unless the attacker also recomputed the entire
+// suffix — which they cannot do without re-inserting every subsequent record) is
+// reported with the exact seq where the chain first breaks.
+//
+// Complexity is O(n) over the records; for very large trails this streams row by
+// row (no full materialization). At cloud's audit volume this is fine; if a trail
+// grows past what an on-demand full walk should touch, verify a seq WINDOW
+// (Verify is easily extended with a bound) or rely on the externally-pinned head.
+func (r *Recorder) Verify(ctx context.Context) (Integrity, error) {
+	rs, err := r.db.QueryContext(ctx,
+		`SELECT `+selectCols+` FROM audit_log ORDER BY seq ASC`)
+	if err != nil {
+		return Integrity{}, fmt.Errorf("audit: verify query: %w", err)
+	}
+	defer func() { _ = rs.Close() }()
+
+	prevHash := genesisPrevHash
+	var expectSeq uint64
+	var count uint64
+	headHash := genesisPrevHash
+
+	for rs.Next() {
+		rec, scanErr := scanRecord(rs)
+		if scanErr != nil {
+			return Integrity{}, fmt.Errorf("audit: verify scan: %w", scanErr)
+		}
+		// Gapless, 0-based ordering.
+		if rec.Seq != expectSeq {
+			return Integrity{
+				OK: false, Count: count, HeadHash: headHash,
+				BrokenAt: int64(rec.Seq),
+				Reason:   fmt.Sprintf("seq gap: expected %d, got %d", expectSeq, rec.Seq),
+			}, nil
+		}
+		// Link continuity: this record must chain to the previous record's hash.
+		if rec.PrevHash != prevHash {
+			return Integrity{
+				OK: false, Count: count, HeadHash: headHash,
+				BrokenAt: int64(rec.Seq),
+				Reason:   "prev_hash discontinuity (a record was deleted, reordered, or altered)",
+			}, nil
+		}
+		// Content integrity: recompute the hash from the record's own fields.
+		want, hErr := computeHash(rec, rec.PrevHash)
+		if hErr != nil {
+			return Integrity{}, fmt.Errorf("audit: verify hash: %w", hErr)
+		}
+		if want != rec.Hash {
+			return Integrity{
+				OK: false, Count: count, HeadHash: headHash,
+				BrokenAt: int64(rec.Seq),
+				Reason:   "hash mismatch (record content was modified after it was written)",
+			}, nil
+		}
+		prevHash = rec.Hash
+		headHash = rec.Hash
+		expectSeq = rec.Seq + 1
+		count++
+	}
+	if err := rs.Err(); err != nil {
+		return Integrity{}, fmt.Errorf("audit: verify rows: %w", err)
+	}
+	return Integrity{OK: true, Count: count, HeadHash: headHash, BrokenAt: -1}, nil
+}

audit/record.go

@@ -0,0 +1,176 @@
+// Package audit is the unified cloud binary's compliance-grade audit trail —
+// tamper-evident, append-only, and complete over the security-relevant request
+// surface (FedRAMP AU-* / SOC 2 CC-* controls).
+//
+// THE CONTROL, IN ONE SENTENCE. Every security-relevant action against this
+// binary is captured as a structured Record, hash-chained to its predecessor so
+// any later deletion or modification is detectable, and written INLINE (never
+// dropped) to an append-only store the application can only INSERT into.
+//
+// THREE PIECES, EACH IN ITS LANE (orthogonal, per the Zen of Hanzo):
+//   - record.go  — the event model + the hash-chain math (what a record IS and
+//     how it links to the one before it). Pure, no I/O.
+//   - store.go   — the append-only sink (SQLite primary, INSERT-only; a
+//     best-effort OLAP mirror) and the serialized Recorder that
+//     owns the chain head. All persistence.
+//   - redact.go  — the secret-stripping allowlist/denylist for any structured
+//     before/after an explicit emit point supplies. No secret ever
+//     reaches a record.
+//
+// The HTTP middleware (Middleware, in the cloud package) and the query/verify
+// endpoints (in clients/admin) are thin callers of this package. This package
+// holds the security logic; it has zero knowledge of routes.
+package audit
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"time"
+)
+
+// Actor identifies WHO performed the action. It is populated ONLY from a
+// validated principal (the sanitized X-User-* headers SanitizeIdentity mints
+// from a verified IAM JWT), never from a raw client header — so an actor can
+// never be forged by the request that is being audited. A service principal
+// (M2M / no user sub) records Org with an empty Sub.
+type Actor struct {
+	// Org is the tenant (IAM `owner`). Empty for an unauthenticated request.
+	Org string `json:"org"`
+	// Sub is the user id (IAM `sub`/`preferred_username`). Empty for a service
+	// principal or an anonymous request.
+	Sub string `json:"sub"`
+	// Email is the validated user email, when present.
+	Email string `json:"email,omitempty"`
+}
+
+// Resource identifies WHAT was acted upon: a type (e.g. "org", "role",
+// "secret", "provider-config", "credit") and its id. For a plain HTTP mutation
+// with no finer resource semantics, Type is the route family and ID is empty —
+// the Action verb + path already pin the object.
+type Resource struct {
+	Type string `json:"type"`
+	ID   string `json:"id,omitempty"`
+}
+
+// AuthContext records HOW the actor authenticated and what authority they held
+// at decision time — the AC-* evidence (was this a global admin? by what
+// credential?). Method is "jwt" | "api-key" | "none". IsAdmin is the VALIDATED
+// global-admin bit (owner == AdminOrg), never a raw X-User-IsAdmin.
+type AuthContext struct {
+	Method  string `json:"method"`
+	IsAdmin bool   `json:"isAdmin"`
+}
+
+// Outcome is the result of the action: whether it was allowed and what
+// happened. Result is "success" | "deny" | "error". Status is the HTTP status.
+// Reason is a short, non-sensitive explanation for a deny/error (e.g.
+// "global admin required", "insufficient_balance") — never a secret, never a
+// raw upstream error body.
+type Outcome struct {
+	Result string `json:"result"`
+	Status int    `json:"status"`
+	Reason string `json:"reason,omitempty"`
+}
+
+// Record is one audit event. The JSON tags ARE the on-disk and on-wire contract.
+//
+// Field order in the struct is deliberate but IRRELEVANT to the hash: the chain
+// hashes the CANONICAL (sorted-key) JSON of the record with Hash/PrevHash zeroed
+// (see canonicalBytes), so re-ordering fields or adding an omitempty field can
+// never change an existing record's hash.
+type Record struct {
+	// Seq is the strictly-increasing chain position (0-based). It is assigned by
+	// the Recorder under its lock, so it is a true total order with no gaps.
+	Seq uint64 `json:"seq"`
+
+	// Time is the UTC event timestamp (RFC3339Nano).
+	Time time.Time `json:"time"`
+
+	// Actor / Action / Resource / Auth / Outcome — the AU-3 "content of audit
+	// records" core: who, what, on what, how-authenticated, with what result.
+	Actor    Actor       `json:"actor"`
+	Action   string      `json:"action"`
+	Resource Resource    `json:"resource"`
+	Auth     AuthContext `json:"auth"`
+	Outcome  Outcome     `json:"outcome"`
+
+	// SourceIP + UserAgent — the AU-3 "source of the event" fields.
+	SourceIP  string `json:"sourceIp,omitempty"`
+	UserAgent string `json:"userAgent,omitempty"`
+
+	// RequestID correlates the record to the request-line log and any downstream
+	// trace (the X-Request-Id the pipeline mints).
+	RequestID string `json:"requestId,omitempty"`
+
+	// Method + Path are the HTTP verb and route for a request-sourced event.
+	Method string `json:"method,omitempty"`
+	Path   string `json:"path,omitempty"`
+
+	// Before / After capture a mutation's prior and resulting state for the
+	// AU-required "before/after" on config-affecting changes. They are populated
+	// ONLY by explicit emit points and ONLY after Redact has stripped secrets —
+	// the HTTP middleware never sets them (it never reads bodies), so a secret in
+	// a request body can never leak here. Raw JSON so any shape round-trips.
+	Before json.RawMessage `json:"before,omitempty"`
+	After  json.RawMessage `json:"after,omitempty"`
+
+	// PrevHash is the hash of record Seq-1 (hex). For the genesis record (Seq 0)
+	// it is genesisPrevHash. Hash is this record's hash. Neither participates in
+	// its own hash computation (both are zeroed in canonicalBytes).
+	PrevHash string `json:"prevHash"`
+	Hash     string `json:"hash"`
+}
+
+// genesisPrevHash is the PrevHash of the first record in a fresh chain: 32 zero
+// bytes, hex-encoded. A non-empty, fixed anchor so the genesis record's hash is
+// still a function of a known constant (not the empty string, which would be
+// indistinguishable from "field omitted").
+const genesisPrevHash = "0000000000000000000000000000000000000000000000000000000000000000"
+
+// canonicalBytes returns the deterministic byte string a record hashes over: the
+// record with Hash AND PrevHash zeroed, marshaled by encoding/json (which sorts
+// struct fields in declaration order and, critically, is stable for a given
+// struct — the SAME bytes on every machine and every run). Zeroing PrevHash here
+// means the hash covers only the record's OWN content; the link to the previous
+// record is added explicitly in computeHash by appending prevHash. This keeps
+// the two concerns separable and the math obvious.
+//
+// We marshal a copy with the two hash fields cleared rather than a parallel
+// struct so there is exactly ONE definition of a record's fields (DRY): add a
+// field to Record and it is covered by the hash automatically.
+func canonicalBytes(r Record) ([]byte, error) {
+	r.Hash = ""
+	r.PrevHash = ""
+	return json.Marshal(r)
+}
+
+// computeHash returns the hex SHA-256 of (canonical(record) || prevHash-bytes).
+// The prevHash is folded in as its RAW hex string bytes — the exact value stored
+// in the record's PrevHash field — so the verifier reproduces it byte-for-byte
+// from stored data alone. Any change to the record's content OR to which record
+// precedes it changes this output, which is the whole tamper-evidence property.
+func computeHash(r Record, prevHash string) (string, error) {
+	body, err := canonicalBytes(r)
+	if err != nil {
+		return "", err
+	}
+	h := sha256.New()
+	h.Write(body)
+	h.Write([]byte(prevHash))
+	return hex.EncodeToString(h.Sum(nil)), nil
+}
+
+// seal finalizes a record into position seq linked to prevHash: it stamps Seq
+// and PrevHash, computes Hash, and returns the sealed record ready to append.
+// The Recorder calls this under its lock so seq/prevHash reflect the true head.
+func seal(r Record, seq uint64, prevHash string) (Record, error) {
+	r.Seq = seq
+	r.PrevHash = prevHash
+	hash, err := computeHash(r, prevHash)
+	if err != nil {
+		return Record{}, err
+	}
+	r.Hash = hash
+	return r, nil
+}

audit/redact.go

@@ -0,0 +1,140 @@
+package audit
+
+// Secret redaction for the before/after captured on a mutation.
+//
+// THE RULE. An audit record must NEVER contain a credential — no password,
+// token, API key, private key, card number, or session secret. Two layers
+// enforce this:
+//
+//  1. The HTTP middleware captures METADATA ONLY (actor/action/resource/outcome).
+//     It NEVER reads a request or response body, so a secret in a POST body can
+//     never reach a record through the automatic path. This is the primary
+//     guarantee: the code that can't see a secret can't leak one.
+//
+//  2. An EXPLICIT emit point that supplies structured before/after (e.g. a config
+//     change diff) runs it through Redact first. Redact walks the JSON and
+//     replaces the VALUE of any key whose name matches the secret denylist with a
+//     fixed marker, recursively. It is deny-by-key-name — the same allowlist
+//     PATTERN cloud already uses for user-secret redaction — chosen because a
+//     mutation diff has arbitrary shape and key-name matching is the robust,
+//     well-understood control (vs. trying to detect "secret-looking" values).
+//
+// Redact is conservative: on any structural surprise it returns the redaction
+// marker rather than the input, so a parser edge case fails CLOSED (no raw
+// passthrough).
+
+import (
+	"encoding/json"
+	"strings"
+)
+
+// redactedMarker replaces every redacted value. A constant so tests and the
+// query UI recognize it unambiguously.
+const redactedMarker = "[REDACTED]"
+
+// secretKeyParts are substrings that, when contained (case-insensitively) in a
+// JSON object key, mark that key's value as secret. Kept as a small, auditable
+// denylist of the credential-bearing field names that actually occur across the
+// Hanzo surface (IAM, KMS, commerce, provider config). Matching is substring so
+// "clientSecret", "api_key", "PRIVATE_KEY", "accessToken" all match.
+var secretKeyParts = []string{
+	"password",
+	"passwd",
+	"secret",
+	"token",
+	"apikey",
+	"api_key",
+	"api-key",
+	"authorization",
+	"auth_token",
+	"private_key",
+	"privatekey",
+	"privkey", // privkey, wgPrivKey
+	"passphrase",
+	"client_secret",
+	"credential",
+	"session",
+	"cookie",
+	"card", // card_number, cardNumber
+	"cvv",
+	"cvc",
+	"pin",
+	"ssn",
+	"social_security", // socialSecurityNumber (lower-cased match covers camelCase)
+	"socialsecurity",
+	"otp",
+	"mnemonic",
+	"phrase", // seed_phrase, seedPhrase, recoveryPhrase
+	"access_key",
+	"secret_key",
+	"refresh_token",
+	"id_token",
+	"bearer",
+	"signing_key",
+	"encryption_key",
+}
+
+// isSecretKey reports whether a JSON key names a credential-bearing field.
+func isSecretKey(key string) bool {
+	k := strings.ToLower(key)
+	for _, part := range secretKeyParts {
+		if strings.Contains(k, part) {
+			return true
+		}
+	}
+	return false
+}
+
+// Redact returns a copy of the JSON value with every secret-keyed value replaced
+// by the redaction marker, recursively through objects and arrays. Non-JSON or
+// empty input yields nil (nothing to record). On a JSON parse error the input is
+// dropped (returns the marker as a JSON string) rather than passed through —
+// fail closed.
+//
+// Use it at any explicit emit point that supplies before/after:
+//
+//	audit.Emit(ctx, rec.WithChange(audit.Redact(before), audit.Redact(after)))
+func Redact(raw json.RawMessage) json.RawMessage {
+	if len(raw) == 0 {
+		return nil
+	}
+	var v any
+	if err := json.Unmarshal(raw, &v); err != nil {
+		// Unparseable — never echo it back verbatim; record a marker instead.
+		b, _ := json.Marshal(redactedMarker)
+		return b
+	}
+	cleaned := redactValue("", v)
+	out, err := json.Marshal(cleaned)
+	if err != nil {
+		b, _ := json.Marshal(redactedMarker)
+		return b
+	}
+	return out
+}
+
+// redactValue walks a decoded JSON value. key is the object key under which v
+// sits (empty at the root and for array elements); when key is a secret key, the
+// ENTIRE value v is replaced (whether it is a scalar, object, or array — a secret
+// nested object is redacted whole). Otherwise objects/arrays are recursed.
+func redactValue(key string, v any) any {
+	if key != "" && isSecretKey(key) {
+		return redactedMarker
+	}
+	switch t := v.(type) {
+	case map[string]any:
+		out := make(map[string]any, len(t))
+		for k, val := range t {
+			out[k] = redactValue(k, val)
+		}
+		return out
+	case []any:
+		out := make([]any, len(t))
+		for i, val := range t {
+			out[i] = redactValue("", val) // array elements inherit no key
+		}
+		return out
+	default:
+		return v
+	}
+}

audit/store.go

@@ -0,0 +1,380 @@
+package audit
+
+// The append-only sink + the serialized Recorder that owns the hash-chain head.
+//
+// WHY SQLITE IS THE PRIMARY, DURABLE STORE (not ClickHouse). The chain is only
+// tamper-EVIDENT if records are appended in a strict, gapless total order and
+// each record's PrevHash is the immediately-preceding record's Hash. That demands
+// a single serializing writer with a synchronous, read-your-write head. cloud's
+// canonical store is embedded SQLite (one store per the storagelock lockdown;
+// pricing/provisioning already persist to {DataDir}/*.db). A local SQLite table
+// the application can only INSERT into gives us: (a) a real total order under one
+// connection, (b) synchronous durability so NO record is ever lost on the request
+// path (unlike a fire-and-forget mirror), and (c) an append-only surface — the
+// app issues no UPDATE/DELETE, and the hash-chain detects any out-of-band edit to
+// the file. That is the compliance-grade primary control.
+//
+// THE CLICKHOUSE MIRROR IS A PROJECTION, NOT THE SOURCE OF TRUTH. The datastore
+// (ClickHouse MergeTree — insert-only, mutation-rejected at parse time) is the
+// fleet-wide OLAP mirror for long-retention, cross-deployment query. It is
+// best-effort and asynchronous: a mirror outage must never block or fail an
+// audited request, and the local chain remains the authority the verifier walks.
+// Losing a mirror row is a query-completeness issue, not an integrity one.
+//
+// FAIL MODE. Append is INLINE and its error is RETURNED to the middleware, which
+// fails the request CLOSED (a security-relevant action that cannot be recorded is
+// not permitted to silently succeed). This is the AU-5 "response to audit logging
+// process failure": deny rather than act-unlogged.
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	// github.com/hanzoai/sqlite is the ONE Hanzo SQLite driver: it registers
+	// the "sqlite" database/sql name under both build tags (cgo →
+	// mattn+SQLCipher, encrypted at rest; !cgo → pure-Go modernc). Importing
+	// modernc directly instead would double-register "sqlite" under CGO and
+	// panic at init. Blank import registers the driver.
+	_ "github.com/hanzoai/sqlite"
+)
+
+// Mirror is the optional OLAP projection sink (the datastore/ClickHouse). It is
+// deliberately a tiny interface, not a concrete client, so the Recorder has no
+// compile-time dependency on ClickHouse and tests can supply a fake. Append is
+// called best-effort, asynchronously, off the request path.
+type Mirror interface {
+	// Append writes one sealed record to the projection. A returned error is
+	// logged and dropped by the Recorder — the mirror never gates a request.
+	Append(ctx context.Context, r Record) error
+}
+
+// Checkpoint is a periodic, tamper-EVIDENCE digest of the chain head: the record
+// count and the head hash at a moment in time. It is the AU-9 anchor for
+// TAIL-TRUNCATION detection — an internal chain walk cannot notice that the last
+// K records were deleted (the surviving prefix still verifies), but a durable,
+// INDEPENDENT series of head checkpoints can: Count is monotonic, so any decrease
+// between two consecutive checkpoints is deletion, and an attacker cannot forge a
+// higher count without appending records whose hashes the chain walk would reject.
+type Checkpoint struct {
+	Time  time.Time `json:"time"`
+	Count uint64    `json:"count"`
+	Head  string    `json:"head"`
+}
+
+// CheckpointSink is an optional capability a Mirror may implement to persist the
+// head digest series to an INDEPENDENT store (so truncating the local SQLite
+// cannot also rewrite the anchor history). A Mirror that does not implement it
+// still gets its records; checkpoints then flow only to the structured log.
+type CheckpointSink interface {
+	Checkpoint(ctx context.Context, cp Checkpoint) error
+}
+
+// Recorder is the single serialized writer that owns the audit chain head and
+// the append-only store. Every Record flows through Append, which under one lock
+// assigns the next Seq, links PrevHash to the current head, seals (hashes), and
+// synchronously persists to SQLite before returning. Concurrency is serialized
+// by mu AND by the single-connection SQLite pool, so the on-disk order equals
+// the chain order with no gaps.
+type Recorder struct {
+	db     *sql.DB
+	mirror Mirror // nil when no OLAP mirror is configured.
+
+	mu       sync.Mutex // guards nextSeq/headHash and serializes appends.
+	nextSeq  uint64     // Seq to assign to the next record.
+	headHash string     // Hash of the last-appended record (PrevHash for the next).
+
+	// Checkpoint emission (AU-9 tail-truncation anchor). logCheckpoint, when set,
+	// receives each head digest so it lands in the append-only observability log;
+	// stopCh/wg manage the periodic emitter goroutine's lifecycle; started guards
+	// against a second StartCheckpoints call (the field write + WaitGroup use are
+	// not safe to race). All three are set once, before any concurrent Append.
+	logCheckpoint func(cp Checkpoint)
+	stopCh        chan struct{}
+	wg            sync.WaitGroup
+	started       bool
+}
+
+// CheckpointFunc receives a head digest for the structured (o11y) log. It is a
+// plain func so the pure audit package stays free of any concrete logger type;
+// the cloud wiring adapts luxlog to it.
+type CheckpointFunc func(cp Checkpoint)
+
+// Open opens (creating if needed) the append-only audit DB at path and recovers
+// the chain head from it, so a restart continues the SAME chain rather than
+// forking a new one. path may be ":memory:" for tests. mirror may be nil.
+//
+// the hanzoai/sqlite "sqlite" driver; MaxOpenConns(1) serializes every statement against
+// the file lock — the same single-writer discipline pricing/provisioning use,
+// here doubling as the chain's serialization guarantee.
+func Open(path string, mirror Mirror) (*Recorder, error) {
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		return nil, fmt.Errorf("audit: open sqlite %q: %w", path, err)
+	}
+	db.SetMaxOpenConns(1)
+	for _, pragma := range []string{
+		"PRAGMA busy_timeout=5000",
+		"PRAGMA journal_mode=WAL",
+		"PRAGMA synchronous=NORMAL",
+	} {
+		if _, err := db.Exec(pragma); err != nil {
+			_ = db.Close()
+			return nil, fmt.Errorf("audit: pragma %q: %w", pragma, err)
+		}
+	}
+	r := &Recorder{db: db, mirror: mirror}
+	if err := r.migrate(); err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+	if err := r.recoverHead(); err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+	return r, nil
+}
+
+// migrate creates the append-only audit table. It is INSERT-only by application
+// discipline: this package issues no UPDATE or DELETE against it, and seq is the
+// PRIMARY KEY so a replayed/duplicated seq is rejected by the engine. The hash
+// columns make any out-of-band row edit detectable by Verify regardless of the
+// storage layer's own guarantees.
+func (r *Recorder) migrate() error {
+	const ddl = `
+CREATE TABLE IF NOT EXISTS audit_log (
+  seq        INTEGER PRIMARY KEY,       -- chain position; gapless, assigned under lock
+  ts         TEXT    NOT NULL,          -- RFC3339Nano UTC event time
+  actor_org  TEXT    NOT NULL DEFAULT '',
+  actor_sub  TEXT    NOT NULL DEFAULT '',
+  actor_email TEXT   NOT NULL DEFAULT '',
+  action     TEXT    NOT NULL,
+  res_type   TEXT    NOT NULL DEFAULT '',
+  res_id     TEXT    NOT NULL DEFAULT '',
+  auth_method TEXT   NOT NULL DEFAULT '',
+  is_admin   INTEGER NOT NULL DEFAULT 0,
+  result     TEXT    NOT NULL,          -- success|deny|error
+  status     INTEGER NOT NULL DEFAULT 0,
+  reason     TEXT    NOT NULL DEFAULT '',
+  source_ip  TEXT    NOT NULL DEFAULT '',
+  user_agent TEXT    NOT NULL DEFAULT '',
+  request_id TEXT    NOT NULL DEFAULT '',
+  method     TEXT    NOT NULL DEFAULT '',
+  path       TEXT    NOT NULL DEFAULT '',
+  before     TEXT    NOT NULL DEFAULT '',  -- redacted JSON (explicit emit only)
+  after      TEXT    NOT NULL DEFAULT '',  -- redacted JSON (explicit emit only)
+  prev_hash  TEXT    NOT NULL,
+  hash       TEXT    NOT NULL
+);
+-- Query indexes for the /v1/admin/audit filters (actor/action/resource/time).
+CREATE INDEX IF NOT EXISTS ix_audit_org_seq    ON audit_log(actor_org, seq);
+CREATE INDEX IF NOT EXISTS ix_audit_action_seq ON audit_log(action, seq);
+CREATE INDEX IF NOT EXISTS ix_audit_result_seq ON audit_log(result, seq);
+CREATE INDEX IF NOT EXISTS ix_audit_ts         ON audit_log(ts);
+`
+	if _, err := r.db.Exec(ddl); err != nil {
+		return fmt.Errorf("audit: migrate: %w", err)
+	}
+	return nil
+}
+
+// recoverHead loads the highest-seq record so a restarted process continues the
+// existing chain (nextSeq = maxSeq+1, headHash = its hash). An empty table starts
+// the genesis chain (nextSeq 0, headHash = genesisPrevHash).
+func (r *Recorder) recoverHead() error {
+	var (
+		maxSeq sql.NullInt64
+		hash   sql.NullString
+	)
+	row := r.db.QueryRow(`SELECT seq, hash FROM audit_log WHERE seq = (SELECT MAX(seq) FROM audit_log)`)
+	if err := row.Scan(&maxSeq, &hash); err != nil {
+		if errors.Is(err, sql.ErrNoRows) {
+			r.nextSeq = 0
+			r.headHash = genesisPrevHash
+			return nil
+		}
+		return fmt.Errorf("audit: recover head: %w", err)
+	}
+	if !maxSeq.Valid { // empty table (MAX over zero rows is NULL)
+		r.nextSeq = 0
+		r.headHash = genesisPrevHash
+		return nil
+	}
+	r.nextSeq = uint64(maxSeq.Int64) + 1
+	r.headHash = hash.String
+	return nil
+}
+
+// Append seals r into the next chain position and persists it. It fills Seq,
+// PrevHash, and Hash (the caller sets everything else), advances the in-memory
+// head only AFTER the durable INSERT succeeds, and mirrors best-effort. A
+// persistence error is returned so the caller can fail the request CLOSED — the
+// head is NOT advanced on failure, so the chain never gaps.
+//
+// The whole critical section (assign seq → seal → INSERT → advance head) holds
+// mu, so two concurrent requests can never claim the same seq or race the head.
+func (r *Recorder) Append(ctx context.Context, rec Record) (Record, error) {
+	if rec.Time.IsZero() {
+		rec.Time = time.Now().UTC()
+	} else {
+		rec.Time = rec.Time.UTC()
+	}
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	sealed, err := seal(rec, r.nextSeq, r.headHash)
+	if err != nil {
+		return Record{}, fmt.Errorf("audit: seal: %w", err)
+	}
+	if err := r.insert(ctx, sealed); err != nil {
+		// Head not advanced; the next append reuses this seq. Fail closed upstream.
+		return Record{}, fmt.Errorf("audit: persist: %w", err)
+	}
+	// Durable — advance the chain head.
+	r.nextSeq = sealed.Seq + 1
+	r.headHash = sealed.Hash
+
+	// Best-effort OLAP mirror, detached so a slow/failed mirror never blocks the
+	// request or corrupts the reply. The local chain is already durable and is the
+	// authority; a lost mirror row is a query-completeness gap, not an integrity
+	// one. Copy by value: the record is immutable and safe to hand to a goroutine.
+	if r.mirror != nil {
+		m, out := r.mirror, sealed
+		go func() { _ = m.Append(context.Background(), out) }()
+	}
+	return sealed, nil
+}
+
+// insert writes one sealed record. INSERT-only — the sole write statement in this
+// package. A duplicate seq (PRIMARY KEY) fails here, which is the desired
+// invariant: the chain never overwrites a position.
+func (r *Recorder) insert(ctx context.Context, rec Record) error {
+	_, err := r.db.ExecContext(ctx, `
+INSERT INTO audit_log (
+  seq, ts, actor_org, actor_sub, actor_email, action, res_type, res_id,
+  auth_method, is_admin, result, status, reason, source_ip, user_agent,
+  request_id, method, path, before, after, prev_hash, hash
+) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
+		rec.Seq, rec.Time.Format(time.RFC3339Nano),
+		rec.Actor.Org, rec.Actor.Sub, rec.Actor.Email,
+		rec.Action, rec.Resource.Type, rec.Resource.ID,
+		rec.Auth.Method, boolToInt(rec.Auth.IsAdmin),
+		rec.Outcome.Result, rec.Outcome.Status, rec.Outcome.Reason,
+		rec.SourceIP, rec.UserAgent, rec.RequestID, rec.Method, rec.Path,
+		string(rec.Before), string(rec.After), rec.PrevHash, rec.Hash,
+	)
+	return err
+}
+
+// checkpointCloseTimeout bounds the final (synchronous) checkpoint write to the
+// independent sink at shutdown, so Close cannot hang on an unreachable datastore.
+const checkpointCloseTimeout = 5 * time.Second
+
+// Close stops the periodic checkpoint emitter, emits a FINAL checkpoint
+// SYNCHRONOUSLY (so the head at shutdown reaches both the o11y log and the
+// independent digest store before the process exits — the AU-9 anchor must be
+// current exactly when an attacker might trigger shutdown then truncate), and
+// closes the underlying database.
+func (r *Recorder) Close() error {
+	if r == nil || r.db == nil {
+		return nil
+	}
+	if r.stopCh != nil {
+		close(r.stopCh)
+		r.wg.Wait()
+		r.stopCh = nil
+	}
+	// Anchor the final head before the DB closes — independent of whether the
+	// periodic ticker was running (every<=0 still gets a shutdown checkpoint).
+	// Synchronous to the sink (bounded), so the independent store's last count is
+	// as fresh as the local chain at the moment of shutdown.
+	r.emitCheckpoint(true)
+	return r.db.Close()
+}
+
+// StartCheckpoints begins periodic head-digest emission every `every` (no ticker
+// if every<=0; the on-Close checkpoint still fires). logFn, when non-nil,
+// receives each digest for the append-only observability log (o11y), and a mirror
+// implementing CheckpointSink also gets it persisted to an INDEPENDENT store —
+// together the AU-9 anchor an external monitor compares to detect tail-truncation
+// (count regression). MUST be called at most once, before any concurrent Append
+// (a second call is ignored); the emitter stops on Close.
+func (r *Recorder) StartCheckpoints(every time.Duration, logFn CheckpointFunc) {
+	// Guard the check-and-set under mu so a (mis)use that calls this concurrently
+	// is race-free, not just the single-call production path.
+	r.mu.Lock()
+	if r.started {
+		r.mu.Unlock()
+		return // already started — do not re-arm (avoids a field/WaitGroup race).
+	}
+	r.started = true
+	r.logCheckpoint = logFn
+	r.mu.Unlock()
+	if every <= 0 {
+		return
+	}
+	r.stopCh = make(chan struct{})
+	stop := r.stopCh
+	r.wg.Add(1)
+	go func() {
+		defer r.wg.Done()
+		t := time.NewTicker(every)
+		defer t.Stop()
+		for {
+			select {
+			case <-stop:
+				return
+			case <-t.C:
+				r.emitCheckpoint(false)
+			}
+		}
+	}()
+}
+
+// emitCheckpoint snapshots the head and emits it to the log and (if the mirror is
+// a CheckpointSink) the independent digest store. sync controls the sink write:
+// on the periodic path (sync=false) it is detached so a slow sink never delays
+// the ticker; on the Close path (sync=true) it BLOCKS on a bounded context so the
+// final anchor is durable before shutdown. The log emission is always synchronous
+// (it is the primary anchor and o11y ingests it append-only).
+func (r *Recorder) emitCheckpoint(sync bool) {
+	count, head := r.Head()
+	cp := Checkpoint{Time: time.Now().UTC(), Count: count, Head: head}
+	if r.logCheckpoint != nil {
+		r.logCheckpoint(cp)
+	}
+	cs, ok := r.mirror.(CheckpointSink)
+	if !ok || cs == nil {
+		return
+	}
+	if sync {
+		ctx, cancel := context.WithTimeout(context.Background(), checkpointCloseTimeout)
+		defer cancel()
+		_ = cs.Checkpoint(ctx, cp)
+		return
+	}
+	go func() { _ = cs.Checkpoint(context.Background(), cp) }()
+}
+
+// Head returns the current chain head (count of records, and the head hash). A
+// count of 0 means the genesis (empty) chain, headHash == genesisPrevHash. An
+// external monitor can pin (count, headHash) over time to detect tail-truncation
+// — which an internal chain walk alone cannot catch (a truncated prefix still
+// verifies). This is the anchor point for AU-9 protection against deletion of the
+// most-recent records.
+func (r *Recorder) Head() (count uint64, headHash string) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.nextSeq, r.headHash
+}
+
+func boolToInt(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}

audit_middleware.go

@@ -0,0 +1,677 @@
+package cloud
+
+// The audit middleware — the ONE place every security-relevant request is
+// recorded to the tamper-evident trail (decomplected: one function, every route).
+//
+// PLACEMENT (why it sits exactly where serve.go puts it). The pipeline is
+// Recover → RequestID → Logger → SanitizeIdentity → AuditTrail → BillingGate →
+// subsystems. AuditTrail runs:
+//   - AFTER SanitizeIdentity, so the actor/isAdmin it records come from a
+//     VALIDATED IAM principal (the sanitized X-User-* headers), never a raw
+//     client header. The request being audited cannot forge its own actor.
+//   - BEFORE BillingGate and every subsystem, so it WRAPS the whole handler
+//     chain and observes the FINAL outcome — including a 402/503 billing denial
+//     and a 403 admin-guard denial (both security-relevant) — via the response
+//     status after Continue(), exactly like the Logger middleware reads it.
+//
+// WHAT IT CAPTURES: metadata only — actor, action (method+route family),
+// resource, source ip, user agent, request id, auth context, and the outcome
+// (result/status/reason). It NEVER reads the request or response BODY, so a
+// secret in a POST body can never reach a record through this path. before/after
+// diffs are the job of explicit emit points (audit.Recorder.Append with a
+// redacted diff), not this middleware.
+//
+// WHAT IT RECORDS (the coverage predicate, auditable in one place — see
+// isSecurityRelevant): every mutating request (POST/PUT/PATCH/DELETE), every
+// /v1/admin/* request (read or write — admin reads are AC-relevant), and every
+// auth-failure outcome (401/403) on ANY method (a denied GET is an access-control
+// event). Safe, unauthenticated reads (a 200 GET on a public route) are NOT
+// audited — that is request-log noise, not a security event, and auditing it
+// would bury the signal and balloon the trail.
+//
+// FAIL MODE (AU-5): if the trail write fails on a request we decided to audit,
+// the CLIENT gets a fail-closed 503 rather than a success it can rely on — the
+// AU-5 "response to an audit logging process failure" is to interrupt, not to
+// operate silently unlogged. A write to local SQLite is sub-millisecond, so this
+// is a real integrity stance, not a latency tax. When no Recorder is configured
+// the middleware is a no-op passthrough (an unconfigured deployment is never
+// blocked), exactly like BillingGate.
+//
+// PRECISE SEMANTIC (do not over-read the 503). This is POST-RESPONSE audit: the
+// handler has already run when the record is written, so a 503 here means "this
+// event could not be RECORDED", NOT "the action did not execute". PREVENTION is
+// the access-control layer's job and runs BEFORE the action — SanitizeIdentity
+// (identity can't be forged) + the per-route admin guard both execute inside
+// c.Next() ahead of any side effect. The audit trail's job is DETECTION and
+// ACCOUNTABILITY (tamper-evident record of what happened), which it does. On a
+// persistent audit-store outage every mutation returns 503 (loud, logged), so
+// the system degrades to read-only rather than mutating unaudited — the intended
+// compliance posture.
+//
+// PANIC BOUND. If a handler PANICS, the outermost middleware.Recover catches it
+// and renders 500 with a full stack trace (loud, never silent); the panic unwinds
+// PAST this middleware's post-c.Next() code, so a panicking request is not written
+// to the trail. This is an accepted bound, not an evasion: an attacker cannot turn
+// a panic into a SUCCESSFUL-but-unaudited mutation (a panic yields 500, not a
+// completed action), and every panic is already captured by Recover's logging.
+// Normal outcomes — including billing 402/503 and admin 403 denials, which return
+// through c.Next() rather than panicking — are always audited.
+
+import (
+	"errors"
+	"net/url"
+	"strings"
+
+	"github.com/hanzoai/cloud/audit"
+	"github.com/hanzoai/cloud/clients/principal"
+	"github.com/zap-proto/zip"
+)
+
+// AuditTrail returns the audit middleware bound to rec. A nil rec makes it a
+// no-op passthrough so callers always Use() it unconditionally.
+func AuditTrail(rec *audit.Recorder) zip.Handler {
+	if rec == nil {
+		return func(c *zip.Ctx) error { return c.Next() }
+	}
+	return func(c *zip.Ctx) error {
+		// Capture the pre-decision inputs BEFORE running the chain: the request
+		// context is recycled by Fiber after the handler returns, so identity and
+		// request fields must be read now (mirrors BillingGate capturing usage by
+		// value pre-Record).
+		method := c.Method()
+		path := c.Path()
+
+		err := c.Next()
+
+		// Resolve the EFFECTIVE status. A handler may set it on the response
+		// directly (c.Status(...).JSON(...)) OR return a *zip.HTTPError that the
+		// framework's error handler renders AFTER this middleware unwinds — in the
+		// latter case the response still reads 200 here, so the returned error is
+		// the authoritative source of a 401/403. Prefer the error's status when it
+		// carries one; this is what makes admin-guard denials (which return
+		// ErrForbidden) get audited as 403.
+		status := effectiveStatus(c.Fiber().Response().StatusCode(), err)
+		if !isSecurityRelevant(method, path, status) {
+			return err // not an audited event; pass the handler result through.
+		}
+
+		record := audit.Record{
+			Actor:    actorFromCtx(c),
+			Action:   method + " " + routeFamily(path),
+			Resource: resourceFromPath(path),
+			Auth:     authFromCtx(c),
+			Outcome:  outcomeOf(status, err),
+			SourceIP: ClientIP(c),
+			// User-Agent is client-controlled free text; a misconfigured/malicious
+			// client could embed a bearer token in it. Scrub credential-shaped runs
+			// (and cap length) so the UA can never carry a secret into the record.
+			UserAgent: scrubFreeText(c.Header("User-Agent")),
+			RequestID: c.RequestID(),
+			Method:    method,
+			// Path is scrubbed of any credential-shaped segment: Hanzo routes use
+			// identifiers (:name/:slug/:id), not secrets, but a token that ever
+			// rides in the path (an hk-/sk-/pk-/fw_/hz_ key) must never be recorded
+			// verbatim. resourceFromPath applies the same scrub to the resource id.
+			Path: scrubCredentialSegments(path),
+		}
+
+		if _, aerr := rec.Append(c.Context(), record); aerr != nil {
+			// AU-5: could not record a security-relevant event — fail the request
+			// closed. Do NOT leak the audit error to the client; log it loud.
+			c.Log().Error("audit append failed — failing request closed",
+				"path", path, "method", method, "err", aerr)
+			return c.JSON(503, map[string]any{
+				"error": map[string]string{
+					"code":    "audit_unavailable",
+					"message": "Request could not be securely recorded",
+				},
+			})
+		}
+		return err
+	}
+}
+
+// isSecurityRelevant is the coverage predicate — the ONE place that decides which
+// requests enter the audit trail (AC/AU scope). Kept tiny and total so the
+// coverage matrix is reviewable at a glance. Order matters and is deliberate:
+// the security-relevant conditions are checked FIRST and are unconditional, so
+// the health-probe exemption can NEVER be used to evade audit of a mutation or a
+// denial (a POST/DELETE, or any 401/403, is always audited whatever the path).
+//   - any auth-failure outcome (401/403) on any method (a denied access attempt),
+//   - any /v1/admin/* request (admin reads are access-control-relevant),
+//   - any mutating request (POST/PUT/PATCH/DELETE).
+//
+// Only then, a genuine liveness probe (a GET to an exact health route) is
+// exempted — it is neither a mutation, a denial, nor an admin call, so it is pure
+// request-log noise. The exemption matches EXACT probe paths, never an arbitrary
+// path that merely ends in "/health" (which a wildcard/attacker-named segment
+// like POST /v1/admin/orgs/x/health could otherwise abuse to slip past audit).
+func isSecurityRelevant(method, path string, status int) bool {
+	// Unconditional security signals — never suppressed by any path shape. A
+	// denial, an admin call, or a mutation is ALWAYS audited, whatever the path
+	// (so a wildcard/attacker-named "/health" tail cannot evade it).
+	if status == 401 || status == 403 {
+		return true
+	}
+	if strings.HasPrefix(path, "/v1/admin/") {
+		return true
+	}
+	if isMutation(method) {
+		return true
+	}
+	// Everything left is a safe read (non-mutating, non-admin, non-denied). None
+	// are audited — they are request-log noise, not security events. (Liveness
+	// probes fall here too; there is no separate case because the answer is the
+	// same: not recorded.)
+	return false
+}
+
+// isMutation reports whether a method changes state.
+func isMutation(method string) bool {
+	switch method {
+	case "POST", "PUT", "PATCH", "DELETE":
+		return true
+	default:
+		return false
+	}
+}
+
+// actorFromCtx builds the Actor from the sanitized identity headers, gating
+// ANTI-FORGERY of the recorded actor on a VALIDATED principal.
+//
+// The authoritative "this request carried a validated principal" signal is a
+// non-empty X-User-Id (c.User()): SanitizeIdentity sets X-User-Id ONLY from a
+// JWT it verified, and strips any client-supplied copy on ingress. A request
+// with no principal — anonymous, OR one bearing an INVALID/garbage bearer that
+// failed validation — has an empty c.User().
+//
+// In that unvalidated case the org header is NOT trustworthy: SanitizeIdentity's
+// Phase-1 residual restores a client-supplied X-Org-Id for the data path, so an
+// anonymous attacker could send X-Org-Id: victim-org and, if we recorded it,
+// forge a FALSE ATTRIBUTION (an event stamped with a victim's org). So when there
+// is no validated sub, the actor is left EMPTY — the record stands as an honest
+// anonymous event identified by SourceIP, never mis-attributed to a claimed org.
+//
+// With a validated sub, org/sub/email all reflect the verified principal and are
+// recorded authoritatively.
+func actorFromCtx(c *zip.Ctx) audit.Actor {
+	if !principal.Validated(c) {
+		// No validated principal — do not trust the client-asserted org.
+		return audit.Actor{}
+	}
+	return audit.Actor{
+		Org:   strings.TrimSpace(c.Org()),
+		Sub:   strings.TrimSpace(c.User()),
+		Email: strings.TrimSpace(c.UserEmail()),
+	}
+}
+
+// authFromCtx records HOW the caller authenticated and the VALIDATED admin bit.
+// IsAdmin comes from c.IsAdmin() (the sanitized X-User-IsAdmin, true only for a
+// verified global admin), never a raw header. Method is inferred from the
+// presence/shape of a credential: an Authorization/X-Authorization bearer or a
+// session cookie ⇒ "jwt" (or "api-key" for an opaque hk-/sk- token); none ⇒
+// "none".
+func authFromCtx(c *zip.Ctx) audit.AuthContext {
+	return audit.AuthContext{
+		Method:  authMethodOf(c),
+		IsAdmin: c.IsAdmin(),
+	}
+}
+
+// authMethodOf classifies the credential kind WITHOUT capturing it — it inspects
+// only the token PREFIX (never stores the value). Order mirrors the sanitizer's
+// extraction (bearer, then cookie).
+func authMethodOf(c *zip.Ctx) string {
+	auth := c.Header("Authorization")
+	if auth == "" {
+		auth = c.Header("X-Authorization")
+	}
+	if tok := bearerFromAuth(auth); tok != "" {
+		if isAPIKey(tok) {
+			return "api-key"
+		}
+		return "jwt"
+	}
+	if basicFromAuth(auth) != "" {
+		return "basic"
+	}
+	for _, name := range cookieTokenNames {
+		if c.Fiber().Cookies(name) != "" {
+			return "jwt"
+		}
+	}
+	return "none"
+}
+
+// effectiveStatus reconciles the response status with a returned error. If the
+// handler returned a *zip.HTTPError (e.g. ErrForbidden), its Status is
+// authoritative — the framework renders it after this middleware unwinds, so the
+// live response status does not yet reflect it. A non-HTTPError returned error
+// with a still-2xx response means the framework will render a 500. Otherwise the
+// response status stands.
+func effectiveStatus(respStatus int, err error) int {
+	if err != nil {
+		var he *zip.HTTPError
+		if errors.As(err, &he) && he.Status != 0 {
+			return he.Status
+		}
+		// A non-HTTPError propagating up renders as 500, unless the handler already
+		// set an explicit error status on the response.
+		if respStatus < 400 {
+			return 500
+		}
+	}
+	return respStatus
+}
+
+// outcomeOf maps the final HTTP status + handler error to an audit Outcome.
+// 2xx/3xx ⇒ success; 401/403 ⇒ deny; everything else (4xx/5xx) ⇒ error. The
+// reason is a short, non-sensitive label derived from the status class — never a
+// raw upstream error body (which could echo sensitive detail).
+func outcomeOf(status int, err error) audit.Outcome {
+	switch {
+	case status == 401:
+		return audit.Outcome{Result: "deny", Status: status, Reason: "unauthenticated"}
+	case status == 403:
+		return audit.Outcome{Result: "deny", Status: status, Reason: "forbidden"}
+	case status >= 500:
+		return audit.Outcome{Result: "error", Status: status, Reason: "server_error"}
+	case status >= 400:
+		return audit.Outcome{Result: "error", Status: status, Reason: "client_error"}
+	default:
+		return audit.Outcome{Result: "success", Status: status}
+	}
+}
+
+// routeFamily reduces a concrete path to its stable route family for the action
+// verb, dropping trailing high-cardinality id segments so "DELETE /v1/admin/
+// orgs/acme" and "DELETE /v1/admin/orgs/globex" share the action "DELETE
+// /v1/admin/orgs". The full concrete path is preserved separately in Record.Path.
+func routeFamily(path string) string {
+	segs := strings.Split(strings.Trim(path, "/"), "/")
+	// Keep the leading "v1/<subsystem>/<noun>" and stop before an id-looking tail.
+	out := make([]string, 0, len(segs))
+	for _, s := range segs {
+		if looksLikeID(s) {
+			break
+		}
+		out = append(out, s)
+	}
+	if len(out) == 0 {
+		return "/" + strings.Join(segs, "/")
+	}
+	return "/" + strings.Join(out, "/")
+}
+
+// resourceFromPath derives the {type,id} resource from a /v1/<subsystem>/<type>/
+// [<id>] path. Type is the noun after the subsystem; ID is the following segment
+// when it looks like an identifier. Best-effort — the Action verb + Path are the
+// authoritative locator; this is a convenience for filtering by resource type.
+func resourceFromPath(path string) audit.Resource {
+	segs := strings.Split(strings.Trim(path, "/"), "/")
+	// segs: [v1, <subsystem>, <type>, <id?>, ...]
+	if len(segs) < 3 {
+		return audit.Resource{}
+	}
+	res := audit.Resource{Type: segs[2]}
+	if len(segs) >= 4 && looksLikeID(segs[3]) {
+		res.ID = scrubToken(segs[3])
+	}
+	return res
+}
+
+// scrubToken replaces a path segment that is a credential-shaped token with a
+// fixed marker, so a secret that ever appears in a URL is never recorded
+// verbatim. It catches, in increasing generality:
+//   - a known API-key prefix (hk-/sk-/pk-/fw_/hz_ — what isAPIKey recognizes),
+//     ALSO after percent-decoding, so hk%2DKEY can't slip the prefix check,
+//   - a JWT (three base64url parts split by '.', starting eyJ),
+//   - a long, high-entropy base64url/hex run (>=24) — a raw API key / access
+//     token / hex secret that carries no telltale prefix.
+//
+// A normal identifier passes through unchanged: a UUID (5 hyphen-split groups,
+// each short), a slug, a numeric id, a dotted model name — none is a long
+// unbroken high-entropy blob. See TestScrubToken_NoFalsePositives.
+//
+// RED-review hardening (finding: scrub bypass): the entropy test now (a) accepts
+// the FULL base64url alphabet incl. '-' and '_' (RFC 4648 §5), (b) does NOT
+// require a digit (an all-alpha opaque key is still a secret), (c) percent-
+// decodes first so %2D/%5F can't hide structure, and (d) drops the threshold to
+// 24 (short enough for a 128-bit base64 or a 24-hex key, long enough that no
+// human-readable slug reaches it).
+func scrubToken(seg string) string {
+	dec := percentDecode(seg)
+	if isAPIKey(seg) || isAPIKey(dec) ||
+		looksLikeJWT(seg) || looksLikeJWT(dec) ||
+		looksLikeHighEntropyToken(seg) || looksLikeHighEntropyToken(dec) {
+		return "[REDACTED-TOKEN]"
+	}
+	return seg
+}
+
+// percentDecode best-effort URL-decodes s so a percent-encoded credential
+// (hk%2DKEY, sk%5Flive%5F…) is normalized before the credential tests run. On a
+// malformed escape it returns s unchanged (the raw form is then tested as-is).
+func percentDecode(s string) string {
+	// Decode repeatedly (bounded) so a NESTED encoding (%252D -> %2D -> -) is
+	// fully normalized before the credential tests run. Stop when a pass makes no
+	// change, on a malformed escape, or after a small cap (defeats a decode bomb).
+	for i := 0; i < 3 && strings.Contains(s, "%"); i++ {
+		dec, err := url.PathUnescape(s)
+		if err != nil || dec == s {
+			break
+		}
+		s = dec
+	}
+	return s
+}
+
+// looksLikeJWT reports whether s is a JSON Web Token OR a JWT header segment. A
+// full JWT is three base64url parts split by '.', header starting "eyJ". But when
+// free text (a UA) is tokenized on '.', a dotted JWT splits into parts; a real
+// header part is >=24 chars (caught by the high-entropy run), yet to be safe we
+// ALSO flag any lone segment starting with the canonical base64url header prefix
+// "eyJ" (which decodes to '{"') regardless of length — a JWT header can never be
+// a legitimate resource id, so redacting it has no false-positive cost.
+func looksLikeJWT(s string) bool {
+	if !strings.HasPrefix(s, "eyJ") {
+		return false
+	}
+	// A full token (two dots) or a bare header segment — either way, redact.
+	return true
+}
+
+// highEntropyMinLen is the length at/above which an UNBROKEN run of base64/hex
+// chars is treated as an opaque secret. 24 covers a 128-bit base64 token, a
+// 24-nibble hex key, and short API keys, while every human-readable path
+// segment/slug/model-name stays under it once split on its separators — no
+// single run of a name like "text-embedding-3-large" reaches 24.
+const highEntropyMinLen = 24
+
+// looksLikeHighEntropyToken reports whether s CONTAINS an unbroken run of
+// >= highEntropyMinLen base64/hex chars — the shape of a raw API key / access
+// token / hex secret — UNLESS s is a structured human identifier.
+//
+// The run alphabet is [A-Za-z0-9_+/-] — the base64 alphabets (url-safe §5 '-”_'
+// and standard §4 '+”/') and hex. '-' is INCLUDED so a url-safe-base64 token
+// that embeds '-' is still caught by its run (excluding '-' left an ~11% bypass
+// for 32-byte url-safe tokens whose '-' happened to break every 24-run — measured).
+//
+// TWO-STAGE DETECTION:
+//
+//  1. UNCONDITIONAL run scan — a >= highEntropyMinLen UNBROKEN run over
+//     [A-Za-z0-9_+/-] flags the value REGARDLESS of the structured-id exemption.
+//     This catches every raw secret WITHOUT internal separators (hex, base64) —
+//     the realistic "a client bug put a raw key in the URL" case — at 100%. A
+//     structured identifier never has a 24-char unbroken run, so this stage never
+//     over-scrubs one.
+//
+//  2. STRUCTURED-ID EXEMPTION for the rest (values with separators that DON'T have
+//     a 24-run): exempt a clearly hyphen-joined human id, judged by LEXICAL
+//     content — every group WORD-LIKE (single-case word or decimal number), which
+//     a mixed-case base64 chunk or a long hex-with-letters chunk is NOT (those are
+//     redacted). Shape alone is attacker-satisfiable (RED found a 3x12-chunked
+//     secret slipped a shape-only check); the lexical test rejects the common
+//     secret encodings.
+//
+// ACCEPTED RESIDUAL BOUND (documented, per RED review): the ONLY residual is a
+// secret deliberately chunked into >=3 SINGLE-CASE-ALPHABETIC groups of <=12 chars
+// with no hex-letter runs >= hexChunkMinLen (a lowercase- or uppercase-only base32
+// alphabet, e.g. abcdefghijkl-mnopqrstuvwx-…). Such a value is lexically
+// indistinguishable from a hyphenated model id (deepseek-r1-distill-qwen-32b
+// carries the SAME 24-char entropy budget), so it passes stage 2. This is NOT
+// closable by any length/case/count rule without a word dictionary
+// (over-engineering for a defense-in-depth URL/UA backstop). Mixed-case base64
+// chunks AND small-hex chunks (md5/sha display grouping) ARE now caught
+// (isWordLikeGroup). The residual does not widen exposure for any REAL credential:
+// Hanzo keys are hk-/sk-/pk-/fw_/hz_-prefixed (isAPIKey, caught at any
+// length/shape), JWTs are eyJ-prefixed (looksLikeJWT), and request/response BODIES
+// are never read. It is an adversary DELIBERATELY base32-chunking their OWN secret
+// into a URL path to seed an admin-only audit row — contrived, low-value. The
+// realistic accidental leak (an unbroken raw key) is caught by stage 1.
+//
+// A canonical UUID is exempt (its longest run is 12; the check documents intent).
+func looksLikeHighEntropyToken(s string) bool {
+	if len(s) < highEntropyMinLen {
+		return false
+	}
+	// Stage 1 — unconditional: a long unbroken run is always a secret.
+	if hasHighEntropyRun(s) {
+		return true
+	}
+	// Stage 2 — separated values: redact unless it's a lexical structured id.
+	if isUUID(s) || isStructuredID(s) {
+		return false
+	}
+	// A >=24-length value with separators, not a UUID, not a structured id — e.g.
+	// "sk.live.LONGSECRET…" dotted, or a chunk pattern that is not word-like.
+	return true
+}
+
+// hasHighEntropyRun reports whether s contains an unbroken run of
+// >= highEntropyMinLen high-entropy chars where the run alphabet EXCLUDES '-'
+// (and '.'): a real separator breaks the run. This is stage 1 — it fires only on
+// a genuinely UNBROKEN opaque blob (a raw hex/base64 key with no separators), so
+// it never catches a hyphenated identifier (which stage 2 then classifies). A
+// url-safe-base64 secret that embeds '-' still trips because the run on ONE side
+// of the hyphen is >= 24 (verified: "AbCdEf-GhIjKl_MnOpQrStUvWxYz012345" -> 27).
+func hasHighEntropyRun(s string) bool {
+	run := 0
+	for _, r := range s {
+		if isUnbrokenTokenChar(r) {
+			run++
+			if run >= highEntropyMinLen {
+				return true
+			}
+		} else {
+			run = 0
+		}
+	}
+	return false
+}
+
+// isUnbrokenTokenChar is the stage-1 run alphabet: base64/hex MINUS '-' (and the
+// implicit exclusion of '.', space, etc.). '_' '+' '/' are kept — they appear
+// inside opaque tokens and are not identifier separators.
+func isUnbrokenTokenChar(r rune) bool {
+	return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') ||
+		(r >= '0' && r <= '9') || r == '_' || r == '+' || r == '/'
+}
+
+// idPartMaxLen bounds a hyphen-group length in the structured-id exemption. 12
+// covers the longest word in real model ids ("embedding", "20241022", "preview")
+// while a raw secret's random hyphen groups routinely exceed it.
+const idPartMaxLen = 12
+
+// isStructuredID reports whether s is a hyphen-joined human identifier (a model
+// name, slug): >= 3 hyphen groups where EVERY group is non-empty, <= idPartMaxLen
+// chars, and WORD-LIKE. The word-like test is the anti-bypass core — a raw secret
+// chunk cannot satisfy it — so the exemption is not attacker-satisfiable by shape.
+func isStructuredID(s string) bool {
+	groups := strings.Split(s, "-")
+	if len(groups) < 3 {
+		return false
+	}
+	for _, g := range groups {
+		if g == "" || len(g) > idPartMaxLen || !isWordLikeGroup(g) {
+			return false
+		}
+	}
+	return true
+}
+
+// isWordLikeGroup reports whether a hyphen group looks like a model-id token (a
+// dictionary word or a decimal number) rather than a random secret chunk. Two
+// lexical signals reject a secret chunk:
+//   - MIXED CASE (both upper and lower letters) — base64 tokens are dense
+//     mixed-case; real model tokens are single-case ("sonnet", "Instruct", "3").
+//   - an all-hex-with-letters run (>= hexChunkMinLen chars, all [0-9a-fA-F], and
+//     not all-digits) — a hex secret chunk ("dead", "beef", "cafebabe"); a version
+//     date ("20241022", all digits) is NOT hex-with-letters, so it stays
+//     word-like. The threshold is 4: RED re-review verified that NO real model-id
+//     group is all-hex-with-letters of length >= 4, so 4 (vs the prior 8) closes
+//     the small-hex-chunk leak (md5/sha shown in "xxxx-xxxx" display grouping)
+//     with zero model-id over-scrub.
+func isWordLikeGroup(g string) bool {
+	var hasUpper, hasLower, allHex, allDigit bool = false, false, true, true
+	for _, r := range g {
+		switch {
+		case r >= 'A' && r <= 'Z':
+			hasUpper = true
+		case r >= 'a' && r <= 'z':
+			hasLower = true
+		}
+		if r < '0' || r > '9' {
+			allDigit = false
+		}
+		isHex := (r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')
+		if !isHex {
+			allHex = false
+		}
+	}
+	if hasUpper && hasLower {
+		return false // dense mixed-case → base64 secret chunk, not a word.
+	}
+	if allHex && !allDigit && len(g) >= hexChunkMinLen {
+		return false // hex-with-letters → hex secret chunk (not a version date).
+	}
+	return true
+}
+
+// hexChunkMinLen is the length at/above which an all-hex-with-letters group is
+// treated as a secret chunk rather than a word. 4 is the tightest bound that does
+// not over-scrub any real model-id group (RED-verified across 19 model ids) while
+// catching hex secrets displayed in short groups (md5 "xxxx-xxxx", uuid-ish).
+const hexChunkMinLen = 4
+
+// isUUID reports whether s is a canonical 8-4-4-4-12 hex UUID (case-insensitive).
+// Used to exempt uuids from the high-entropy secret test — a uuid is a legitimate
+// resource id, not a credential.
+func isUUID(s string) bool {
+	if len(s) != 36 {
+		return false
+	}
+	for i, r := range s {
+		switch i {
+		case 8, 13, 18, 23:
+			if r != '-' {
+				return false
+			}
+		default:
+			if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+// maxUserAgentLen caps the recorded User-Agent so an oversized UA can neither
+// bloat the trail nor smuggle a long payload. 512 chars covers every real UA.
+const maxUserAgentLen = 512
+
+// scrubFreeText scrubs credential-shaped words out of client-controlled free
+// text (the User-Agent) and caps its length. It tokenizes on a broad delimiter
+// superset — whitespace and the punctuation that commonly glues a token into a
+// UA/header value (= ; , : / ( ) [ ] { } " ' < > | and backslash) — scrubs each
+// token, and rebuilds the string preserving the exact delimiters between tokens.
+//
+// RED-review hardening (finding: UA tokenizer split on too few delimiters, and
+// strings.ReplaceAll was substring-fragile): this walks the string in one pass
+// (token-run, delimiter-run, …) and replaces each credential token IN PLACE, so a
+// secret delimited by ':' '/' '(' etc. is caught and a token that is a substring
+// of another is never mis-replaced. A normal UA ("Mozilla/5.0 (Macintosh …)") is
+// unchanged because none of its words is credential-shaped.
+func scrubFreeText(s string) string {
+	if s == "" {
+		return ""
+	}
+	if len(s) > maxUserAgentLen {
+		s = s[:maxUserAgentLen]
+	}
+	var b strings.Builder
+	b.Grow(len(s))
+	start := -1 // start index of the current token run, or -1 in a delimiter run.
+	flush := func(end int) {
+		if start >= 0 {
+			b.WriteString(scrubToken(s[start:end]))
+			start = -1
+		}
+	}
+	for i, r := range s {
+		if isFreeTextDelimiter(r) {
+			flush(i)
+			b.WriteRune(r)
+		} else if start < 0 {
+			start = i
+		}
+	}
+	flush(len(s))
+	return b.String()
+}
+
+// isFreeTextDelimiter reports whether r separates tokens in free text (a UA /
+// header value). Deliberately broad so a credential can't hide behind an unusual
+// separator.
+//
+// RED re-review (UA bypass persists): '.' '@' '#' '~' are INCLUDED — a prefixed
+// key glued by one of them (client@sk-live-KEY, app.sk-live-KEY, build#hk-KEY)
+// otherwise stayed one token whose PREFIX was no longer sk-/hk-, so isAPIKey
+// missed it. Splitting on them exposes the bare key to scrubToken. Real UA dots
+// are numeric version separators (<24, safe) and are split harmlessly. A JWT
+// (eyJ.h.p.s) is handled up-front by scrubToken via looksLikeJWT before any
+// tokenizer runs on a URL path segment; in free text a dotted JWT will split,
+// but each ~40-char base64 part is itself a >=24 high-entropy run, so every part
+// is still redacted.
+func isFreeTextDelimiter(r rune) bool {
+	switch r {
+	case ' ', '\t', '\n', '\r', '=', ';', ',', ':', '/', '\\',
+		'(', ')', '[', ']', '{', '}', '"', '\'', '<', '>', '|', '&', '?',
+		'.', '@', '#', '~':
+		return true
+	}
+	return false
+}
+
+// scrubCredentialSegments applies scrubToken to every segment of a path, so the
+// recorded Path can never carry a credential even if a future route embeds one.
+// Route nouns/ids pass through unchanged (they are not isAPIKey-shaped).
+func scrubCredentialSegments(path string) string {
+	if !strings.ContainsAny(path, "/") {
+		return scrubToken(path)
+	}
+	segs := strings.Split(path, "/")
+	changed := false
+	for i, s := range segs {
+		if scrubbed := scrubToken(s); scrubbed != s {
+			segs[i] = scrubbed
+			changed = true
+		}
+	}
+	if !changed {
+		return path
+	}
+	return strings.Join(segs, "/")
+}
+
+// looksLikeID heuristically flags a path segment as a high-cardinality id (a
+// uuid, a long hex/opaque token, or a numeric id) vs a fixed route noun. Used to
+// collapse route families; false positives only make the action slightly more
+// specific, never leak anything.
+func looksLikeID(s string) bool {
+	if s == "" {
+		return false
+	}
+	if len(s) >= 16 { // long opaque/uuid-ish segment
+		return true
+	}
+	allDigits := true
+	for _, r := range s {
+		if r < '0' || r > '9' {
+			allDigits = false
+			break
+		}
+	}
+	return allDigits && len(s) > 0
+}

audit_middleware_test.go

@@ -0,0 +1,579 @@
+package cloud
+
+// Integration tests for the audit middleware. They drive REAL requests through
+// the zip/fiber stack (app.Fiber().Test) with the audit middleware in front of a
+// handler, backed by a REAL on-disk audit store, then read the store back to
+// assert what was (and was not) recorded. No mocks — the whole capture path runs.
+//
+// The middleware trusts SanitizeIdentity to have already validated identity, so
+// these tests set the sanitized X-User-* headers directly (as SanitizeIdentity
+// would after verifying a JWT) — that is the contract boundary under test here.
+// The forgery/bypass properties of SanitizeIdentity itself are proven in
+// middleware_identity_test.go; here we prove the middleware records the VALIDATED
+// identity and the correct outcome for every security-relevant request.
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/hanzoai/cloud/audit"
+	"github.com/zap-proto/zip"
+)
+
+// newAuditApp wires a zip app with the audit middleware in front of a small set
+// of routes covering the coverage matrix: a mutating POST, a safe GET, an
+// admin-gated route that 403s, and a route that echoes a (secret-bearing) body
+// so we can prove the body never reaches a record.
+func newAuditApp(t *testing.T) (*zip.App, *audit.Recorder) {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "audit.db")
+	rec, err := audit.Open(path, nil)
+	if err != nil {
+		t.Fatalf("audit.Open: %v", err)
+	}
+	t.Cleanup(func() { _ = rec.Close() })
+
+	app := zip.New(zip.Config{})
+	app.Use(AuditTrail(rec))
+
+	// Mutating route — must be audited.
+	app.Post("/v1/kms/secrets", func(c *zip.Ctx) error {
+		return c.JSON(http.StatusCreated, map[string]string{"id": "sec_1"})
+	})
+	// Safe read — must NOT be audited (not a mutation, not admin, not a denial).
+	app.Get("/v1/pricing/models", func(c *zip.Ctx) error {
+		return c.JSON(http.StatusOK, map[string]string{"ok": "true"})
+	})
+	// Admin route that denies (as the real admin guard would) — the 403 is a
+	// security event and MUST be audited even though it is a GET.
+	app.Get("/v1/admin/orgs", func(c *zip.Ctx) error {
+		if !c.IsAdmin() {
+			return zip.ErrForbidden("global admin required")
+		}
+		return c.JSON(http.StatusOK, map[string]string{"ok": "true"})
+	})
+	// A mutation whose request body carries secrets — used to prove the body is
+	// never captured. The handler ignores the body; the point is what the
+	// middleware records (metadata only).
+	app.Post("/v1/iam/users", func(c *zip.Ctx) error {
+		return c.JSON(http.StatusOK, map[string]string{"ok": "true"})
+	})
+	return app, rec
+}
+
+// asAdmin sets the sanitized identity headers a VALIDATED global admin would
+// carry after SanitizeIdentity (X-User-IsAdmin=true, org=admin).
+func asAdmin(req *http.Request) {
+	req.Header.Set("X-User-Id", "z@hanzo.ai")
+	req.Header.Set("X-User-Email", "z@hanzo.ai")
+	req.Header.Set("X-Org-Id", "admin")
+	req.Header.Set("X-User-IsAdmin", "true")
+	req.Header.Set("Authorization", "Bearer eyJ.validated.jwt") // shape only; classifies as jwt
+}
+
+// asUser sets sanitized headers for a normal (non-admin) validated principal.
+func asUser(req *http.Request) {
+	req.Header.Set("X-User-Id", "alice")
+	req.Header.Set("X-Org-Id", "acme")
+	req.Header.Set("Authorization", "Bearer eyJ.validated.jwt")
+}
+
+func mustTest(t *testing.T, app *zip.App, req *http.Request) *http.Response {
+	t.Helper()
+	resp, err := app.Fiber().Test(req)
+	if err != nil {
+		t.Fatalf("Test %s %s: %v", req.Method, req.URL.Path, err)
+	}
+	return resp
+}
+
+// TestAudit_RecordsMutation proves a mutating request is captured with the
+// correct validated actor, action, resource, outcome, and auth context — and the
+// record is hash-chained (has a hash) and verifies.
+func TestAudit_RecordsMutation(t *testing.T) {
+	app, rec := newAuditApp(t)
+
+	req := httptest.NewRequest(http.MethodPost, "/v1/kms/secrets", nil)
+	asUser(req)
+	req.Header.Set("X-Forwarded-For", "203.0.113.9, 10.0.0.1")
+	req.Header.Set("User-Agent", "test-agent/1.0")
+	resp := mustTest(t, app, req)
+	if resp.StatusCode != http.StatusCreated {
+		t.Fatalf("status = %d, want 201", resp.StatusCode)
+	}
+
+	rows, total, err := rec.Query(t.Context(), audit.Filter{})
+	if err != nil {
+		t.Fatalf("Query: %v", err)
+	}
+	if total != 1 || len(rows) != 1 {
+		t.Fatalf("recorded %d events, want exactly 1", total)
+	}
+	r := rows[0]
+	if r.Actor.Org != "acme" || r.Actor.Sub != "alice" {
+		t.Errorf("actor = %+v, want org=acme sub=alice (the VALIDATED identity)", r.Actor)
+	}
+	if r.Method != "POST" || r.Path != "/v1/kms/secrets" {
+		t.Errorf("method/path = %s %s, want POST /v1/kms/secrets", r.Method, r.Path)
+	}
+	if r.Resource.Type != "secrets" {
+		t.Errorf("resource type = %q, want secrets", r.Resource.Type)
+	}
+	if r.Outcome.Result != "success" || r.Outcome.Status != 201 {
+		t.Errorf("outcome = %+v, want success/201", r.Outcome)
+	}
+	if r.Auth.Method != "jwt" {
+		t.Errorf("auth method = %q, want jwt", r.Auth.Method)
+	}
+	if r.SourceIP != "203.0.113.9" {
+		t.Errorf("source ip = %q, want the left-most XFF entry", r.SourceIP)
+	}
+	if r.UserAgent != "test-agent/1.0" {
+		t.Errorf("user agent = %q, want test-agent/1.0", r.UserAgent)
+	}
+	if r.Hash == "" {
+		t.Error("record has no hash — not chained")
+	}
+	if iv, _ := rec.Verify(t.Context()); !iv.OK {
+		t.Errorf("chain broke after one record at %d (%s)", iv.BrokenAt, iv.Reason)
+	}
+}
+
+// TestAudit_RecordsDenial proves a 403 (access-control denial) is audited even on
+// a GET, with outcome result="deny", and records the actor who was denied.
+func TestAudit_RecordsDenial(t *testing.T) {
+	app, rec := newAuditApp(t)
+
+	// A NON-admin hits the admin route → 403.
+	req := httptest.NewRequest(http.MethodGet, "/v1/admin/orgs", nil)
+	asUser(req) // not admin
+	resp := mustTest(t, app, req)
+	if resp.StatusCode != http.StatusForbidden {
+		t.Fatalf("status = %d, want 403", resp.StatusCode)
+	}
+
+	rows, total, err := rec.Query(t.Context(), audit.Filter{Result: "deny"})
+	if err != nil {
+		t.Fatalf("Query: %v", err)
+	}
+	if total != 1 {
+		t.Fatalf("recorded %d denials, want 1", total)
+	}
+	r := rows[0]
+	if r.Outcome.Result != "deny" || r.Outcome.Status != 403 {
+		t.Errorf("outcome = %+v, want deny/403", r.Outcome)
+	}
+	if r.Actor.Sub != "alice" {
+		t.Errorf("denied actor sub = %q, want alice", r.Actor.Sub)
+	}
+	if r.Auth.IsAdmin {
+		t.Error("denied non-admin recorded as admin")
+	}
+}
+
+// TestAudit_SkipsSafeReads proves an ordinary successful GET on a non-admin route
+// is NOT audited — the trail captures security events, not read-log noise.
+func TestAudit_SkipsSafeReads(t *testing.T) {
+	app, rec := newAuditApp(t)
+
+	req := httptest.NewRequest(http.MethodGet, "/v1/pricing/models", nil)
+	asUser(req)
+	resp := mustTest(t, app, req)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("status = %d, want 200", resp.StatusCode)
+	}
+	_, total, err := rec.Query(t.Context(), audit.Filter{})
+	if err != nil {
+		t.Fatalf("Query: %v", err)
+	}
+	if total != 0 {
+		t.Fatalf("a safe GET was audited (%d rows) — trail should skip non-security reads", total)
+	}
+}
+
+// TestAudit_AdminReadIsAudited proves a SUCCESSFUL admin read is audited (admin
+// access itself is an AC-relevant event), distinguishing it from a normal read.
+func TestAudit_AdminReadIsAudited(t *testing.T) {
+	app, rec := newAuditApp(t)
+
+	req := httptest.NewRequest(http.MethodGet, "/v1/admin/orgs", nil)
+	asAdmin(req)
+	resp := mustTest(t, app, req)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("status = %d, want 200", resp.StatusCode)
+	}
+	rows, total, err := rec.Query(t.Context(), audit.Filter{})
+	if err != nil {
+		t.Fatalf("Query: %v", err)
+	}
+	if total != 1 {
+		t.Fatalf("admin read recorded %d, want 1", total)
+	}
+	if !rows[0].Auth.IsAdmin || rows[0].Outcome.Result != "success" {
+		t.Errorf("admin read record = auth.isAdmin=%v outcome=%+v, want admin+success", rows[0].Auth.IsAdmin, rows[0].Outcome)
+	}
+}
+
+// TestAudit_NeverCapturesRequestBody is the secret-safety proof: a mutation whose
+// body is FULL of credentials is audited, but the stored record contains NONE of
+// the body — the middleware captures metadata only, so a secret in a body can
+// never leak into the trail. This is the "reuse RedactUserSecrets" guarantee at
+// its strongest: the code that could leak a secret never reads it.
+func TestAudit_NeverCapturesRequestBody(t *testing.T) {
+	app, rec := newAuditApp(t)
+
+	secretBody := `{"username":"bob","password":"hunter2","apiKey":"sk-live-DEADBEEF","token":"ghp_SECRET"}`
+	req := httptest.NewRequest(http.MethodPost, "/v1/iam/users", strings.NewReader(secretBody))
+	req.Header.Set("Content-Type", "application/json")
+	asAdmin(req)
+	// Also stuff a secret into a header value that is NOT an identity header — it
+	// must not be captured either (we only record User-Agent + XFF, never arbitrary
+	// headers, and never Authorization's value).
+	req.Header.Set("Authorization", "Bearer eyJsuper.secret.token.value")
+	resp := mustTest(t, app, req)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("status = %d, want 200", resp.StatusCode)
+	}
+
+	rows, total, err := rec.Query(t.Context(), audit.Filter{})
+	if err != nil {
+		t.Fatalf("Query: %v", err)
+	}
+	if total != 1 {
+		t.Fatalf("recorded %d, want 1", total)
+	}
+	// Serialize the WHOLE record and scan for any secret substring.
+	blob, _ := json.Marshal(rows[0])
+	for _, secret := range []string{"hunter2", "sk-live-DEADBEEF", "ghp_SECRET", "super.secret.token.value"} {
+		if strings.Contains(string(blob), secret) {
+			t.Fatalf("SECRET LEAKED into audit record: %q found in %s", secret, blob)
+		}
+	}
+	// But the metadata IS there: the auth method is classified without the token.
+	if rows[0].Auth.Method != "jwt" {
+		t.Errorf("auth method = %q, want jwt (classified from prefix, token not stored)", rows[0].Auth.Method)
+	}
+	if rows[0].Before != nil || rows[0].After != nil {
+		t.Errorf("middleware set before/after (%s / %s) — it must never read bodies", rows[0].Before, rows[0].After)
+	}
+}
+
+// TestAudit_ScrubsCredentialInPath proves a credential-shaped token that rides in
+// the URL PATH (e.g. a KMS route where a caller wrongly puts an sk-/hk- key in the
+// path) is never recorded verbatim in either Path or resource.ID — defense in
+// depth beyond "bodies are never read". A normal identifier is untouched.
+func TestAudit_ScrubsCredentialInPath(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "audit.db")
+	rec, err := audit.Open(path, nil)
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	t.Cleanup(func() { _ = rec.Close() })
+	app := zip.New(zip.Config{})
+	app.Use(AuditTrail(rec))
+	app.Delete("/v1/kms/secrets/*", func(c *zip.Ctx) error {
+		return c.JSON(http.StatusOK, map[string]string{"ok": "true"})
+	})
+
+	// Three credential shapes smuggled into the path across three requests: a
+	// prefixed key, a raw high-entropy hex secret (NO telltale prefix), and a JWT.
+	paths := []struct{ path, secret string }{
+		{"/v1/kms/secrets/sk-live-SUPERSECRETKEY1234567890", "SUPERSECRETKEY"},
+		{"/v1/kms/secrets/deadbeefcafe0123456789abcdef0123456789abcdef0123", "deadbeefcafe0123"},
+		{"/v1/kms/secrets/eyJhbGciOiJIUzI1NiJ9.cGF5bG9hZA.c2ln", "eyJhbGciOiJIUzI1NiJ9"},
+	}
+	for _, tc := range paths {
+		req := httptest.NewRequest(http.MethodDelete, tc.path, nil)
+		asAdmin(req)
+		if resp := mustTest(t, app, req); resp.StatusCode != http.StatusOK {
+			t.Fatalf("%s: status = %d, want 200", tc.path, resp.StatusCode)
+		}
+	}
+
+	rows, total, err := rec.Query(t.Context(), audit.Filter{})
+	if err != nil {
+		t.Fatalf("Query: %v", err)
+	}
+	if total != len(paths) {
+		t.Fatalf("recorded %d, want %d", total, len(paths))
+	}
+	blob, _ := json.Marshal(rows)
+	for _, tc := range paths {
+		if strings.Contains(string(blob), tc.secret) {
+			t.Fatalf("credential in path leaked into record: %q present in %s", tc.secret, blob)
+		}
+	}
+	for _, r := range rows {
+		if !strings.Contains(r.Path, "[REDACTED-TOKEN]") {
+			t.Errorf("path token not scrubbed: %q", r.Path)
+		}
+	}
+}
+
+// TestAudit_ScrubsSecretInUserAgent proves a bearer/API-key embedded in the
+// client-controlled User-Agent is scrubbed, and a normal UA is untouched.
+func TestAudit_ScrubsSecretInUserAgent(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "audit.db")
+	rec, err := audit.Open(path, nil)
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	t.Cleanup(func() { _ = rec.Close() })
+	app := zip.New(zip.Config{})
+	app.Use(AuditTrail(rec))
+	app.Post("/v1/kms/secrets", func(c *zip.Ctx) error { return c.JSON(http.StatusOK, map[string]string{"ok": "1"}) })
+
+	req := httptest.NewRequest(http.MethodPost, "/v1/kms/secrets", nil)
+	asAdmin(req)
+	req.Header.Set("User-Agent", "myclient/1.0 Bearer eyJhbGciOiJI.pay.sig key=sk-live-LEAKME99999")
+	mustTest(t, app, req)
+
+	rows, _, _ := rec.Query(t.Context(), audit.Filter{})
+	if len(rows) != 1 {
+		t.Fatalf("got %d rows, want 1", len(rows))
+	}
+	for _, secret := range []string{"eyJhbGciOiJI", "sk-live-LEAKME99999"} {
+		if strings.Contains(rows[0].UserAgent, secret) {
+			t.Errorf("UA secret leaked: %q in %q", secret, rows[0].UserAgent)
+		}
+	}
+	// The non-secret UA prefix survives (audit usefulness preserved).
+	if !strings.Contains(rows[0].UserAgent, "myclient/1.0") {
+		t.Errorf("UA over-scrubbed, lost the client name: %q", rows[0].UserAgent)
+	}
+}
+
+// TestScrubToken_NoFalsePositives proves legitimate identifiers are NEVER
+// scrubbed — the guard fires only on genuinely secret-shaped segments, so the
+// audit trail keeps its query precision for normal resource ids.
+func TestScrubToken_NoFalsePositives(t *testing.T) {
+	for _, id := range []string{
+		"acme-corp", "gpt-4o-mini", "my_project_123", "user@example.com",
+		"550e8400-e29b-41d4-a716-446655440000", // uuid (hyphens)
+		"claude-opus-4-20250514", "text-embedding-3-large",
+		"my-cool-project-name", "feature-branch-xyz",
+		"deployment-2024-01-15", "report.pdf", "data.json",
+		// Red re-review round 2 — hyphenated model ids MUST pass (AU-3: an auditor
+		// must still see WHICH model a config change touched).
+		"claude-3-5-sonnet-20241022", "claude-3-5-haiku-20241022",
+		"claude-sonnet-4-20250514", "claude-3-7-sonnet-20250219",
+		"claude-3-5-sonnet-latest", "deepseek-r1-distill-qwen-32b",
+		"claude-3-opus-20240229", "stable-diffusion-xl-base",
+		"mixtral-8x7b-instruct", "llama-3-1-8b-instruct", "whisper-large-v3-turbo",
+		"v1", "models", "sync", "12345", "a", "",
+	} {
+		if got := scrubToken(id); got != id {
+			t.Errorf("false scrub: legit id %q → %q", id, got)
+		}
+	}
+	// And genuine secrets ARE scrubbed.
+	for _, sec := range []string{
+		"sk-live-abcdef", "hk-1234567890abcdef", "eyJhbG.payload.signature",
+		"deadbeefcafe0123456789abcdef0123456789abcdef0123", // 48-char raw hex
+	} {
+		if scrubToken(sec) == sec {
+			t.Errorf("missed secret: %q not scrubbed", sec)
+		}
+	}
+}
+
+// TestScrubToken_RedReviewBypassClasses is the regression for the 4 scrub-bypass
+// classes Red found: base64url with -/_, all-alpha opaque >=len, percent-encoded
+// prefixes, and delimiter-glued UA tokens. Each MUST now be redacted.
+func TestScrubToken_RedReviewBypassClasses(t *testing.T) {
+	for _, sec := range []string{
+		"AbCdEf-GhIjKl_MnOpQrStUvWxYz012345",       // base64url with - and _
+		"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN", // all-alpha opaque, no digit
+		"hk%2DROTATEKEY0001SECRETKEY",              // percent-encoded hk-
+		"sk%5Flive%5FBYPASS0001SECRETKEY",          // percent-encoded sk_
+		// Red re-review round 2 — dotted-exemption + encoding + standard-base64:
+		"deadbeefcafe0123456789abcdef0123456789abcdef.x", // ".x" tail forces dotted exemption
+		"AbCdEfGhIjKlMnOpQrStUvWxYz012345.json",          // secret with a filename-ish suffix
+		"hk%252DROTATEKEY0001SECRETKEY",                  // double percent-encoded hk-
+		"AbCdEfGhIjKlMnOpQrStUvWx0123456789",             // 34-char opaque run (standard/url b64)
+		// Red re-review round 3 — interior-hyphen raw secret (NOT a structured id:
+		// only 1 hyphen, long parts) must still redact despite '-' in the run.
+		"aaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbbbbbbbbbb",
+		// Red final re-review — a secret CHUNKED to satisfy the structured-id shape
+		// (>=3 groups <=12 chars) must STILL redact: the lexical (word-like) test
+		// rejects mixed-case base64 chunks and hex chunks (>= hexChunkMinLen=4).
+		"AbCdEfGhIjKl-MnOpQrStUvWx-YzAbCdEfGhIj", // mixed-case base64 chunks
+		"A1b2C3d4-E5f6G7h8-I9j0K1l2",             // 128-bit mixed-case chunks
+		"deadbeef-cafebabe-01234567-89abcdef",    // all-hex chunks (8-char groups)
+		"abcdef01-23456789-abcdef01",             // hex chunks
+		// Red final polish — SMALL hex chunks (md5/sha "xxxx-xxxx" display) now
+		// caught by the len>=4 hex rule.
+		"abcd-ef01-2345-6789-abcd-ef01", // 4-char hex groups
+		"dead-beef-cafe-babe-0123-4567", // 4-char hex groups
+	} {
+		if got := scrubToken(sec); got != "[REDACTED-TOKEN]" {
+			t.Errorf("Red bypass STILL OPEN: %q → %q (want redacted)", sec, got)
+		}
+	}
+	// UA with a secret glued by :/()[]= — must be scrubbed, client name kept.
+	ua := scrubFreeText("myapp/1.0 (token:sk-live-BYPASS0001) [key=hk-1234567890abcdef]")
+	for _, leak := range []string{"sk-live-BYPASS0001", "hk-1234567890abcdef"} {
+		if strings.Contains(ua, leak) {
+			t.Errorf("UA bypass: %q leaked in %q", leak, ua)
+		}
+	}
+	if !strings.Contains(ua, "myapp/1.0") {
+		t.Errorf("UA over-scrubbed, lost client name: %q", ua)
+	}
+	// Red re-review round 2 — a key glued by . @ # ~ must NOT survive.
+	for _, glued := range []string{
+		"client@sk-live-SECRETKEY00001", "app.sk-live-SECRETKEY00001",
+		"build#hk-SECRETKEY000000001", "v1~sk-live-SECRETKEY00001",
+	} {
+		if got := scrubFreeText(glued); strings.Contains(got, "SECRETKEY") {
+			t.Errorf("UA glue-char bypass: %q → %q (secret survives)", glued, got)
+		}
+	}
+	// Normal UAs must be byte-identical (no false scrub).
+	for _, ua := range []string{
+		"console2", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
+		"curl/8.1.2", "Go-http-client/2.0",
+	} {
+		if got := scrubFreeText(ua); got != ua {
+			t.Errorf("UA false positive: %q → %q", ua, got)
+		}
+	}
+}
+
+// TestAudit_HealthSuffixCannotEvadeAudit proves a mutating request whose path
+// ENDS in /health (an attacker-named wildcard segment) is STILL audited — the
+// liveness exemption is exact and never suppresses a mutation or a denial. This
+// closes an audit-evasion hole where POST /v1/admin/orgs/x/health would slip past.
+func TestAudit_HealthSuffixCannotEvadeAudit(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "audit.db")
+	rec, err := audit.Open(path, nil)
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	t.Cleanup(func() { _ = rec.Close() })
+	app := zip.New(zip.Config{})
+	app.Use(AuditTrail(rec))
+	app.Post("/v1/admin/orgs/*", func(c *zip.Ctx) error { return c.JSON(http.StatusOK, map[string]string{"ok": "1"}) })
+	app.Get("/v1/kms/health", func(c *zip.Ctx) error { return c.JSON(http.StatusOK, map[string]string{"status": "ok"}) })
+
+	// (a) A mutating POST ending in /health MUST be audited.
+	req := httptest.NewRequest(http.MethodPost, "/v1/admin/orgs/evil/health", nil)
+	asAdmin(req)
+	mustTest(t, app, req)
+	_, total, _ := rec.Query(t.Context(), audit.Filter{})
+	if total != 1 {
+		t.Fatalf("EVASION: mutating POST ending /health audited %d times, want 1", total)
+	}
+
+	// (b) A genuine liveness GET /v1/kms/health MUST still be skipped.
+	req2 := httptest.NewRequest(http.MethodGet, "/v1/kms/health", nil)
+	mustTest(t, app, req2)
+	_, total2, _ := rec.Query(t.Context(), audit.Filter{})
+	if total2 != 1 {
+		t.Fatalf("liveness probe was audited (total went %d→%d) — should be exempt", total, total2)
+	}
+}
+
+// TestAudit_AnonRequestNotAttributedToForgedOrg proves an UNAUTHENTICATED
+// attacker cannot forge a false attribution: sending X-Org-Id/X-User-Id/
+// X-User-IsAdmin with no validated principal records an ANONYMOUS actor (empty
+// org+sub, not admin), never the claimed victim org. Runs the REAL SanitizeIdentity
+// (nil validator ⇒ strips authority, restores client X-Org-Id for the data path)
+// ahead of AuditTrail, exactly as serve.go wires them.
+func TestAudit_AnonRequestNotAttributedToForgedOrg(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "audit.db")
+	rec, err := audit.Open(path, nil)
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	t.Cleanup(func() { _ = rec.Close() })
+	app := zip.New(zip.Config{})
+	app.Use(SanitizeIdentity(nil, "admin")) // trust boundary
+	app.Use(AuditTrail(rec))
+	app.Post("/v1/kms/secrets", func(c *zip.Ctx) error { return c.JSON(http.StatusOK, map[string]string{"ok": "1"}) })
+
+	req := httptest.NewRequest(http.MethodPost, "/v1/kms/secrets", nil)
+	// Anonymous attacker forging every identity header.
+	req.Header.Set("X-Org-Id", "victim-org")
+	req.Header.Set("X-User-Id", "victim-user")
+	req.Header.Set("X-User-IsAdmin", "true")
+	mustTest(t, app, req)
+
+	rows, _, _ := rec.Query(t.Context(), audit.Filter{})
+	if len(rows) != 1 {
+		t.Fatalf("got %d rows, want 1", len(rows))
+	}
+	r := rows[0]
+	if r.Auth.IsAdmin {
+		t.Error("forged X-User-IsAdmin survived into the record")
+	}
+	if r.Actor.Sub != "" {
+		t.Errorf("forged X-User-Id recorded as actor.Sub = %q", r.Actor.Sub)
+	}
+	if r.Actor.Org == "victim-org" {
+		t.Errorf("FALSE ATTRIBUTION: anonymous request stamped with claimed org %q", r.Actor.Org)
+	}
+	if r.Auth.Method != "none" {
+		t.Errorf("auth method = %q, want none (no valid credential)", r.Auth.Method)
+	}
+	// The event is still recorded (a mutation), honestly anonymous.
+	if r.Outcome.Result != "success" {
+		t.Errorf("outcome = %+v, want the anonymous mutation recorded", r.Outcome)
+	}
+}
+
+// TestAudit_NoopWhenUnconfigured proves a nil Recorder makes the middleware a
+// pass-through (an unconfigured deployment is never blocked), exactly like
+// BillingGate's nil-client behavior.
+func TestAudit_NoopWhenUnconfigured(t *testing.T) {
+	app := zip.New(zip.Config{})
+	app.Use(AuditTrail(nil))
+	var ran bool
+	app.Post("/v1/kms/secrets", func(c *zip.Ctx) error {
+		ran = true
+		return c.JSON(http.StatusOK, map[string]string{"ok": "true"})
+	})
+	req := httptest.NewRequest(http.MethodPost, "/v1/kms/secrets", nil)
+	resp := mustTest(t, app, req)
+	if resp.StatusCode != http.StatusOK || !ran {
+		t.Fatalf("nil-recorder gate must pass through: status=%d ran=%v", resp.StatusCode, ran)
+	}
+}
+
+// TestAudit_FailsClosedOnWriteError proves that when the audit store cannot
+// record a security-relevant event, the request is failed CLOSED (503) rather
+// than allowed to succeed unlogged (AU-5). We force the failure by closing the
+// store's DB before the request, so Append errors.
+func TestAudit_FailsClosedOnWriteError(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "audit.db")
+	rec, err := audit.Open(path, nil)
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	// Close the underlying store so every subsequent Append fails.
+	_ = rec.Close()
+
+	app := zip.New(zip.Config{})
+	app.Use(AuditTrail(rec))
+	var ran bool
+	app.Post("/v1/kms/secrets", func(c *zip.Ctx) error {
+		ran = true
+		return c.JSON(http.StatusCreated, map[string]string{"id": "x"})
+	})
+
+	req := httptest.NewRequest(http.MethodPost, "/v1/kms/secrets", nil)
+	asAdmin(req)
+	resp := mustTest(t, app, req)
+	// The handler may have run (audit wraps AFTER the chain), but the response the
+	// CLIENT sees must be the fail-closed 503, not the handler's 201 — a
+	// security-relevant action that could not be recorded is not acknowledged as
+	// success.
+	if resp.StatusCode != http.StatusServiceUnavailable {
+		t.Fatalf("status = %d, want 503 (fail-closed when audit write fails)", resp.StatusCode)
+	}
+	_ = ran
+}

audit_mirror.go

@@ -0,0 +1,199 @@
+package cloud
+
+// The datastore (ClickHouse) OLAP mirror — a best-effort projection of the audit
+// trail for fleet-wide, long-retention, cross-deployment query. It implements
+// audit.Mirror.
+//
+// The datastore is the natural OLAP audit sink: the table is a MergeTree, which
+// is INSERT-ONLY by engine — ClickHouse rejects UPDATE/DELETE against it at parse
+// time ("MergeTree does not support mutations"), so the mirror is append-only at
+// the storage layer, matching the local chain's discipline. We create the table
+// idempotently on first connect (CREATE TABLE IF NOT EXISTS) and insert via the
+// canonical clickhouse-go PrepareBatch → Append → Send idiom (the same the
+// provisioning subsystem uses; the driver is already in cloud's module graph, so
+// this adds no dependency).
+//
+// This mirror is NEVER the integrity authority — the local SQLite hash-chain is.
+// Its rows carry the same seq + hash so an operator CAN cross-check the OLAP copy
+// against the chain, but a mirror gap is a query-completeness issue, not a
+// tamper-evidence one. Every mirror error is logged and dropped by the Recorder;
+// the request path never sees it.
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	clickhouse "github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/hanzoai/cloud/audit"
+	luxlog "github.com/luxfi/log"
+)
+
+// clickhouseMirror writes audit records to a ClickHouse MergeTree table.
+type clickhouseMirror struct {
+	conn  clickhouse.Conn
+	table string
+	log   luxlog.Logger
+}
+
+// newAuditMirror builds the OLAP mirror from operator config, or returns nil when
+// no datastore is configured (mirroring is optional — the local chain is the
+// authority). It connects lazily-validated (a Ping) and ensures the table exists.
+//
+// Config (all from env / KMS-injected secrets, never hard-coded):
+//
+//	CLOUD_AUDIT_CLICKHOUSE_ADDR   host:9000 of the datastore native port
+//	CLOUD_AUDIT_CLICKHOUSE_DB     database (default "hanzo")
+//	CLOUD_AUDIT_CLICKHOUSE_TABLE  table (default "audit_log")
+//	CLOUD_AUDIT_CLICKHOUSE_USER   user
+//	CLOUD_AUDIT_CLICKHOUSE_PASSWORD  password (KMS-backed secret)
+func newAuditMirror(log luxlog.Logger) (audit.Mirror, error) {
+	addr := strings.TrimSpace(os.Getenv("CLOUD_AUDIT_CLICKHOUSE_ADDR"))
+	if addr == "" {
+		return nil, nil // no datastore configured — local chain only.
+	}
+	db := getenv("CLOUD_AUDIT_CLICKHOUSE_DB", "hanzo")
+	table := getenv("CLOUD_AUDIT_CLICKHOUSE_TABLE", "audit_log")
+
+	conn, err := clickhouse.Open(&clickhouse.Options{
+		Addr: []string{addr},
+		Auth: clickhouse.Auth{
+			Database: db,
+			Username: os.Getenv("CLOUD_AUDIT_CLICKHOUSE_USER"),
+			Password: os.Getenv("CLOUD_AUDIT_CLICKHOUSE_PASSWORD"),
+		},
+		DialTimeout: 5 * time.Second,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("audit mirror: open: %w", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if err := conn.Ping(ctx); err != nil {
+		_ = conn.Close()
+		return nil, fmt.Errorf("audit mirror: ping %s: %w", addr, err)
+	}
+
+	qualified := db + "." + table
+	m := &clickhouseMirror{conn: conn, table: qualified, log: log}
+	if err := m.ensureTable(ctx); err != nil {
+		_ = conn.Close()
+		return nil, err
+	}
+	if log != nil {
+		log.Info("audit OLAP mirror connected", "addr", addr, "table", qualified)
+	}
+	return m, nil
+}
+
+// ensureTable creates the append-only audit table if it does not exist. MergeTree
+// = insert-only (mutations rejected at parse time). Partitioned by month and
+// ordered for the (org, time) query pattern; seq + hash are carried so the OLAP
+// copy is cross-checkable against the local chain.
+func (m *clickhouseMirror) ensureTable(ctx context.Context) error {
+	ddl := fmt.Sprintf(`
+CREATE TABLE IF NOT EXISTS %s (
+  seq        UInt64,
+  ts         DateTime64(3, 'UTC'),
+  actor_org  LowCardinality(String),
+  actor_sub  String,
+  actor_email String,
+  action     LowCardinality(String),
+  res_type   LowCardinality(String),
+  res_id     String,
+  auth_method LowCardinality(String),
+  is_admin   UInt8,
+  result     LowCardinality(String),
+  status     UInt16,
+  reason     String,
+  source_ip  String,
+  user_agent String,
+  request_id String,
+  method     LowCardinality(String),
+  path       String,
+  prev_hash  String,
+  hash       String
+) ENGINE = MergeTree
+PARTITION BY toYYYYMM(ts)
+ORDER BY (actor_org, ts, seq)`, m.table)
+	if err := m.conn.Exec(ctx, ddl); err != nil {
+		return fmt.Errorf("audit mirror: ensure table: %w", err)
+	}
+	return nil
+}
+
+// Append writes one record to the OLAP mirror via the canonical batch idiom. The
+// before/after diffs are DELIBERATELY not mirrored — the OLAP copy is for
+// query/analytics over the event stream, and keeping the (already-redacted but
+// still payload-bearing) diffs out of the fleet warehouse minimizes the blast
+// radius of a warehouse compromise. The full record (with diffs) lives only in
+// the local, access-controlled chain.
+func (m *clickhouseMirror) Append(ctx context.Context, r audit.Record) error {
+	batch, err := m.conn.PrepareBatch(ctx, "INSERT INTO "+m.table+` (
+  seq, ts, actor_org, actor_sub, actor_email, action, res_type, res_id,
+  auth_method, is_admin, result, status, reason, source_ip, user_agent,
+  request_id, method, path, prev_hash, hash)`)
+	if err != nil {
+		return fmt.Errorf("audit mirror: prepare: %w", err)
+	}
+	if err := batch.Append(
+		r.Seq, r.Time.UTC(), r.Actor.Org, r.Actor.Sub, r.Actor.Email,
+		r.Action, r.Resource.Type, r.Resource.ID,
+		r.Auth.Method, boolToUint8(r.Auth.IsAdmin),
+		r.Outcome.Result, uint16(r.Outcome.Status), r.Outcome.Reason,
+		r.SourceIP, r.UserAgent, r.RequestID, r.Method, r.Path,
+		r.PrevHash, r.Hash,
+	); err != nil {
+		_ = batch.Abort()
+		return fmt.Errorf("audit mirror: append: %w", err)
+	}
+	return batch.Send()
+}
+
+// Checkpoint persists a head-digest checkpoint to an INDEPENDENT digest table in
+// the datastore — the AU-9 tail-truncation anchor. Because this lives in a store
+// SEPARATE from the local SQLite chain, truncating the chain cannot also rewrite
+// the checkpoint history: an external monitor querying this table sees the count
+// series and alerts on any regression. Best-effort; a failure is dropped by the
+// Recorder (the structured log carries the same digest). Implements
+// audit.CheckpointSink.
+func (m *clickhouseMirror) Checkpoint(ctx context.Context, cp audit.Checkpoint) error {
+	if err := m.ensureCheckpointTable(ctx); err != nil {
+		return err
+	}
+	batch, err := m.conn.PrepareBatch(ctx, "INSERT INTO "+m.table+"_checkpoints (ts, count, head)")
+	if err != nil {
+		return fmt.Errorf("audit mirror: checkpoint prepare: %w", err)
+	}
+	if err := batch.Append(cp.Time.UTC(), cp.Count, cp.Head); err != nil {
+		_ = batch.Abort()
+		return fmt.Errorf("audit mirror: checkpoint append: %w", err)
+	}
+	return batch.Send()
+}
+
+// ensureCheckpointTable creates the append-only checkpoint digest table. A plain
+// MergeTree ordered by time — the monitor reads the latest rows and checks that
+// count never decreases.
+func (m *clickhouseMirror) ensureCheckpointTable(ctx context.Context) error {
+	ddl := fmt.Sprintf(`
+CREATE TABLE IF NOT EXISTS %s_checkpoints (
+  ts    DateTime64(3, 'UTC'),
+  count UInt64,
+  head  String
+) ENGINE = MergeTree
+ORDER BY ts`, m.table)
+	if err := m.conn.Exec(ctx, ddl); err != nil {
+		return fmt.Errorf("audit mirror: ensure checkpoint table: %w", err)
+	}
+	return nil
+}
+
+func boolToUint8(b bool) uint8 {
+	if b {
+		return 1
+	}
+	return 0
+}

audit_serve.go

@@ -0,0 +1,95 @@
+package cloud
+
+// Audit trail construction — the wiring Serve calls to stand up the Recorder.
+//
+// The audit store is a COMPLIANCE CONTROL, so its persistence is treated like the
+// pricing catalog overlay's: a non-persistent (in-memory) audit trail would
+// silently lose the record of every prior action on each restart — a fail-OPEN
+// degradation of an integrity control. So an empty DataDir is a hard boot error
+// in a normal run (prod always sets CLOUD_DATA_DIR; provisioning + pricing already
+// require it, so the unified binary always has one). The trail can be turned OFF
+// deliberately (CLOUD_AUDIT_DISABLED=true) for a minimal single-service dev run —
+// an explicit opt-out, never a silent one.
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/hanzoai/cloud/audit"
+	luxlog "github.com/luxfi/log"
+)
+
+// buildAuditRecorder constructs the audit Recorder from cfg: the append-only
+// SQLite chain at {DataDir}/audit.db plus a best-effort ClickHouse OLAP mirror
+// when a datastore is configured. Returns (nil, nil) only when the trail is
+// explicitly disabled — the caller then wires a no-op middleware.
+func buildAuditRecorder(cfg *Config, logger luxlog.Logger) (*audit.Recorder, error) {
+	if getenvBool("CLOUD_AUDIT_DISABLED") {
+		if logger != nil {
+			logger.Warn("audit trail DISABLED by CLOUD_AUDIT_DISABLED — no tamper-evident record will be kept")
+		}
+		return nil, nil
+	}
+	if cfg.DataDir == "" {
+		return nil, fmt.Errorf("empty DataDir — the audit trail is a compliance control and requires a persistent data dir (set CLOUD_DATA_DIR); refusing to boot with a non-persistent trail that would lose all prior records on restart (or set CLOUD_AUDIT_DISABLED=true to opt out explicitly)")
+	}
+	if err := os.MkdirAll(cfg.DataDir, 0o755); err != nil {
+		return nil, fmt.Errorf("data dir: %w", err)
+	}
+
+	// OLAP mirror is optional and best-effort. A mirror that cannot be reached at
+	// boot must NOT stop the binary — the local chain is the authority — so a
+	// mirror construction error is logged and the trail runs local-only.
+	var mirror audit.Mirror
+	if m, err := newAuditMirror(logger); err != nil {
+		if logger != nil {
+			logger.Warn("audit OLAP mirror unavailable — running local-only (chain integrity unaffected)", "err", err)
+		}
+	} else {
+		mirror = m
+	}
+
+	dbPath := filepath.Join(cfg.DataDir, "audit.db")
+	rec, err := audit.Open(dbPath, mirror)
+	if err != nil {
+		return nil, fmt.Errorf("open audit store: %w", err)
+	}
+
+	// AU-9 tail-truncation anchor: emit a periodic head-digest checkpoint to the
+	// append-only observability log (and, when a mirror supports it, an
+	// independent digest store). An external o11y monitor compares consecutive
+	// checkpoints and alerts on a count regression — the only way to detect that
+	// the most-recent records were deleted (an internal chain walk cannot). The
+	// interval is CLOUD_AUDIT_CHECKPOINT_INTERVAL (default 5m; 0 disables).
+	interval := auditCheckpointInterval()
+	if logger != nil {
+		rec.StartCheckpoints(interval, func(cp audit.Checkpoint) {
+			logger.Info("audit_head_checkpoint",
+				"count", cp.Count, "head", cp.Head, "ts", cp.Time.Format(time.RFC3339Nano))
+		})
+	} else {
+		rec.StartCheckpoints(interval, nil)
+	}
+
+	if logger != nil {
+		count, head := rec.Head()
+		logger.Info("audit trail ready (tamper-evident, append-only)",
+			"store", dbPath, "records", count, "head", head,
+			"mirror", mirror != nil, "checkpoint_interval", interval.String())
+	}
+	return rec, nil
+}
+
+// auditCheckpointInterval resolves the head-digest checkpoint cadence.
+// CLOUD_AUDIT_CHECKPOINT_INTERVAL is a Go duration (e.g. "5m", "1h"); default 5m;
+// "0" disables periodic checkpoints (the on-close checkpoint still fires).
+func auditCheckpointInterval() time.Duration {
+	if v := getenv("CLOUD_AUDIT_CHECKPOINT_INTERVAL", ""); v != "" {
+		if d, err := time.ParseDuration(v); err == nil {
+			return d
+		}
+	}
+	return 5 * time.Minute
+}

auth_identity.go

@@ -0,0 +1,407 @@
+package cloud
+
+// In-binary IAM JWT validation — the trust anchor for SanitizeIdentity.
+//
+// This MIRRORS github.com/hanzoai/gateway/v2/iamauth, the canonical edge
+// validator, but cloud deliberately does NOT import that package: iamauth lives
+// in the heavyweight gateway module (KrakenD/gin/traefik) AND gateway/v2 already
+// imports github.com/hanzoai/cloud, so importing it back would braid a module
+// cycle and pull the gateway's whole dependency tree into cloud for ~150 lines
+// of validation. The gateway remains the PRIMARY edge authority — in production
+// it fronts cloud-api (universe routes.yaml). This validator is the in-binary
+// defense-in-depth layer for the in-cluster / direct path, kept tiny and
+// auditable on go-jose alone (already in cloud's module graph).
+//
+// What it enforces, exactly like iamauth.ValidateToken: signature against the
+// IAM JWKS, issuer (strict), audience (allowlist, OR semantics), and expiry —
+// always. A token missing the issuer is rejected.
+
+import (
+	"context"
+	"crypto/rsa"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	gojose "github.com/go-jose/go-jose/v4"
+	"github.com/go-jose/go-jose/v4/jwt"
+)
+
+// idClaims is the subset of Hanzo IAM JWT claims the identity sanitizer needs.
+// Shape mirrors iamauth.Claims so a token resolves identically at both layers.
+type idClaims struct {
+	jwt.Claims
+
+	Owner             string `json:"owner"`              // org slug (the tenant)
+	Name              string `json:"name"`               // display name (id fallback)
+	PreferredUsername string `json:"preferred_username"` // id fallback
+	Email             string `json:"email"`
+	IsAdmin           bool   `json:"isAdmin"`
+}
+
+// userID resolves the canonical user id: sub, then preferred_username, then
+// name. IAM may leave sub empty.
+func (c *idClaims) userID() string {
+	if c.Subject != "" {
+		return c.Subject
+	}
+	if c.PreferredUsername != "" {
+		return c.PreferredUsername
+	}
+	return c.Name
+}
+
+// jwtSigAlgs is the accepted signature-algorithm allowlist passed to
+// jwt.ParseSigned (go-jose v4 requires it explicitly). RSA + ECDSA + PSS, the
+// set IAM may sign with — never "none".
+var jwtSigAlgs = []gojose.SignatureAlgorithm{
+	gojose.RS256, gojose.RS384, gojose.RS512,
+	gojose.ES256, gojose.ES384, gojose.ES512,
+	gojose.PS256, gojose.PS384, gojose.PS512,
+}
+
+// identityValidator validates an IAM JWT against a cached JWKS. Issuer (any of a
+// trusted SET) + audience + expiry are always enforced.
+//
+// The issuer is a SET so ONE cloud binary validates every white-label brand's
+// tokens (hanzo iss=hanzo.id AND lux iss=lux.id, ...). Signature verification is
+// unaffected: the in-cluster IAM serves EVERY brand's signing cert in one JWKS
+// (cert-hanzo/cert-lux/cert-zoo/...), keyed by the token kid, so a single
+// jwksURL verifies all brands. Only the issuer-string comparison had to widen.
+type identityValidator struct {
+	issuers   []string
+	audiences []string
+	cache     *jwksCache
+}
+
+// newIdentityValidator builds a validator whose trusted-issuer set is the primary
+// issuer UNIONED with every white-label brand issuer (BrandIssuers) plus any
+// WHITELABEL_ISSUERS override. ttl<=0 uses the 15m JWKS default. The union is
+// fail-secure: it only ADDS the known-good brand issuers, never an arbitrary one.
+func newIdentityValidator(issuer, jwksURL string, audiences []string, ttl time.Duration) *identityValidator {
+	return &identityValidator{
+		issuers:   trustedIssuers(issuer),
+		audiences: audiences,
+		cache:     newJWKSCache(jwksURL, ttl),
+	}
+}
+
+// kmsMachineAudSuffix is the fixed suffix of a per-tenant PaaS-KMS sync machine
+// identity's audience. Each tenant's KMS sync authenticates as a dedicated,
+// NON-shared IAM application named "<org>-platform-kms" (Organization=<org>,
+// client_credentials grant), so IAM stamps the token's aud == the app's own
+// clientId == "<org>-platform-kms" (a non-shared app's audience is its clientId,
+// object/token_jwt.go tokenAudience) and owner == <org>
+// (object/token_oauth.go GetClientCredentialsToken sets owner = app.Organization).
+//
+// That audience is, by construction, absent from CLOUD_JWT_AUDIENCES — it is
+// per-tenant, not a fixed app — which is EXACTLY why the sync stayed pending: the
+// machine token failed the audience check below, SanitizeIdentity treated it as
+// anonymous, and the /v1/kms org-scope guard 403'd it before the store. The fix is
+// to accept this one audience, but ONLY when it equals the token's OWN owner claim
+// plus this suffix, so it certifies "the KMS sync identity for its own org" and
+// grants nothing wider. Tenancy is still enforced downstream by owner at the guard
+// (owner == :org); this only lets a legitimately-minted, owner-scoped machine token
+// clear validation. A per-tenant application means a per-tenant clientSecret — never
+// a shared platform-wide reader, which would be a cross-tenant hole.
+const kmsMachineAudSuffix = "-platform-kms"
+
+// kmsMachineAudience returns the audience a tenant org's PaaS-KMS sync identity
+// carries: "<owner>-platform-kms". An empty owner yields empty — no machine
+// audience is ever granted to an org-less token (fail closed).
+func kmsMachineAudience(owner string) string {
+	if owner == "" {
+		return ""
+	}
+	return owner + kmsMachineAudSuffix
+}
+
+// isKMSMachinePrincipal reports whether a validated token is a per-tenant KMS-sync
+// machine identity: its audience set contains the owner-bound machine audience
+// (<owner>-platform-kms). Such a principal is a client_credentials machine identity
+// scoped to exactly one org. SanitizeIdentity uses this to DENY it global-admin
+// authority even if it somehow carries isAdmin=true and owner==adminOrg, so V6's
+// audience widening can never be leveraged (via an admin-org machine token) into a
+// cross-tenant read. Its org-scoped data access is unaffected — this gates ONLY the
+// admin grant, keeping the machine path decoupled from admin inside cloud (rather
+// than resting on the external invariant "IAM never stamps isAdmin=true on a
+// machine-aud token", which cloud cannot see or enforce).
+func isKMSMachinePrincipal(claims *idClaims) bool {
+	mach := kmsMachineAudience(claims.Owner)
+	if mach == "" {
+		return false
+	}
+	for _, a := range claims.Audience {
+		if a == mach {
+			return true
+		}
+	}
+	return false
+}
+
+// validate parses raw, verifies its signature against the JWKS, and enforces
+// issuer/audience/expiry. Returns the claims on success, an error otherwise.
+func (v *identityValidator) validate(raw string) (*idClaims, error) {
+	tok, err := jwt.ParseSigned(raw, jwtSigAlgs)
+	if err != nil {
+		return nil, fmt.Errorf("parse: %w", err)
+	}
+	keys, err := v.cache.get()
+	if err != nil {
+		return nil, fmt.Errorf("jwks: %w", err)
+	}
+
+	var claims idClaims
+	if err := verifyAgainstKeys(tok, keys, &claims); err != nil {
+		return nil, err
+	}
+
+	// Reject a missing issuer: an empty issuer must never pass the set check.
+	if claims.Issuer == "" {
+		return nil, fmt.Errorf("missing issuer")
+	}
+	// Reject a missing expiry: ValidateWithLeeway only enforces exp when present
+	// (it checks `if c.Expiry != nil`), so a token with NO exp would never expire.
+	// An IAM access token always carries exp; require it.
+	if claims.Expiry == nil {
+		return nil, fmt.Errorf("missing expiry")
+	}
+	// Issuer must be one of the trusted brand issuers. go-jose's jwt.Expected
+	// checks a SINGLE issuer, so the issuer is validated here against the set and
+	// left out of Expected (audience + expiry stay with Expected).
+	if !issuerAllowed(claims.Issuer, v.issuers) {
+		return nil, fmt.Errorf("untrusted issuer %q", claims.Issuer)
+	}
+	// Audience: the static allowlist (CLOUD_JWT_AUDIENCES / brand app client_ids)
+	// PLUS the per-tenant PaaS-KMS sync machine audience bound to THIS token's own
+	// owner (<owner>-platform-kms). The machine audience is added only when the
+	// allowlist is active (non-empty — always so in production) and only for the
+	// token's own org, so accepting it never widens tenancy: the /v1/kms guard still
+	// gates on owner == :org. Without this, a real client_credentials machine token
+	// (aud == its per-tenant clientId, never in the allowlist) fails here and the
+	// sync silently stays pending — the activation blocker.
+	expected := jwt.Expected{}
+	if len(v.audiences) > 0 {
+		auds := v.audiences
+		if mach := kmsMachineAudience(claims.Owner); mach != "" {
+			auds = append(append(make([]string, 0, len(v.audiences)+1), v.audiences...), mach)
+		}
+		expected.AnyAudience = jwt.Audience(auds)
+	}
+	if err := claims.Claims.ValidateWithLeeway(expected, 2*time.Minute); err != nil {
+		return nil, fmt.Errorf("claims: %w", err)
+	}
+	return &claims, nil
+}
+
+// verifyAgainstKeys tries the kid-matched key first, then any RSA signing key —
+// mirrors iamauth's selection so a token verifies the same way at both layers.
+func verifyAgainstKeys(tok *jwt.JSONWebToken, keys *gojose.JSONWebKeySet, claims *idClaims) error {
+	var lastErr error
+	for _, h := range tok.Headers {
+		if h.KeyID == "" {
+			continue
+		}
+		for _, k := range keys.Key(h.KeyID) {
+			if err := tok.Claims(k.Key, claims); err == nil {
+				return nil
+			} else {
+				lastErr = err
+			}
+		}
+	}
+	for _, k := range keys.Keys {
+		if k.Use != "sig" && k.Use != "" {
+			continue
+		}
+		if _, ok := k.Key.(*rsa.PublicKey); !ok {
+			continue
+		}
+		if err := tok.Claims(k.Key, claims); err == nil {
+			return nil
+		} else {
+			lastErr = err
+		}
+	}
+	if lastErr != nil {
+		return fmt.Errorf("no matching key: %w", lastErr)
+	}
+	return fmt.Errorf("no matching key in JWKS")
+}
+
+// ----------------------------------------------------------------------------
+// JWKS cache (TTL refresh, stale-on-error) — mirrors iamauth.JWKSCache.
+// ----------------------------------------------------------------------------
+
+type jwksCache struct {
+	mu        sync.RWMutex
+	keys      *gojose.JSONWebKeySet
+	fetchedAt time.Time
+	ttl       time.Duration
+	url       string
+	client    *http.Client
+}
+
+func newJWKSCache(url string, ttl time.Duration) *jwksCache {
+	if ttl <= 0 {
+		ttl = 15 * time.Minute
+	}
+	return &jwksCache{url: url, ttl: ttl, client: &http.Client{Timeout: 10 * time.Second}}
+}
+
+// get returns the cached key set, refreshing past TTL. On a fetch error with a
+// previously-cached set, the stale set is returned rather than failing — a
+// transient JWKS blip must not flap validation (and so admin auth) closed.
+func (c *jwksCache) get() (*gojose.JSONWebKeySet, error) {
+	c.mu.RLock()
+	if c.keys != nil && time.Since(c.fetchedAt) < c.ttl {
+		k := c.keys
+		c.mu.RUnlock()
+		return k, nil
+	}
+	c.mu.RUnlock()
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.keys != nil && time.Since(c.fetchedAt) < c.ttl {
+		return c.keys, nil
+	}
+	keys, err := c.fetch()
+	if err != nil {
+		if c.keys != nil {
+			return c.keys, nil
+		}
+		return nil, err
+	}
+	c.keys = keys
+	c.fetchedAt = time.Now()
+	return keys, nil
+}
+
+func (c *jwksCache) fetch() (*gojose.JSONWebKeySet, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("request: %w", err)
+	}
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("fetch: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("status %d", resp.StatusCode)
+	}
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	if err != nil {
+		return nil, fmt.Errorf("read: %w", err)
+	}
+	var set gojose.JSONWebKeySet
+	if err := json.Unmarshal(body, &set); err != nil {
+		return nil, fmt.Errorf("parse: %w", err)
+	}
+	return &set, nil
+}
+
+// ----------------------------------------------------------------------------
+// Token extraction — mirrors iamauth's Bearer / Basic / API-key helpers.
+// ----------------------------------------------------------------------------
+
+// isAPIKey reports whether tok is an opaque, backend-validated key (hk-/sk-/…)
+// rather than a JWT, so the sanitizer skips JWT parsing for it.
+func isAPIKey(tok string) bool {
+	return strings.HasPrefix(tok, "hk-") ||
+		strings.HasPrefix(tok, "sk-") ||
+		strings.HasPrefix(tok, "pk-") ||
+		strings.HasPrefix(tok, "fw_") ||
+		strings.HasPrefix(tok, "hz_")
+}
+
+// bearerFromAuth extracts the token from a "Bearer <token>" header value.
+func bearerFromAuth(auth string) string {
+	if auth == "" {
+		return ""
+	}
+	parts := strings.SplitN(auth, " ", 2)
+	if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") {
+		return ""
+	}
+	return strings.TrimSpace(parts[1])
+}
+
+// basicFromAuth extracts the token from an HTTP Basic header value: the password
+// field (the go/.netrc proxy idiom), falling back to the username when empty.
+func basicFromAuth(auth string) string {
+	if auth == "" {
+		return ""
+	}
+	parts := strings.SplitN(auth, " ", 2)
+	if len(parts) != 2 || !strings.EqualFold(parts[0], "Basic") {
+		return ""
+	}
+	raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(parts[1]))
+	if err != nil {
+		return ""
+	}
+	user, pass, ok := strings.Cut(string(raw), ":")
+	if !ok {
+		return ""
+	}
+	if pass != "" {
+		return pass
+	}
+	return user
+}
+
+// trustedIssuers returns the full trusted-issuer set for the in-binary validator:
+// the PRIMARY issuer (the deployment's own brand, cfg.IAMIssuer) UNIONED with every
+// white-label brand issuer (BrandIssuers) and any WHITELABEL_ISSUERS override
+// (comma-separated). Fail-secure: it only ADDS known-good issuers; a nil/empty
+// result is impossible when a primary is set, so the issuer check is always
+// enforced. Duplicates are removed; order is primary-first.
+func trustedIssuers(primary string) []string {
+	out := make([]string, 0, 6)
+	add := func(v string) {
+		v = strings.TrimSpace(v)
+		if v == "" {
+			return
+		}
+		for _, e := range out {
+			if e == v {
+				return
+			}
+		}
+		out = append(out, v)
+	}
+	add(primary)
+	for _, iss := range BrandIssuers() {
+		add(iss)
+	}
+	for _, iss := range splitTrim(os.Getenv("WHITELABEL_ISSUERS")) {
+		add(iss)
+	}
+	return out
+}
+
+// issuerAllowed reports whether iss is one of the trusted issuers. An empty set
+// (no primary, no brands — never the case in production) skips the check, matching
+// the prior "empty issuer disables the check" behavior; a non-empty set is
+// fail-secure (a token whose iss is not in the set is rejected).
+func issuerAllowed(iss string, trusted []string) bool {
+	if len(trusted) == 0 {
+		return true
+	}
+	for _, t := range trusted {
+		if iss == t {
+			return true
+		}
+	}
+	return false
+}

auth_identity_kms_test.go

@@ -0,0 +1,87 @@
+package cloud
+
+// V6 (the activation blocker) — the identity validator must accept a per-tenant
+// PaaS-KMS sync machine token: a client_credentials JWT whose aud is the tenant's
+// own IAM application clientId "<owner>-platform-kms" (a per-tenant value, NEVER in
+// CLOUD_JWT_AUDIENCES) — but ONLY when that audience is bound to the token's OWN
+// owner claim. Before the fix the machine token failed the audience check,
+// SanitizeIdentity resolved anonymous, and the /v1/kms guard 403'd it, so the sync
+// silently stayed pending. These are white-box unit tests of validate() itself;
+// the end-to-end proof through SanitizeIdentity + the real guard lives in
+// clients/kmssvc (v6_aud_e2e_test.go). Reuses the jwksServer/signWith/tokenClaims
+// helpers from middleware_identity_test.go (same package).
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"testing"
+	"time"
+)
+
+func TestIdentityValidator_KMSMachineAudience(t *testing.T) {
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("genkey: %v", err)
+	}
+	jwks := jwksServer(t, &key.PublicKey)
+	// The static allowlist deliberately contains NO *-platform-kms audience, so any
+	// acceptance below can come ONLY from the owner-bound machine-aud rule, not the
+	// allowlist — this is what makes it a fix and not a config workaround.
+	v := newIdentityValidator(testIssuer, jwks.URL, []string{"hanzo-console"}, 0)
+	future := time.Now().Add(time.Hour)
+
+	t.Run("machine token for its own org is accepted", func(t *testing.T) {
+		c, err := v.validate(signWith(t, key, tokenClaims("maxpower-platform-kms", "maxpower", "", false, future)))
+		if err != nil {
+			t.Fatalf("machine token rejected: %v", err)
+		}
+		if c.Owner != "maxpower" {
+			t.Fatalf("owner=%q, want maxpower", c.Owner)
+		}
+	})
+
+	t.Run("machine aud for a DIFFERENT org is rejected (owner-bound)", func(t *testing.T) {
+		// owner=maxpower but aud=acme-platform-kms: the accepted machine aud is bound
+		// to the token's OWN owner (maxpower-platform-kms), so this must fail — it is
+		// not a blanket "*-platform-kms" wildcard.
+		if _, err := v.validate(signWith(t, key, tokenClaims("acme-platform-kms", "maxpower", "", false, future))); err == nil {
+			t.Fatal("cross-org machine audience must be rejected (owner-bound)")
+		}
+	})
+
+	t.Run("arbitrary audience still rejected (fix is scoped, not a disable)", func(t *testing.T) {
+		if _, err := v.validate(signWith(t, key, tokenClaims("some-random-app", "maxpower", "", false, future))); err == nil {
+			t.Fatal("an arbitrary audience must still be rejected")
+		}
+	})
+
+	t.Run("machine aud with empty owner is rejected (fail closed)", func(t *testing.T) {
+		// aud="-platform-kms" with owner="": kmsMachineAudience("")=="" so no machine
+		// audience is granted and the bare suffix is not in the allowlist.
+		if _, err := v.validate(signWith(t, key, tokenClaims("-platform-kms", "", "", false, future))); err == nil {
+			t.Fatal("machine aud with empty owner must be rejected")
+		}
+	})
+
+	t.Run("normal static-allowlist token still accepted (regression)", func(t *testing.T) {
+		if _, err := v.validate(signWith(t, key, tokenClaims("hanzo-console", "maxpower", "", false, future))); err != nil {
+			t.Fatalf("static-allowlist token rejected: %v", err)
+		}
+	})
+
+	t.Run("machine token expiry still enforced", func(t *testing.T) {
+		if _, err := v.validate(signWith(t, key, tokenClaims("maxpower-platform-kms", "maxpower", "", false, time.Now().Add(-time.Hour)))); err == nil {
+			t.Fatal("expired machine token must be rejected")
+		}
+	})
+}
+
+// kmsMachineAudience is a pure helper; lock its contract directly.
+func TestKMSMachineAudience(t *testing.T) {
+	if got := kmsMachineAudience("maxpower"); got != "maxpower-platform-kms" {
+		t.Fatalf("kmsMachineAudience(maxpower)=%q, want maxpower-platform-kms", got)
+	}
+	if got := kmsMachineAudience(""); got != "" {
+		t.Fatalf("kmsMachineAudience(\"\")=%q, want \"\" (no machine aud for an org-less token)", got)
+	}
+}

auth_identity_whitelabel_test.go

@@ -0,0 +1,150 @@
+package cloud
+
+import (
+	"os"
+	"testing"
+)
+
+// TestTrustedIssuers_WhiteLabel proves the in-binary validator's trusted-issuer
+// set is the primary issuer UNIONED with every white-label brand issuer plus the
+// WHITELABEL_ISSUERS override, deduped, primary-first.
+func TestTrustedIssuers_WhiteLabel(t *testing.T) {
+	os.Unsetenv("WHITELABEL_ISSUERS")
+	got := trustedIssuers("https://hanzo.id")
+	want := map[string]bool{
+		"https://hanzo.id":     true,
+		"https://lux.id":       true,
+		"https://zoo.id":       true, // per cloud brand.go registry
+		"https://pars.id":      true,
+		"https://id.bootno.de": true, // bootnode brand also in the registry
+	}
+	set := map[string]bool{}
+	for _, g := range got {
+		set[g] = true
+	}
+	for w := range want {
+		if !set[w] {
+			t.Errorf("trusted set %v missing %q", got, w)
+		}
+	}
+	if got[0] != "https://hanzo.id" {
+		t.Errorf("primary issuer must be first, got %q", got[0])
+	}
+
+	// Override adds a brand without a rebuild.
+	t.Setenv("WHITELABEL_ISSUERS", "https://custom.id, https://another.id")
+	got2 := trustedIssuers("https://hanzo.id")
+	if !issuerAllowed("https://custom.id", got2) || !issuerAllowed("https://another.id", got2) {
+		t.Errorf("WHITELABEL_ISSUERS override must add issuers, got %v", got2)
+	}
+}
+
+// TestIssuerAllowed proves the set membership check: brand issuers pass, an
+// outsider is rejected, and an empty set (never in prod) skips the check.
+func TestIssuerAllowed(t *testing.T) {
+	set := []string{"https://hanzo.id", "https://lux.id"}
+	if !issuerAllowed("https://lux.id", set) {
+		t.Error("lux.id must be allowed")
+	}
+	if issuerAllowed("https://attacker.id", set) {
+		t.Error("attacker.id must be rejected")
+	}
+	if !issuerAllowed("anything", nil) {
+		t.Error("empty set must skip the check (matches prior empty-issuer behavior)")
+	}
+}
+
+// TestBrandIssuers proves the issuer list is derived from the brands registry and
+// covers every configured brand (one source of truth).
+func TestBrandIssuers(t *testing.T) {
+	got := BrandIssuers()
+	for _, want := range []string{"https://hanzo.id", "https://lux.id", "https://zoo.id", "https://pars.id", "https://id.bootno.de"} {
+		found := false
+		for _, g := range got {
+			if g == want {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Errorf("BrandIssuers()=%v missing %q", got, want)
+		}
+	}
+}
+
+// TestNewIdentityValidator_MultiIssuer proves the constructed validator carries the
+// full brand set, so a lux token would pass the issuer gate on the hanzo binary.
+func TestNewIdentityValidator_MultiIssuer(t *testing.T) {
+	os.Unsetenv("WHITELABEL_ISSUERS")
+	v := newIdentityValidator("https://hanzo.id", "http://iam.hanzo.svc/v1/iam/.well-known/jwks", []string{"hanzo-cloud", "lux-cloud"}, 0)
+	if !issuerAllowed("https://lux.id", v.issuers) {
+		t.Fatalf("validator must trust the lux issuer, set=%v", v.issuers)
+	}
+	if !issuerAllowed("https://hanzo.id", v.issuers) {
+		t.Fatalf("validator must still trust hanzo (no regression), set=%v", v.issuers)
+	}
+	if issuerAllowed("https://evil.id", v.issuers) {
+		t.Fatalf("validator must reject an untrusted issuer, set=%v", v.issuers)
+	}
+}
+
+// TestBrandAudiences proves every brand's cloud audience (<brand>-cloud) is derived
+// from the brands registry — one source of truth, mirroring BrandIssuers.
+func TestBrandAudiences(t *testing.T) {
+	got := BrandAudiences()
+	for _, want := range []string{"hanzo-cloud", "lux-cloud", "zoo-cloud", "pars-cloud", "bootnode-cloud"} {
+		found := false
+		for _, g := range got {
+			if g == want {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Errorf("BrandAudiences()=%v missing %q", got, want)
+		}
+	}
+}
+
+// TestJWTAudiencesFromEnv_BrandUnion proves the resolved audience allowlist ALWAYS
+// includes every brand's <brand>-cloud aud (so a lux token validates), whether the
+// list comes from the baked default or a hanzo-only env override — and that an
+// env-supplied entry is not duplicated.
+func TestJWTAudiencesFromEnv_BrandUnion(t *testing.T) {
+	has := func(list []string, v string) bool {
+		for _, s := range list {
+			if s == v {
+				return true
+			}
+		}
+		return false
+	}
+
+	// Baked default path (no env).
+	os.Unsetenv("CLOUD_JWT_AUDIENCES")
+	os.Unsetenv("GATEWAY_ALLOWED_AUDIENCES")
+	def := jwtAudiencesFromEnv()
+	for _, want := range []string{"hanzo-cloud", "lux-cloud", "zoo-cloud", "pars-cloud"} {
+		if !has(def, want) {
+			t.Errorf("baked audiences %v missing brand aud %q", def, want)
+		}
+	}
+
+	// A legacy hanzo-only env override must STILL accept lux-cloud (brand union),
+	// with no duplicate of the env-supplied hanzo-cloud.
+	os.Setenv("GATEWAY_ALLOWED_AUDIENCES", "hanzo-app,hanzo-console,hanzo-cloud")
+	defer os.Unsetenv("GATEWAY_ALLOWED_AUDIENCES")
+	got := jwtAudiencesFromEnv()
+	if !has(got, "lux-cloud") {
+		t.Fatalf("hanzo-only env override must still accept lux-cloud, got %v", got)
+	}
+	n := 0
+	for _, s := range got {
+		if s == "hanzo-cloud" {
+			n++
+		}
+	}
+	if n != 1 {
+		t.Fatalf("hanzo-cloud must appear exactly once (no duplicate), got %d in %v", n, got)
+	}
+}

brand.go

@@ -0,0 +1,96 @@
+package cloud
+
+import "strings"
+
+// Brand white-label registry (HIP-0111).
+//
+// The cloud binary is one artifact serving every brand's API host
+// (api.hanzo.ai, api.lux.cloud, api.zoo.cloud, api.cloud.pars.network, ...).
+// Brand is a per-deployment value (CLOUD_BRAND / --brand). This registry maps a
+// brand to its PUBLIC IAM facts — the canonical OIDC issuer the deployment must
+// validate JWTs against. These are public (issuer host + brand domain), so they
+// live in code, not in KMS.
+//
+// One source of truth: nothing else in the binary hardcodes a per-brand issuer.
+// Config.IAMIssuer is derived from here when the operator does not pin one, so a
+// lux deployment validates against lux.id, a zoo deployment against zoo.id, etc.,
+// instead of silently defaulting every brand to iam.hanzo.ai.
+
+// BrandInfo is the PUBLIC per-brand identity used for token validation + URL
+// scoping. No secrets.
+type BrandInfo struct {
+	// ID is the canonical brand key.
+	ID string
+	// IAMIssuer is the OIDC issuer (JWKS source) for this brand — the value the
+	// JWT `iss` claim must equal and whose /v1/iam/.well-known/jwks signs tokens.
+	IAMIssuer string
+	// Domain is the brand's primary marketing/site domain (for response scoping).
+	Domain string
+}
+
+// brands is the brand→IAM registry. Keys are the canonical brand IDs accepted
+// by CLOUD_BRAND. Per HIP-0111 §Brands: hanzo→hanzo.id, lux→lux.id,
+// zoo→zoo.id, pars→pars.id, bootnode→id.bootno.de.
+//
+// IAMIssuer MUST equal the `iss` IAM actually stamps AND host the signing JWKS.
+// For hanzo the live .well-known/openid-configuration on BOTH hanzo.id and
+// iam.hanzo.ai reports issuer=https://hanzo.id + jwks_uri=
+// https://hanzo.id/v1/iam/.well-known/jwks (iam.hanzo.ai is a routing alias, not
+// the issuer), and the cloud CLI already defaults to hanzo.id. Pinning
+// iam.hanzo.ai here would fail the issuer check on every real token, anonymizing
+// every principal — global admin would 403 platform-wide (fail-secure, but
+// broken). lux/zoo/pars already correctly point at their own .id issuers.
+var brands = map[string]BrandInfo{
+	"hanzo":    {ID: "hanzo", IAMIssuer: "https://hanzo.id", Domain: "hanzo.ai"},
+	"lux":      {ID: "lux", IAMIssuer: "https://lux.id", Domain: "lux.network"},
+	"zoo":      {ID: "zoo", IAMIssuer: "https://zoo.id", Domain: "zoo.ngo"},
+	"pars":     {ID: "pars", IAMIssuer: "https://pars.id", Domain: "pars.network"},
+	"bootnode": {ID: "bootnode", IAMIssuer: "https://id.bootno.de", Domain: "bootno.de"},
+}
+
+// DefaultBrand is the fallback brand when CLOUD_BRAND is unknown.
+const DefaultBrand = "hanzo"
+
+// BrandFor returns the BrandInfo for id, falling back to the Hanzo brand for an
+// unknown id. Lookup is case-insensitive.
+func BrandFor(id string) BrandInfo {
+	if b, ok := brands[strings.ToLower(strings.TrimSpace(id))]; ok {
+		return b
+	}
+	return brands[DefaultBrand]
+}
+
+// IssuerForBrand returns the canonical OIDC issuer for a brand id.
+func IssuerForBrand(id string) string {
+	return BrandFor(id).IAMIssuer
+}
+
+// BrandIssuers returns the OIDC issuer of every configured white-label brand. The
+// in-binary identity validator (auth_identity.go) trusts a token whose `iss` is
+// any of these, so ONE cloud binary validates hanzo AND lux/zoo/pars tokens. One
+// source of truth: derived from the same `brands` registry above.
+func BrandIssuers() []string {
+	out := make([]string, 0, len(brands))
+	for _, b := range brands {
+		if b.IAMIssuer != "" {
+			out = append(out, b.IAMIssuer)
+		}
+	}
+	return out
+}
+
+// BrandAudiences returns the OAuth `aud` (== IAM client_id == app name) of every
+// white-label brand's cloud login app: `<brand>-cloud` (hanzo-cloud, lux-cloud,
+// zoo-cloud, pars-cloud, bootnode-cloud). A brand's session token carries
+// aud=<brand>-cloud (HIP-0111: client_id == app == aud), so the audience allowlist
+// must include each to accept a lux/zoo/pars token on the ONE binary. Derived from
+// the same `brands` registry as BrandIssuers — one source of truth, no hand-listing.
+func BrandAudiences() []string {
+	out := make([]string, 0, len(brands))
+	for id := range brands {
+		if id != "" {
+			out = append(out, id+"-cloud")
+		}
+	}
+	return out
+}

brand_test.go

@@ -0,0 +1,45 @@
+package cloud
+
+import (
+	"os"
+	"testing"
+)
+
+func TestBrandFor(t *testing.T) {
+	// hanzo → hanzo.id (NOT iam.hanzo.ai): brand.go was pinned to the real OIDC
+	// issuer in fddaeb14 ("pin hanzo IAM issuer to hanzo.id") — iam.hanzo.ai is a
+	// routing alias, and the live .well-known reports iss=https://hanzo.id, so a
+	// token would fail the issuer check against iam.hanzo.ai. This stale
+	// assertion predated that pin; aligned here (drive-by, brand.go unchanged).
+	cases := map[string]string{
+		"hanzo":    "https://hanzo.id",
+		"lux":      "https://lux.id",
+		"zoo":      "https://zoo.id",
+		"pars":     "https://pars.id",
+		"bootnode": "https://id.bootno.de",
+		"LUX":      "https://lux.id",   // case-insensitive
+		"  zoo  ":  "https://zoo.id",   // trimmed
+		"unknown":  "https://hanzo.id", // falls back to hanzo
+		"":         "https://hanzo.id", // empty → hanzo default
+	}
+	for brand, want := range cases {
+		if got := IssuerForBrand(brand); got != want {
+			t.Errorf("IssuerForBrand(%q) = %q, want %q", brand, got, want)
+		}
+	}
+}
+
+// TestLoadConfig_IssuerDerivedFromBrand asserts that when CLOUD_IAM_ISSUER is
+// unset, the issuer is derived from CLOUD_BRAND — so a non-hanzo brand does not
+// silently validate against iam.hanzo.ai.
+func TestLoadConfig_IssuerDerivedFromBrand(t *testing.T) {
+	t.Setenv("CLOUD_BRAND", "lux")
+	os.Unsetenv("CLOUD_IAM_ISSUER")
+	cfg := LoadConfig()
+	if cfg.IAMIssuer != "https://lux.id" {
+		t.Fatalf("derived issuer = %q, want https://lux.id", cfg.IAMIssuer)
+	}
+	if cfg.Brand != "lux" {
+		t.Fatalf("brand = %q, want lux", cfg.Brand)
+	}
+}

build.go

@@ -1,11 +1,15 @@
 package cloud
 
 import (
+	"context"
 	"fmt"
+	"strings"
 
+	"github.com/hanzoai/commerce/metering"
 	luxlog "github.com/luxfi/log"
 
 	"github.com/hanzoai/cloud/clients"
+	"github.com/hanzoai/cloud/clients/kms"
 )
 
 // BuildDeps constructs the Deps used by every subsystem's Mount(app, deps).
@@ -33,7 +37,7 @@ import (
 // JSON does not appear in any of these paths. Inter-subsystem calls
 // are ZAP-typed Go values either via direct method dispatch (mode 1)
 // or via ZAP RPC over the wire (mode 2). JSON happens only at the
-// gateway/ingress edge, through the hanzoai/zip jsonenc helper.
+// gateway/ingress edge, through the zip jsonenc helper.
 //
 // Payments and Vault are special: they are NEVER in-process per
 // HIP-0106 solo-vault CDE. Their clients always resolve via
@@ -44,15 +48,18 @@ func BuildDeps(cfg *Config) Deps {
 	logger.Info("building deps",
 		"brand", cfg.Brand,
 		"domain", cfg.Domain,
+		"iam_issuer", cfg.IAMIssuer,
 		"data_dir", cfg.DataDir,
 		"enabled", cfg.Enable,
 	)
 
 	deps := Deps{
-		Logger:  logger,
-		Brand:   cfg.Brand,
-		Domain:  cfg.Domain,
-		DataDir: cfg.DataDir,
+		Logger:    logger,
+		Brand:     cfg.Brand,
+		Env:       cfg.Env,
+		Domain:    cfg.Domain,
+		IAMIssuer: cfg.IAMIssuer,
+		DataDir:   cfg.DataDir,
 	}
 
 	// For each subsystem: enabled → leave nil (Mount fills it); not
@@ -72,9 +79,40 @@ func BuildDeps(cfg *Config) Deps {
 	deps.Payments = pickPaymentsClient(cfg, logger)
 	deps.Vault = pickVaultClient(cfg, logger)
 
+	// Billing metering client for the request-edge gate. nil-safe: when no
+	// commerce URL is configured the resulting client is !Enabled() and the
+	// gate is a no-op.
+	deps.Metering = buildMeteringClient(cfg, logger)
+
 	return deps
 }
 
+// buildMeteringClient constructs the commerce metering client for BillingGate.
+// An empty CommerceHTTPURL yields a not-Enabled() client (allow + no-op),
+// matching the metering package's "not configured" mode, so an unconfigured
+// deployment is never blocked. The token is a KMS-sourced secret supplied via
+// config; it is never logged.
+func buildMeteringClient(cfg *Config, log luxlog.Logger) *metering.Client {
+	m, err := metering.New(metering.Config{
+		BaseURL:  cfg.CommerceHTTPURL,
+		Token:    cfg.CommerceServiceToken,
+		Org:      cfg.Brand, // X-Org-Id default for S2S; per-request org overrides.
+		FailOpen: cfg.BillingFailOpen,
+	})
+	if err != nil {
+		// Only an unparseable URL reaches here. Fall back to a not-configured
+		// client (no-op gate) rather than failing boot over billing wiring.
+		log.Error("billing: invalid commerce URL, gate disabled", "err", err)
+		m, _ = metering.New(metering.Config{})
+	}
+	if m.Enabled() {
+		log.Info("billing gate enabled", "commerce_url", cfg.CommerceHTTPURL, "fail_open", cfg.BillingFailOpen)
+	} else {
+		log.Info("billing gate disabled (no commerce URL)")
+	}
+	return m
+}
+
 // pickIAMClient returns the canonical IAMClient for this process.
 // nil = enabled here, Mount will fill it. RPC = remote endpoint
 // configured. Disabled = not enabled, no endpoint.
@@ -89,9 +127,31 @@ func pickIAMClient(cfg *Config, log luxlog.Logger) IAMClient {
 	return clients.DisabledIAM()
 }
 
+// pickKMSClient resolves deps.KMS. When the kms subsystem is co-resident
+// (Enabled("kmssvc")) it returns the IN-PROCESS Client backed by the embedded
+// luxfi/kms SecretStore under CLOUD_DATA_DIR — no external RPC. A store-open
+// failure is NOT fatal to the whole binary: it falls back to the disabled stub
+// (fail-closed) and logs, so a bad data dir degrades KMS rather than crashing
+// every subsystem. Absent co-residency the legacy ZAP-RPC + disabled fallbacks
+// apply (out-of-process KMS, or not wired).
+//
+// The internal subsystem name is "kmssvc" (see clients/kmssvc.init — it avoids the
+// serve.go generic-health shadow on /v1/kms/health); the client gate keys on the
+// same name so "enabled" is one concept.
 func pickKMSClient(cfg *Config, log luxlog.Logger) KMSClient {
-	if cfg.Enabled("kms") {
-		return nil
+	if cfg.Enabled("kmssvc") {
+		c, err := kms.New(kms.Config{
+			DataDir:      cfg.DataDir,
+			MasterKeyB64: cfg.KMSMasterKeyRef,
+			MPCAddr:      cfg.KMSMPCAddr,
+			MPCVaultID:   cfg.KMSMPCVaultID,
+		}, log)
+		if err != nil {
+			log.Error("deps.KMS: embedded KMS unavailable, failing closed", "err", err)
+			return clients.DisabledKMS()
+		}
+		log.Info("deps.KMS → in-process (embedded luxfi/kms)", "ready", c.Ready(), "signing", c.SigningConfigured())
+		return c
 	}
 	if cfg.KMSZAPAddr != "" {
 		log.Info("deps.KMS → ZAP RPC", "addr", cfg.KMSZAPAddr)
@@ -122,14 +182,39 @@ func pickCommerceClient(cfg *Config, log luxlog.Logger) CommerceClient {
 	return clients.DisabledCommerce()
 }
 
+// pickAIClient resolves deps.AI — the client the agents subsystem runs chat
+// completions through. Unlike the co-resident subsystems, there is NO in-process
+// "ai" mount that fills a nil deps.AI: inference is an external gateway, so this
+// must return a concrete client, never nil. (A nil deps.AI was the live bug —
+// the default all-enabled config returned nil here and nothing ever filled it,
+// so every /v1/agents/:name/run 503'd "inference is not configured".)
+//
+// Preference order:
+//  1. Static-key HTTP gateway when a base URL AND a static key are configured —
+//     an operator override / pre-provisioned key. The key is a KMS-injected
+//     secret; only the base URL and default model are ever logged.
+//  2. M2M HTTP gateway when a base URL AND the binary's IAM identity are present
+//     (the durable Hanzo default): the client mints+refreshes a client-
+//     credentials token from IAM_CLIENT_ID/SECRET — no static key to rotate. The
+//     secret is never logged.
+//  3. ZAP RPC when an addr is configured (split-deploy of a future ai subsystem).
+//  4. Fail-closed stub otherwise — a run records an honest error, never fakes one.
 func pickAIClient(cfg *Config, log luxlog.Logger) AIClient {
-	if cfg.Enabled("ai") {
-		return nil
+	if cfg.AIBaseURL != "" && cfg.AIAPIKey != "" {
+		log.Info("deps.AI → HTTP gateway (static key)", "base_url", cfg.AIBaseURL, "default_model", cfg.AIDefaultModel)
+		return clients.AIHTTPAt(cfg.AIBaseURL, cfg.AIAPIKey, cfg.AIDefaultModel)
+	}
+	if cfg.AIBaseURL != "" && cfg.AIAuthClientID != "" && cfg.AIAuthClientSecret != "" && cfg.IAMIssuer != "" {
+		tokenURL := strings.TrimRight(cfg.IAMIssuer, "/") + "/v1/iam/oauth/token"
+		log.Info("deps.AI → HTTP gateway (IAM M2M)", "base_url", cfg.AIBaseURL,
+			"token_url", tokenURL, "client_id", cfg.AIAuthClientID, "default_model", cfg.AIDefaultModel)
+		return clients.AIHTTPM2M(cfg.AIBaseURL, tokenURL, cfg.AIAuthClientID, cfg.AIAuthClientSecret, cfg.AIDefaultModel)
 	}
 	if cfg.AIZAPAddr != "" {
 		log.Info("deps.AI → ZAP RPC", "addr", cfg.AIZAPAddr)
 		return clients.AIRPCAt(cfg.AIZAPAddr)
 	}
+	log.Info("deps.AI → disabled (no CLOUD_AI_API_KEY, no IAM M2M identity, no gateway configured)")
 	return clients.DisabledAI()
 }
 
@@ -190,13 +275,20 @@ func pickVaultClient(cfg *Config, log luxlog.Logger) VaultClient {
 // calls it.
 type MountFunc func(app any, deps Deps) error // app is *zip.App; using any here to avoid an import cycle in pkg/cloud
 
+// ShutdownFunc releases a subsystem's process-lifetime resources (background
+// goroutines, open DB handles) on graceful shutdown. It must be idempotent and
+// bounded — Serve calls it within the shutdown deadline. ctx carries that
+// deadline so a slow teardown is cut off rather than hanging SIGTERM.
+type ShutdownFunc func(ctx context.Context) error
+
 // MountSpec describes one subsystem registered for mounting. The Order
 // is used when ordering matters for inter-subsystem deps (e.g. iam
 // before authz before commerce).
 type MountSpec struct {
-	Name  string
-	Order int
-	Mount MountFunc
+	Name     string
+	Order    int
+	Mount    MountFunc
+	Shutdown ShutdownFunc // optional; nil means the subsystem has nothing to tear down.
 }
 
 // Registry is the in-process subsystem registry. Subsystems register via
@@ -209,6 +301,32 @@ func Register(name string, order int, mount MountFunc) {
 	Registry = append(Registry, MountSpec{Name: name, Order: order, Mount: mount})
 }
 
+// RegisterWithShutdown adds a subsystem that owns process-lifetime resources: a
+// background worker (e.g. the agents scheduler) or a DB handle that must be
+// flushed. shutdown is invoked by ShutdownAll on graceful stop. This is the ONE
+// way a subsystem gets a teardown — Register stays the zero-teardown default.
+func RegisterWithShutdown(name string, order int, mount MountFunc, shutdown ShutdownFunc) {
+	Registry = append(Registry, MountSpec{Name: name, Order: order, Mount: mount, Shutdown: shutdown})
+}
+
+// ShutdownAll tears down every ENABLED subsystem that registered a ShutdownFunc,
+// in REVERSE mount order (a dependency is torn down after its dependents), best
+// effort: a failure is collected and the rest still run, so one stuck subsystem
+// can't strand another's flush. Serve calls this inside the shutdown deadline.
+func ShutdownAll(ctx context.Context, cfg *Config) error {
+	var firstErr error
+	for i := len(Registry) - 1; i >= 0; i-- {
+		spec := Registry[i]
+		if spec.Shutdown == nil || !cfg.Enabled(spec.Name) {
+			continue
+		}
+		if err := spec.Shutdown(ctx); err != nil && firstErr == nil {
+			firstErr = fmt.Errorf("shutdown %s: %w", spec.Name, err)
+		}
+	}
+	return firstErr
+}
+
 // MountAll iterates the registry in order and calls Mount() on each
 // enabled subsystem.
 func MountAll(app any, cfg *Config, deps Deps) error {

build_test.go

@@ -8,23 +8,50 @@ import (
 	"github.com/hanzoai/cloud/clients"
 )
 
-// TestBuildDeps_EnabledLeavesNil verifies that BuildDeps leaves an
-// enabled subsystem's Client field nil — the subsystem Mount() is
-// responsible for filling it in.
+// TestBuildDeps_EnabledLeavesNil verifies that BuildDeps leaves an enabled
+// Mount-fills-it subsystem's Client field nil — the subsystem Mount() installs
+// it. KMS is the exception (see TestBuildDeps_KMSEnabledIsInProcess): it is
+// constructed eagerly in BuildDeps because its store must exist before any
+// dependent subsystem mounts.
 func TestBuildDeps_EnabledLeavesNil(t *testing.T) {
 	cfg := &cloud.Config{
 		Brand:   "hanzo",
 		Domain:  "api.hanzo.ai",
-		DataDir: "/tmp",
-		Enable:  []string{"iam", "kms", "base", "commerce", "ai", "o11y", "vfs", "mq"},
+		DataDir: t.TempDir(),
+		Enable:  []string{"iam", "base", "commerce", "ai", "o11y", "vfs", "mq"},
 	}
 	deps := cloud.BuildDeps(cfg)
 
 	if deps.IAM != nil {
 		t.Errorf("deps.IAM: enabled subsystem must leave Client nil, got %T", deps.IAM)
 	}
-	if deps.KMS != nil {
-		t.Errorf("deps.KMS: enabled subsystem must leave Client nil, got %T", deps.KMS)
+}
+
+// TestBuildDeps_KMSEnabledIsInProcess verifies the HIP-0106 "embed KMS in cloud"
+// contract: when the kms subsystem (kmssvc) is enabled, deps.KMS is a live
+// in-process client (never nil, never a disabled stub) so other subsystems get a
+// working KMS via direct Go dispatch with no RPC. Absent a master key it still
+// resolves (health-only, fail-closed) — the point is that deps.KMS is populated.
+func TestBuildDeps_KMSEnabledIsInProcess(t *testing.T) {
+	cfg := &cloud.Config{
+		Brand:   "hanzo",
+		Domain:  "api.hanzo.ai",
+		DataDir: t.TempDir(),
+		Enable:  []string{"kmssvc"},
+	}
+	deps := cloud.BuildDeps(cfg)
+
+	if deps.KMS == nil {
+		t.Fatal("deps.KMS: enabled kmssvc must give an in-process client, got nil")
+	}
+	// It must NOT be the fail-closed disabled stub — that stub returns IsDisabled
+	// errors; an in-process client (no master key) returns a master-key error.
+	_, err := deps.KMS.GetSecret(context.Background(), "any")
+	if err == nil {
+		t.Fatal("GetSecret with no master key must fail closed")
+	}
+	if clients.IsDisabled(err) {
+		t.Errorf("deps.KMS resolved to the DISABLED stub, want the in-process client: %v", err)
 	}
 }
 

cli/auth.go

@@ -0,0 +1,381 @@
+package cli
+
+import (
+	"bufio"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+	"golang.org/x/term"
+)
+
+// iamClient is a thin client over the Hanzo IAM OAuth2 surface at
+// {issuer}/v1/iam/oauth/*. It speaks only the standard token + userinfo
+// endpoints; it holds no IAM business logic.
+type iamClient struct {
+	issuer   string
+	clientID string
+	http     *http.Client
+}
+
+func newIAMClient(issuer, clientID string) *iamClient {
+	return &iamClient{
+		issuer:   strings.TrimRight(issuer, "/"),
+		clientID: clientID,
+		http:     &http.Client{Timeout: 30 * time.Second},
+	}
+}
+
+// tokenResp is the OAuth2 token endpoint response (success or RFC-6749 error).
+type tokenResp struct {
+	AccessToken  string `json:"access_token"`
+	RefreshToken string `json:"refresh_token"`
+	TokenType    string `json:"token_type"`
+	ExpiresIn    int64  `json:"expires_in"`
+	Scope        string `json:"scope"`
+	Error        string `json:"error"`
+	ErrorDesc    string `json:"error_description"`
+}
+
+// postForm performs an x-www-form-urlencoded POST to an oauth endpoint and
+// decodes the token response, surfacing OAuth errors as Go errors.
+func (c *iamClient) postForm(ctx context.Context, endpoint string, form url.Values) (*tokenResp, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.issuer+endpoint, strings.NewReader(form.Encode()))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "hanzo-cli/"+Version)
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+
+	var tr tokenResp
+	if err := json.Unmarshal(body, &tr); err != nil {
+		return nil, fmt.Errorf("iam %s: HTTP %d: %s", endpoint, resp.StatusCode, strings.TrimSpace(string(body)))
+	}
+	if tr.Error != "" {
+		return nil, fmt.Errorf("iam %s: %s: %s", endpoint, tr.Error, tr.ErrorDesc)
+	}
+	if tr.AccessToken == "" {
+		return nil, fmt.Errorf("iam %s: HTTP %d: no access_token in response", endpoint, resp.StatusCode)
+	}
+	return &tr, nil
+}
+
+// passwordGrant exchanges username+password for a token (the live IAM client
+// supports password grant; device_code is hard-rejected server-side).
+func (c *iamClient) passwordGrant(ctx context.Context, username, password, scope string) (*tokenResp, error) {
+	return c.postForm(ctx, "/v1/iam/oauth/access_token", url.Values{
+		"grant_type": {"password"},
+		"client_id":  {c.clientID},
+		"username":   {username},
+		"password":   {password},
+		"scope":      {scope},
+	})
+}
+
+// refreshGrant exchanges a refresh token for a fresh access token.
+func (c *iamClient) refreshGrant(ctx context.Context, refreshToken string) (*tokenResp, error) {
+	return c.postForm(ctx, "/v1/iam/oauth/refresh_token", url.Values{
+		"grant_type":    {"refresh_token"},
+		"client_id":     {c.clientID},
+		"refresh_token": {refreshToken},
+	})
+}
+
+// decodeJWTClaims base64url-decodes a JWT's payload segment WITHOUT verifying
+// the signature — used only to display the user's own token claims locally.
+func decodeJWTClaims(token string) (map[string]any, error) {
+	parts := strings.Split(token, ".")
+	if len(parts) < 2 {
+		return nil, fmt.Errorf("not a JWT (need 3 dot-separated segments)")
+	}
+	payload, err := base64.RawURLEncoding.DecodeString(strings.TrimRight(parts[1], "="))
+	if err != nil {
+		return nil, fmt.Errorf("decode JWT payload: %w", err)
+	}
+	var claims map[string]any
+	if err := json.Unmarshal(payload, &claims); err != nil {
+		return nil, fmt.Errorf("parse JWT claims: %w", err)
+	}
+	return claims, nil
+}
+
+// claimString reads a string claim, tolerating absence.
+func claimString(claims map[string]any, key string) string {
+	if v, ok := claims[key].(string); ok {
+		return v
+	}
+	return ""
+}
+
+// credsFromToken builds a Credentials carrying the token plus the identity
+// fields decoded from its claims. expiresIn (token endpoint) wins for expiry;
+// otherwise the JWT `exp` claim is used.
+func credsFromToken(tr *tokenResp) *Credentials {
+	c := &Credentials{
+		AccessToken:  tr.AccessToken,
+		RefreshToken: tr.RefreshToken,
+		TokenType:    firstNonEmpty(tr.TokenType, "Bearer"),
+	}
+	if claims, err := decodeJWTClaims(tr.AccessToken); err == nil {
+		c.Subject = firstNonEmpty(claimString(claims, "email"), claimString(claims, "sub"))
+		c.Owner = claimString(claims, "owner")
+		if exp, ok := claims["exp"].(float64); ok {
+			c.Expiry = int64(exp)
+		}
+	}
+	if tr.ExpiresIn > 0 {
+		c.Expiry = time.Now().Add(time.Duration(tr.ExpiresIn) * time.Second).Unix()
+	}
+	return c
+}
+
+// ---------------------------------------------------------------------------
+// Commands: login / logout / whoami, grouped under `auth`.
+// ---------------------------------------------------------------------------
+
+// loginFlags are shared by `hanzo login` and `hanzo auth login`.
+type loginFlags struct {
+	username      string
+	passwordStdin bool
+	token         string
+	platformToken string
+	buildToken    string
+	scope         string
+}
+
+func runLogin(env *Env, lf *loginFlags, cmd *cobra.Command) error {
+	creds, err := LoadCredentials()
+	if err != nil {
+		return err
+	}
+
+	switch {
+	case lf.token != "":
+		// Paste an externally-minted token. Decode claims for identity.
+		tr := &tokenResp{AccessToken: lf.token, TokenType: "Bearer"}
+		creds = credsFromToken(tr)
+	default:
+		username := lf.username
+		if username == "" {
+			username, err = prompt(cmd, "Email: ")
+			if err != nil {
+				return err
+			}
+		}
+		password, err := readPassword(cmd, lf.passwordStdin)
+		if err != nil {
+			return err
+		}
+		iam := newIAMClient(env.IAMIssuer, env.ClientID)
+		tr, err := iam.passwordGrant(cmd.Context(), username, password, lf.scope)
+		if err != nil {
+			return err
+		}
+		creds = credsFromToken(tr)
+	}
+
+	// Optional machine-to-machine tokens for the platform control plane,
+	// stored alongside the identity so apps/deploy work post-login.
+	if lf.platformToken != "" {
+		creds.PlatformToken = lf.platformToken
+	}
+	if lf.buildToken != "" {
+		creds.BuildToken = lf.buildToken
+	}
+
+	if err := creds.Save(); err != nil {
+		return err
+	}
+	who := firstNonEmpty(creds.Subject, "(unknown)")
+	if creds.Owner != "" {
+		who += " @ " + creds.Owner
+	}
+	fmt.Fprintf(cmd.OutOrStdout(), "Logged in as %s (token expires %s)\n", who, shortTime(creds.Expiry))
+	return nil
+}
+
+func newLoginCmd(envOf func() *Env, _ *globalFlags) *cobra.Command {
+	lf := &loginFlags{}
+	cmd := &cobra.Command{
+		Use:   "login",
+		Short: "Authenticate against Hanzo IAM and store a token",
+		Long: "Authenticate against Hanzo IAM (hanzo.id) via the password grant and store\n" +
+			"the token in ~/.hanzo/credentials.json (mode 0600). Use --token to store an\n" +
+			"externally-minted token instead, and --platform-token to store the platform\n" +
+			"control-plane service token needed by apps/deploy/clusters.",
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error { return runLogin(envOf(), lf, cmd) },
+	}
+	bindLoginFlags(cmd, lf)
+	return cmd
+}
+
+func bindLoginFlags(cmd *cobra.Command, lf *loginFlags) {
+	f := cmd.Flags()
+	f.StringVarP(&lf.username, "username", "u", "", "IAM username/email")
+	f.BoolVar(&lf.passwordStdin, "password-stdin", false, "read the password from stdin (for automation)")
+	f.StringVar(&lf.token, "token", "", "store this access token directly (skip the password grant)")
+	f.StringVar(&lf.platformToken, "platform-token", "", "also store the platform control-plane service token")
+	f.StringVar(&lf.buildToken, "build-token", "", "also store the platform build-enqueue token")
+	f.StringVar(&lf.scope, "scope", "openid profile email", "OAuth scope")
+}
+
+func newLogoutCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:               "logout",
+		Short:             "Remove stored credentials",
+		Args:              cobra.NoArgs,
+		PersistentPreRunE: func(*cobra.Command, []string) error { return nil },
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			if err := DeleteCredentials(); err != nil {
+				return err
+			}
+			fmt.Fprintln(cmd.OutOrStdout(), "Logged out.")
+			return nil
+		},
+	}
+}
+
+func newWhoamiCmd(envOf func() *Env) *cobra.Command {
+	var verify bool
+	cmd := &cobra.Command{
+		Use:   "whoami",
+		Short: "Show the current identity from the stored token",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			env := envOf()
+			tok := env.accessToken()
+			if tok == "" {
+				return fmt.Errorf("not logged in: run `hanzo login`")
+			}
+			claims, err := decodeJWTClaims(tok)
+			if err != nil {
+				return err
+			}
+			if verify {
+				if err := verifyUserInfo(cmd.Context(), env, tok); err != nil {
+					return fmt.Errorf("token rejected by IAM: %w", err)
+				}
+			}
+			return env.emit(claims, func(w io.Writer) {
+				fmt.Fprintf(w, "email:   %s\n", claimString(claims, "email"))
+				fmt.Fprintf(w, "name:    %s\n", firstNonEmpty(claimString(claims, "displayName"), claimString(claims, "name")))
+				fmt.Fprintf(w, "org:     %s\n", claimString(claims, "owner"))
+				fmt.Fprintf(w, "subject: %s\n", claimString(claims, "sub"))
+				fmt.Fprintf(w, "issuer:  %s\n", claimString(claims, "iss"))
+				if exp, ok := claims["exp"].(float64); ok {
+					fmt.Fprintf(w, "expires: %s\n", shortTime(int64(exp)))
+				}
+				if verify {
+					fmt.Fprintln(w, "verified: yes (IAM userinfo accepted the token)")
+				}
+			})
+		},
+	}
+	cmd.Flags().BoolVar(&verify, "verify", false, "verify the token against the IAM userinfo endpoint")
+	return cmd
+}
+
+// verifyUserInfo calls the IAM userinfo endpoint with the bearer token; a 2xx
+// means IAM accepts the token as live.
+func verifyUserInfo(ctx context.Context, env *Env, token string) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, env.IAMIssuer+"/v1/iam/oauth/userinfo", nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	req.Header.Set("User-Agent", "hanzo-cli/"+Version)
+	resp, err := (&http.Client{Timeout: 20 * time.Second}).Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode/100 != 2 {
+		body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16))
+		return fmt.Errorf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
+	}
+	return nil
+}
+
+// newAuthCmd is the `auth` group: login/logout/whoami plus `token` (print the
+// stored access token, for piping into other tools).
+func newAuthCmd(envOf func() *Env, gf *globalFlags) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "auth",
+		Short: "Manage authentication",
+	}
+	tokenCmd := &cobra.Command{
+		Use:   "token",
+		Short: "Print the stored access token",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			tok := envOf().accessToken()
+			if tok == "" {
+				return fmt.Errorf("not logged in: run `hanzo login`")
+			}
+			fmt.Fprintln(cmd.OutOrStdout(), tok)
+			return nil
+		},
+	}
+	cmd.AddCommand(newLoginCmd(envOf, gf), newLogoutCmd(), newWhoamiCmd(envOf), tokenCmd)
+	return cmd
+}
+
+// ---------------------------------------------------------------------------
+// Terminal helpers.
+// ---------------------------------------------------------------------------
+
+// prompt writes a prompt to stderr and reads a trimmed line from stdin.
+func prompt(cmd *cobra.Command, label string) (string, error) {
+	fmt.Fprint(cmd.ErrOrStderr(), label)
+	r := bufio.NewReader(cmd.InOrStdin())
+	line, err := r.ReadString('\n')
+	if err != nil && err != io.EOF {
+		return "", err
+	}
+	return strings.TrimSpace(line), nil
+}
+
+// readPassword reads a password without echo from the terminal, or as a plain
+// line from stdin when --password-stdin is set (automation) or stdin is not a
+// terminal.
+func readPassword(cmd *cobra.Command, fromStdin bool) (string, error) {
+	if fromStdin {
+		r := bufio.NewReader(cmd.InOrStdin())
+		line, err := r.ReadString('\n')
+		if err != nil && err != io.EOF {
+			return "", err
+		}
+		return strings.TrimRight(line, "\r\n"), nil
+	}
+	if f, ok := cmd.InOrStdin().(*os.File); ok && term.IsTerminal(int(f.Fd())) {
+		fmt.Fprint(cmd.ErrOrStderr(), "Password: ")
+		b, err := term.ReadPassword(int(f.Fd()))
+		fmt.Fprintln(cmd.ErrOrStderr())
+		return string(b), err
+	}
+	// Non-terminal stdin without --password-stdin: read a line so piped input
+	// still works, but nudge toward the explicit flag.
+	r := bufio.NewReader(cmd.InOrStdin())
+	line, err := r.ReadString('\n')
+	if err != nil && err != io.EOF {
+		return "", err
+	}
+	return strings.TrimRight(line, "\r\n"), nil
+}

cli/auth_test.go

@@ -0,0 +1,210 @@
+package cli
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// makeJWT builds an unsigned JWT (alg=none) carrying claims — enough to test
+// the local, signature-free claim decode the CLI uses for display.
+func makeJWT(claims map[string]any) string {
+	hdr := base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"none","typ":"JWT"}`))
+	p, _ := json.Marshal(claims)
+	return hdr + "." + base64.RawURLEncoding.EncodeToString(p) + ".sig"
+}
+
+func TestDecodeJWTClaims(t *testing.T) {
+	tok := makeJWT(map[string]any{"email": "z@hanzo.ai", "owner": "hanzo", "sub": "abc", "exp": float64(1783110016)})
+	claims, err := decodeJWTClaims(tok)
+	if err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if claimString(claims, "email") != "z@hanzo.ai" || claimString(claims, "owner") != "hanzo" {
+		t.Fatalf("claims wrong: %+v", claims)
+	}
+	if _, err := decodeJWTClaims("not-a-jwt"); err == nil {
+		t.Fatalf("expected error for non-JWT")
+	}
+}
+
+func TestCredsFromToken(t *testing.T) {
+	tok := makeJWT(map[string]any{"email": "z@hanzo.ai", "owner": "hanzo", "exp": float64(2000000000)})
+	// expires_in present → wins over exp.
+	c := credsFromToken(&tokenResp{AccessToken: tok, RefreshToken: "r", ExpiresIn: 3600})
+	if c.Subject != "z@hanzo.ai" || c.Owner != "hanzo" || c.RefreshToken != "r" {
+		t.Fatalf("identity not extracted: %+v", c)
+	}
+	if c.Expiry == 2000000000 {
+		t.Fatalf("expires_in should win over exp claim")
+	}
+	// No expires_in → falls back to exp claim.
+	c2 := credsFromToken(&tokenResp{AccessToken: tok})
+	if c2.Expiry != 2000000000 {
+		t.Fatalf("exp claim fallback failed: %d", c2.Expiry)
+	}
+}
+
+func TestPasswordGrant(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v1/iam/oauth/access_token" {
+			t.Errorf("path = %s", r.URL.Path)
+		}
+		_ = r.ParseForm()
+		if r.Form.Get("grant_type") != "password" || r.Form.Get("client_id") != "hanzo-console" ||
+			r.Form.Get("username") != "z@hanzo.ai" || r.Form.Get("password") != "pw" {
+			t.Errorf("bad form: %v", r.Form)
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"access_token": makeJWT(map[string]any{"email": "z@hanzo.ai"}),
+			"token_type":   "Bearer",
+			"expires_in":   3600,
+			"refresh_token": "r",
+		})
+	}))
+	defer srv.Close()
+
+	c := newIAMClient(srv.URL, "hanzo-console")
+	tr, err := c.passwordGrant(context.Background(), "z@hanzo.ai", "pw", "openid")
+	if err != nil {
+		t.Fatalf("passwordGrant: %v", err)
+	}
+	if tr.AccessToken == "" || tr.RefreshToken != "r" {
+		t.Fatalf("token resp bad: %+v", tr)
+	}
+}
+
+func TestPasswordGrantError(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(400)
+		_ = json.NewEncoder(w).Encode(map[string]any{"error": "invalid_grant", "error_description": "bad password"})
+	}))
+	defer srv.Close()
+	c := newIAMClient(srv.URL, "hanzo-console")
+	_, err := c.passwordGrant(context.Background(), "u", "p", "openid")
+	if err == nil || !strings.Contains(err.Error(), "invalid_grant") {
+		t.Fatalf("expected invalid_grant error, got %v", err)
+	}
+}
+
+func TestRefreshGrant(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = r.ParseForm()
+		if r.URL.Path != "/v1/iam/oauth/refresh_token" || r.Form.Get("grant_type") != "refresh_token" || r.Form.Get("refresh_token") != "rt" {
+			t.Errorf("bad refresh request: %s %v", r.URL.Path, r.Form)
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{"access_token": makeJWT(nil), "token_type": "Bearer"})
+	}))
+	defer srv.Close()
+	c := newIAMClient(srv.URL, "hanzo-console")
+	if _, err := c.refreshGrant(context.Background(), "rt"); err != nil {
+		t.Fatalf("refreshGrant: %v", err)
+	}
+}
+
+// runRoot executes the cobra root with args, returning stdout and any error.
+// stderr is discarded; stdin is provided for password prompts.
+func runRoot(t *testing.T, stdin string, args ...string) (string, error) {
+	t.Helper()
+	root := newRootCmd()
+	var out bytes.Buffer
+	root.SetOut(&out)
+	root.SetErr(new(bytes.Buffer))
+	root.SetIn(strings.NewReader(stdin))
+	root.SetArgs(args)
+	err := root.Execute()
+	return out.String(), err
+}
+
+func TestLoginCommandPasswordStdin(t *testing.T) {
+	sandbox(t)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"access_token": makeJWT(map[string]any{"email": "z@hanzo.ai", "owner": "hanzo"}),
+			"token_type":   "Bearer", "expires_in": 3600,
+		})
+	}))
+	defer srv.Close()
+
+	out, err := runRoot(t, "pw\n", "login", "-u", "z@hanzo.ai", "--password-stdin", "--iam-issuer", srv.URL)
+	if err != nil {
+		t.Fatalf("login: %v", err)
+	}
+	if !strings.Contains(out, "Logged in as z@hanzo.ai @ hanzo") {
+		t.Fatalf("login output: %q", out)
+	}
+	creds, _ := LoadCredentials()
+	if creds.AccessToken == "" || creds.Subject != "z@hanzo.ai" {
+		t.Fatalf("credentials not persisted: %+v", creds)
+	}
+}
+
+func TestLoginTokenPasteAndPlatformToken(t *testing.T) {
+	sandbox(t)
+	tok := makeJWT(map[string]any{"email": "ops@hanzo.ai", "owner": "hanzo"})
+	out, err := runRoot(t, "", "login", "--token", tok, "--platform-token", "svc-123")
+	if err != nil {
+		t.Fatalf("login --token: %v", err)
+	}
+	if !strings.Contains(out, "ops@hanzo.ai") {
+		t.Fatalf("login output: %q", out)
+	}
+	creds, _ := LoadCredentials()
+	if creds.AccessToken != tok || creds.PlatformToken != "svc-123" {
+		t.Fatalf("creds not stored: %+v", creds)
+	}
+}
+
+func TestWhoamiCommand(t *testing.T) {
+	sandbox(t)
+	creds := credsFromToken(&tokenResp{AccessToken: makeJWT(map[string]any{
+		"email": "z@hanzo.ai", "name": "z", "owner": "hanzo", "sub": "u-1", "iss": "https://hanzo.id",
+	})})
+	if err := creds.Save(); err != nil {
+		t.Fatal(err)
+	}
+	out, err := runRoot(t, "", "whoami")
+	if err != nil {
+		t.Fatalf("whoami: %v", err)
+	}
+	for _, want := range []string{"z@hanzo.ai", "hanzo", "u-1", "https://hanzo.id"} {
+		if !strings.Contains(out, want) {
+			t.Fatalf("whoami missing %q in %q", want, out)
+		}
+	}
+}
+
+func TestWhoamiLoggedOut(t *testing.T) {
+	sandbox(t)
+	if _, err := runRoot(t, "", "whoami"); err == nil {
+		t.Fatalf("expected error when logged out")
+	}
+}
+
+func TestLogoutCommand(t *testing.T) {
+	sandbox(t)
+	(&Credentials{AccessToken: "x"}).Save()
+	if _, err := runRoot(t, "", "logout"); err != nil {
+		t.Fatalf("logout: %v", err)
+	}
+	if c, _ := LoadCredentials(); c.AccessToken != "" {
+		t.Fatalf("logout did not clear credentials")
+	}
+}
+
+func TestAuthTokenCommand(t *testing.T) {
+	sandbox(t)
+	(&Credentials{AccessToken: "the-token"}).Save()
+	out, err := runRoot(t, "", "auth", "token")
+	if err != nil {
+		t.Fatalf("auth token: %v", err)
+	}
+	if strings.TrimSpace(out) != "the-token" {
+		t.Fatalf("auth token output: %q", out)
+	}
+}

cli/cli.go

@@ -0,0 +1,554 @@
+// Package cli is the Hanzo cloud-control CLI — the gcloud/doctl-class client
+// half of the `hanzo` binary.
+//
+// `hanzo <subsystem>` SERVES a subsystem (server mode, cmd/hanzo dispatch);
+// `hanzo <verb>` CONTROLS the live estate (client mode, this package):
+//
+//	hanzo login | auth        identity against hanzo.id (IAM)
+//	hanzo apps  list|get      the platform apps board (declared/running/drift)
+//	hanzo deploy              drive a platform redeploy (rolling, zero-downtime)
+//	hanzo clusters …          provision/list/select dedicated DOKS clusters
+//	hanzo build               enqueue a platform-native (arcd) build
+//	hanzo k8s …               current deploy target helpers
+//	hanzo config …            ~/.hanzo/config preferences
+//
+// It is a THIN client over surfaces that already exist — Hanzo IAM
+// (hanzo.id /v1/iam/oauth/*), the platform REST control plane
+// (platform.hanzo.ai /v1/*), and the cloud /v1 API. It invents no parallel
+// API and holds no business logic; every command is one HTTP call shaped by
+// resolved configuration. Secrets live only in ~/.hanzo (0600) or the
+// environment — never in source, never logged.
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+// Version is the binary version, set by cmd/hanzo from its -ldflags value so
+// the CLI and the server report one string. Used in the User-Agent.
+var Version = "dev"
+
+// Default endpoints. Overridable per-field via config / env / flag.
+const (
+	defaultIAMIssuer   = "https://hanzo.id"
+	defaultPlatformURL = "https://platform.hanzo.ai"
+	defaultCloudURL    = "https://api.hanzo.ai"
+	// hanzo-console is the only live IAM client that accepts the password
+	// grant today; a dedicated `hanzo-cli` client is a one-line IAM seed
+	// follow-up. Override with `--client-id` / HANZO_CLIENT_ID / config.
+	defaultClientID = "hanzo-console"
+)
+
+// controlCommands maps every client-mode verb to its one-line help. cmd/hanzo
+// reads this both to ROUTE (a first token in here means client mode) and to
+// list the commands in `hanzo help`, so the verb set is defined exactly once.
+var controlCommands = map[string]string{
+	"login":    "authenticate against Hanzo IAM (hanzo.id) and store a token",
+	"logout":   "remove stored credentials",
+	"whoami":   "show the current identity from the stored token",
+	"auth":     "manage authentication (login, logout, whoami, token)",
+	"apps":     "list/get the platform apps board (declared/running/drift)",
+	"deploy":   "drive a platform redeploy (rolling restart, zero-downtime)",
+	"clusters": "provision/list/select dedicated DOKS clusters",
+	"build":    "enqueue a platform-native (arcd) build",
+	"k8s":      "deploy-target helpers (current target)",
+	"config":   "view/edit ~/.hanzo/config preferences",
+	"security": "scan files for hardcoded secrets (local guardrail; no server/auth)",
+}
+
+// IsControlVerb reports whether sub is a client-mode command (and therefore
+// must be routed to this package, not the server dispatcher).
+func IsControlVerb(sub string) bool {
+	_, ok := controlCommands[sub]
+	return ok
+}
+
+// ControlCommands returns the verb→description map for `hanzo help`.
+func ControlCommands() map[string]string { return controlCommands }
+
+// Execute runs the control CLI with args (already stripped of "hanzo"). It is
+// the single entrypoint cmd/hanzo calls for client-mode verbs.
+func Execute(args []string) error {
+	root := newRootCmd()
+	root.SetArgs(args)
+	return root.Execute()
+}
+
+// ---------------------------------------------------------------------------
+// Config — non-secret preferences, ~/.hanzo/config (JSON).
+// ---------------------------------------------------------------------------
+
+// Config holds non-secret CLI preferences. Every field is optional; empty
+// fields fall back to the built-in defaults at resolution time.
+type Config struct {
+	Org         string `json:"org,omitempty"`
+	Output      string `json:"output,omitempty"` // "table" (default) | "json"
+	IAMIssuer   string `json:"iam_issuer,omitempty"`
+	PlatformURL string `json:"platform_url,omitempty"`
+	CloudURL    string `json:"cloud_url,omitempty"`
+	ClientID    string `json:"client_id,omitempty"`
+}
+
+// Credentials holds secret material, ~/.hanzo/credentials.json, mode 0600.
+// AccessToken/RefreshToken are the IAM user identity (from `hanzo login`);
+// PlatformToken/BuildToken are the machine-to-machine tokens the platform
+// REST control plane requires (it cannot validate IAM user tokens).
+type Credentials struct {
+	AccessToken   string `json:"access_token,omitempty"`
+	RefreshToken  string `json:"refresh_token,omitempty"`
+	TokenType     string `json:"token_type,omitempty"`
+	Expiry        int64  `json:"expiry,omitempty"` // unix seconds
+	Subject       string `json:"subject,omitempty"`
+	Owner         string `json:"owner,omitempty"` // org slug from the token
+	PlatformToken string `json:"platform_token,omitempty"`
+	BuildToken    string `json:"build_token,omitempty"`
+}
+
+// hanzoDir is ~/.hanzo, created 0700 if missing. Overridable with HANZO_HOME
+// (used by tests to sandbox the credential store).
+func hanzoDir() (string, error) {
+	if h := os.Getenv("HANZO_HOME"); h != "" {
+		return h, os.MkdirAll(h, 0o700)
+	}
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return "", err
+	}
+	dir := filepath.Join(home, ".hanzo")
+	return dir, os.MkdirAll(dir, 0o700)
+}
+
+func configPath() (string, error) {
+	dir, err := hanzoDir()
+	if err != nil {
+		return "", err
+	}
+	if p := os.Getenv("HANZO_CONFIG"); p != "" {
+		return p, nil
+	}
+	return filepath.Join(dir, "config"), nil
+}
+
+func credentialsPath() (string, error) {
+	dir, err := hanzoDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, "credentials.json"), nil
+}
+
+// loadJSON reads a JSON file into v; a missing file is not an error (v is left
+// at its zero value) so first-run with no config/credentials just works.
+func loadJSON(path string, v any) error {
+	b, err := os.ReadFile(path)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	if len(b) == 0 {
+		return nil
+	}
+	return json.Unmarshal(b, v)
+}
+
+// writeJSON writes v as indented JSON at path with the given mode, via a
+// temp-file rename so a crash mid-write never truncates the store.
+func writeJSON(path string, v any, mode os.FileMode) error {
+	b, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		return err
+	}
+	tmp := path + ".tmp"
+	if err := os.WriteFile(tmp, append(b, '\n'), mode); err != nil {
+		return err
+	}
+	return os.Rename(tmp, path)
+}
+
+// LoadConfig reads ~/.hanzo/config (or HANZO_CONFIG).
+func LoadConfig() (*Config, error) {
+	p, err := configPath()
+	if err != nil {
+		return nil, err
+	}
+	c := &Config{}
+	return c, loadJSON(p, c)
+}
+
+// Save persists the config (mode 0644 — non-secret).
+func (c *Config) Save() error {
+	p, err := configPath()
+	if err != nil {
+		return err
+	}
+	return writeJSON(p, c, 0o644)
+}
+
+// LoadCredentials reads ~/.hanzo/credentials.json.
+func LoadCredentials() (*Credentials, error) {
+	p, err := credentialsPath()
+	if err != nil {
+		return nil, err
+	}
+	c := &Credentials{}
+	return c, loadJSON(p, c)
+}
+
+// Save persists credentials with mode 0600 (owner read/write only).
+func (c *Credentials) Save() error {
+	p, err := credentialsPath()
+	if err != nil {
+		return err
+	}
+	return writeJSON(p, c, 0o600)
+}
+
+// DeleteCredentials removes the credential store (used by logout).
+func DeleteCredentials() error {
+	p, err := credentialsPath()
+	if err != nil {
+		return err
+	}
+	if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+// ---------------------------------------------------------------------------
+// Env — the effective, resolved settings a command operates with.
+// ---------------------------------------------------------------------------
+
+// Env is the fully-resolved runtime context for a command: config + creds
+// merged with environment and the global flags. Built once in the root's
+// PersistentPreRunE and read by every subcommand.
+type Env struct {
+	cfg   *Config
+	creds *Credentials
+
+	Org         string
+	Output      string
+	IAMIssuer   string
+	PlatformURL string
+	CloudURL    string
+	ClientID    string
+
+	out io.Writer
+}
+
+// flag values bound by the persistent flags (empty == unset, fall through).
+type globalFlags struct {
+	org, output, platformURL, iamIssuer, cloudURL, clientID, platformToken string
+}
+
+// firstNonEmpty returns the first non-empty argument, or "".
+func firstNonEmpty(vs ...string) string {
+	for _, v := range vs {
+		if v != "" {
+			return v
+		}
+	}
+	return ""
+}
+
+// resolve merges flags > env > config > built-in defaults into an Env. It is
+// pure given its inputs (config/creds are loaded by the caller) so it is
+// directly unit-testable.
+func resolve(cfg *Config, creds *Credentials, f globalFlags) *Env {
+	e := &Env{cfg: cfg, creds: creds, out: os.Stdout}
+	e.Output = firstNonEmpty(f.output, os.Getenv("HANZO_OUTPUT"), cfg.Output, "table")
+	e.IAMIssuer = strings.TrimRight(firstNonEmpty(f.iamIssuer, os.Getenv("HANZO_IAM_ISSUER"), cfg.IAMIssuer, defaultIAMIssuer), "/")
+	e.PlatformURL = strings.TrimRight(firstNonEmpty(f.platformURL, os.Getenv("HANZO_PLATFORM_URL"), cfg.PlatformURL, defaultPlatformURL), "/")
+	e.CloudURL = strings.TrimRight(firstNonEmpty(f.cloudURL, os.Getenv("HANZO_CLOUD_URL"), cfg.CloudURL, defaultCloudURL), "/")
+	e.ClientID = firstNonEmpty(f.clientID, os.Getenv("HANZO_CLIENT_ID"), cfg.ClientID, defaultClientID)
+	// Org for platform calls is the platform organization id (a distinct
+	// namespace from the IAM token's `owner` slug), so it comes only from
+	// flag/env/config — never silently from the token.
+	e.Org = firstNonEmpty(f.org, os.Getenv("HANZO_ORG"), cfg.Org)
+	return e
+}
+
+// accessToken is the IAM user token (identity / cloud calls).
+func (e *Env) accessToken() string {
+	return firstNonEmpty(os.Getenv("HANZO_TOKEN"), e.creds.AccessToken)
+}
+
+// platformToken resolves the platform control-plane service token. The
+// platform REST surface is machine-to-machine (it cannot validate IAM user
+// tokens), so apps/clusters/redeploy authenticate with this, sourced from
+// (in precedence) the bound --platform-token flag, the environment, then the
+// credential store. Never hardcoded.
+func (e *Env) platformToken(flagVal string) string {
+	return firstNonEmpty(
+		flagVal,
+		os.Getenv("HANZO_PLATFORM_TOKEN"),
+		os.Getenv("PLATFORM_SERVICE_TOKEN"),
+		os.Getenv("PAAS_SERVICE_TOKEN"),
+		e.creds.PlatformToken,
+	)
+}
+
+// buildToken resolves the platform build-enqueue token (a distinct credential
+// from the service token — see /v1/arcd/enqueue).
+func (e *Env) buildToken(flagVal string) string {
+	return firstNonEmpty(
+		flagVal,
+		os.Getenv("HANZO_BUILD_TOKEN"),
+		os.Getenv("PLATFORM_BUILD_CALLBACK_TOKEN"),
+		e.creds.BuildToken,
+	)
+}
+
+// requireOrg returns the resolved org or a clear error telling the user how to
+// set it.
+func (e *Env) requireOrg() (string, error) {
+	if e.Org == "" {
+		return "", fmt.Errorf("no org set: pass --org, set HANZO_ORG, or run `hanzo config set org <org>`")
+	}
+	return e.Org, nil
+}
+
+// ---------------------------------------------------------------------------
+// Output helpers — one place decides JSON vs human-readable tables.
+// ---------------------------------------------------------------------------
+
+// emit prints v as JSON when --output=json, otherwise calls table to render a
+// human view. This is the single output branch for every command.
+func (e *Env) emit(v any, table func(w io.Writer)) error {
+	if e.Output == "json" {
+		enc := json.NewEncoder(e.out)
+		enc.SetIndent("", "  ")
+		return enc.Encode(v)
+	}
+	table(e.out)
+	return nil
+}
+
+// ---------------------------------------------------------------------------
+// Root command + global flags.
+// ---------------------------------------------------------------------------
+
+func newRootCmd() *cobra.Command {
+	var f globalFlags
+	var env *Env
+
+	root := &cobra.Command{
+		Use:           "hanzo",
+		Short:         "Hanzo cloud control — manage the live Hanzo estate",
+		Long:          "hanzo — gcloud/doctl-class control for the Hanzo platform (IAM, apps, deploys, clusters, builds).",
+		SilenceUsage:  true,
+		SilenceErrors: false,
+		PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {
+			cfg, err := LoadConfig()
+			if err != nil {
+				return fmt.Errorf("load config: %w", err)
+			}
+			creds, err := LoadCredentials()
+			if err != nil {
+				return fmt.Errorf("load credentials: %w", err)
+			}
+			env = resolve(cfg, creds, f)
+			env.out = cmd.OutOrStdout()
+			return nil
+		},
+	}
+
+	pf := root.PersistentFlags()
+	pf.StringVar(&f.org, "org", "", "organization (overrides config / HANZO_ORG)")
+	pf.StringVarP(&f.output, "output", "o", "", "output format: table|json")
+	pf.StringVar(&f.platformURL, "platform-url", "", "platform base URL (default "+defaultPlatformURL+")")
+	pf.StringVar(&f.iamIssuer, "iam-issuer", "", "IAM issuer (default "+defaultIAMIssuer+")")
+	pf.StringVar(&f.cloudURL, "cloud-url", "", "cloud API base URL (default "+defaultCloudURL+")")
+	pf.StringVar(&f.clientID, "client-id", "", "IAM OAuth client id (default "+defaultClientID+")")
+	pf.StringVar(&f.platformToken, "platform-token", "", "platform control-plane service token (else env/credential store)")
+
+	// envOf returns the resolved Env for a command's RunE (always non-nil after
+	// PersistentPreRunE).
+	envOf := func() *Env { return env }
+
+	root.AddCommand(
+		newVersionCmd(),
+		newAuthCmd(envOf, &f),
+		newLoginCmd(envOf, &f),
+		newLogoutCmd(),
+		newWhoamiCmd(envOf),
+		newAppsCmd(envOf, &f),
+		newDeployCmd(envOf, &f),
+		newClustersCmd(envOf, &f),
+		newBuildCmd(envOf, &f),
+		newK8sCmd(envOf, &f),
+		newConfigCmd(),
+		newSecurityCmd(envOf),
+	)
+	return root
+}
+
+func newVersionCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:               "version",
+		Short:             "Print the hanzo version",
+		Args:              cobra.NoArgs,
+		PersistentPreRunE: func(*cobra.Command, []string) error { return nil },
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			fmt.Fprintf(cmd.OutOrStdout(), "hanzo %s\n", Version)
+			return nil
+		},
+	}
+}
+
+// ---------------------------------------------------------------------------
+// config command — view/edit the non-secret preference file.
+// ---------------------------------------------------------------------------
+
+func newConfigCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               "config",
+		Short:             "View/edit ~/.hanzo/config preferences",
+		PersistentPreRunE: func(*cobra.Command, []string) error { return nil },
+	}
+
+	configKeys := []string{"org", "output", "iam_issuer", "platform_url", "cloud_url", "client_id"}
+
+	get := &cobra.Command{
+		Use:   "get <key>",
+		Short: "Print one config value",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := LoadConfig()
+			if err != nil {
+				return err
+			}
+			v, err := cfg.field(args[0])
+			if err != nil {
+				return err
+			}
+			fmt.Fprintln(cmd.OutOrStdout(), v)
+			return nil
+		},
+	}
+
+	set := &cobra.Command{
+		Use:   "set <key> <value>",
+		Short: "Set one config value (keys: " + strings.Join(configKeys, ", ") + ")",
+		Args:  cobra.ExactArgs(2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			cfg, err := LoadConfig()
+			if err != nil {
+				return err
+			}
+			if err := cfg.setField(args[0], args[1]); err != nil {
+				return err
+			}
+			if err := cfg.Save(); err != nil {
+				return err
+			}
+			fmt.Fprintf(cmd.OutOrStdout(), "set %s = %s\n", args[0], args[1])
+			return nil
+		},
+	}
+
+	list := &cobra.Command{
+		Use:   "list",
+		Short: "Print the full config",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			cfg, err := LoadConfig()
+			if err != nil {
+				return err
+			}
+			enc := json.NewEncoder(cmd.OutOrStdout())
+			enc.SetIndent("", "  ")
+			return enc.Encode(cfg)
+		},
+	}
+
+	path := &cobra.Command{
+		Use:   "path",
+		Short: "Print the config file path",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			p, err := configPath()
+			if err != nil {
+				return err
+			}
+			fmt.Fprintln(cmd.OutOrStdout(), p)
+			return nil
+		},
+	}
+
+	cmd.AddCommand(get, set, list, path)
+	return cmd
+}
+
+// field returns the named config value as a string.
+func (c *Config) field(key string) (string, error) {
+	switch key {
+	case "org":
+		return c.Org, nil
+	case "output":
+		return c.Output, nil
+	case "iam_issuer":
+		return c.IAMIssuer, nil
+	case "platform_url":
+		return c.PlatformURL, nil
+	case "cloud_url":
+		return c.CloudURL, nil
+	case "client_id":
+		return c.ClientID, nil
+	default:
+		return "", fmt.Errorf("unknown config key %q", key)
+	}
+}
+
+// setField sets the named config value.
+func (c *Config) setField(key, val string) error {
+	switch key {
+	case "org":
+		c.Org = val
+	case "output":
+		if val != "table" && val != "json" {
+			return fmt.Errorf("output must be table|json")
+		}
+		c.Output = val
+	case "iam_issuer":
+		c.IAMIssuer = val
+	case "platform_url":
+		c.PlatformURL = val
+	case "cloud_url":
+		c.CloudURL = val
+	case "client_id":
+		c.ClientID = val
+	default:
+		return fmt.Errorf("unknown config key %q", key)
+	}
+	return nil
+}
+
+// shortTime renders a unix timestamp for human tables; "" for zero.
+func shortTime(unix int64) string {
+	if unix == 0 {
+		return ""
+	}
+	return time.Unix(unix, 0).Format(time.RFC3339)
+}
+
+// sortedKeys returns the keys of m, sorted — for deterministic help output.
+func sortedKeys(m map[string]string) []string {
+	ks := make([]string, 0, len(m))
+	for k := range m {
+		ks = append(ks, k)
+	}
+	sort.Strings(ks)
+	return ks
+}

cli/cli_test.go

@@ -0,0 +1,240 @@
+package cli
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// TestMain restores stdout (quiet.go redirected it to stderr at init) so go
+// test's own reporting stays on stdout.
+func TestMain(m *testing.M) {
+	RestoreStdout()
+	os.Exit(m.Run())
+}
+
+// sandbox isolates the credential/config store in a temp dir and clears every
+// env var resolve() consults, so tests are deterministic and never touch the
+// developer's real ~/.hanzo.
+func sandbox(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	t.Setenv("HANZO_HOME", dir)
+	for _, k := range []string{
+		"HANZO_CONFIG", "HANZO_OUTPUT", "HANZO_IAM_ISSUER", "HANZO_PLATFORM_URL",
+		"HANZO_CLOUD_URL", "HANZO_CLIENT_ID", "HANZO_ORG", "HANZO_TOKEN",
+		"HANZO_PLATFORM_TOKEN", "PLATFORM_SERVICE_TOKEN", "PAAS_SERVICE_TOKEN",
+		"HANZO_BUILD_TOKEN", "PLATFORM_BUILD_CALLBACK_TOKEN",
+	} {
+		t.Setenv(k, "")
+	}
+	return dir
+}
+
+func TestConfigRoundTrip(t *testing.T) {
+	sandbox(t)
+	in := &Config{Org: "acme", Output: "json", PlatformURL: "https://p.example", ClientID: "hanzo-console"}
+	if err := in.Save(); err != nil {
+		t.Fatalf("save: %v", err)
+	}
+	out, err := LoadConfig()
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+	if *out != *in {
+		t.Fatalf("round-trip mismatch: %+v != %+v", out, in)
+	}
+}
+
+func TestCredentialsRoundTripAndPerms(t *testing.T) {
+	dir := sandbox(t)
+	in := &Credentials{AccessToken: "tok", RefreshToken: "ref", TokenType: "Bearer", Subject: "z@hanzo.ai", Owner: "hanzo", PlatformToken: "pt"}
+	if err := in.Save(); err != nil {
+		t.Fatalf("save: %v", err)
+	}
+	fi, err := os.Stat(filepath.Join(dir, "credentials.json"))
+	if err != nil {
+		t.Fatalf("stat: %v", err)
+	}
+	if perm := fi.Mode().Perm(); perm != 0o600 {
+		t.Fatalf("credentials perm = %o, want 0600", perm)
+	}
+	out, err := LoadCredentials()
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+	if *out != *in {
+		t.Fatalf("round-trip mismatch: %+v != %+v", out, in)
+	}
+	if err := DeleteCredentials(); err != nil {
+		t.Fatalf("delete: %v", err)
+	}
+	if out, _ := LoadCredentials(); out.AccessToken != "" {
+		t.Fatalf("credentials not deleted")
+	}
+}
+
+func TestLoadMissingFilesIsZeroValue(t *testing.T) {
+	sandbox(t)
+	cfg, err := LoadConfig()
+	if err != nil || cfg.Org != "" {
+		t.Fatalf("missing config should be zero value, got %+v err %v", cfg, err)
+	}
+	creds, err := LoadCredentials()
+	if err != nil || creds.AccessToken != "" {
+		t.Fatalf("missing credentials should be zero value, got %+v err %v", creds, err)
+	}
+}
+
+func TestResolveDefaults(t *testing.T) {
+	sandbox(t)
+	e := resolve(&Config{}, &Credentials{}, globalFlags{})
+	if e.IAMIssuer != defaultIAMIssuer || e.PlatformURL != defaultPlatformURL ||
+		e.CloudURL != defaultCloudURL || e.ClientID != defaultClientID || e.Output != "table" {
+		t.Fatalf("defaults not applied: %+v", e)
+	}
+}
+
+func TestResolvePrecedenceFlagOverEnvOverConfig(t *testing.T) {
+	sandbox(t)
+	t.Setenv("HANZO_ORG", "env-org")
+	cfg := &Config{Org: "cfg-org", Output: "json"}
+	// Flag wins.
+	if e := resolve(cfg, &Credentials{}, globalFlags{org: "flag-org"}); e.Org != "flag-org" {
+		t.Fatalf("flag should win: %q", e.Org)
+	}
+	// Env beats config.
+	if e := resolve(cfg, &Credentials{}, globalFlags{}); e.Org != "env-org" {
+		t.Fatalf("env should beat config: %q", e.Org)
+	}
+	// Config used when no flag/env.
+	t.Setenv("HANZO_ORG", "")
+	if e := resolve(cfg, &Credentials{}, globalFlags{}); e.Org != "cfg-org" {
+		t.Fatalf("config should be used: %q", e.Org)
+	}
+}
+
+func TestPlatformTokenPrecedence(t *testing.T) {
+	sandbox(t)
+	e := resolve(&Config{}, &Credentials{PlatformToken: "from-creds"}, globalFlags{})
+	if got := e.platformToken(""); got != "from-creds" {
+		t.Fatalf("creds token: %q", got)
+	}
+	t.Setenv("PAAS_SERVICE_TOKEN", "from-paas")
+	if got := e.platformToken(""); got != "from-paas" {
+		t.Fatalf("PAAS env should beat creds: %q", got)
+	}
+	t.Setenv("PLATFORM_SERVICE_TOKEN", "from-platform")
+	if got := e.platformToken(""); got != "from-platform" {
+		t.Fatalf("PLATFORM env should beat PAAS: %q", got)
+	}
+	t.Setenv("HANZO_PLATFORM_TOKEN", "from-hanzo")
+	if got := e.platformToken(""); got != "from-hanzo" {
+		t.Fatalf("HANZO_PLATFORM_TOKEN should beat all envs: %q", got)
+	}
+	if got := e.platformToken("from-flag"); got != "from-flag" {
+		t.Fatalf("flag should beat everything: %q", got)
+	}
+}
+
+func TestBuildTokenPrecedence(t *testing.T) {
+	sandbox(t)
+	e := resolve(&Config{}, &Credentials{BuildToken: "creds"}, globalFlags{})
+	if got := e.buildToken(""); got != "creds" {
+		t.Fatalf("creds build token: %q", got)
+	}
+	t.Setenv("PLATFORM_BUILD_CALLBACK_TOKEN", "cb")
+	if got := e.buildToken(""); got != "cb" {
+		t.Fatalf("callback env: %q", got)
+	}
+	if got := e.buildToken("flag"); got != "flag" {
+		t.Fatalf("flag wins: %q", got)
+	}
+}
+
+func TestAccessTokenFromEnvOverCreds(t *testing.T) {
+	sandbox(t)
+	e := resolve(&Config{}, &Credentials{AccessToken: "creds"}, globalFlags{})
+	if got := e.accessToken(); got != "creds" {
+		t.Fatalf("creds token: %q", got)
+	}
+	t.Setenv("HANZO_TOKEN", "env")
+	if got := e.accessToken(); got != "env" {
+		t.Fatalf("env token should win: %q", got)
+	}
+}
+
+func TestRequireOrg(t *testing.T) {
+	sandbox(t)
+	e := resolve(&Config{}, &Credentials{}, globalFlags{})
+	if _, err := e.requireOrg(); err == nil {
+		t.Fatalf("expected error when org unset")
+	}
+	e = resolve(&Config{Org: "acme"}, &Credentials{}, globalFlags{})
+	if org, err := e.requireOrg(); err != nil || org != "acme" {
+		t.Fatalf("org=%q err=%v", org, err)
+	}
+}
+
+func TestConfigFieldGetSet(t *testing.T) {
+	c := &Config{}
+	if err := c.setField("org", "acme"); err != nil || c.Org != "acme" {
+		t.Fatalf("set org: %v", err)
+	}
+	if v, _ := c.field("org"); v != "acme" {
+		t.Fatalf("get org: %q", v)
+	}
+	if err := c.setField("output", "xml"); err == nil {
+		t.Fatalf("invalid output should error")
+	}
+	if err := c.setField("nope", "x"); err == nil {
+		t.Fatalf("unknown key should error")
+	}
+	if _, err := c.field("nope"); err == nil {
+		t.Fatalf("unknown key get should error")
+	}
+}
+
+func TestIsControlVerb(t *testing.T) {
+	for _, v := range []string{"login", "apps", "deploy", "clusters", "build", "k8s", "config", "auth", "whoami", "logout"} {
+		if !IsControlVerb(v) {
+			t.Errorf("%q should be a control verb", v)
+		}
+	}
+	for _, v := range []string{"iam", "kms", "cloud", "gateway", "datastore", "nope"} {
+		if IsControlVerb(v) {
+			t.Errorf("%q must NOT be a control verb (server mode)", v)
+		}
+	}
+}
+
+func TestEmitJSONvsTable(t *testing.T) {
+	// JSON branch: encodes the value, ignores the table func.
+	var jbuf bytes.Buffer
+	ej := &Env{Output: "json", out: &jbuf}
+	called := false
+	if err := ej.emit(map[string]string{"k": "v"}, func(_ io.Writer) { called = true }); err != nil {
+		t.Fatalf("emit json: %v", err)
+	}
+	if called {
+		t.Fatalf("table func must not run in json mode")
+	}
+	var got map[string]string
+	if err := json.Unmarshal(jbuf.Bytes(), &got); err != nil || got["k"] != "v" {
+		t.Fatalf("json output bad: %q (%v)", jbuf.String(), err)
+	}
+
+	// Table branch: runs the table func, does not emit JSON.
+	var tbuf bytes.Buffer
+	et := &Env{Output: "table", out: &tbuf}
+	if err := et.emit(map[string]string{"k": "v"}, func(w io.Writer) { _, _ = w.Write([]byte("ROW")) }); err != nil {
+		t.Fatalf("emit table: %v", err)
+	}
+	if !strings.Contains(tbuf.String(), "ROW") {
+		t.Fatalf("table output missing: %q", tbuf.String())
+	}
+}

cli/commands.go

@@ -0,0 +1,450 @@
+package cli
+
+import (
+	"fmt"
+	"io"
+	"text/tabwriter"
+
+	"github.com/spf13/cobra"
+)
+
+// platform builds a platform REST client from the resolved env + the global
+// --platform-token flag. The token may be empty here; the client surfaces a
+// precise error on first use.
+func (e *Env) platform(gf *globalFlags) *Platform {
+	return newPlatform(e.PlatformURL, e.platformToken(gf.platformToken))
+}
+
+// deref renders a *string for a table cell, "-" when nil/empty.
+func deref(p *string) string {
+	if p == nil || *p == "" {
+		return "-"
+	}
+	return *p
+}
+
+// yesno renders a bool for a table cell.
+func yesno(b bool) string {
+	if b {
+		return "yes"
+	}
+	return "no"
+}
+
+// newTab returns a tabwriter writing to w with a 2-space gutter.
+func newTab(w io.Writer) *tabwriter.Writer {
+	return tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
+}
+
+// ---------------------------------------------------------------------------
+// apps — the observe surface.
+// ---------------------------------------------------------------------------
+
+func newAppsCmd(envOf func() *Env, gf *globalFlags) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "apps",
+		Short: "List/get the platform apps board (declared/running/latest/drift)",
+	}
+
+	var envFilter, healthFilter string
+	var driftOnly bool
+
+	list := &cobra.Command{
+		Use:   "list",
+		Short: "List apps with declared/running tags, health and drift",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			e := envOf()
+			res, err := e.platform(gf).Apps(cmd.Context(), AppsQuery{
+				Org:    e.Org, // empty == all (single-tenant default)
+				Env:    envFilter,
+				Health: healthFilter,
+				Drift:  driftOnly,
+			})
+			if err != nil {
+				return err
+			}
+			return e.emit(res, func(w io.Writer) {
+				tw := newTab(w)
+				fmt.Fprintln(tw, "ORG\tAPP\tENV\tDECLARED\tRUNNING\tHEALTH\tDRIFT")
+				for _, a := range res.Apps {
+					fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
+						a.Org, a.App, a.Env, deref(a.DeclaredTag), deref(a.RunningTag),
+						deref(a.Health), driftSeverity(a.Drift))
+				}
+				tw.Flush()
+				fmt.Fprintf(w, "\n%d apps  (ok=%d yellow=%d red=%d)\n",
+					res.Summary.Total, res.Summary.ByDrift["ok"],
+					res.Summary.ByDrift["yellow"], res.Summary.ByDrift["red"])
+			})
+		},
+	}
+	list.Flags().StringVar(&envFilter, "env", "", "filter by env: dev|test|main")
+	list.Flags().StringVar(&healthFilter, "health", "", "filter by health: green|yellow|red")
+	list.Flags().BoolVar(&driftOnly, "drift", false, "only rows that are drifting")
+
+	get := &cobra.Command{
+		Use:   "get <org/app/env>",
+		Short: "Get one app row by its <org>/<app>/<env> id",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			e := envOf()
+			a, err := e.platform(gf).App(cmd.Context(), args[0], e.Org)
+			if err != nil {
+				return err
+			}
+			return e.emit(a, func(w io.Writer) {
+				tw := newTab(w)
+				fmt.Fprintf(tw, "id:\t%s\n", a.ID)
+				fmt.Fprintf(tw, "org:\t%s\n", a.Org)
+				fmt.Fprintf(tw, "app:\t%s\n", a.App)
+				fmt.Fprintf(tw, "env:\t%s\n", a.Env)
+				fmt.Fprintf(tw, "repo:\t%s\n", a.Repo)
+				fmt.Fprintf(tw, "registry:\t%s\n", a.Registry)
+				fmt.Fprintf(tw, "declared:\t%s\n", deref(a.DeclaredTag))
+				fmt.Fprintf(tw, "running:\t%s\n", deref(a.RunningTag))
+				fmt.Fprintf(tw, "latest:\t%s\n", deref(a.LatestTag))
+				fmt.Fprintf(tw, "health:\t%s\n", deref(a.Health))
+				fmt.Fprintf(tw, "drift:\t%s\n", driftSeverity(a.Drift))
+				fmt.Fprintf(tw, "cluster:\t%s\n", deref(a.Cluster))
+				fmt.Fprintf(tw, "namespace:\t%s\n", deref(a.Namespace))
+				fmt.Fprintf(tw, "updated:\t%s\n", a.UpdatedAt)
+				tw.Flush()
+			})
+		},
+	}
+
+	sync := &cobra.Command{
+		Use:   "sync",
+		Short: "Trigger an inventory refresh of the apps board",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			e := envOf()
+			if err := e.platform(gf).SyncApps(cmd.Context()); err != nil {
+				return err
+			}
+			fmt.Fprintln(cmd.OutOrStdout(), "apps sync triggered")
+			return nil
+		},
+	}
+
+	cmd.AddCommand(list, get, sync)
+	return cmd
+}
+
+// ---------------------------------------------------------------------------
+// deploy — the drive surface (rolling restart, zero-downtime).
+// ---------------------------------------------------------------------------
+
+func newDeployCmd(envOf func() *Env, gf *globalFlags) *cobra.Command {
+	var project, environment string
+	cmd := &cobra.Command{
+		Use:   "deploy <container>",
+		Short: "Redeploy a container (rolling restart, zero-downtime)",
+		Long: "Drive a platform redeploy: a rolling restart of the container's k8s\n" +
+			"Deployment (re-pulls the image, recreates pods, zero downtime). Coordinates\n" +
+			"are exact — org (--org/config), project (--project), env (--env) and the\n" +
+			"container id (positional). This is the canonical PaaS-driven deploy.",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			e := envOf()
+			org, err := e.requireOrg()
+			if err != nil {
+				return err
+			}
+			if project == "" || environment == "" {
+				return fmt.Errorf("--project and --env are required (the container's project/environment ids)")
+			}
+			container := args[0]
+			if err := e.platform(gf).Redeploy(cmd.Context(), org, project, environment, container); err != nil {
+				return err
+			}
+			fmt.Fprintf(cmd.OutOrStdout(), "redeployed %s (org=%s project=%s env=%s)\n", container, org, project, environment)
+			return nil
+		},
+	}
+	cmd.Flags().StringVar(&project, "project", "", "project id")
+	cmd.Flags().StringVar(&environment, "env", "", "environment id")
+	return cmd
+}
+
+// ---------------------------------------------------------------------------
+// clusters — dedicated DOKS cluster lifecycle.
+// ---------------------------------------------------------------------------
+
+func newClustersCmd(envOf func() *Env, gf *globalFlags) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "clusters",
+		Aliases: []string{"cluster"},
+		Short:   "Provision/list/select dedicated DOKS clusters",
+	}
+
+	list := &cobra.Command{
+		Use:   "list",
+		Short: "List the org's dedicated clusters",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			e := envOf()
+			org, err := e.requireOrg()
+			if err != nil {
+				return err
+			}
+			cs, err := e.platform(gf).Clusters(cmd.Context(), org)
+			if err != nil {
+				return err
+			}
+			return e.emit(cs, func(w io.Writer) {
+				tw := newTab(w)
+				fmt.Fprintln(tw, "NAME\tID\tREGION\tSTATUS\tPHASE\tACTIVE\tOPERATOR\tBASELINE")
+				for _, c := range cs {
+					fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
+						c.Name, c.DoksClusterID, c.Region, c.Status, c.Phase,
+						yesno(c.Active), yesno(c.OperatorInstalled), yesno(c.BaselineInstalled))
+				}
+				tw.Flush()
+				if len(cs) == 0 {
+					fmt.Fprintln(w, "(no dedicated clusters)")
+				}
+			})
+		},
+	}
+
+	get := &cobra.Command{
+		Use:   "get <cluster-id>",
+		Short: "Show one cluster",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			e := envOf()
+			org, err := e.requireOrg()
+			if err != nil {
+				return err
+			}
+			cs, err := e.platform(gf).Clusters(cmd.Context(), org)
+			if err != nil {
+				return err
+			}
+			for _, c := range cs {
+				if c.DoksClusterID == args[0] || c.Name == args[0] {
+					return e.emit(c, func(w io.Writer) { printCluster(w, c) })
+				}
+			}
+			return fmt.Errorf("cluster %q not found in org %s", args[0], org)
+		},
+	}
+
+	var region, nodeSize string
+	var ha bool
+	var nodeCount int
+	create := &cobra.Command{
+		Use:   "create",
+		Short: "Provision a new dedicated DOKS cluster for the org",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			e := envOf()
+			org, err := e.requireOrg()
+			if err != nil {
+				return err
+			}
+			c, err := e.platform(gf).ProvisionCluster(cmd.Context(), org, ProvisionReq{
+				Region: region, HA: ha, NodeSize: nodeSize, NodeCount: nodeCount,
+			})
+			if err != nil {
+				return err
+			}
+			return e.emit(c, func(w io.Writer) {
+				fmt.Fprintf(w, "provisioning cluster %s (%s)\n", c.Name, c.DoksClusterID)
+				printCluster(w, *c)
+			})
+		},
+	}
+	create.Flags().StringVar(&region, "region", "", "DO region (default sfo3)")
+	create.Flags().BoolVar(&ha, "ha", false, "highly-available control plane")
+	create.Flags().StringVar(&nodeSize, "node-size", "", "node size slug (e.g. s-2vcpu-4gb)")
+	create.Flags().IntVar(&nodeCount, "node-count", 0, "node count")
+
+	var shared bool
+	selectCmd := &cobra.Command{
+		Use:   "select <cluster-id>",
+		Short: "Set the org's active deploy target (or --shared to revert)",
+		Args:  cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			e := envOf()
+			org, err := e.requireOrg()
+			if err != nil {
+				return err
+			}
+			var clusterID *string
+			switch {
+			case shared:
+				clusterID = nil
+			case len(args) == 1:
+				clusterID = &args[0]
+			default:
+				return fmt.Errorf("give a cluster id, or --shared to revert to the shared cluster")
+			}
+			t, err := e.platform(gf).SelectTarget(cmd.Context(), org, clusterID)
+			if err != nil {
+				return err
+			}
+			return e.emit(t, func(w io.Writer) { printTarget(w, t) })
+		},
+	}
+	selectCmd.Flags().BoolVar(&shared, "shared", false, "revert to the shared cluster")
+
+	installBaseline := &cobra.Command{
+		Use:   "install-baseline <cluster-id>",
+		Short: "Install the hanzo-operator + per-tenant baseline on a cluster",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			e := envOf()
+			org, err := e.requireOrg()
+			if err != nil {
+				return err
+			}
+			if err := e.platform(gf).InstallBaseline(cmd.Context(), org, args[0]); err != nil {
+				return err
+			}
+			fmt.Fprintf(cmd.OutOrStdout(), "baseline install requested for %s\n", args[0])
+			return nil
+		},
+	}
+
+	target := &cobra.Command{
+		Use:   "target",
+		Short: "Show the org's current resolved deploy target",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			e := envOf()
+			org, err := e.requireOrg()
+			if err != nil {
+				return err
+			}
+			t, err := e.platform(gf).Target(cmd.Context(), org)
+			if err != nil {
+				return err
+			}
+			return e.emit(t, func(w io.Writer) { printTarget(w, t) })
+		},
+	}
+
+	cmd.AddCommand(list, get, create, selectCmd, installBaseline, target)
+	return cmd
+}
+
+func printCluster(w io.Writer, c Cluster) {
+	tw := newTab(w)
+	fmt.Fprintf(tw, "id:\t%s\n", c.DoksClusterID)
+	fmt.Fprintf(tw, "name:\t%s\n", c.Name)
+	fmt.Fprintf(tw, "region:\t%s\n", c.Region)
+	fmt.Fprintf(tw, "status:\t%s\n", c.Status)
+	fmt.Fprintf(tw, "phase:\t%s\n", c.Phase)
+	fmt.Fprintf(tw, "active:\t%s\n", yesno(c.Active))
+	fmt.Fprintf(tw, "operatorInstalled:\t%s\n", yesno(c.OperatorInstalled))
+	fmt.Fprintf(tw, "baselineInstalled:\t%s\n", yesno(c.BaselineInstalled))
+	fmt.Fprintf(tw, "endpoint:\t%s\n", deref(c.Endpoint))
+	fmt.Fprintf(tw, "k8sVersion:\t%s\n", deref(c.K8sVersion))
+	fmt.Fprintf(tw, "created:\t%s\n", c.CreatedAt)
+	if c.BaselineError != nil && *c.BaselineError != "" {
+		fmt.Fprintf(tw, "baselineError:\t%s\n", *c.BaselineError)
+	}
+	tw.Flush()
+}
+
+func printTarget(w io.Writer, t *Target) {
+	tw := newTab(w)
+	kind := "shared"
+	if t.Dedicated {
+		kind = "dedicated"
+	}
+	fmt.Fprintf(tw, "cluster:\t%s\n", t.Cluster)
+	fmt.Fprintf(tw, "kind:\t%s\n", kind)
+	for ns, env := range t.Namespaces {
+		fmt.Fprintf(tw, "namespace:\t%s -> %s\n", ns, env)
+	}
+	tw.Flush()
+}
+
+// ---------------------------------------------------------------------------
+// build — platform-native (arcd) build enqueue.
+// ---------------------------------------------------------------------------
+
+func newBuildCmd(envOf func() *Env, gf *globalFlags) *cobra.Command {
+	var br BuildReq
+	var buildToken string
+	cmd := &cobra.Command{
+		Use:   "build <repo>",
+		Short: "Enqueue a platform-native (arcd) build (no GitHub builders)",
+		Long: "Enqueue a build on the platform's native CI fabric (arcd). Builds and pushes\n" +
+			"the named image at a SHA; on completion the platform patches the operator\n" +
+			"Service CR (build-job → deploy). Requires a live registered runner for the\n" +
+			"target pool (409 otherwise).",
+		Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			e := envOf()
+			if len(args) == 1 {
+				br.Repo = args[0]
+			}
+			if br.Repo == "" || br.SHA == "" || br.Image == "" {
+				return fmt.Errorf("--repo (or positional), --sha and --image are required")
+			}
+			if br.OrganizationID == "" {
+				br.OrganizationID = e.Org // optional; server defaults to DEFAULT_BUILD_ORG_ID
+			}
+			job, err := e.platform(gf).EnqueueBuild(cmd.Context(), br, e.buildToken(buildToken))
+			if err != nil {
+				return err
+			}
+			return e.emit(job, func(w io.Writer) {
+				tw := newTab(w)
+				fmt.Fprintf(tw, "buildJobId:\t%s\n", job.BuildJobID)
+				fmt.Fprintf(tw, "status:\t%s\n", job.Status)
+				fmt.Fprintf(tw, "runnerPool:\t%s\n", job.RunnerPool)
+				fmt.Fprintf(tw, "image:\t%s\n", job.Image)
+				fmt.Fprintf(tw, "target:\t%s\n", job.Target)
+				tw.Flush()
+			})
+		},
+	}
+	f := cmd.Flags()
+	f.StringVar(&br.Repo, "repo", "", "owner/name (e.g. hanzoai/pricing)")
+	f.StringVar(&br.SHA, "sha", "", "commit SHA to build")
+	f.StringVar(&br.Image, "image", "", "image to build+push (e.g. ghcr.io/hanzoai/pricing:<tag>)")
+	f.StringVar(&br.Branch, "branch", "", "branch (default main)")
+	f.StringVar(&br.Dockerfile, "dockerfile", "", "Dockerfile path")
+	f.StringVar(&br.Context, "context", "", "build context")
+	f.StringVar(&br.DockerTarget, "target", "", "Docker build stage (--target)")
+	f.StringVar(&br.OS, "os", "", "linux|darwin|windows (default linux)")
+	f.StringVar(&br.Arch, "arch", "", "amd64|arm64 (default amd64)")
+	f.StringVar(&buildToken, "build-token", "", "platform build-enqueue token (else env/credential store)")
+	return cmd
+}
+
+// ---------------------------------------------------------------------------
+// k8s — deploy-target helpers.
+// ---------------------------------------------------------------------------
+
+func newK8sCmd(envOf func() *Env, gf *globalFlags) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "k8s",
+		Short: "Kubernetes deploy-target helpers",
+	}
+	target := &cobra.Command{
+		Use:   "target",
+		Short: "Show the org's current resolved deploy target (cluster + namespaces)",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			e := envOf()
+			org, err := e.requireOrg()
+			if err != nil {
+				return err
+			}
+			t, err := e.platform(gf).Target(cmd.Context(), org)
+			if err != nil {
+				return err
+			}
+			return e.emit(t, func(w io.Writer) { printTarget(w, t) })
+		},
+	}
+	cmd.AddCommand(target)
+	return cmd
+}

cli/commands_test.go

@@ -0,0 +1,171 @@
+package cli
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// withPlatform points the CLI at an httptest platform via env (HANZO_PLATFORM_URL
+// + HANZO_PLATFORM_TOKEN), the same resolution path the real binary uses.
+func withPlatform(t *testing.T, h http.HandlerFunc) string {
+	t.Helper()
+	sandbox(t)
+	srv := httptest.NewServer(h)
+	t.Cleanup(srv.Close)
+	t.Setenv("HANZO_PLATFORM_URL", srv.URL)
+	t.Setenv("HANZO_PLATFORM_TOKEN", "svc-tok")
+	return srv.URL
+}
+
+func TestAppsListCommandTable(t *testing.T) {
+	withPlatform(t, func(w http.ResponseWriter, _ *http.Request) {
+		_ = json.NewEncoder(w).Encode(AppsList{
+			Apps: []AppView{
+				{Org: "hanzoai", App: "iam", Env: "main", DeclaredTag: strptr("v1.2.3"), RunningTag: strptr("v1.2.3"), Health: strptr("green"), Drift: json.RawMessage(`{"severity":"ok"}`)},
+			},
+			Summary: struct {
+				Total   int            `json:"total"`
+				ByDrift map[string]int `json:"byDrift"`
+			}{Total: 1, ByDrift: map[string]int{"ok": 1}},
+		})
+	})
+	out, err := runRoot(t, "", "apps", "list")
+	if err != nil {
+		t.Fatalf("apps list: %v", err)
+	}
+	for _, want := range []string{"APP", "iam", "v1.2.3", "green", "ok", "1 apps"} {
+		if !strings.Contains(out, want) {
+			t.Fatalf("apps list table missing %q in:\n%s", want, out)
+		}
+	}
+}
+
+func TestAppsListCommandJSON(t *testing.T) {
+	withPlatform(t, func(w http.ResponseWriter, _ *http.Request) {
+		_ = json.NewEncoder(w).Encode(AppsList{Apps: []AppView{{Org: "hanzoai", App: "iam", Env: "main"}}})
+	})
+	out, err := runRoot(t, "", "apps", "list", "-o", "json")
+	if err != nil {
+		t.Fatalf("apps list json: %v", err)
+	}
+	var res AppsList
+	if err := json.Unmarshal([]byte(out), &res); err != nil {
+		t.Fatalf("output is not valid JSON: %v\n%s", err, out)
+	}
+	if len(res.Apps) != 1 || res.Apps[0].App != "iam" {
+		t.Fatalf("json decode wrong: %+v", res.Apps)
+	}
+}
+
+func TestDeployCommand(t *testing.T) {
+	withPlatform(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v1/org/acme/project/p1/env/e1/container/app-x/redeploy" {
+			t.Errorf("redeploy path = %s", r.URL.Path)
+		}
+		_ = json.NewEncoder(w).Encode(map[string]bool{"ok": true})
+	})
+	out, err := runRoot(t, "", "deploy", "app-x", "--org", "acme", "--project", "p1", "--env", "e1")
+	if err != nil {
+		t.Fatalf("deploy: %v", err)
+	}
+	if !strings.Contains(out, "redeployed app-x") {
+		t.Fatalf("deploy output: %q", out)
+	}
+}
+
+func TestDeployRequiresProjectEnv(t *testing.T) {
+	withPlatform(t, func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(200) })
+	if _, err := runRoot(t, "", "deploy", "app-x", "--org", "acme"); err == nil {
+		t.Fatalf("deploy must require --project/--env")
+	}
+}
+
+func TestDeployRequiresOrg(t *testing.T) {
+	withPlatform(t, func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(200) })
+	if _, err := runRoot(t, "", "deploy", "app-x", "--project", "p1", "--env", "e1"); err == nil {
+		t.Fatalf("deploy must require an org")
+	}
+}
+
+func TestClustersListCommand(t *testing.T) {
+	withPlatform(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v1/org/acme/cluster" {
+			t.Errorf("path = %s", r.URL.Path)
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{"clusters": []Cluster{
+			{DoksClusterID: "c1", Name: "hanzo-acme", Region: "sfo3", Status: "running", Phase: "ready", Active: true, OperatorInstalled: true, BaselineInstalled: true},
+		}})
+	})
+	out, err := runRoot(t, "", "clusters", "list", "--org", "acme")
+	if err != nil {
+		t.Fatalf("clusters list: %v", err)
+	}
+	for _, want := range []string{"NAME", "hanzo-acme", "c1", "ready", "yes"} {
+		if !strings.Contains(out, want) {
+			t.Fatalf("clusters list missing %q in:\n%s", want, out)
+		}
+	}
+}
+
+func TestK8sTargetCommand(t *testing.T) {
+	withPlatform(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v1/org/acme/cluster/select" {
+			t.Errorf("path = %s", r.URL.Path)
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{"target": Target{Cluster: "hanzo-k8s", Dedicated: false, Namespaces: map[string]string{"hanzo": "main"}}})
+	})
+	out, err := runRoot(t, "", "k8s", "target", "--org", "acme")
+	if err != nil {
+		t.Fatalf("k8s target: %v", err)
+	}
+	if !strings.Contains(out, "hanzo-k8s") || !strings.Contains(out, "shared") {
+		t.Fatalf("k8s target output: %q", out)
+	}
+}
+
+func TestBuildCommandValidation(t *testing.T) {
+	withPlatform(t, func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(202) })
+	// Missing --sha/--image → validation error before any HTTP call.
+	if _, err := runRoot(t, "", "build", "hanzoai/pricing"); err == nil {
+		t.Fatalf("build must require --sha and --image")
+	}
+}
+
+func TestBuildCommand(t *testing.T) {
+	withPlatform(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v1/arcd/enqueue" {
+			t.Errorf("path = %s", r.URL.Path)
+		}
+		if got := r.Header.Get("Authorization"); got != "Bearer bt" {
+			t.Errorf("build auth = %q", got)
+		}
+		w.WriteHeader(202)
+		_ = json.NewEncoder(w).Encode(BuildJob{BuildJobID: "bj-9", Status: "queued", Image: "ghcr.io/hanzoai/pricing:t"})
+	})
+	out, err := runRoot(t, "", "build", "hanzoai/pricing", "--sha", "abc", "--image", "ghcr.io/hanzoai/pricing:t", "--build-token", "bt")
+	if err != nil {
+		t.Fatalf("build: %v", err)
+	}
+	if !strings.Contains(out, "bj-9") {
+		t.Fatalf("build output: %q", out)
+	}
+}
+
+func TestConfigSetGetCommand(t *testing.T) {
+	sandbox(t)
+	if _, err := runRoot(t, "", "config", "set", "org", "acme"); err != nil {
+		t.Fatalf("config set: %v", err)
+	}
+	out, err := runRoot(t, "", "config", "get", "org")
+	if err != nil {
+		t.Fatalf("config get: %v", err)
+	}
+	if strings.TrimSpace(out) != "acme" {
+		t.Fatalf("config get = %q", out)
+	}
+}
+
+func strptr(s string) *string { return &s }

cli/platform.go

@@ -0,0 +1,348 @@
+package cli
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+// Platform is a thin client over the platform.hanzo.ai /v1 control plane. That
+// surface is machine-to-machine (service-token, "No OIDC" — it cannot validate
+// IAM user tokens), so the token here is the platform service token, resolved
+// from flag/env/credential store by the caller; the build endpoint takes its
+// own token per call.
+type Platform struct {
+	baseURL string
+	token   string
+	http    *http.Client
+}
+
+func newPlatform(baseURL, token string) *Platform {
+	return &Platform{
+		baseURL: strings.TrimRight(baseURL, "/"),
+		token:   token,
+		http:    &http.Client{Timeout: 60 * time.Second},
+	}
+}
+
+// apiError carries the HTTP status + server message for a failed call so
+// commands can give precise diagnostics (e.g. 401 → token problem).
+type apiError struct {
+	status  int
+	message string
+	path    string
+}
+
+func (e *apiError) Error() string {
+	msg := e.message
+	if msg == "" {
+		msg = http.StatusText(e.status)
+	}
+	hint := ""
+	if e.status == http.StatusUnauthorized {
+		hint = " (set the platform service token: --platform-token, HANZO_PLATFORM_TOKEN, or `hanzo login --platform-token`)"
+	}
+	return fmt.Sprintf("platform %s: HTTP %d: %s%s", e.path, e.status, msg, hint)
+}
+
+// do performs one JSON request with the given bearer token, decoding a 2xx body
+// into out (when non-nil) and mapping a non-2xx into an *apiError.
+func (p *Platform) do(ctx context.Context, method, path, token string, body, out any) error {
+	if token == "" {
+		return fmt.Errorf("no platform token: pass --platform-token, set HANZO_PLATFORM_TOKEN, or run `hanzo login --platform-token <tok>`")
+	}
+	var rdr io.Reader
+	if body != nil {
+		b, err := json.Marshal(body)
+		if err != nil {
+			return err
+		}
+		rdr = bytes.NewReader(b)
+	}
+	req, err := http.NewRequestWithContext(ctx, method, p.baseURL+path, rdr)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("User-Agent", "hanzo-cli/"+Version)
+	if body != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+
+	resp, err := p.http.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	raw, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
+
+	if resp.StatusCode/100 != 2 {
+		return &apiError{status: resp.StatusCode, message: serverMessage(raw), path: path}
+	}
+	if out != nil && len(raw) > 0 {
+		if err := json.Unmarshal(raw, out); err != nil {
+			return fmt.Errorf("platform %s: decode response: %w", path, err)
+		}
+	}
+	return nil
+}
+
+// serverMessage pulls the `{ "message": … }` field platform errors use, falling
+// back to the raw (truncated) body.
+func serverMessage(raw []byte) string {
+	var e struct {
+		Message string `json:"message"`
+		Error   string `json:"error"`
+	}
+	if json.Unmarshal(raw, &e) == nil {
+		if e.Message != "" {
+			return e.Message
+		}
+		if e.Error != "" {
+			return e.Error
+		}
+	}
+	s := strings.TrimSpace(string(raw))
+	if len(s) > 240 {
+		s = s[:240] + "…"
+	}
+	return s
+}
+
+// ---------------------------------------------------------------------------
+// Apps board — GET /v1/apps, GET /v1/apps/{id}, POST /v1/apps/sync.
+// ---------------------------------------------------------------------------
+
+// AppView mirrors the platform apps-lifecycle DTO. Nullable columns are *string
+// so JSON null round-trips; Drift is kept raw so --json is byte-faithful and
+// the drift schema can evolve without a client bump.
+type AppView struct {
+	ID            string          `json:"id"`
+	Org           string          `json:"org"`
+	App           string          `json:"app"`
+	Env           string          `json:"env"`
+	Repo          string          `json:"repo"`
+	Registry      string          `json:"registry"`
+	DeclaredTag   *string         `json:"declaredTag"`
+	RunningTag    *string         `json:"runningTag"`
+	LatestTag     *string         `json:"latestTag"`
+	ReleaseURL    *string         `json:"releaseUrl"`
+	ReleaseAssets int             `json:"releaseAssets"`
+	Health        *string         `json:"health"`
+	Cluster       *string         `json:"cluster"`
+	Namespace     *string         `json:"namespace"`
+	LastObserved  *string         `json:"lastObserved"`
+	UpdatedAt     string          `json:"updatedAt"`
+	Drift         json.RawMessage `json:"drift"`
+}
+
+// AppsList is the /v1/apps envelope: ordered rows + a drift summary.
+type AppsList struct {
+	Apps    []AppView `json:"apps"`
+	Summary struct {
+		Total   int            `json:"total"`
+		ByDrift map[string]int `json:"byDrift"`
+	} `json:"summary"`
+}
+
+// AppsQuery are the optional /v1/apps filters.
+type AppsQuery struct {
+	Org    string
+	Env    string
+	Health string
+	Drift  bool
+}
+
+func (p *Platform) Apps(ctx context.Context, q AppsQuery) (*AppsList, error) {
+	v := url.Values{}
+	if q.Org != "" {
+		v.Set("org", q.Org)
+	}
+	if q.Env != "" {
+		v.Set("env", q.Env)
+	}
+	if q.Health != "" {
+		v.Set("health", q.Health)
+	}
+	if q.Drift {
+		v.Set("drift", "1")
+	}
+	path := "/v1/apps"
+	if len(v) > 0 {
+		path += "?" + v.Encode()
+	}
+	out := &AppsList{}
+	return out, p.do(ctx, http.MethodGet, path, p.token, nil, out)
+}
+
+func (p *Platform) App(ctx context.Context, id, org string) (*AppView, error) {
+	path := "/v1/apps/" + id
+	if org != "" {
+		path += "?org=" + url.QueryEscape(org)
+	}
+	out := &AppView{}
+	return out, p.do(ctx, http.MethodGet, path, p.token, nil, out)
+}
+
+func (p *Platform) SyncApps(ctx context.Context) error {
+	return p.do(ctx, http.MethodPost, "/v1/apps/sync", p.token, nil, nil)
+}
+
+// driftSeverity extracts the severity string from the raw drift object.
+func driftSeverity(raw json.RawMessage) string {
+	var d struct {
+		Severity string `json:"severity"`
+	}
+	if json.Unmarshal(raw, &d) == nil && d.Severity != "" {
+		return d.Severity
+	}
+	return "-"
+}
+
+// ---------------------------------------------------------------------------
+// Dedicated clusters — /v1/org/{org}/cluster[ /select | /{id}/install-baseline ].
+// ---------------------------------------------------------------------------
+
+// Cluster mirrors a doks_cluster record. `status` is DigitalOcean state; `phase`
+// is the platform provisioning lifecycle — orthogonal (a DO-running cluster is
+// not a usable target until phase=ready).
+type Cluster struct {
+	DoksClusterID     string          `json:"doksClusterId"`
+	Name              string          `json:"name"`
+	DoClusterID       *string         `json:"doClusterId"`
+	Region            string          `json:"region"`
+	Status            string          `json:"status"`
+	Endpoint          *string         `json:"endpoint"`
+	K8sVersion        *string         `json:"k8sVersion"`
+	HA                bool            `json:"ha"`
+	Phase             string          `json:"phase"`
+	OperatorInstalled bool            `json:"operatorInstalled"`
+	BaselineInstalled bool            `json:"baselineInstalled"`
+	Active            bool            `json:"active"`
+	BaselineError     *string         `json:"baselineError"`
+	OrganizationID    string          `json:"organizationId"`
+	CreatedAt         string          `json:"createdAt"`
+	Tags              []string        `json:"tags"`
+	MaintenancePolicy json.RawMessage `json:"maintenancePolicy,omitempty"`
+}
+
+// ProvisionReq is the dedicated-cluster provisioning body (org forced by path).
+type ProvisionReq struct {
+	Region    string `json:"region,omitempty"`
+	HA        bool   `json:"ha,omitempty"`
+	NodeSize  string `json:"nodeSize,omitempty"`
+	NodeCount int    `json:"nodeCount,omitempty"`
+}
+
+// Target is the redacted ClusterTargetView — the kubeconfig is never present.
+type Target struct {
+	Cluster    string            `json:"cluster"`
+	Namespaces map[string]string `json:"namespaces"`
+	Dedicated  bool              `json:"dedicated"`
+}
+
+func (p *Platform) Clusters(ctx context.Context, org string) ([]Cluster, error) {
+	var out struct {
+		Clusters []Cluster `json:"clusters"`
+	}
+	err := p.do(ctx, http.MethodGet, "/v1/org/"+url.PathEscape(org)+"/cluster", p.token, nil, &out)
+	return out.Clusters, err
+}
+
+func (p *Platform) ProvisionCluster(ctx context.Context, org string, req ProvisionReq) (*Cluster, error) {
+	var out struct {
+		Cluster Cluster `json:"cluster"`
+	}
+	err := p.do(ctx, http.MethodPost, "/v1/org/"+url.PathEscape(org)+"/cluster", p.token, req, &out)
+	return &out.Cluster, err
+}
+
+func (p *Platform) Target(ctx context.Context, org string) (*Target, error) {
+	var out struct {
+		Target Target `json:"target"`
+	}
+	err := p.do(ctx, http.MethodGet, "/v1/org/"+url.PathEscape(org)+"/cluster/select", p.token, nil, &out)
+	return &out.Target, err
+}
+
+// SelectTarget activates a dedicated cluster as the org's deploy target, or
+// reverts to the shared cluster when clusterID is nil.
+func (p *Platform) SelectTarget(ctx context.Context, org string, clusterID *string) (*Target, error) {
+	var out struct {
+		Target Target `json:"target"`
+	}
+	body := map[string]any{"doksClusterId": clusterID}
+	err := p.do(ctx, http.MethodPost, "/v1/org/"+url.PathEscape(org)+"/cluster/select", p.token, body, &out)
+	return &out.Target, err
+}
+
+func (p *Platform) InstallBaseline(ctx context.Context, org, clusterID string) error {
+	path := "/v1/org/" + url.PathEscape(org) + "/cluster/" + url.PathEscape(clusterID) + "/install-baseline"
+	return p.do(ctx, http.MethodPost, path, p.token, nil, nil)
+}
+
+// ---------------------------------------------------------------------------
+// Deploy — POST …/container/{id}/redeploy (rolling restart, zero-downtime).
+// ---------------------------------------------------------------------------
+
+// Redeploy triggers a rolling restart of the container's k8s Deployment. The
+// coordinates are exact (the platform validates org+project+env+container scope).
+func (p *Platform) Redeploy(ctx context.Context, org, project, env, container string) error {
+	path := fmt.Sprintf("/v1/org/%s/project/%s/env/%s/container/%s/redeploy",
+		url.PathEscape(org), url.PathEscape(project), url.PathEscape(env), url.PathEscape(container))
+	var out struct {
+		OK bool `json:"ok"`
+	}
+	if err := p.do(ctx, http.MethodPost, path, p.token, nil, &out); err != nil {
+		return err
+	}
+	if !out.OK {
+		return fmt.Errorf("redeploy did not report ok")
+	}
+	return nil
+}
+
+// ---------------------------------------------------------------------------
+// Build — POST /v1/arcd/enqueue (platform-native CI, no GitHub builders).
+// ---------------------------------------------------------------------------
+
+// BuildReq is the direct-enqueue body. Repo/SHA/Image are required.
+type BuildReq struct {
+	Repo           string `json:"repo"`
+	SHA            string `json:"sha"`
+	Image          string `json:"image"`
+	Branch         string `json:"branch,omitempty"`
+	Ref            string `json:"ref,omitempty"`
+	Dockerfile     string `json:"dockerfile,omitempty"`
+	Context        string `json:"context,omitempty"`
+	DockerTarget   string `json:"dockerTarget,omitempty"`
+	OS             string `json:"os,omitempty"`
+	Arch           string `json:"arch,omitempty"`
+	OrganizationID string `json:"organizationId,omitempty"`
+}
+
+// BuildJob is the enqueue acceptance (HTTP 202).
+type BuildJob struct {
+	BuildJobID string `json:"buildJobId"`
+	Status     string `json:"status"`
+	RunnerPool string `json:"runnerPool"`
+	Image      string `json:"image"`
+	Target     string `json:"target"`
+}
+
+// EnqueueBuild enqueues a native build. It authenticates with the dedicated
+// build-callback token, not the service token.
+func (p *Platform) EnqueueBuild(ctx context.Context, req BuildReq, buildToken string) (*BuildJob, error) {
+	if buildToken == "" {
+		return nil, fmt.Errorf("no build token: set HANZO_BUILD_TOKEN / PLATFORM_BUILD_CALLBACK_TOKEN or `hanzo login --build-token <tok>`")
+	}
+	out := &BuildJob{}
+	return out, p.do(ctx, http.MethodPost, "/v1/arcd/enqueue", buildToken, req, out)
+}

cli/platform_test.go

@@ -0,0 +1,220 @@
+package cli
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// platformStub spins an httptest server whose handler is provided by the test,
+// plus a client pointed at it with the given token.
+func platformStub(t *testing.T, token string, h http.HandlerFunc) (*Platform, func()) {
+	t.Helper()
+	srv := httptest.NewServer(h)
+	return newPlatform(srv.URL, token), srv.Close
+}
+
+func TestPlatformAuthHeaderAndApps(t *testing.T) {
+	p, done := platformStub(t, "svc-tok", func(w http.ResponseWriter, r *http.Request) {
+		if got := r.Header.Get("Authorization"); got != "Bearer svc-tok" {
+			t.Errorf("auth header = %q", got)
+		}
+		if r.URL.Path != "/v1/apps" {
+			t.Errorf("path = %s", r.URL.Path)
+		}
+		if r.URL.Query().Get("env") != "main" || r.URL.Query().Get("drift") != "1" {
+			t.Errorf("query = %s", r.URL.RawQuery)
+		}
+		_ = json.NewEncoder(w).Encode(AppsList{
+			Apps: []AppView{{ID: "hanzoai/iam/main", Org: "hanzoai", App: "iam", Env: "main", Drift: json.RawMessage(`{"severity":"red"}`)}},
+		})
+	})
+	defer done()
+
+	res, err := p.Apps(context.Background(), AppsQuery{Env: "main", Drift: true})
+	if err != nil {
+		t.Fatalf("Apps: %v", err)
+	}
+	if len(res.Apps) != 1 || res.Apps[0].App != "iam" {
+		t.Fatalf("apps wrong: %+v", res.Apps)
+	}
+	if driftSeverity(res.Apps[0].Drift) != "red" {
+		t.Fatalf("drift severity = %q", driftSeverity(res.Apps[0].Drift))
+	}
+}
+
+func TestPlatformApp(t *testing.T) {
+	p, done := platformStub(t, "t", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v1/apps/hanzoai/iam/main" {
+			t.Errorf("path = %s", r.URL.Path)
+		}
+		if r.URL.Query().Get("org") != "hanzoai" {
+			t.Errorf("org query = %s", r.URL.RawQuery)
+		}
+		_ = json.NewEncoder(w).Encode(AppView{ID: "hanzoai/iam/main", App: "iam"})
+	})
+	defer done()
+	a, err := p.App(context.Background(), "hanzoai/iam/main", "hanzoai")
+	if err != nil || a.App != "iam" {
+		t.Fatalf("App: %v %+v", err, a)
+	}
+}
+
+func TestPlatformSyncApps(t *testing.T) {
+	p, done := platformStub(t, "t", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPost || r.URL.Path != "/v1/apps/sync" {
+			t.Errorf("sync = %s %s", r.Method, r.URL.Path)
+		}
+		w.WriteHeader(200)
+	})
+	defer done()
+	if err := p.SyncApps(context.Background()); err != nil {
+		t.Fatalf("SyncApps: %v", err)
+	}
+}
+
+func TestPlatformClustersAndProvision(t *testing.T) {
+	p, done := platformStub(t, "t", func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.Method == http.MethodGet && r.URL.Path == "/v1/org/acme/cluster":
+			_ = json.NewEncoder(w).Encode(map[string]any{"clusters": []Cluster{{DoksClusterID: "c1", Name: "hanzo-acme", Region: "sfo3", Status: "running", Phase: "ready", Active: true}}})
+		case r.Method == http.MethodPost && r.URL.Path == "/v1/org/acme/cluster":
+			body, _ := io.ReadAll(r.Body)
+			var req ProvisionReq
+			_ = json.Unmarshal(body, &req)
+			if req.Region != "sfo3" || !req.HA {
+				t.Errorf("provision body = %+v", req)
+			}
+			w.WriteHeader(201)
+			_ = json.NewEncoder(w).Encode(map[string]any{"cluster": Cluster{DoksClusterID: "c2", Name: "new", Phase: "requested"}})
+		default:
+			t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
+		}
+	})
+	defer done()
+
+	cs, err := p.Clusters(context.Background(), "acme")
+	if err != nil || len(cs) != 1 || cs[0].DoksClusterID != "c1" {
+		t.Fatalf("Clusters: %v %+v", err, cs)
+	}
+	c, err := p.ProvisionCluster(context.Background(), "acme", ProvisionReq{Region: "sfo3", HA: true})
+	if err != nil || c.DoksClusterID != "c2" {
+		t.Fatalf("ProvisionCluster: %v %+v", err, c)
+	}
+}
+
+func TestPlatformTargetAndSelect(t *testing.T) {
+	p, done := platformStub(t, "t", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v1/org/acme/cluster/select" {
+			t.Errorf("path = %s", r.URL.Path)
+		}
+		if r.Method == http.MethodPost {
+			body, _ := io.ReadAll(r.Body)
+			var m map[string]any
+			_ = json.Unmarshal(body, &m)
+			if m["doksClusterId"] != "c1" {
+				t.Errorf("select body = %v", m)
+			}
+		}
+		_ = json.NewEncoder(w).Encode(map[string]any{"target": Target{Cluster: "hanzo-acme", Dedicated: true, Namespaces: map[string]string{"acme": "main"}}})
+	})
+	defer done()
+
+	tg, err := p.Target(context.Background(), "acme")
+	if err != nil || tg.Cluster != "hanzo-acme" || !tg.Dedicated {
+		t.Fatalf("Target: %v %+v", err, tg)
+	}
+	id := "c1"
+	if _, err := p.SelectTarget(context.Background(), "acme", &id); err != nil {
+		t.Fatalf("SelectTarget: %v", err)
+	}
+}
+
+func TestPlatformInstallBaseline(t *testing.T) {
+	p, done := platformStub(t, "t", func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPost || r.URL.Path != "/v1/org/acme/cluster/c1/install-baseline" {
+			t.Errorf("install-baseline = %s %s", r.Method, r.URL.Path)
+		}
+		w.WriteHeader(200)
+	})
+	defer done()
+	if err := p.InstallBaseline(context.Background(), "acme", "c1"); err != nil {
+		t.Fatalf("InstallBaseline: %v", err)
+	}
+}
+
+func TestPlatformRedeploy(t *testing.T) {
+	p, done := platformStub(t, "t", func(w http.ResponseWriter, r *http.Request) {
+		want := "/v1/org/acme/project/p1/env/e1/container/app-x/redeploy"
+		if r.Method != http.MethodPost || r.URL.Path != want {
+			t.Errorf("redeploy path = %s %s", r.Method, r.URL.Path)
+		}
+		_ = json.NewEncoder(w).Encode(map[string]bool{"ok": true})
+	})
+	defer done()
+	if err := p.Redeploy(context.Background(), "acme", "p1", "e1", "app-x"); err != nil {
+		t.Fatalf("Redeploy: %v", err)
+	}
+}
+
+func TestPlatformRedeployNotOK(t *testing.T) {
+	p, done := platformStub(t, "t", func(w http.ResponseWriter, _ *http.Request) {
+		_ = json.NewEncoder(w).Encode(map[string]bool{"ok": false})
+	})
+	defer done()
+	if err := p.Redeploy(context.Background(), "o", "p", "e", "c"); err == nil {
+		t.Fatalf("expected error when ok=false")
+	}
+}
+
+func TestPlatformEnqueueBuild(t *testing.T) {
+	p, done := platformStub(t, "svc-tok", func(w http.ResponseWriter, r *http.Request) {
+		// The build endpoint must use the BUILD token, not the service token.
+		if got := r.Header.Get("Authorization"); got != "Bearer build-tok" {
+			t.Errorf("build auth header = %q (must use build token)", got)
+		}
+		if r.URL.Path != "/v1/arcd/enqueue" {
+			t.Errorf("path = %s", r.URL.Path)
+		}
+		body, _ := io.ReadAll(r.Body)
+		var req BuildReq
+		_ = json.Unmarshal(body, &req)
+		if req.Repo != "hanzoai/pricing" || req.SHA != "abc123" || req.Image == "" {
+			t.Errorf("build body = %+v", req)
+		}
+		w.WriteHeader(202)
+		_ = json.NewEncoder(w).Encode(BuildJob{BuildJobID: "bj-1", Status: "queued", RunnerPool: "runner-pool-32g", Image: req.Image})
+	})
+	defer done()
+
+	job, err := p.EnqueueBuild(context.Background(), BuildReq{Repo: "hanzoai/pricing", SHA: "abc123", Image: "ghcr.io/hanzoai/pricing:t"}, "build-tok")
+	if err != nil || job.BuildJobID != "bj-1" {
+		t.Fatalf("EnqueueBuild: %v %+v", err, job)
+	}
+	if _, err := p.EnqueueBuild(context.Background(), BuildReq{Repo: "r", SHA: "s", Image: "i"}, ""); err == nil {
+		t.Fatalf("expected error with empty build token")
+	}
+}
+
+func TestPlatformError401Hint(t *testing.T) {
+	p, done := platformStub(t, "bad", func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(401)
+		_ = json.NewEncoder(w).Encode(map[string]string{"message": "Unauthorized"})
+	})
+	defer done()
+	_, err := p.Apps(context.Background(), AppsQuery{})
+	if err == nil || !strings.Contains(err.Error(), "HTTP 401") || !strings.Contains(err.Error(), "platform service token") {
+		t.Fatalf("401 error should carry a token hint, got %v", err)
+	}
+}
+
+func TestPlatformNoTokenError(t *testing.T) {
+	p := newPlatform("https://platform.hanzo.ai", "")
+	if _, err := p.Apps(context.Background(), AppsQuery{}); err == nil || !strings.Contains(err.Error(), "no platform token") {
+		t.Fatalf("expected no-token error, got %v", err)
+	}
+}

cli/quiet.go

@@ -0,0 +1,24 @@
+package cli
+
+import "os"
+
+// realStdout is the process's true stdout, captured at this package's init
+// before the server-graph dependencies (iam/beego, kms) run their own init()
+// functions — several of which emit warnings to stdout (e.g. the IAM registry
+// signing-key loader). To keep the CLI's stdout machine-readable (so
+// `hanzo apps list -o json | jq` is never corrupted by a dependency's startup
+// chatter), this init redirects stdout to stderr for the duration of process
+// initialization; RestoreStdout puts the real stdout back before any command
+// writes a byte.
+//
+// This is best-effort: it only helps when this package initializes before the
+// noisy dependency (cmd/hanzo imports cli first) AND that dependency reads the
+// os.Stdout variable at log time rather than capturing it earlier. main always
+// calls RestoreStdout, so correctness never depends on the redirect taking.
+var realStdout = os.Stdout
+
+func init() { os.Stdout = os.Stderr }
+
+// RestoreStdout restores the real process stdout. cmd/hanzo calls this as its
+// first statement so every command writes to the genuine stdout.
+func RestoreStdout() { os.Stdout = realStdout }

cli/security.go

@@ -0,0 +1,259 @@
+package cli
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/hanzoai/cloud/clients/security/detect"
+	"github.com/spf13/cobra"
+)
+
+// `hanzo security scan` is the LOCAL guardrail-at-generation: it walks a path,
+// runs the SAME detect engine the /v1/security server surface uses (one engine,
+// two surfaces — see clients/security/detect), and exits non-zero when a finding
+// at or above the fail threshold is present. No server, no auth, no network — so
+// it drops straight into a pre-commit hook, a CI step, or an agent's shell the
+// moment code is written. It never prints a raw secret: findings carry the
+// engine's masked preview only.
+
+// skipDirs are never descended into — vendored / generated / VCS trees that
+// would drown real findings in noise (and, for node_modules, minified blobs
+// that trip entropy heuristics).
+var skipDirs = map[string]bool{
+	".git": true, "node_modules": true, "vendor": true, "dist": true,
+	"build": true, "out": true, ".vscode-test": true, "target": true,
+	".next": true, "__pycache__": true, ".venv": true, "venv": true,
+}
+
+// maxScanFileBytes caps a single file read; a source file over this is almost
+// certainly a data/blob artifact, not code a human wrote a secret into.
+const maxScanFileBytes = 2 << 20 // 2 MiB
+
+func newSecurityCmd(envOf func() *Env) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "security",
+		Short: "Code-security tools (local secret scanning)",
+		// No PersistentPreRunE override: the root's runs, resolving config/creds
+		// into env so envOf() (used for -o json) is non-nil. It is local-only —
+		// scan requires no login or network.
+	}
+
+	var failOn string
+	scan := &cobra.Command{
+		Use:   "scan [path ...]",
+		Short: "Scan files/directories for hardcoded secrets (default: current dir)",
+		Long: "Walk each path and report hardcoded secrets using the native Hanzo\n" +
+			"detection engine (the same one behind /v1/security). Exits non-zero when a\n" +
+			"finding at or above --fail-on is present, so it gates a pre-commit hook or CI\n" +
+			"step. Secrets are never printed — only a masked preview.",
+		// We print our own findings + a clean "N secrets" error; no cobra usage
+		// dump on a policy failure.
+		SilenceUsage:  true,
+		SilenceErrors: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			paths := args
+			if len(paths) == 0 {
+				paths = []string{"."}
+			}
+			floor := strings.ToLower(strings.TrimSpace(failOn))
+			if floor == "" {
+				floor = "low"
+			}
+			if floor != "none" && detect.SeverityRank(floor) == 0 {
+				return fmt.Errorf("invalid --fail-on %q (want critical|high|medium|low|none)", failOn)
+			}
+
+			findings, scanned, err := scanPaths(paths)
+			if err != nil {
+				return err
+			}
+
+			e := envOf()
+			result := scanResult{
+				FilesScanned: scanned,
+				Findings:     findings,
+				Summary:      tally(findings),
+			}
+			if err := e.emit(result, func(w io.Writer) { renderScan(w, result) }); err != nil {
+				return err
+			}
+
+			// Policy gate: fail when any finding is at/above the floor.
+			if floor != "none" {
+				bad := 0
+				for _, f := range findings {
+					if detect.SeverityRank(f.Severity) >= detect.SeverityRank(floor) {
+						bad++
+					}
+				}
+				if bad > 0 {
+					return fmt.Errorf("%d secret(s) at or above %q severity", bad, floor)
+				}
+			}
+			return nil
+		},
+	}
+	scan.Flags().StringVar(&failOn, "fail-on", "low",
+		"minimum severity that fails the command: critical|high|medium|low|none")
+
+	rules := &cobra.Command{
+		Use:   "rules",
+		Short: "List the detection rules the scanner applies",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			e := envOf()
+			rv := detect.Rules()
+			return e.emit(rv, func(w io.Writer) {
+				tw := newTab(w)
+				fmt.Fprintln(tw, "SEVERITY\tID\tNAME")
+				for _, r := range rv {
+					fmt.Fprintf(tw, "%s\t%s\t%s\n", r.Severity, r.ID, r.Name)
+				}
+				tw.Flush()
+				fmt.Fprintf(w, "\n%d rules\n", len(rv))
+			})
+		},
+	}
+
+	cmd.AddCommand(scan, rules)
+	return cmd
+}
+
+// scanFinding is a CLI finding: the engine finding with the file path it was
+// found in (the engine echoes the path it was handed, which is what we want).
+type scanResult struct {
+	FilesScanned int              `json:"filesScanned"`
+	Findings     []detect.Finding `json:"findings"`
+	Summary      scanSummary      `json:"summary"`
+}
+
+type scanSummary struct {
+	Total    int `json:"total"`
+	Critical int `json:"critical"`
+	High     int `json:"high"`
+	Medium   int `json:"medium"`
+	Low      int `json:"low"`
+}
+
+func tally(fs []detect.Finding) scanSummary {
+	var s scanSummary
+	for _, f := range fs {
+		s.Total++
+		switch f.Severity {
+		case detect.SeverityCritical:
+			s.Critical++
+		case detect.SeverityHigh:
+			s.High++
+		case detect.SeverityMedium:
+			s.Medium++
+		case detect.SeverityLow:
+			s.Low++
+		}
+	}
+	return s
+}
+
+// scanPaths walks each path and runs the engine over every readable text file,
+// returning findings sorted worst-first (by severity, then path, then line).
+func scanPaths(paths []string) ([]detect.Finding, int, error) {
+	var findings []detect.Finding
+	scanned := 0
+	for _, root := range paths {
+		info, err := os.Stat(root)
+		if err != nil {
+			return nil, 0, fmt.Errorf("stat %q: %w", root, err)
+		}
+		if !info.IsDir() {
+			fs, ok := scanOneFile(root)
+			if ok {
+				scanned++
+				findings = append(findings, fs...)
+			}
+			continue
+		}
+		walkErr := filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error {
+			if err != nil {
+				return nil // unreadable entry — skip, don't abort the whole walk
+			}
+			if d.IsDir() {
+				if skipDirs[d.Name()] {
+					return filepath.SkipDir
+				}
+				return nil
+			}
+			if fs, ok := scanOneFile(p); ok {
+				scanned++
+				findings = append(findings, fs...)
+			}
+			return nil
+		})
+		if walkErr != nil {
+			return nil, 0, fmt.Errorf("walk %q: %w", root, walkErr)
+		}
+	}
+	sort.SliceStable(findings, func(i, j int) bool {
+		a, b := findings[i], findings[j]
+		if ra, rb := detect.SeverityRank(a.Severity), detect.SeverityRank(b.Severity); ra != rb {
+			return ra > rb
+		}
+		if a.Path != b.Path {
+			return a.Path < b.Path
+		}
+		return a.Line < b.Line
+	})
+	return findings, scanned, nil
+}
+
+// scanOneFile reads a file (bounded, text-only) and runs the engine. Returns
+// ok=false for a file that was skipped (too big, binary, unreadable) so it is
+// not counted as scanned.
+func scanOneFile(path string) ([]detect.Finding, bool) {
+	info, err := os.Stat(path)
+	if err != nil || info.Size() > maxScanFileBytes {
+		return nil, false
+	}
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return nil, false
+	}
+	if isBinary(b) {
+		return nil, false
+	}
+	return detect.ScanContent(path, string(b)), true
+}
+
+// isBinary reports whether b looks like a non-text blob — a NUL byte in the
+// first 8 KiB is the same heuristic git uses. Skipping binaries avoids both
+// false positives and wasted work on assets.
+func isBinary(b []byte) bool {
+	n := len(b)
+	if n > 8192 {
+		n = 8192
+	}
+	for i := 0; i < n; i++ {
+		if b[i] == 0 {
+			return true
+		}
+	}
+	return false
+}
+
+func renderScan(w io.Writer, r scanResult) {
+	if len(r.Findings) == 0 {
+		fmt.Fprintf(w, "✓ no secrets found (%d files scanned)\n", r.FilesScanned)
+		return
+	}
+	tw := newTab(w)
+	fmt.Fprintln(tw, "SEVERITY\tRULE\tLOCATION\tPREVIEW")
+	for _, f := range r.Findings {
+		fmt.Fprintf(tw, "%s\t%s\t%s:%d\t%s\n", f.Severity, f.RuleID, f.Path, f.Line, f.Preview)
+	}
+	tw.Flush()
+	fmt.Fprintf(w, "\n%d finding(s) in %d files  (critical=%d high=%d medium=%d low=%d)\n",
+		r.Summary.Total, r.FilesScanned, r.Summary.Critical, r.Summary.High,
+		r.Summary.Medium, r.Summary.Low)
+}

cli/security_test.go

@@ -0,0 +1,176 @@
+package cli
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// runSecurity executes `security <args>` through the real root command with an
+// isolated $HOME, capturing stdout. Returns output + the Execute error (the
+// non-zero-exit signal).
+func runSecurity(t *testing.T, args ...string) (string, error) {
+	t.Helper()
+	sandbox(t)
+	root := newRootCmd()
+	var out bytes.Buffer
+	root.SetOut(&out)
+	root.SetErr(&out)
+	root.SetArgs(append([]string{"security"}, args...))
+	err := root.Execute()
+	return out.String(), err
+}
+
+// TestSecurityScanFindsAndFails proves scan detects a planted secret, exits
+// non-zero at the default fail threshold, and never prints the raw secret.
+func TestSecurityScanFindsAndFails(t *testing.T) {
+	dir := t.TempDir()
+	secret := "AKIAIOSFODNN7EXAMPLE"
+	must(t, os.WriteFile(filepath.Join(dir, "config.py"),
+		[]byte("aws_key = \""+secret+"\"\nok = 1\n"), 0o644))
+	// a clean file that must NOT trip anything
+	must(t, os.WriteFile(filepath.Join(dir, "clean.go"),
+		[]byte("package main\nfunc main() {}\n"), 0o644))
+
+	out, err := runSecurity(t, "scan", dir)
+	if err == nil {
+		t.Fatalf("expected non-zero exit on a found secret; out:\n%s", out)
+	}
+	if !strings.Contains(err.Error(), "at or above") {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if strings.Contains(out, secret) {
+		t.Fatalf("output leaked the raw secret:\n%s", out)
+	}
+	if !strings.Contains(out, "aws-access-key-id") {
+		t.Fatalf("expected the aws rule id in output:\n%s", out)
+	}
+}
+
+// TestSecurityScanCleanPasses proves a clean tree exits zero with the ok line.
+func TestSecurityScanCleanPasses(t *testing.T) {
+	dir := t.TempDir()
+	must(t, os.WriteFile(filepath.Join(dir, "app.go"),
+		[]byte("package main\nfunc main() { println(\"hi\") }\n"), 0o644))
+
+	out, err := runSecurity(t, "scan", dir)
+	if err != nil {
+		t.Fatalf("clean tree should exit zero, got %v\n%s", err, out)
+	}
+	if !strings.Contains(out, "no secrets found") {
+		t.Fatalf("expected the clean message:\n%s", out)
+	}
+}
+
+// TestSecurityScanFailOnNone proves --fail-on=none reports findings but exits
+// zero (report-only mode).
+func TestSecurityScanFailOnNone(t *testing.T) {
+	dir := t.TempDir()
+	must(t, os.WriteFile(filepath.Join(dir, "s.py"),
+		[]byte(`k = "AKIAIOSFODNN7EXAMPLE"`), 0o644))
+
+	out, err := runSecurity(t, "scan", "--fail-on", "none", dir)
+	if err != nil {
+		t.Fatalf("--fail-on=none should exit zero, got %v\n%s", err, out)
+	}
+	if !strings.Contains(out, "aws-access-key-id") {
+		t.Fatalf("report-only should still list the finding:\n%s", out)
+	}
+}
+
+// TestSecurityScanFailOnThreshold proves a medium finding does NOT fail when
+// --fail-on=critical, but a critical one does.
+func TestSecurityScanFailOnThreshold(t *testing.T) {
+	dir := t.TempDir()
+	// a jwt is medium severity
+	must(t, os.WriteFile(filepath.Join(dir, "t.txt"),
+		[]byte("tok = eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhYmMifQ.SGVsbG9TaWduYXR1cmU\n"), 0o644))
+
+	if out, err := runSecurity(t, "scan", "--fail-on", "critical", dir); err != nil {
+		t.Fatalf("medium finding must not fail at --fail-on=critical, got %v\n%s", err, out)
+	}
+	// now add a critical
+	must(t, os.WriteFile(filepath.Join(dir, "k.py"),
+		[]byte(`k = "AKIAIOSFODNN7EXAMPLE"`), 0o644))
+	if _, err := runSecurity(t, "scan", "--fail-on", "critical", dir); err == nil {
+		t.Fatal("a critical finding must fail at --fail-on=critical")
+	}
+}
+
+// TestSecurityScanJSON proves -o json emits a machine-readable result with the
+// summary and no raw secret.
+func TestSecurityScanJSON(t *testing.T) {
+	dir := t.TempDir()
+	secret := "AKIAIOSFODNN7EXAMPLE"
+	must(t, os.WriteFile(filepath.Join(dir, "s.py"), []byte(`k = "`+secret+`"`), 0o644))
+
+	out, _ := runSecurity(t, "scan", "-o", "json", "--fail-on", "none", dir)
+	var res scanResult
+	if err := json.Unmarshal([]byte(out), &res); err != nil {
+		t.Fatalf("json parse: %v\n%s", err, out)
+	}
+	if res.Summary.Critical < 1 || res.Summary.Total < 1 {
+		t.Fatalf("summary missing the critical: %+v", res.Summary)
+	}
+	if strings.Contains(out, secret) {
+		t.Fatalf("json leaked the secret:\n%s", out)
+	}
+}
+
+// TestSecurityScanSkipsVendorAndBinary proves the walker skips skipDirs and
+// binary files (a secret inside node_modules or a NUL-laden blob is ignored).
+func TestSecurityScanSkipsVendorAndBinary(t *testing.T) {
+	dir := t.TempDir()
+	nm := filepath.Join(dir, "node_modules")
+	must(t, os.MkdirAll(nm, 0o755))
+	must(t, os.WriteFile(filepath.Join(nm, "dep.js"),
+		[]byte(`k = "AKIAIOSFODNN7EXAMPLE"`), 0o644))
+	must(t, os.WriteFile(filepath.Join(dir, "blob.bin"),
+		append([]byte{0, 1, 2}, []byte(`AKIAIOSFODNN7EXAMPLE`)...), 0o644))
+
+	out, err := runSecurity(t, "scan", dir)
+	if err != nil {
+		t.Fatalf("vendored + binary secrets should be skipped → clean exit, got %v\n%s", err, out)
+	}
+	if !strings.Contains(out, "no secrets found") {
+		t.Fatalf("expected clean (skipped) result:\n%s", out)
+	}
+}
+
+// TestSecurityScanBadFailOn proves an invalid --fail-on is a clean error.
+func TestSecurityScanBadFailOn(t *testing.T) {
+	dir := t.TempDir()
+	_, err := runSecurity(t, "scan", "--fail-on", "nope", dir)
+	if err == nil || !strings.Contains(err.Error(), "invalid --fail-on") {
+		t.Fatalf("want invalid --fail-on error, got %v", err)
+	}
+}
+
+// TestSecurityRules proves the rules subcommand lists the catalog.
+func TestSecurityRules(t *testing.T) {
+	out, err := runSecurity(t, "rules")
+	if err != nil {
+		t.Fatalf("rules failed: %v", err)
+	}
+	if !strings.Contains(out, "aws-access-key-id") || !strings.Contains(out, "rules") {
+		t.Fatalf("rules output missing content:\n%s", out)
+	}
+}
+
+// TestSecurityIsControlVerb proves `security` routes to the CLI, not the server
+// dispatcher.
+func TestSecurityIsControlVerb(t *testing.T) {
+	if !IsControlVerb("security") {
+		t.Fatal("security must be a control verb")
+	}
+}
+
+func must(t *testing.T, err error) {
+	t.Helper()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

clients/admin/admin.go

@@ -0,0 +1,556 @@
+// Package admin mounts the god-mode admin surface (/v1/admin/*) the Hanzo
+// Admin Console (admin.hanzo.ai, apps/operator) calls, per the api.ts contract.
+//
+// It is an AGGREGATOR, not a new store: identity (orgs/users/roles/applications/
+// audit/me) is read from IAM, the money panels (spend/tokens/credits) from
+// commerce, and System Health from o11y — every one a real upstream, none fused
+// into this binary (see subsystems.go). The facade fans out over HTTP exactly
+// like o11ysvc / productsvc: it holds no business logic, it shapes the reads into
+// the /v1 envelope { status, msg, data, data2 } the operator's transport
+// decodes (get<T> reads data; getList<T> reads data + data2 total).
+//
+// SECURITY — every route is GLOBAL-ADMIN ONLY, fail-closed. The gate is the
+// SAME predicate the rest of cloud uses: c.IsAdmin(), which after SanitizeIdentity
+// (serve.go) is true ONLY for a JWT-validated principal whose org is the admin org
+// (owner == AdminOrg — IAM's IsGlobalAdmin), matching the gateway's admin-guard.
+// No principal → 403; a tenant-admin (owner != AdminOrg) → 403; a forged
+// X-User-IsAdmin never survives ingress. admin adds no service credential to
+// the IAM fan-out — it replays the caller's own cookie/bearer, so it can never
+// read more than the caller already could, and IAM re-checks IsGlobalAdmin too.
+//
+// Panels with no in-binary feed yet (the Usage & Costs timeseries + per-product
+// breakdown live in insights/datastore; the product/workload registry + infra
+// tiles live in platform.hanzo.ai / the operator inventory) return the real,
+// honest empty state — never a fabricated number. The operator UI renders those
+// as an em-dash / empty table by design.
+package admin
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/hanzoai/cloud"
+	"github.com/hanzoai/cloud/audit"
+	"github.com/zap-proto/zip"
+)
+
+// svc holds the resolved upstream clients + the admin org for this deployment.
+type svc struct {
+	iam      *iamClient
+	commerce *commerceClient
+	health   *healthClient
+	do       *doClient
+	adminOrg string
+	// auditStore is cloud's OWN tamper-evident audit store (nil when unconfigured,
+	// in which case /v1/admin/audit falls back to the IAM get-records proxy). Serve
+	// builds it and hands it over via deps.Audit. See audit.go.
+	auditStore *audit.Recorder
+}
+
+// Mount registers the /v1/admin/* surface on app. Every handler gates on
+// c.IsAdmin() first (global-admin only), then aggregates real upstream data.
+func Mount(app *zip.App, deps cloud.Deps) error {
+	if app == nil {
+		return fmt.Errorf("admin.Mount: nil zip.App")
+	}
+	logger := deps.Logger
+	if logger == nil {
+		return fmt.Errorf("admin.Mount: nil deps.Logger")
+	}
+	logger = logger.New("subsystem", "admin")
+
+	s := &svc{
+		iam:        newIAMClient(iamBase(deps)),
+		commerce:   newCommerceClient(os.Getenv("CLOUD_COMMERCE_HTTP_URL"), os.Getenv("COMMERCE_SERVICE_TOKEN")),
+		health:     newHealthClient(o11yHealthURL()),
+		do:         newDOClient(doTokenFromEnv()),
+		adminOrg:   adminOrgOf(deps),
+		auditStore: deps.Audit,
+	}
+
+	app.Get("/v1/admin/me", s.guard(s.me))
+	app.Get("/v1/admin/overview", s.guard(s.overview))
+	app.Get("/v1/admin/orgs", s.guard(s.orgs))
+	app.Get("/v1/admin/users", s.guard(s.users))
+	app.Get("/v1/admin/roles", s.guard(s.roles))
+	app.Get("/v1/admin/applications", s.guard(s.applications))
+	app.Get("/v1/admin/audit", s.guard(s.audit))
+	app.Get("/v1/admin/audit/verify", s.guard(s.auditVerify))
+	app.Get("/v1/admin/usage", s.guard(s.usage))
+	app.Get("/v1/admin/products", s.guard(s.products))
+	app.Get("/v1/admin/finance", s.guard(s.finance))
+	app.Get("/v1/admin/compute", s.guard(s.compute))
+	app.Get("/v1/admin/o11y", s.guard(s.o11y))
+	app.Post("/v1/admin/sync", s.guard(s.sync))
+
+	// Customer management — the operator cockpit. List (static) precedes the :org
+	// param route; the write actions are POST (distinct method), so none collide.
+	app.Get("/v1/admin/customers", s.guard(s.customers))
+	app.Get("/v1/admin/customers/:org", s.guard(s.customerDetail))
+	app.Post("/v1/admin/customers/:org/credit", s.guard(s.grantCredit))
+	app.Post("/v1/admin/customers/:org/suspend", s.guard(s.suspendCustomer))
+	app.Post("/v1/admin/customers/:org/reactivate", s.guard(s.reactivateCustomer))
+
+	// Fleet revenue aggregate + native SaaS analytics (retention/growth/churn).
+	app.Get("/v1/admin/revenue", s.guard(s.revenue))
+	app.Get("/v1/admin/analytics", s.guard(s.analytics))
+
+	logger.Info("admin surface mounted",
+		"prefix", "/v1/admin",
+		"iam", s.iam.configured(),
+		"commerce", s.commerce.configured(),
+		"digitalocean", s.do.configured(),
+		"adminOrg", s.adminOrg,
+	)
+	return nil
+}
+
+// guard wraps a handler with the global-admin gate. Fail-closed: any request
+// whose validated identity is not a global admin (X-User-IsAdmin != "true",
+// which SanitizeIdentity sets only for owner == AdminOrg) is refused 403 before
+// the handler — no upstream is touched, no data leaks.
+func (s *svc) guard(h func(*zip.Ctx) error) zip.Handler {
+	return func(c *zip.Ctx) error {
+		if !c.IsAdmin() {
+			return zip.ErrForbidden("global admin required")
+		}
+		return h(c)
+	}
+}
+
+// callerCreds captures the caller's replayed authorization context for the IAM
+// fan-out: the raw Cookie header (session model) and the Authorization bearer.
+func callerCreds(c *zip.Ctx) creds {
+	return creds{
+		cookie: string(c.Fiber().Request().Header.Peek("Cookie")),
+		auth:   c.Header("Authorization"),
+	}
+}
+
+// ── /v1 envelope writers ────────────────────────────────────────────────
+
+// ok writes a { status:"ok", data } envelope (the get<T> shape).
+func ok(c *zip.Ctx, data any) error {
+	return c.JSON(200, map[string]any{"status": "ok", "msg": "", "data": data})
+}
+
+// okList writes a { status:"ok", data:[...], data2:total } envelope (getList<T>).
+func okList(c *zip.Ctx, rows any, total int) error {
+	return c.JSON(200, map[string]any{"status": "ok", "msg": "", "data": rows, "data2": total})
+}
+
+// okRaw writes a { status:"ok", data:<raw>, data2:total } envelope, forwarding an
+// IAM payload verbatim so its exact wire shape (Role, Application, Record, User)
+// reaches the operator field-for-field.
+func okRaw(c *zip.Ctx, rows json.RawMessage, total int) error {
+	if len(rows) == 0 {
+		rows = json.RawMessage("[]")
+	}
+	return c.JSON(200, map[string]any{"status": "ok", "msg": "", "data": rows, "data2": total})
+}
+
+// fail writes a { status:"error", msg } envelope. The operator's transport maps
+// a non-ok envelope to a surfaced error (never a fabricated value).
+func fail(c *zip.Ctx, msg string) error {
+	return c.JSON(200, map[string]any{"status": "error", "msg": msg, "data": nil})
+}
+
+// ── /v1/admin/me — operator identity (AdminMe) ───────────────────────────────
+
+// me answers with the validated operator identity. The gate already proved this
+// is a global admin, so the fields come from the sanitized identity headers —
+// authoritative and never client-forgeable.
+func (s *svc) me(c *zip.Ctx) error {
+	owner := s.adminOrg
+	if o := strings.TrimSpace(c.Org()); o != "" {
+		owner = o
+	}
+	name := strings.TrimSpace(c.User())
+	return ok(c, adminMe{
+		Owner:         owner,
+		Name:          name,
+		Email:         strings.TrimSpace(c.UserEmail()),
+		DisplayName:   name,
+		IsGlobalAdmin: true,
+	})
+}
+
+// ── /v1/admin/orgs — tenant directory (OrgRow[]) ─────────────────────────────
+
+func (s *svc) orgs(c *zip.Ctx) error {
+	ctx := c.Context()
+	cr := callerCreds(c)
+	orgs, err := s.listOrgs(ctx, cr)
+	if err != nil {
+		return fail(c, err.Error())
+	}
+	rows := make([]orgRow, 0, len(orgs))
+	for _, o := range orgs {
+		users := s.orgUserCount(ctx, cr, o.Name)
+		spend, credits := s.orgMoney(ctx, o.Name)
+		rows = append(rows, orgRow{
+			Org:          o.Name,
+			Display:      display(o.DisplayName, o.Name),
+			Users:        users,
+			Products:     0, // workload registry feed pending (platform apps table)
+			SpendCents:   spend,
+			CreditsCents: credits,
+			Tokens:       0, // fleet token counters pending (insights/datastore)
+			Created:      o.CreatedTime,
+		})
+	}
+	sort.Slice(rows, func(i, j int) bool { return rows[i].Org < rows[j].Org })
+	return okList(c, rows, len(rows))
+}
+
+// ── /v1/admin/users — cross-org directory (OperatorUser[]) ───────────────────
+
+func (s *svc) users(c *zip.Ctx) error {
+	ctx := c.Context()
+	cr := callerCreds(c)
+	q := url.Values{}
+	if owner := strings.TrimSpace(c.Query("org")); owner != "" {
+		q.Set("owner", owner)
+	}
+	if p := strings.TrimSpace(c.Query("p")); p != "" {
+		q.Set("p", p)
+	}
+	if ps := strings.TrimSpace(c.Query("pageSize")); ps != "" {
+		q.Set("pageSize", ps)
+	}
+	if term := strings.TrimSpace(c.Query("q")); term != "" {
+		// IAM's list uses field/value contains-matching for the free-text filter.
+		q.Set("field", "name")
+		q.Set("value", term)
+	}
+	res, err := s.iam.getList(ctx, cr, "/v1/iam/get-users", q)
+	if err != nil {
+		return fail(c, err.Error())
+	}
+	var raw []iamUser
+	if len(res.rows) > 0 {
+		if err := json.Unmarshal(res.rows, &raw); err != nil {
+			return fail(c, "users decode: "+err.Error())
+		}
+	}
+	rows := make([]operatorUser, 0, len(raw))
+	for _, u := range raw {
+		rows = append(rows, operatorUser{
+			Owner:         u.Owner,
+			Name:          u.Name,
+			Email:         u.Email,
+			DisplayName:   u.DisplayName,
+			IsAdmin:       u.IsAdmin,
+			IsGlobalAdmin: u.Owner == s.adminOrg,
+			Tag:           u.Tag,
+			Created:       u.CreatedTime,
+			LastSignin:    u.LastSigninTime,
+			Forbidden:     u.IsForbidden,
+		})
+	}
+	total := res.total
+	if total < len(rows) {
+		total = len(rows)
+	}
+	return okList(c, rows, total)
+}
+
+// ── /v1/admin/roles and /applications — verbatim IAM passthrough ─────────────
+
+func (s *svc) roles(c *zip.Ctx) error {
+	return s.iamPassthrough(c, "/v1/iam/get-roles")
+}
+
+func (s *svc) applications(c *zip.Ctx) error {
+	return s.iamPassthrough(c, "/v1/iam/get-applications")
+}
+
+// iamPassthrough forwards a paginated IAM read verbatim (the operator decodes
+// Role / Application as the raw IAM wire shape). `owner` defaults to the admin
+// org, which owns the platform applications.
+func (s *svc) iamPassthrough(c *zip.Ctx, path string) error {
+	q := url.Values{}
+	owner := strings.TrimSpace(c.Query("owner"))
+	if owner == "" {
+		owner = s.adminOrg
+	}
+	q.Set("owner", owner)
+	if p := strings.TrimSpace(c.Query("p")); p != "" {
+		q.Set("p", p)
+	}
+	if ps := strings.TrimSpace(c.Query("pageSize")); ps != "" {
+		q.Set("pageSize", ps)
+	}
+	res, err := s.iam.getList(c.Context(), callerCreds(c), path, q)
+	if err != nil {
+		return fail(c, err.Error())
+	}
+	return okRaw(c, res.rows, res.total)
+}
+
+// ── /v1/admin/audit — records directory (AuditRow[]) ─────────────────────────
+//
+// The handler lives in audit.go (it reads cloud's OWN tamper-evident store).
+// iamAuditQuery builds the IAM get-records query for the federated fallback
+// auditFromIAM uses when no local store is configured.
+
+func iamAuditQuery(c *zip.Ctx) url.Values {
+	q := url.Values{}
+	if org := strings.TrimSpace(c.Query("org")); org != "" {
+		q.Set("organizationName", org)
+	}
+	q.Set("p", "1")
+	ps := strings.TrimSpace(c.Query("pageSize"))
+	if ps == "" {
+		ps = "100"
+	}
+	q.Set("pageSize", ps)
+	q.Set("sortField", "createdTime")
+	q.Set("sortOrder", "descend")
+	return q
+}
+
+// ── /v1/admin/usage — fleet usage roll-up (UsageData) ────────────────────────
+
+// usage returns the real fleet money totals from commerce. The daily series and
+// the per-product breakdown are NOT derivable from the commerce billing API
+// (they live in insights/datastore, owned separately); admin returns the
+// honest empty series/byProduct rather than fabricating a trend — the operator
+// renders that as an empty chart, never a fake line.
+func (s *svc) usage(c *zip.Ctx) error {
+	ctx := c.Context()
+	cr := callerCreds(c)
+	org := strings.TrimSpace(c.Query("org"))
+
+	var spend int64
+	if org != "" {
+		r, err := s.commerce.usageRollup(ctx, org, orgSubject(org))
+		if err == nil {
+			spend = r.ConsumedCents
+		}
+	} else {
+		// Fleet: sum month-to-date consumption across every org.
+		orgs, err := s.listOrgs(ctx, cr)
+		if err == nil {
+			for _, o := range orgs {
+				if r, e := s.commerce.usageRollup(ctx, o.Name, orgSubject(o.Name)); e == nil {
+					spend += r.ConsumedCents
+				}
+			}
+		}
+	}
+
+	return ok(c, usageData{
+		Totals:    usageTotals{SpendCents: spend, Tokens: 0, Requests: 0},
+		Series:    []usagePoint{},
+		ByProduct: []usageByProduct{},
+	})
+}
+
+// ── /v1/admin/products — workload registry (ProductRow[]) ────────────────────
+
+// products is the workload/drift registry (declared vs running tag, health).
+// That inventory is the platform.hanzo.ai apps table / operator reconcile state,
+// NOT an in-binary source. admin exposes the gated endpoint and returns the
+// real empty registry until that feed is wired — it never fabricates workload
+// rows. The operator renders an empty table, not fake products.
+func (s *svc) products(c *zip.Ctx) error {
+	return okList(c, []productRow{}, 0)
+}
+
+// ── /v1/admin/overview — Platform Overview tiles (OverviewData) ───────────────
+
+func (s *svc) overview(c *zip.Ctx) error {
+	ctx := c.Context()
+	cr := callerCreds(c)
+	now := time.Now().UTC().Format(time.RFC3339)
+
+	var sources []sourceStatus
+	orgCount, userCount, spend, credits := 0, 0, int64(0), int64(0)
+
+	orgs, orgErr := s.listOrgs(ctx, cr)
+	sources = append(sources, srcOf("iam", orgErr, len(orgs), now))
+	if orgErr == nil {
+		orgCount = len(orgs)
+		for _, o := range orgs {
+			userCount += s.orgUserCount(ctx, cr, o.Name)
+			sp, cr2 := s.orgMoney(ctx, o.Name)
+			spend += sp
+			credits += cr2
+		}
+	}
+
+	// Commerce freshness: probe one org's rollup so the tile reflects a real read.
+	commerceRows := 0
+	var commerceErr error
+	if s.commerce.configured() {
+		probe := s.adminOrg
+		if len(orgs) > 0 {
+			probe = orgs[0].Name
+		}
+		if _, err := s.commerce.usageRollup(ctx, probe, orgSubject(probe)); err != nil {
+			commerceErr = err
+		} else {
+			commerceRows = 1
+		}
+	} else {
+		commerceErr = fmt.Errorf("commerce endpoint not configured")
+	}
+	sources = append(sources, srcOf("commerce", commerceErr, commerceRows, now))
+
+	// o11y System Health.
+	o11yRows := 0
+	oOK, oErr := s.health.ok(ctx)
+	if oOK {
+		o11yRows = 1
+	}
+	sources = append(sources, srcOf("o11y", oErr, o11yRows, now))
+
+	return ok(c, overviewData{
+		Orgs:           orgCount,
+		Users:          userCount,
+		Products:       0, // workload registry feed pending (platform apps table)
+		ActiveProducts: 0,
+		Drift:          0,
+		SpendCents30d:  spend,
+		Tokens30d:      0, // fleet token counters pending (insights/datastore)
+		CreditsCents:   credits,
+		LastSync:       now,
+		Sources:        sources,
+	})
+}
+
+// ── /v1/admin/sync — refresh trigger ─────────────────────────────────────────
+
+// sync answers the operator's "Sync now" button. admin aggregates LIVE on
+// every read (there is no cached fleet snapshot in-binary), so there is no batch
+// job to kick — the button simply re-reads. We acknowledge honestly with
+// { started: true } so the UI re-fetches the (freshly-computed) overview.
+func (s *svc) sync(c *zip.Ctx) error {
+	return ok(c, map[string]bool{"started": true})
+}
+
+// ── aggregation helpers ──────────────────────────────────────────────────────
+
+// listOrgs reads the org directory (owner = admin org) as the typed shape the
+// overview/orgs/usage aggregators fold over.
+func (s *svc) listOrgs(ctx context.Context, cr creds) ([]iamOrg, error) {
+	q := url.Values{}
+	q.Set("owner", s.adminOrg)
+	res, err := s.iam.getList(ctx, cr, "/v1/iam/get-organizations", q)
+	if err != nil {
+		return nil, err
+	}
+	var orgs []iamOrg
+	if len(res.rows) > 0 {
+		if err := json.Unmarshal(res.rows, &orgs); err != nil {
+			return nil, fmt.Errorf("orgs decode: %w", err)
+		}
+	}
+	return orgs, nil
+}
+
+// orgUserCount returns the member count for one org from the IAM list total
+// (data2). Best-effort: an error yields 0 rather than failing the whole row.
+func (s *svc) orgUserCount(ctx context.Context, cr creds, org string) int {
+	q := url.Values{}
+	q.Set("owner", org)
+	q.Set("p", "1")
+	q.Set("pageSize", "1")
+	res, err := s.iam.getList(ctx, cr, "/v1/iam/get-users", q)
+	if err != nil {
+		return 0
+	}
+	return res.total
+}
+
+// orgMoney returns (spendCents, creditsCents) for one org from commerce.
+// Best-effort: unreachable/unconfigured commerce yields zeros.
+func (s *svc) orgMoney(ctx context.Context, org string) (int64, int64) {
+	subj := orgSubject(org)
+	var spend, credits int64
+	if r, err := s.commerce.usageRollup(ctx, org, subj); err == nil {
+		spend = r.ConsumedCents
+	}
+	if c, err := s.commerce.creditsCents(ctx, org, subj); err == nil {
+		credits = c
+	}
+	return spend, credits
+}
+
+// orgSubject is the billing subject commerce keys an org's wallet on. Commerce's
+// per-org billing store (the 2026-07 durability rework, commerce >=1.46.8)
+// namespaces by the TRUSTED X-Org-Id header (set by commerceClient.get from this
+// same org) and keys the org wallet under the BARE org slug as the `user` subject —
+// NOT "org/user". The prior "org/org" subject (with the wrong X-IAM-Org-Id header)
+// resolved to an EMPTY wallet, so every per-org money panel read $0 while real
+// balances existed (lux $10,000, maxpower $20,498). Verified live against commerce
+// /v1/billing/{balance,usage-rollup}: user=<org> + X-Org-Id=<org> returns the real
+// wallet; user="org/org" or a missing/other org header returns $0.
+func orgSubject(org string) string { return org }
+
+// srcOf builds a SourceStatus freshness row for the overview.
+func srcOf(name string, err error, rows int, at string) sourceStatus {
+	s := sourceStatus{Name: name, OK: err == nil, Rows: rows, At: at}
+	if err != nil {
+		s.Error = err.Error()
+	}
+	return s
+}
+
+func display(displayName, fallback string) string {
+	if strings.TrimSpace(displayName) != "" {
+		return displayName
+	}
+	return fallback
+}
+
+// ── config resolution ────────────────────────────────────────────────────────
+
+// iamBase resolves the IAM management HTTP base. CLOUD_IAM_HTTP_URL wins (the
+// in-cluster Service, e.g. http://iam.hanzo.svc.cluster.local:8000); otherwise
+// the public issuer (deps.IAMIssuer, e.g. https://hanzo.id) which also serves
+// /v1/iam/*. Empty only when neither is set (endpoint reports not-configured).
+func iamBase(deps cloud.Deps) string {
+	if v := strings.TrimSpace(os.Getenv("CLOUD_IAM_HTTP_URL")); v != "" {
+		return v
+	}
+	return strings.TrimSpace(deps.IAMIssuer)
+}
+
+// o11yHealthURL resolves the o11y health probe URL for the System Health source.
+// CLOUD_O11Y_HEALTH_URL wins; else the in-cluster o11y Service default.
+func o11yHealthURL() string {
+	if v := strings.TrimSpace(os.Getenv("CLOUD_O11Y_HEALTH_URL")); v != "" {
+		return v
+	}
+	return "http://o11y.hanzo.svc.cluster.local:80/v1/o11y/health"
+}
+
+// adminOrgOf resolves the admin org slug (IAM's IsGlobalAdmin owner). IAM_ADMIN_ORG
+// mirrors config.go's default; "admin" is the fleet-wide default.
+func adminOrgOf(_ cloud.Deps) string {
+	if v := strings.TrimSpace(os.Getenv("IAM_ADMIN_ORG")); v != "" {
+		return v
+	}
+	return "admin"
+}
+
+func init() {
+	// Order 146: after productsvc (145); the admin surface has no ordering
+	// dependency (it fans out over HTTP), placed adjacent to the other console
+	// read facades.
+	cloud.Register("admin", 146, func(app any, deps cloud.Deps) error {
+		a, ok := app.(*zip.App)
+		if !ok {
+			return fmt.Errorf("admin.Mount: app is %T, want *zip.App", app)
+		}
+		return Mount(a, deps)
+	})
+}

clients/admin/admin_test.go

@@ -0,0 +1,586 @@
+package admin
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	fiber "github.com/gofiber/fiber/v3"
+	"github.com/hanzoai/cloud"
+	"github.com/zap-proto/zip"
+	luxlog "github.com/luxfi/log"
+)
+
+// mount builds a zip app with admin mounted against the given upstream bases,
+// and returns a `do` helper that issues test requests through the whole app.
+func mount(t *testing.T, iamURL, commerceURL, healthURL string) func(method, path string, hdr map[string]string) (*http.Response, []byte) {
+	do, _, _ := mountSvc(t, iamURL, commerceURL, healthURL)
+	return do
+}
+
+// mountSvc is mount but also returns the underlying svc (so finance tests can swap
+// in a fake DigitalOcean client, and the cockpit tests can attach an audit store)
+// AND the raw fiber app (so tests that need a request BODY can drive it directly —
+// the returned `do` sends a nil body). The handlers read s.* live at request time,
+// so an override before issuing a request takes effect.
+func mountSvc(t *testing.T, iamURL, commerceURL, healthURL string) (func(method, path string, hdr map[string]string) (*http.Response, []byte), *svc, *fiber.App) {
+	t.Helper()
+	app := zip.New(zip.Config{Logger: luxlog.New("test")})
+	s := &svc{
+		iam:      newIAMClient(iamURL),
+		commerce: newCommerceClient(commerceURL, "test-token"),
+		health:   newHealthClient(healthURL),
+		do:       newDOClient(""), // no token → honest not-configured unless a test overrides s.do
+		adminOrg: "admin",
+	}
+	app.Get("/v1/admin/me", s.guard(s.me))
+	app.Get("/v1/admin/overview", s.guard(s.overview))
+	app.Get("/v1/admin/orgs", s.guard(s.orgs))
+	app.Get("/v1/admin/users", s.guard(s.users))
+	app.Get("/v1/admin/roles", s.guard(s.roles))
+	app.Get("/v1/admin/applications", s.guard(s.applications))
+	app.Get("/v1/admin/audit", s.guard(s.audit))
+	app.Get("/v1/admin/audit/verify", s.guard(s.auditVerify))
+	app.Get("/v1/admin/usage", s.guard(s.usage))
+	app.Get("/v1/admin/products", s.guard(s.products))
+	app.Get("/v1/admin/finance", s.guard(s.finance))
+	app.Post("/v1/admin/sync", s.guard(s.sync))
+	app.Get("/v1/admin/customers", s.guard(s.customers))
+	app.Get("/v1/admin/customers/:org", s.guard(s.customerDetail))
+	app.Post("/v1/admin/customers/:org/credit", s.guard(s.grantCredit))
+	app.Post("/v1/admin/customers/:org/suspend", s.guard(s.suspendCustomer))
+	app.Post("/v1/admin/customers/:org/reactivate", s.guard(s.reactivateCustomer))
+	app.Get("/v1/admin/revenue", s.guard(s.revenue))
+	app.Get("/v1/admin/analytics", s.guard(s.analytics))
+	fa := app.Fiber()
+
+	return func(method, path string, hdr map[string]string) (*http.Response, []byte) {
+		t.Helper()
+		req := httptest.NewRequest(method, path, nil)
+		for k, v := range hdr {
+			req.Header.Set(k, v)
+		}
+		resp, err := fa.Test(req, fiber.TestConfig{Timeout: 30 * time.Second})
+		if err != nil {
+			t.Fatalf("%s %s: %v", method, path, err)
+		}
+		b, _ := io.ReadAll(resp.Body)
+		return resp, b
+	}, s, fa
+}
+
+// adminRoutes is every mounted /v1/admin route + its method — the full god-mode
+// surface the gate must fail-close on for a non-global-admin.
+var adminRoutes = []struct{ method, path string }{
+	{"GET", "/v1/admin/me"},
+	{"GET", "/v1/admin/overview"},
+	{"GET", "/v1/admin/orgs"},
+	{"GET", "/v1/admin/users"},
+	{"GET", "/v1/admin/roles"},
+	{"GET", "/v1/admin/applications"},
+	{"GET", "/v1/admin/audit"},
+	{"GET", "/v1/admin/audit/verify"},
+	{"GET", "/v1/admin/usage"},
+	{"GET", "/v1/admin/products"},
+	{"GET", "/v1/admin/finance"},
+	{"POST", "/v1/admin/sync"},
+	{"GET", "/v1/admin/customers"},
+	{"GET", "/v1/admin/customers/acme"},
+	{"POST", "/v1/admin/customers/acme/credit"},
+	{"POST", "/v1/admin/customers/acme/suspend"},
+	{"POST", "/v1/admin/customers/acme/reactivate"},
+	{"GET", "/v1/admin/revenue"},
+	{"GET", "/v1/admin/analytics"},
+}
+
+// TestGate_DeniesEveryRoute proves the non-negotiable: EVERY /v1/admin/* route is
+// global-admin only, fail-closed. An anonymous caller and a tenant-admin (whose
+// identity carries an org but NOT the sanitizer-minted X-User-IsAdmin) are BOTH
+// denied 403 on every route — no upstream is even reached. admin mirrors the
+// gateway's admin-guard: SanitizeIdentity sets X-User-IsAdmin only for a
+// validated principal whose owner == AdminOrg, so a forged header never survives
+// ingress and the c.IsAdmin() read here is authoritative.
+func TestGate_DeniesEveryRoute(t *testing.T) {
+	// Upstreams point nowhere reachable; the gate must reject BEFORE any call.
+	do := mount(t, "http://127.0.0.1:0", "http://127.0.0.1:0", "http://127.0.0.1:0")
+
+	cases := []struct {
+		name string
+		hdr  map[string]string
+	}{
+		{"anonymous", nil},
+		{"tenant-admin (owner set, not global-admin)", map[string]string{"X-Org-Id": "acme"}},
+		{"tenant-user with email but no admin", map[string]string{"X-Org-Id": "acme", "X-User-Id": "acme/bob", "X-User-Email": "bob@acme.test"}},
+	}
+	for _, tc := range cases {
+		for _, r := range adminRoutes {
+			resp, body := do(r.method, r.path, tc.hdr)
+			if resp.StatusCode != http.StatusForbidden {
+				t.Errorf("%s %s [%s]: got %d, want 403 (body=%s)", r.method, r.path, tc.name, resp.StatusCode, body)
+			}
+		}
+	}
+}
+
+// TestGate_AllowsGlobalAdmin proves the flip side: a validated global admin
+// (X-User-IsAdmin=true, minted only for owner==AdminOrg) is admitted — the gate
+// is not vacuously closed. Reaches /v1/admin/me, which needs no upstream.
+func TestGate_AllowsGlobalAdmin(t *testing.T) {
+	do := mount(t, "http://127.0.0.1:0", "http://127.0.0.1:0", "http://127.0.0.1:0")
+	admin := map[string]string{"X-User-IsAdmin": "true", "X-Org-Id": "admin", "X-User-Id": "admin/z", "X-User-Email": "z@hanzo.ai"}
+	resp, body := do("GET", "/v1/admin/me", admin)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("global-admin GET /v1/admin/me: got %d, want 200 (body=%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Status string  `json:"status"`
+		Data   adminMe `json:"data"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode me envelope: %v", err)
+	}
+	if env.Status != "ok" {
+		t.Fatalf("me status = %q, want ok", env.Status)
+	}
+	if env.Data.Owner != "admin" || env.Data.Email != "z@hanzo.ai" || !env.Data.IsGlobalAdmin {
+		t.Errorf("me identity wrong: %+v", env.Data)
+	}
+}
+
+// fakeIAM stands in for the IAM management surface. It records whether the
+// caller's credential was replayed and returns /v1 envelopes.
+type fakeIAM struct {
+	server  *httptest.Server
+	gotAuth string
+	gotCook string
+}
+
+func newFakeIAM() *fakeIAM {
+	f := &fakeIAM{}
+	f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		f.gotAuth = r.Header.Get("Authorization")
+		f.gotCook = r.Header.Get("Cookie")
+		w.Header().Set("Content-Type", "application/json")
+		switch {
+		case strings.HasSuffix(r.URL.Path, "/get-organizations"):
+			io.WriteString(w, `{"status":"ok","msg":"","data":[
+				{"owner":"admin","name":"hanzo","displayName":"Hanzo","createdTime":"2020-01-01T00:00:00Z"},
+				{"owner":"admin","name":"acme","displayName":"Acme Inc","createdTime":"2021-02-02T00:00:00Z"}
+			],"data2":2}`)
+		case strings.HasSuffix(r.URL.Path, "/get-users"):
+			// A single-page count probe (pageSize=1) still reports data2 total.
+			io.WriteString(w, `{"status":"ok","msg":"","data":[
+				{"owner":"hanzo","name":"alice","email":"alice@hanzo.ai","displayName":"Alice","tag":"staff","createdTime":"2020-03-01T00:00:00Z","lastSigninTime":"2026-06-01T00:00:00Z","isAdmin":true,"isForbidden":false}
+			],"data2":7}`)
+		case strings.HasSuffix(r.URL.Path, "/get-roles"):
+			io.WriteString(w, `{"status":"ok","msg":"","data":[{"owner":"admin","name":"ops","displayName":"Ops"}],"data2":1}`)
+		case strings.HasSuffix(r.URL.Path, "/get-applications"):
+			io.WriteString(w, `{"status":"ok","msg":"","data":[{"owner":"admin","name":"hanzo-cloud","clientId":"cid"}],"data2":1}`)
+		case strings.HasSuffix(r.URL.Path, "/get-records"):
+			io.WriteString(w, `{"status":"ok","msg":"","data":[{"createdTime":"2026-06-29T00:00:00Z","organization":"hanzo","user":"alice","clientIp":"1.2.3.4","method":"POST","action":"login","requestUri":"/v1/iam/login"}],"data2":1}`)
+		default:
+			w.WriteHeader(404)
+			io.WriteString(w, `{"status":"error","msg":"not found"}`)
+		}
+	}))
+	return f
+}
+
+// fakeCommerce mimics the LIVE commerce billing contract (commerce >=1.46.8, the
+// 2026-07 per-org durability rework): the per-org wallet is resolved from the
+// TRUSTED X-Org-Id header (set only with the service-token bearer) and keyed under
+// the BARE org slug as the `user` subject. A wrong header (X-IAM-Org-Id) or a wrong
+// subject ("org/org") resolves to an EMPTY wallet — so this fake is a regression
+// guard for the reconciliation bug that made every admin money panel read $0 while
+// real balances existed (lux $10,000, maxpower $20,498). Verified against live
+// commerce /v1/billing/{balance,usage-rollup}.
+type fakeCommerce struct {
+	server          *httptest.Server
+	balances        map[string]int64 // org slug -> availableCents (credits)
+	spend           map[string]int64 // org slug -> consumedCents (month-to-date)
+	sawIAMOrgHeader bool             // true if the stale X-IAM-Org-Id header was ever sent
+}
+
+func newFakeCommerce() *fakeCommerce {
+	f := &fakeCommerce{
+		balances: map[string]int64{"acme": 5000, "hanzo": 5000},
+		spend:    map[string]int64{"acme": 1500, "hanzo": 1500},
+	}
+	f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		if r.Header.Get("X-IAM-Org-Id") != "" {
+			f.sawIAMOrgHeader = true
+		}
+		// Live commerce trusts ONLY X-Org-Id (with the service-token bearer) for the
+		// org namespace and keys the wallet under the bare org slug. Anything else
+		// (missing X-Org-Id, or user != org) resolves to an empty wallet.
+		org := r.Header.Get("X-Org-Id")
+		user := r.URL.Query().Get("user")
+		bal, spend := int64(0), int64(0)
+		if org != "" && user == org {
+			bal, spend = f.balances[org], f.spend[org]
+		}
+		switch {
+		case strings.HasSuffix(r.URL.Path, "/usage-rollup"):
+			fmt.Fprintf(w, `{"consumedCents":%d,"overageCents":0,"balance":{"balanceCents":%d,"availableCents":%d}}`, spend, bal, bal)
+		case strings.HasSuffix(r.URL.Path, "/balance"):
+			fmt.Fprintf(w, `{"user":%q,"currency":"usd","balance":%d,"holds":0,"available":%d}`, user, bal, bal)
+		case strings.HasSuffix(r.URL.Path, "/subscriptions"):
+			io.WriteString(w, `{"subscriptions":[]}`)
+		default:
+			w.WriteHeader(404)
+		}
+	}))
+	return f
+}
+
+// TestCommerce_ReconcilesWithXOrgIdBareSlug pins the exact live-commerce contract
+// the admin money aggregation depends on: the org selector is the TRUSTED X-Org-Id
+// header and the wallet subject is the BARE org slug (user=<org>) — NOT
+// X-IAM-Org-Id and NOT "org/org". This is the regression guard for the $0-fleet-
+// revenue bug (commerce.go had X-IAM-Org-Id; admin.go orgSubject had "org/org", so
+// every real balance read $0). /v1/admin/orgs must surface acme's real $50.00.
+func TestCommerce_ReconcilesWithXOrgIdBareSlug(t *testing.T) {
+	// orgSubject MUST be the bare slug (not "org/org").
+	if got := orgSubject("acme"); got != "acme" {
+		t.Fatalf("orgSubject(\"acme\") = %q, want \"acme\" (bare slug; \"acme/acme\" reads an empty commerce wallet)", got)
+	}
+
+	iam := newFakeIAM()
+	defer iam.server.Close()
+	commerce := newFakeCommerce()
+	defer commerce.server.Close()
+
+	do := mount(t, iam.server.URL, commerce.server.URL, "")
+	admin := map[string]string{"X-User-IsAdmin": "true", "X-Org-Id": "admin"}
+	resp, body := do("GET", "/v1/admin/orgs", admin)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("orgs: got %d (body=%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Data []orgRow `json:"data"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	// acme (sorted first) must show its REAL money, proving the header + subject key.
+	var acme *orgRow
+	for i := range env.Data {
+		if env.Data[i].Org == "acme" {
+			acme = &env.Data[i]
+		}
+	}
+	if acme == nil {
+		t.Fatalf("acme org missing from %+v", env.Data)
+	}
+	if acme.CreditsCents != 5000 || acme.SpendCents != 1500 {
+		t.Errorf("acme money = credits %d / spend %d, want 5000/1500 — the money did NOT reconcile (stale X-IAM-Org-Id or org/org subject reads $0)", acme.CreditsCents, acme.SpendCents)
+	}
+	// The stale header must NEVER be sent.
+	if commerce.sawIAMOrgHeader {
+		t.Error("admin sent the stale X-IAM-Org-Id header — commerce reads X-Org-Id only")
+	}
+}
+
+// TestOrgs_RealAggregation drives /v1/admin/orgs against fake IAM + commerce and
+// verifies the envelope, the field mapping, the per-org user count (from IAM
+// data2), the money (from commerce), and that the caller's credential is
+// replayed to IAM (admin never forges a service credential for the fan-out).
+func TestOrgs_RealAggregation(t *testing.T) {
+	iam := newFakeIAM()
+	defer iam.server.Close()
+	commerce := newFakeCommerce()
+	defer commerce.server.Close()
+
+	do := mount(t, iam.server.URL, commerce.server.URL, "")
+	admin := map[string]string{
+		"X-User-IsAdmin": "true", "X-Org-Id": "admin",
+		"Authorization": "Bearer operator-jwt", "Cookie": "iam_access_token=operator-jwt",
+	}
+	resp, body := do("GET", "/v1/admin/orgs", admin)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("orgs: got %d, want 200 (body=%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Status string   `json:"status"`
+		Data   []orgRow `json:"data"`
+		Data2  int      `json:"data2"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if env.Status != "ok" || env.Data2 != 2 || len(env.Data) != 2 {
+		t.Fatalf("orgs envelope wrong: status=%q data2=%d rows=%d", env.Status, env.Data2, len(env.Data))
+	}
+	// Rows are sorted by org name: acme, hanzo.
+	acme := env.Data[0]
+	if acme.Org != "acme" || acme.Display != "Acme Inc" {
+		t.Errorf("org row[0] = %+v, want acme/Acme Inc", acme)
+	}
+	if acme.Users != 7 {
+		t.Errorf("org acme users = %d, want 7 (IAM data2)", acme.Users)
+	}
+	if acme.SpendCents != 1500 || acme.CreditsCents != 5000 {
+		t.Errorf("org acme money = spend %d credits %d, want 1500/5000", acme.SpendCents, acme.CreditsCents)
+	}
+	// The operator's own credential MUST have been replayed to IAM.
+	if iam.gotAuth != "Bearer operator-jwt" {
+		t.Errorf("IAM did not receive the caller's Authorization: got %q", iam.gotAuth)
+	}
+	if !strings.Contains(iam.gotCook, "operator-jwt") {
+		t.Errorf("IAM did not receive the caller's Cookie: got %q", iam.gotCook)
+	}
+}
+
+// TestUsers_MapsIAMToOperatorUser verifies the cross-org directory mapping,
+// including the derived isGlobalAdmin (owner == adminOrg) and the data2 total.
+func TestUsers_MapsIAMToOperatorUser(t *testing.T) {
+	iam := newFakeIAM()
+	defer iam.server.Close()
+	do := mount(t, iam.server.URL, "", "")
+	admin := map[string]string{"X-User-IsAdmin": "true", "X-Org-Id": "admin"}
+
+	resp, body := do("GET", "/v1/admin/users?org=hanzo", admin)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("users: got %d (body=%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Data  []operatorUser `json:"data"`
+		Data2 int            `json:"data2"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if env.Data2 != 7 || len(env.Data) != 1 {
+		t.Fatalf("users total=%d rows=%d, want 7/1", env.Data2, len(env.Data))
+	}
+	u := env.Data[0]
+	if u.Name != "alice" || u.Email != "alice@hanzo.ai" || !u.IsAdmin || u.LastSignin == "" {
+		t.Errorf("user mapping wrong: %+v", u)
+	}
+	// owner "hanzo" != adminOrg "admin" → not a global admin.
+	if u.IsGlobalAdmin {
+		t.Errorf("user owner=hanzo must not be flagged global admin")
+	}
+}
+
+// TestRolesAndApplications_PassthroughShape verifies the verbatim IAM passthrough
+// keeps the exact wire fields (clientId on Application, etc.) the operator decodes.
+func TestRolesAndApplications_PassthroughShape(t *testing.T) {
+	iam := newFakeIAM()
+	defer iam.server.Close()
+	do := mount(t, iam.server.URL, "", "")
+	admin := map[string]string{"X-User-IsAdmin": "true", "X-Org-Id": "admin"}
+
+	_, appsBody := do("GET", "/v1/admin/applications", admin)
+	var appsEnv struct {
+		Data []struct {
+			Name     string `json:"name"`
+			ClientId string `json:"clientId"`
+		} `json:"data"`
+		Data2 int `json:"data2"`
+	}
+	if err := json.Unmarshal(appsBody, &appsEnv); err != nil {
+		t.Fatalf("apps decode: %v", err)
+	}
+	if len(appsEnv.Data) != 1 || appsEnv.Data[0].ClientId != "cid" {
+		t.Errorf("applications passthrough lost clientId: %+v", appsEnv.Data)
+	}
+
+	_, rolesBody := do("GET", "/v1/admin/roles", admin)
+	if !strings.Contains(string(rolesBody), `"ops"`) {
+		t.Errorf("roles passthrough missing role name: %s", rolesBody)
+	}
+}
+
+// TestAudit_MapsRecords verifies the audit directory returns the IAM Record wire
+// shape the operator's AuditRow decodes.
+func TestAudit_MapsRecords(t *testing.T) {
+	iam := newFakeIAM()
+	defer iam.server.Close()
+	do := mount(t, iam.server.URL, "", "")
+	admin := map[string]string{"X-User-IsAdmin": "true", "X-Org-Id": "admin"}
+
+	resp, body := do("GET", "/v1/admin/audit", admin)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("audit: got %d (body=%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Data []struct {
+			CreatedTime  string `json:"createdTime"`
+			Organization string `json:"organization"`
+			RequestUri   string `json:"requestUri"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if len(env.Data) != 1 || env.Data[0].Organization != "hanzo" || env.Data[0].RequestUri != "/v1/iam/login" {
+		t.Errorf("audit record shape wrong: %+v", env.Data)
+	}
+}
+
+// TestOverview_RealTilesAndSources verifies the Platform Overview: real org/user
+// counts + money from the upstreams, and a per-source freshness row that reports
+// the honest state of each feed (iam ok, commerce ok, o11y not-configured here).
+func TestOverview_RealTilesAndSources(t *testing.T) {
+	iam := newFakeIAM()
+	defer iam.server.Close()
+	commerce := newFakeCommerce()
+	defer commerce.server.Close()
+
+	do := mount(t, iam.server.URL, commerce.server.URL, "") // no o11y health → source not-ok
+	admin := map[string]string{"X-User-IsAdmin": "true", "X-Org-Id": "admin"}
+
+	resp, body := do("GET", "/v1/admin/overview", admin)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("overview: got %d (body=%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Data overviewData `json:"data"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	d := env.Data
+	if d.Orgs != 2 {
+		t.Errorf("overview orgs = %d, want 2", d.Orgs)
+	}
+	// 2 orgs × 7 users each (both count probes return data2=7).
+	if d.Users != 14 {
+		t.Errorf("overview users = %d, want 14", d.Users)
+	}
+	// 2 orgs × 1500 consumed cents.
+	if d.SpendCents30d != 3000 {
+		t.Errorf("overview spend = %d, want 3000", d.SpendCents30d)
+	}
+	if d.CreditsCents != 10000 {
+		t.Errorf("overview credits = %d, want 10000", d.CreditsCents)
+	}
+	if d.LastSync == "" {
+		t.Error("overview lastSync must be set")
+	}
+	// Source freshness: iam ok, commerce ok, o11y not-ok (unconfigured).
+	src := map[string]sourceStatus{}
+	for _, s := range d.Sources {
+		src[s.Name] = s
+	}
+	if !src["iam"].OK || src["iam"].Rows != 2 {
+		t.Errorf("iam source = %+v, want ok/2 rows", src["iam"])
+	}
+	if !src["commerce"].OK {
+		t.Errorf("commerce source = %+v, want ok", src["commerce"])
+	}
+	if src["o11y"].OK || src["o11y"].Error == "" {
+		t.Errorf("o11y source must be not-ok with an error when unconfigured: %+v", src["o11y"])
+	}
+}
+
+// TestUsage_RealTotalsHonestEmptySeries proves the usage roll-up returns the REAL
+// fleet spend from commerce but an HONEST empty series/byProduct — the timeseries
+// feed lives in insights/datastore, and admin must never fabricate a trend.
+func TestUsage_RealTotalsHonestEmptySeries(t *testing.T) {
+	iam := newFakeIAM()
+	defer iam.server.Close()
+	commerce := newFakeCommerce()
+	defer commerce.server.Close()
+
+	do := mount(t, iam.server.URL, commerce.server.URL, "")
+	admin := map[string]string{"X-User-IsAdmin": "true", "X-Org-Id": "admin"}
+
+	resp, body := do("GET", "/v1/admin/usage", admin)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("usage: got %d (body=%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Data usageData `json:"data"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if env.Data.Totals.SpendCents != 3000 { // 2 orgs × 1500
+		t.Errorf("usage total spend = %d, want 3000", env.Data.Totals.SpendCents)
+	}
+	// Honest empty — NOT nil (the JSON must be [], which the operator renders as
+	// an empty chart), and NEVER a fabricated point.
+	if env.Data.Series == nil || len(env.Data.Series) != 0 {
+		t.Errorf("usage series must be an empty array (no fabricated trend), got %v", env.Data.Series)
+	}
+	if env.Data.ByProduct == nil || len(env.Data.ByProduct) != 0 {
+		t.Errorf("usage byProduct must be an empty array, got %v", env.Data.ByProduct)
+	}
+}
+
+// TestProductsAndSync_HonestShapes verifies products returns the real empty
+// registry (no fabricated workloads) and sync acknowledges with {started:true}.
+func TestProductsAndSync_HonestShapes(t *testing.T) {
+	do := mount(t, "", "", "")
+	admin := map[string]string{"X-User-IsAdmin": "true", "X-Org-Id": "admin"}
+
+	_, pBody := do("GET", "/v1/admin/products", admin)
+	var pEnv struct {
+		Data  []productRow `json:"data"`
+		Data2 int          `json:"data2"`
+	}
+	if err := json.Unmarshal(pBody, &pEnv); err != nil {
+		t.Fatalf("products decode: %v", err)
+	}
+	if pEnv.Data == nil || len(pEnv.Data) != 0 || pEnv.Data2 != 0 {
+		t.Errorf("products must be an empty registry (no fabricated rows): %+v", pEnv)
+	}
+
+	_, sBody := do("POST", "/v1/admin/sync", admin)
+	var sEnv struct {
+		Status string          `json:"status"`
+		Data   map[string]bool `json:"data"`
+	}
+	if err := json.Unmarshal(sBody, &sEnv); err != nil {
+		t.Fatalf("sync decode: %v", err)
+	}
+	if sEnv.Status != "ok" || !sEnv.Data["started"] {
+		t.Errorf("sync must ack {started:true}: %+v", sEnv)
+	}
+}
+
+// TestIAMError_SurfacedNotFabricated proves a failing upstream yields a real
+// error envelope (status:error), NOT a stubbed/zero success — the operator shows
+// the error state, honoring the api.ts "nothing here fabricates data" contract.
+func TestIAMError_SurfacedNotFabricated(t *testing.T) {
+	// IAM that always 500s.
+	bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(500)
+		io.WriteString(w, `{"status":"error","msg":"iam boom"}`)
+	}))
+	defer bad.Close()
+
+	do := mount(t, bad.URL, "", "")
+	admin := map[string]string{"X-User-IsAdmin": "true", "X-Org-Id": "admin"}
+
+	_, body := do("GET", "/v1/admin/orgs", admin)
+	var env struct {
+		Status string `json:"status"`
+		Msg    string `json:"msg"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if env.Status != "error" || env.Msg == "" {
+		t.Errorf("failing IAM must surface an error envelope, got %+v", env)
+	}
+}
+
+// TestMount_NilGuards keeps the Mount contract honest (nil app / nil logger).
+func TestMount_NilGuards(t *testing.T) {
+	if err := Mount(nil, cloud.Deps{Logger: luxlog.New("test")}); err == nil {
+		t.Error("Mount(nil app) must error")
+	}
+	app := zip.New(zip.Config{Logger: luxlog.New("test")})
+	if err := Mount(app, cloud.Deps{}); err == nil {
+		t.Error("Mount(nil logger) must error")
+	}
+}

clients/admin/analytics.go

@@ -0,0 +1,738 @@
+package admin
+
+// Native SaaS business ANALYTICS (/v1/admin/analytics) — cohort retention, growth,
+// churn, active-customers (DAU/WAU/MAU), revenue (MRR/ARPU) and usage over time,
+// derived from REAL fleet data: IAM org `createdTime` (the signup cohort — always
+// available) + the commerce transaction ledger (usage = `withdraw` rows, the true
+// customer-activity signal). Global-admin only (s.guard), like every admin route.
+//
+// HONEST BY CONSTRUCTION. There is NO fabricated curve anywhere. Growth/cohorts
+// come from real signup timestamps; retention/active/churn/usage come from real
+// consumption events. A metric that cannot yet be computed (LTV needs observed
+// churn; NRR needs MRR history commerce does not expose point-in-time) returns a
+// null / honest-empty series — never an invented trend — and the `computed` map
+// flags exactly which metrics are backed by data, so the console renders honest
+// states and a reviewer can verify no number was made up.
+//
+// The heavy read (every org's ledger) is bounded + fanned out concurrently. Admin
+// is low-QPS; at fleet scale this belongs in the insights/datastore OLAP mirror
+// (same note as the usage series), but the billing ledger is the correct SOURCE
+// OF TRUTH for real per-customer activity today.
+
+import (
+	"context"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/zap-proto/zip"
+)
+
+// ── wire shapes (operator contract) ──────────────────────────────────────────
+
+// seriesPoint is one bucketed point (count OR cents, per the series). T is the
+// bucket key (RFC3339 date / "2006-01" month).
+type seriesPoint struct {
+	T     string `json:"t"`
+	Value int64  `json:"value"`
+}
+
+// retentionCohort is one row of the retention triangle: a signup cohort, its size,
+// and the % of it still ACTIVE at each subsequent period (values[0] = the signup
+// period itself). Percentages are 0..100.
+type retentionCohort struct {
+	Cohort string    `json:"cohort"`
+	Size   int       `json:"size"`
+	Values []float64 `json:"values"`
+}
+
+// retentionGrid is the classic cohort-retention heatmap (cohorts × periods).
+type retentionGrid struct {
+	Interval string            `json:"interval"` // "month"
+	Periods  int               `json:"periods"`
+	Cohorts  []retentionCohort `json:"cohorts"`
+}
+
+// analyticsData is the whole GET /v1/admin/analytics payload.
+type analyticsData struct {
+	Range       string `json:"range"`
+	Interval    string `json:"interval"`
+	GeneratedAt string `json:"generatedAt"`
+
+	// Growth — from IAM createdTime (always real).
+	Signups             []seriesPoint `json:"signups"`
+	CumulativeCustomers []seriesPoint `json:"cumulativeCustomers"`
+	TotalCustomers      int           `json:"totalCustomers"`
+	NewCustomers        int           `json:"newCustomers"`
+	GrowthRatePct       float64       `json:"growthRatePct"`
+
+	// Active customers — from the usage ledger.
+	ActiveCustomers []seriesPoint `json:"activeCustomers"`
+	DAU             int           `json:"dau"`
+	WAU             int           `json:"wau"`
+	MAU             int           `json:"mau"`
+
+	// Retention triangle — signup cohort × active period.
+	Retention retentionGrid `json:"retention"`
+
+	// Churn — logo churn (count) + rate.
+	Churn        []seriesPoint `json:"churn"`
+	ChurnRatePct float64       `json:"churnRatePct"`
+
+	// Revenue analytics.
+	MRRCents      int64    `json:"mrrCents"`
+	Revenue       []seriesPoint `json:"revenue"`
+	ARPUCents     int64    `json:"arpuCents"`
+	LTVCents      *int64   `json:"ltvCents"` // null until churn is observed
+	NRRPct        *float64 `json:"nrrPct"`   // null — needs MRR history
+
+	// Usage analytics.
+	Usage        []seriesPoint  `json:"usage"`
+	TopCustomers []analyticsSlice `json:"topCustomers"`
+
+	// Transparency: which metrics are backed by real data vs honest-empty. A
+	// reviewer/console reads this to know nothing was fabricated.
+	Computed map[string]bool `json:"computed"`
+	Sources  []sourceStatus  `json:"sources"`
+}
+
+// analyticsSlice is a labelled magnitude (top customers by usage cents).
+type analyticsSlice struct {
+	Label string `json:"label"`
+	Value int64  `json:"value"`
+	Hint  string `json:"hint,omitempty"`
+}
+
+// ── the activity model the pure math folds over ──────────────────────────────
+
+// txnPoint is one dated usage event (a commerce `withdraw`, in cents).
+type txnPoint struct {
+	T     time.Time
+	Cents int64
+}
+
+// custActivity is one customer's real analytics input: when they signed up (IAM
+// createdTime) and their consumption events (commerce withdraws). Deposits are
+// NOT activity (a credit grant is not the customer using the product), so only
+// withdraws feed active/retention/churn/usage — the honest "used it" signal.
+type custActivity struct {
+	Org        string
+	Display    string
+	Created    time.Time
+	HasCreated bool
+	Usage      []txnPoint
+	SpendCents int64
+}
+
+func (ca custActivity) activeIn(bucket string, interval string) bool {
+	for _, p := range ca.Usage {
+		if bucketKeyOf(p.T, interval) == bucket {
+			return true
+		}
+	}
+	return false
+}
+
+func (ca custActivity) activeSince(cut time.Time) bool {
+	for _, p := range ca.Usage {
+		if !p.T.Before(cut) {
+			return true
+		}
+	}
+	return false
+}
+
+// ── handler ──────────────────────────────────────────────────────────────────
+
+func (s *svc) analytics(c *zip.Ctx) error {
+	ctx := c.Context()
+	cr := callerCreds(c)
+	now := time.Now().UTC()
+	rangeStr := normalizeRange(c.Query("range"))
+	since, interval, _ := rangeWindow(rangeStr, now)
+
+	var sources []sourceStatus
+
+	orgs, err := s.listOrgs(ctx, cr)
+	if err != nil {
+		return fail(c, err.Error())
+	}
+	sources = append(sources, srcOf("iam", nil, len(orgs), now.Format(time.RFC3339)))
+
+	acts, ledgerOK := s.fleetActivity(ctx, orgs)
+	ledgerRows := 0
+	for _, a := range acts {
+		ledgerRows += len(a.Usage)
+	}
+	var ledgerErr error
+	if !ledgerOK {
+		ledgerErr = errPartialRevenue // partial ledger read — mark degraded
+	}
+	sources = append(sources, srcOf("commerce-ledger", ledgerErr, ledgerRows, now.Format(time.RFC3339)))
+
+	// MRR from subscriptions (point-in-time), fanned out like the money reads.
+	mrr := s.fleetMRR(ctx, orgs)
+
+	data := computeAnalytics(analyticsInput{
+		acts:     acts,
+		mrrCents: mrr,
+		now:      now,
+		since:    since,
+		interval: interval,
+		rangeStr: rangeStr,
+		ledgerOK: ledgerOK && ledgerRows > 0,
+	})
+	data.GeneratedAt = now.Format(time.RFC3339)
+	data.Sources = sources
+	return ok(c, data)
+}
+
+// analyticsInput is everything computeAnalytics needs — no I/O, so the whole SaaS
+// analytics derivation is unit-testable without a network.
+type analyticsInput struct {
+	acts     []custActivity
+	mrrCents int64
+	now      time.Time
+	since    time.Time
+	interval string
+	rangeStr string
+	ledgerOK bool // the ledger read yielded real usage rows
+}
+
+// computeAnalytics is the PURE derivation of every analytics metric from the real
+// activity model. Growth is always computed (signup timestamps); the ledger-backed
+// metrics compute from real usage when present and degrade to honest empty/zero
+// when the fleet has no usage yet — never a fabricated curve. `computed` flags each.
+func computeAnalytics(in analyticsInput) analyticsData {
+	buckets := enumerateBuckets(in.since, in.now, in.interval)
+
+	// ── Growth (IAM createdTime — always real) ──
+	signups := make([]seriesPoint, len(buckets))
+	newCount := 0
+	total := 0
+	for i, b := range buckets {
+		signups[i] = seriesPoint{T: b}
+	}
+	idx := indexOf(buckets)
+	for _, a := range in.acts {
+		if !a.HasCreated {
+			continue
+		}
+		total++
+		if !a.Created.Before(in.since) {
+			newCount++
+		}
+		if i, ok := idx[bucketKeyOf(a.Created, in.interval)]; ok {
+			signups[i].Value++
+		}
+	}
+	// Cumulative customers across the SAME buckets (all-time count at each bucket end).
+	cumulative := make([]seriesPoint, len(buckets))
+	for i, b := range buckets {
+		end := bucketEnd(b, in.interval)
+		n := 0
+		for _, a := range in.acts {
+			if a.HasCreated && !a.Created.After(end) {
+				n++
+			}
+		}
+		cumulative[i] = seriesPoint{T: b, Value: int64(n)}
+	}
+	growthRate := priorWindowGrowth(in.acts, in.since, in.now)
+
+	// ── Active customers + usage (ledger-backed) ──
+	usage := spendSeries(in.acts, in.since, in.now, in.interval)
+	active := make([]seriesPoint, len(buckets))
+	for i, b := range buckets {
+		active[i] = seriesPoint{T: b}
+	}
+	for _, a := range in.acts {
+		// active in a bucket = at least one usage event in it
+		seen := map[string]bool{}
+		for _, p := range a.Usage {
+			if p.T.Before(in.since) || p.T.After(in.now) {
+				continue
+			}
+			seen[bucketKeyOf(p.T, in.interval)] = true
+		}
+		for b := range seen {
+			if i, ok := idx[b]; ok {
+				active[i].Value++
+			}
+		}
+	}
+	dau := activeWithin(in.acts, in.now.AddDate(0, 0, -1))
+	wau := activeWithin(in.acts, in.now.AddDate(0, 0, -7))
+	mau := activeWithin(in.acts, in.now.AddDate(0, 0, -30))
+
+	// ── Retention triangle (monthly cohorts × active month) ──
+	retention := computeRetention(in.acts, in.now, 12)
+
+	// ── Churn (monthly logo churn from active months) + rate ──
+	churn, churnRate := computeChurn(in.acts, in.now, 6)
+
+	// ── Revenue analytics ──
+	var totalSpend int64
+	for _, a := range in.acts {
+		totalSpend += a.SpendCents
+	}
+	arpu := int64(0)
+	if mau > 0 {
+		arpu = totalSpend / int64(mau)
+	} else if total > 0 {
+		arpu = totalSpend / int64(total)
+	}
+	var ltv *int64
+	if churnRate > 0 && arpu > 0 {
+		// LTV ≈ ARPU / monthly churn rate — computed ONLY when real churn is
+		// observed, else honest null (LTV needs churn to mean anything).
+		v := int64(float64(arpu) / (churnRate / 100.0))
+		ltv = &v
+	}
+
+	// ── Top customers by usage ──
+	top := topCustomersByUsage(in.acts, 10)
+
+	// Revenue series = realized usage revenue per bucket (same as usage cents for a
+	// pay-as-you-go fleet; distinct field so the console can theme it as revenue).
+	revenue := make([]seriesPoint, len(usage))
+	copy(revenue, usage)
+
+	return analyticsData{
+		Range:               in.rangeStr,
+		Interval:            in.interval,
+		Signups:             signups,
+		CumulativeCustomers: cumulative,
+		TotalCustomers:      total,
+		NewCustomers:        newCount,
+		GrowthRatePct:       growthRate,
+		ActiveCustomers:     active,
+		DAU:                 dau,
+		WAU:                 wau,
+		MAU:                 mau,
+		Retention:           retention,
+		Churn:               churn,
+		ChurnRatePct:        churnRate,
+		MRRCents:            in.mrrCents,
+		Revenue:             revenue,
+		ARPUCents:           arpu,
+		LTVCents:            ltv,
+		NRRPct:              nil, // honest null — needs MRR history commerce doesn't expose
+		Usage:               usage,
+		TopCustomers:        top,
+		Computed: map[string]bool{
+			"growth":    true, // signup timestamps are always present
+			"retention": in.ledgerOK,
+			"active":    in.ledgerOK,
+			"churn":     in.ledgerOK,
+			"usage":     in.ledgerOK,
+			"revenue":   in.ledgerOK,
+			"mrr":       true,
+			"arpu":      in.ledgerOK,
+			"ltv":       ltv != nil,
+			"nrr":       false,
+		},
+	}
+}
+
+// computeRetention builds the cohort × period retention triangle from real signup
+// months and usage months. retention[c][k] = fraction of cohort c ACTIVE in month
+// c+k. Cohorts are capped to the last `maxCohorts` months (the classic triangle);
+// a cohort with no signups is omitted. Values are 0..100.
+func computeRetention(acts []custActivity, now time.Time, maxCohorts int) retentionGrid {
+	// Group customers by signup month.
+	byCohort := map[string][]custActivity{}
+	for _, a := range acts {
+		if !a.HasCreated {
+			continue
+		}
+		k := monthKey(a.Created)
+		byCohort[k] = append(byCohort[k], a)
+	}
+
+	// Sorted cohort months, newest last, capped.
+	cohorts := make([]string, 0, len(byCohort))
+	for k := range byCohort {
+		cohorts = append(cohorts, k)
+	}
+	sort.Strings(cohorts)
+	if len(cohorts) > maxCohorts {
+		cohorts = cohorts[len(cohorts)-maxCohorts:]
+	}
+
+	nowMonth := monthKey(now)
+	grid := retentionGrid{Interval: "month"}
+	maxPeriods := 0
+	for _, cohort := range cohorts {
+		members := byCohort[cohort]
+		periods := monthsBetween(cohort, nowMonth) + 1
+		if periods < 1 {
+			periods = 1
+		}
+		row := retentionCohort{Cohort: cohort, Size: len(members), Values: make([]float64, periods)}
+		for k := 0; k < periods; k++ {
+			month := addMonths(cohort, k)
+			activeN := 0
+			for _, m := range members {
+				if m.activeIn(month, "month") {
+					activeN++
+				}
+			}
+			if len(members) > 0 {
+				row.Values[k] = pct(activeN, len(members))
+			}
+		}
+		if periods > maxPeriods {
+			maxPeriods = periods
+		}
+		grid.Cohorts = append(grid.Cohorts, row)
+	}
+	grid.Periods = maxPeriods
+	return grid
+}
+
+// computeChurn derives monthly LOGO churn: a customer counts as churned in month M
+// if they were active in M-1 but NOT in M. The rate is the average monthly churn
+// over the observed window (churned / active-at-start). Returns honest zeros when
+// there is no usage history.
+func computeChurn(acts []custActivity, now time.Time, months int) ([]seriesPoint, float64) {
+	// Build the last `months` month keys ending at now.
+	keys := lastMonths(now, months)
+	series := make([]seriesPoint, len(keys))
+	var churnedTotal, baseTotal int
+	for i, m := range keys {
+		series[i] = seriesPoint{T: m}
+		if i == 0 {
+			continue // no prior month to compare
+		}
+		prev := keys[i-1]
+		churned := 0
+		base := 0
+		for _, a := range acts {
+			wasActive := a.activeIn(prev, "month")
+			if wasActive {
+				base++
+				if !a.activeIn(m, "month") {
+					churned++
+				}
+			}
+		}
+		series[i].Value = int64(churned)
+		churnedTotal += churned
+		baseTotal += base
+	}
+	rate := 0.0
+	if baseTotal > 0 {
+		rate = pct(churnedTotal, baseTotal)
+	}
+	return series, rate
+}
+
+// spendSeries buckets fleet usage cents into a continuous series over since..now.
+// Shared by the analytics usage/revenue trend and the revenue board's spend trend
+// (one implementation, DRY). A bucket with no usage is an honest 0, not a gap.
+func spendSeries(acts []custActivity, since, now time.Time, interval string) []seriesPoint {
+	buckets := enumerateBuckets(since, now, interval)
+	idx := indexOf(buckets)
+	out := make([]seriesPoint, len(buckets))
+	for i, b := range buckets {
+		out[i] = seriesPoint{T: b}
+	}
+	for _, a := range acts {
+		for _, p := range a.Usage {
+			if p.T.Before(since) || p.T.After(now) {
+				continue
+			}
+			if i, ok := idx[bucketKeyOf(p.T, interval)]; ok {
+				out[i].Value += p.Cents
+			}
+		}
+	}
+	return out
+}
+
+// topCustomersByUsage returns the top-N customers by total usage cents (desc).
+func topCustomersByUsage(acts []custActivity, n int) []analyticsSlice {
+	rows := make([]analyticsSlice, 0, len(acts))
+	for _, a := range acts {
+		if a.SpendCents <= 0 {
+			continue
+		}
+		rows = append(rows, analyticsSlice{Label: a.Display, Value: a.SpendCents, Hint: a.Org})
+	}
+	sort.Slice(rows, func(i, j int) bool { return rows[i].Value > rows[j].Value })
+	if len(rows) > n {
+		rows = rows[:n]
+	}
+	return rows
+}
+
+// priorWindowGrowth is the signup growth vs the immediately-preceding window: the
+// % change in new signups this window vs last. 0 when the prior window had none.
+func priorWindowGrowth(acts []custActivity, since, now time.Time) float64 {
+	window := now.Sub(since)
+	priorStart := since.Add(-window)
+	cur, prev := 0, 0
+	for _, a := range acts {
+		if !a.HasCreated {
+			continue
+		}
+		if !a.Created.Before(since) && a.Created.Before(now) {
+			cur++
+		} else if !a.Created.Before(priorStart) && a.Created.Before(since) {
+			prev++
+		}
+	}
+	if prev == 0 {
+		return 0
+	}
+	return (float64(cur-prev) / float64(prev)) * 100
+}
+
+// activeWithin counts customers with at least one usage event since `cut`.
+func activeWithin(acts []custActivity, cut time.Time) int {
+	n := 0
+	for _, a := range acts {
+		if a.activeSince(cut) {
+			n++
+		}
+	}
+	return n
+}
+
+// ── fleet readers (I/O; concurrent, bounded) ─────────────────────────────────
+
+// fleetActivity reads every org's signup time (already on the org row) + usage
+// ledger, folded into the pure activity model. Returns (acts, ok) where ok is
+// false if ANY org's ledger read failed (the caller marks the source degraded and
+// flags the ledger-backed metrics as not-fully-computed). Fanned out concurrently
+// with a bound, like the customer list.
+func (s *svc) fleetActivity(ctx context.Context, orgs []iamOrg) ([]custActivity, bool) {
+	acts := make([]custActivity, len(orgs))
+	oks := make([]bool, len(orgs))
+	sem := make(chan struct{}, maxCustomerConcurrency)
+	var wg sync.WaitGroup
+	for i, o := range orgs {
+		wg.Add(1)
+		sem <- struct{}{}
+		go func(i int, o iamOrg) {
+			defer wg.Done()
+			defer func() { <-sem }()
+			ca := custActivity{Org: o.Name, Display: display(o.DisplayName, o.Name)}
+			if t, err := time.Parse(time.RFC3339, o.CreatedTime); err == nil {
+				ca.Created = t.UTC()
+				ca.HasCreated = true
+			}
+			rows, err := s.commerce.transactions(ctx, o.Name, orgSubject(o.Name), 2000)
+			oks[i] = err == nil
+			for _, r := range rows {
+				if strings.ToLower(r.Type) != "withdraw" {
+					continue // only consumption is "activity"; deposits are credits
+				}
+				t, perr := parseTxnTime(r.CreatedAt)
+				if perr != nil {
+					continue
+				}
+				amt := r.Amount
+				if amt < 0 {
+					amt = -amt
+				}
+				ca.Usage = append(ca.Usage, txnPoint{T: t, Cents: amt})
+				ca.SpendCents += amt
+			}
+			acts[i] = ca
+		}(i, o)
+	}
+	wg.Wait()
+	allOK := true
+	for _, ok := range oks {
+		if !ok {
+			allOK = false
+			break
+		}
+	}
+	return acts, allOK
+}
+
+// fleetMRR sums each org's active-subscription MRR concurrently.
+func (s *svc) fleetMRR(ctx context.Context, orgs []iamOrg) int64 {
+	vals := make([]int64, len(orgs))
+	sem := make(chan struct{}, maxCustomerConcurrency)
+	var wg sync.WaitGroup
+	for i, o := range orgs {
+		wg.Add(1)
+		sem <- struct{}{}
+		go func(i int, o iamOrg) {
+			defer wg.Done()
+			defer func() { <-sem }()
+			if sum, err := s.commerce.subscriptionSummary(ctx, o.Name, orgSubject(o.Name)); err == nil {
+				vals[i] = sum.MRR
+			}
+		}(i, o)
+	}
+	wg.Wait()
+	var total int64
+	for _, v := range vals {
+		total += v
+	}
+	return total
+}
+
+// ── pure time-bucket helpers ─────────────────────────────────────────────────
+
+func monthKey(t time.Time) string { return t.UTC().Format("2006-01") }
+func dayKey(t time.Time) string   { return t.UTC().Format("2006-01-02") }
+
+// weekKey buckets to the ISO week's Monday (a stable weekly key).
+func weekKey(t time.Time) string {
+	u := t.UTC()
+	// back up to Monday
+	wd := int(u.Weekday())
+	if wd == 0 {
+		wd = 7
+	}
+	monday := u.AddDate(0, 0, -(wd - 1))
+	return monday.Format("2006-01-02")
+}
+
+func bucketKeyOf(t time.Time, interval string) string {
+	switch interval {
+	case "month":
+		return monthKey(t)
+	case "week":
+		return weekKey(t)
+	default:
+		return dayKey(t)
+	}
+}
+
+// bucketEnd returns the inclusive end instant of a bucket key (for the cumulative
+// count). A day/week/month key advances one unit; the end is one nanosecond before.
+func bucketEnd(key, interval string) time.Time {
+	switch interval {
+	case "month":
+		if t, err := time.Parse("2006-01", key); err == nil {
+			return t.AddDate(0, 1, 0).Add(-time.Nanosecond)
+		}
+	case "week":
+		if t, err := time.Parse("2006-01-02", key); err == nil {
+			return t.AddDate(0, 0, 7).Add(-time.Nanosecond)
+		}
+	default:
+		if t, err := time.Parse("2006-01-02", key); err == nil {
+			return t.AddDate(0, 0, 1).Add(-time.Nanosecond)
+		}
+	}
+	return time.Now().UTC()
+}
+
+// enumerateBuckets lists every bucket key from since..now inclusive so a series has
+// a continuous axis (a zero-usage bucket is an honest 0, not a gap).
+func enumerateBuckets(since, now time.Time, interval string) []string {
+	if since.After(now) {
+		return nil
+	}
+	var out []string
+	seen := map[string]bool{}
+	step := func(t time.Time) time.Time {
+		switch interval {
+		case "month":
+			return t.AddDate(0, 1, 0)
+		case "week":
+			return t.AddDate(0, 0, 7)
+		default:
+			return t.AddDate(0, 0, 1)
+		}
+	}
+	// cap iterations so a bad range can never spin unbounded
+	for t, n := since, 0; !t.After(now) && n < 800; t, n = step(t), n+1 {
+		k := bucketKeyOf(t, interval)
+		if !seen[k] {
+			seen[k] = true
+			out = append(out, k)
+		}
+	}
+	// ensure the final bucket (now) is present
+	last := bucketKeyOf(now, interval)
+	if !seen[last] {
+		out = append(out, last)
+	}
+	return out
+}
+
+func indexOf(buckets []string) map[string]int {
+	m := make(map[string]int, len(buckets))
+	for i, b := range buckets {
+		m[b] = i
+	}
+	return m
+}
+
+// addMonths adds k months to a "2006-01" key.
+func addMonths(month string, k int) string {
+	t, err := time.Parse("2006-01", month)
+	if err != nil {
+		return month
+	}
+	return t.AddDate(0, k, 0).Format("2006-01")
+}
+
+// monthsBetween returns the whole-month distance from a..b ("2006-01" keys).
+func monthsBetween(a, b string) int {
+	ta, ea := time.Parse("2006-01", a)
+	tb, eb := time.Parse("2006-01", b)
+	if ea != nil || eb != nil {
+		return 0
+	}
+	return int(tb.Year()-ta.Year())*12 + int(tb.Month()-ta.Month())
+}
+
+// lastMonths returns the last n month keys ending at `now` (oldest first).
+func lastMonths(now time.Time, n int) []string {
+	out := make([]string, 0, n)
+	for i := n - 1; i >= 0; i-- {
+		out = append(out, monthKey(now.AddDate(0, -i, 0)))
+	}
+	return out
+}
+
+func pct(part, whole int) float64 {
+	if whole <= 0 {
+		return 0
+	}
+	return (float64(part) / float64(whole)) * 100
+}
+
+// parseTxnTime accepts the commerce ledger's RFC3339 forms.
+func parseTxnTime(s string) (time.Time, error) {
+	s = strings.TrimSpace(s)
+	if t, err := time.Parse(time.RFC3339, s); err == nil {
+		return t.UTC(), nil
+	}
+	return time.Parse("2006-01-02T15:04:05Z", s)
+}
+
+// normalizeRange clamps the range param to the supported set (default 30d).
+func normalizeRange(r string) string {
+	switch strings.TrimSpace(r) {
+	case "7d", "30d", "90d", "all":
+		return strings.TrimSpace(r)
+	default:
+		return "30d"
+	}
+}
+
+// rangeWindow maps a range to (since, interval, approxBuckets).
+func rangeWindow(rangeStr string, now time.Time) (time.Time, string, int) {
+	switch rangeStr {
+	case "7d":
+		return now.AddDate(0, 0, -7), "day", 7
+	case "90d":
+		return now.AddDate(0, 0, -90), "week", 13
+	case "all":
+		return now.AddDate(-2, 0, 0), "month", 24
+	default: // 30d
+		return now.AddDate(0, 0, -30), "day", 30
+	}
+}

clients/admin/analytics_math_test.go

@@ -0,0 +1,259 @@
+package admin
+
+import (
+	"math"
+	"testing"
+	"time"
+)
+
+// mkTime is a test helper for an RFC3339-ish instant.
+func mkTime(s string) time.Time {
+	t, err := time.Parse("2006-01-02", s)
+	if err != nil {
+		panic(err)
+	}
+	return t.UTC()
+}
+
+// fleetFixture builds a deterministic 3-customer fleet with known signup cohorts
+// and usage events, so every analytics metric has a hand-computable expectation.
+//
+//	alpha: signup 2024-05-10; usage 2024-05-20 (100c), 2024-06-05 (200c)  [cohort 05, active 05+06]
+//	beta : signup 2024-05-25; usage 2024-05-28 (50c)                       [cohort 05, active 05 only]
+//	gamma: signup 2024-06-15; usage 2024-07-01 (400c)                      [cohort 06, active 07 only]
+func fleetFixture() []custActivity {
+	return []custActivity{
+		{Org: "alpha", Display: "Alpha", Created: mkTime("2024-05-10"), HasCreated: true,
+			Usage: []txnPoint{{T: mkTime("2024-05-20"), Cents: 100}, {T: mkTime("2024-06-05"), Cents: 200}}, SpendCents: 300},
+		{Org: "beta", Display: "Beta", Created: mkTime("2024-05-25"), HasCreated: true,
+			Usage: []txnPoint{{T: mkTime("2024-05-28"), Cents: 50}}, SpendCents: 50},
+		{Org: "gamma", Display: "Gamma", Created: mkTime("2024-06-15"), HasCreated: true,
+			Usage: []txnPoint{{T: mkTime("2024-07-01"), Cents: 400}}, SpendCents: 400},
+	}
+}
+
+// TestComputeRetention_RealCohortTriangle is the headline: the cohort-retention
+// heatmap is REAL math over signup cohorts × active months — never a fabricated
+// curve. Every cell is hand-verified against the fixture.
+func TestComputeRetention_RealCohortTriangle(t *testing.T) {
+	now := mkTime("2024-07-15")
+	grid := computeRetention(fleetFixture(), now, 12)
+
+	if grid.Interval != "month" {
+		t.Fatalf("retention interval = %q, want month", grid.Interval)
+	}
+	byCohort := map[string]retentionCohort{}
+	for _, c := range grid.Cohorts {
+		byCohort[c.Cohort] = c
+	}
+
+	// Cohort 2024-05 (alpha, beta): period0=100% (both active in 05),
+	// period1=50% (only alpha active in 06), period2=0% (neither active in 07).
+	c05, ok := byCohort["2024-05"]
+	if !ok {
+		t.Fatalf("missing cohort 2024-05 in %+v", grid.Cohorts)
+	}
+	if c05.Size != 2 {
+		t.Errorf("cohort 2024-05 size = %d, want 2", c05.Size)
+	}
+	wantC05 := []float64{100, 50, 0}
+	if len(c05.Values) != len(wantC05) {
+		t.Fatalf("cohort 2024-05 periods = %d, want %d (%v)", len(c05.Values), len(wantC05), c05.Values)
+	}
+	for k, want := range wantC05 {
+		if math.Abs(c05.Values[k]-want) > 0.01 {
+			t.Errorf("retention[2024-05][%d] = %.1f, want %.1f", k, c05.Values[k], want)
+		}
+	}
+
+	// Cohort 2024-06 (gamma): period0=0% (no usage in 06), period1=100% (active in 07).
+	c06 := byCohort["2024-06"]
+	wantC06 := []float64{0, 100}
+	if len(c06.Values) != len(wantC06) {
+		t.Fatalf("cohort 2024-06 periods = %d, want %d (%v)", len(c06.Values), len(wantC06), c06.Values)
+	}
+	for k, want := range wantC06 {
+		if math.Abs(c06.Values[k]-want) > 0.01 {
+			t.Errorf("retention[2024-06][%d] = %.1f, want %.1f", k, c06.Values[k], want)
+		}
+	}
+}
+
+// TestComputeChurn_RealLogoChurn proves monthly logo churn + rate are real.
+//
+//	06 vs 05: base {alpha,beta}=2, churned {beta}=1
+//	07 vs 06: base {alpha}=1,      churned {alpha}=1
+//	rate = churned(2) / base(3) = 66.67%
+func TestComputeChurn_RealLogoChurn(t *testing.T) {
+	now := mkTime("2024-07-15")
+	series, rate := computeChurn(fleetFixture(), now, 6)
+
+	got := map[string]int64{}
+	for _, p := range series {
+		got[p.T] = p.Value
+	}
+	if got["2024-06"] != 1 {
+		t.Errorf("churn[2024-06] = %d, want 1 (beta churned)", got["2024-06"])
+	}
+	if got["2024-07"] != 1 {
+		t.Errorf("churn[2024-07] = %d, want 1 (alpha churned)", got["2024-07"])
+	}
+	if math.Abs(rate-66.666) > 0.1 {
+		t.Errorf("churn rate = %.2f, want ~66.67", rate)
+	}
+}
+
+// TestComputeAnalytics_RealFleet drives the whole pure derivation and asserts the
+// growth, active-customer, ARPU, top-customer, LTV, and computed-flag outputs.
+func TestComputeAnalytics_RealFleet(t *testing.T) {
+	now := mkTime("2024-07-15")
+	since, interval, _ := rangeWindow("30d", now) // since 2024-06-15, daily
+	d := computeAnalytics(analyticsInput{
+		acts: fleetFixture(), mrrCents: 0, now: now, since: since, interval: interval, rangeStr: "30d", ledgerOK: true,
+	})
+
+	if d.TotalCustomers != 3 {
+		t.Errorf("total customers = %d, want 3", d.TotalCustomers)
+	}
+	// New in the 30d window [06-15, 07-15]: gamma only (alpha/beta signed up in May).
+	if d.NewCustomers != 1 {
+		t.Errorf("new customers = %d, want 1 (gamma)", d.NewCustomers)
+	}
+	// MAU (30d): only gamma had usage in-window (07-01). DAU/WAU: none in last 1/7d.
+	if d.MAU != 1 {
+		t.Errorf("MAU = %d, want 1", d.MAU)
+	}
+	if d.DAU != 0 || d.WAU != 0 {
+		t.Errorf("DAU/WAU = %d/%d, want 0/0", d.DAU, d.WAU)
+	}
+	// ARPU = totalSpend(750) / MAU(1) = 750.
+	if d.ARPUCents != 750 {
+		t.Errorf("ARPU = %d, want 750", d.ARPUCents)
+	}
+	// Top customer by usage = gamma (400c) first.
+	if len(d.TopCustomers) != 3 || d.TopCustomers[0].Hint != "gamma" || d.TopCustomers[0].Value != 400 {
+		t.Errorf("top customers wrong: %+v", d.TopCustomers)
+	}
+	// LTV computed only because churn is observed (>0).
+	if d.LTVCents == nil {
+		t.Error("LTV must be computed when churn is observed")
+	}
+	// NRR is honest-null (no MRR history).
+	if d.NRRPct != nil {
+		t.Error("NRR must be honest-null (needs MRR history)")
+	}
+	// Computed transparency flags.
+	if !d.Computed["growth"] || !d.Computed["retention"] || !d.Computed["active"] || !d.Computed["churn"] {
+		t.Errorf("computed flags wrong for a real-ledger fleet: %+v", d.Computed)
+	}
+	if d.Computed["nrr"] {
+		t.Error("nrr computed flag must be false")
+	}
+	// The usage series is over a continuous daily axis (honest zeros, not gaps).
+	if len(d.Usage) == 0 {
+		t.Error("usage series must have buckets")
+	}
+	var usageTotal int64
+	for _, p := range d.Usage {
+		usageTotal += p.Value
+	}
+	if usageTotal != 400 { // only gamma's 07-01 400c falls in the 30d window
+		t.Errorf("in-window usage total = %d, want 400", usageTotal)
+	}
+}
+
+// TestComputeAnalytics_HonestEmptyNoLedger proves the no-fabrication contract: with
+// real signups but NO usage ledger, growth is still computed but retention/active/
+// churn/usage are honest-empty and flagged not-computed — NEVER an invented curve.
+func TestComputeAnalytics_HonestEmptyNoLedger(t *testing.T) {
+	now := mkTime("2024-07-15")
+	// Same signups, but strip all usage (ledger empty / unreachable).
+	acts := fleetFixture()
+	for i := range acts {
+		acts[i].Usage = nil
+		acts[i].SpendCents = 0
+	}
+	since, interval, _ := rangeWindow("30d", now)
+	d := computeAnalytics(analyticsInput{acts: acts, now: now, since: since, interval: interval, rangeStr: "30d", ledgerOK: false})
+
+	// Growth is real regardless of the ledger.
+	if d.TotalCustomers != 3 || !d.Computed["growth"] {
+		t.Errorf("growth must still be computed from signups: total=%d computed=%v", d.TotalCustomers, d.Computed["growth"])
+	}
+	// Ledger-backed metrics are flagged NOT computed.
+	for _, k := range []string{"retention", "active", "churn", "usage", "revenue"} {
+		if d.Computed[k] {
+			t.Errorf("computed[%q] must be false with no ledger", k)
+		}
+	}
+	// And the actual values are honest zero — no fabricated activity.
+	if d.MAU != 0 || d.WAU != 0 || d.DAU != 0 {
+		t.Errorf("active must be 0 with no usage: dau=%d wau=%d mau=%d", d.DAU, d.WAU, d.MAU)
+	}
+	var usageTotal int64
+	for _, p := range d.Usage {
+		usageTotal += p.Value
+	}
+	if usageTotal != 0 {
+		t.Errorf("usage must be all-zero with no ledger, got %d", usageTotal)
+	}
+	if d.LTVCents != nil {
+		t.Error("LTV must be null with no churn observed")
+	}
+	// Retention cohorts still exist (from signups) but every cell is 0% (no activity).
+	for _, c := range d.Retention.Cohorts {
+		for k, v := range c.Values {
+			if v != 0 {
+				t.Errorf("retention[%s][%d] = %.1f, want 0 (no usage → no fabricated retention)", c.Cohort, k, v)
+			}
+		}
+	}
+}
+
+// TestSpendSeries_ContinuousHonestBuckets proves the shared spend series buckets
+// real usage onto a continuous axis with honest zeros.
+func TestSpendSeries_ContinuousHonestBuckets(t *testing.T) {
+	now := mkTime("2024-07-05")
+	since := mkTime("2024-07-01")
+	acts := []custActivity{
+		{Usage: []txnPoint{{T: mkTime("2024-07-01"), Cents: 100}, {T: mkTime("2024-07-03"), Cents: 300}}},
+	}
+	series := spendSeries(acts, since, now, "day")
+	// 5 daily buckets 07-01..07-05.
+	if len(series) != 5 {
+		t.Fatalf("series buckets = %d, want 5 (%+v)", len(series), series)
+	}
+	got := map[string]int64{}
+	for _, p := range series {
+		got[p.T] = p.Value
+	}
+	if got["2024-07-01"] != 100 || got["2024-07-03"] != 300 {
+		t.Errorf("spend buckets wrong: %+v", got)
+	}
+	// 07-02, 07-04, 07-05 are honest zeros (present, not missing).
+	if got["2024-07-02"] != 0 || got["2024-07-04"] != 0 {
+		t.Errorf("empty days must be honest 0, got %+v", got)
+	}
+}
+
+// TestBucketHelpers pins the month arithmetic the retention triangle relies on.
+func TestBucketHelpers(t *testing.T) {
+	if addMonths("2024-05", 2) != "2024-07" {
+		t.Errorf("addMonths(2024-05,2) = %q, want 2024-07", addMonths("2024-05", 2))
+	}
+	if addMonths("2024-11", 3) != "2025-02" {
+		t.Errorf("addMonths(2024-11,3) = %q, want 2025-02", addMonths("2024-11", 3))
+	}
+	if monthsBetween("2024-05", "2024-07") != 2 {
+		t.Errorf("monthsBetween(05,07) = %d, want 2", monthsBetween("2024-05", "2024-07"))
+	}
+	if monthsBetween("2024-11", "2025-02") != 3 {
+		t.Errorf("monthsBetween cross-year = %d, want 3", monthsBetween("2024-11", "2025-02"))
+	}
+	if normalizeRange("bogus") != "30d" {
+		t.Errorf("normalizeRange must default to 30d")
+	}
+	if normalizeRange("90d") != "90d" {
+		t.Errorf("normalizeRange must keep 90d")
+	}
+}

clients/admin/audit.go

@@ -0,0 +1,182 @@
+package admin
+
+// The /v1/admin/audit query surface, wired to cloud's REAL tamper-evident audit
+// store (the audit.Recorder Serve builds and hands over via deps.Audit).
+//
+// This REPLACES the previous behavior — proxying IAM get-records — as the primary
+// source: cloud now keeps its OWN append-only, hash-chained trail of every
+// security-relevant request against this binary, and that is what a compliance
+// auditor queries here. IAM's own login/session records remain available in IAM;
+// they are a DIFFERENT trail (IAM's request surface), and admin still federates
+// them as a fallback when cloud's local store is not configured, so no capability
+// is lost.
+//
+// SECURITY. Both handlers are registered behind the SAME s.guard as every other
+// /v1/admin/* route (global-admin only, fail-closed). They are READ-ONLY (Query
+// and Verify issue SELECT only), so exposing them cannot weaken the append-only
+// property. The verify endpoint returns integrity STATUS, never a way to mutate.
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/hanzoai/cloud/audit"
+	"github.com/zap-proto/zip"
+)
+
+// auditRow is one record in the operator's audit table (AuditRow). The JSON tags
+// are the operator contract. It is cloud's OWN record shape — richer than the IAM
+// Record it supersedes: it carries the outcome, the validated auth context, and
+// the hash-chain linkage so the console can show integrity per row.
+type auditRow struct {
+	Seq        uint64 `json:"seq"`
+	Time       string `json:"time"`
+	Org        string `json:"org"`
+	Sub        string `json:"sub"`
+	Email      string `json:"email,omitempty"`
+	Action     string `json:"action"`
+	Resource   string `json:"resource"`
+	ResourceID string `json:"resourceId,omitempty"`
+	Method     string `json:"method,omitempty"`
+	Path       string `json:"path,omitempty"`
+	Result     string `json:"result"`
+	Status     int    `json:"status"`
+	Reason     string `json:"reason,omitempty"`
+	SourceIP   string `json:"sourceIp,omitempty"`
+	UserAgent  string `json:"userAgent,omitempty"`
+	RequestID  string `json:"requestId,omitempty"`
+	IsAdmin    bool   `json:"isAdmin"`
+	Auth       string `json:"authMethod,omitempty"`
+	Hash       string `json:"hash"`
+	PrevHash   string `json:"prevHash"`
+}
+
+// audit answers GET /v1/admin/audit from cloud's local tamper-evident store when
+// configured, else falls back to the IAM get-records proxy (federated view).
+// Filters: org, sub, action, resource, result, since, until, pageSize, p (page).
+// The response is the /v1 list envelope { data:[rows], data2:total } the
+// operator decodes, with the current chain integrity summary attached.
+func (s *svc) audit(c *zip.Ctx) error {
+	// No local store configured → preserve the legacy federated IAM view so the
+	// endpoint never regresses to empty.
+	if s.auditStore == nil {
+		return s.auditFromIAM(c)
+	}
+
+	f := auditFilterFromQuery(c)
+	rows, total, err := s.auditStore.Query(c.Context(), f)
+	if err != nil {
+		return fail(c, err.Error())
+	}
+
+	out := make([]auditRow, 0, len(rows))
+	for _, r := range rows {
+		out = append(out, toAuditRow(r))
+	}
+
+	// Attach the live integrity summary so the console can badge the trail as
+	// verified. Best-effort: a verify error must not fail the listing.
+	integrity, ivErr := s.auditStore.Verify(c.Context())
+	var integrityPayload any
+	if ivErr == nil {
+		integrityPayload = integrity
+	}
+
+	return c.JSON(200, map[string]any{
+		"status":    "ok",
+		"msg":       "",
+		"data":      out,
+		"data2":     total,
+		"integrity": integrityPayload,
+	})
+}
+
+// auditVerify answers GET /v1/admin/audit/verify — the tamper-evidence check. It
+// walks the whole hash chain and returns the integrity result (ok, count, head,
+// and the seq where the chain first breaks if tampered). Global-admin gated like
+// every admin route.
+func (s *svc) auditVerify(c *zip.Ctx) error {
+	if s.auditStore == nil {
+		return fail(c, "audit store not configured")
+	}
+	integrity, err := s.auditStore.Verify(c.Context())
+	if err != nil {
+		return fail(c, err.Error())
+	}
+	return ok(c, integrity)
+}
+
+// auditFilterFromQuery builds an audit.Filter from the request query params. Time
+// bounds accept RFC3339. pageSize (default 100, cap 1000) and p (1-based page)
+// drive Limit/Offset. Unknown/blank params are simply not applied.
+func auditFilterFromQuery(c *zip.Ctx) audit.Filter {
+	f := audit.Filter{
+		Org:      strings.TrimSpace(c.Query("org")),
+		Sub:      strings.TrimSpace(c.Query("sub")),
+		Action:   strings.TrimSpace(c.Query("action")),
+		Resource: strings.TrimSpace(c.Query("resource")),
+		Result:   strings.TrimSpace(c.Query("result")),
+	}
+	if v := strings.TrimSpace(c.Query("since")); v != "" {
+		if t, err := time.Parse(time.RFC3339, v); err == nil {
+			f.Since = t
+		}
+	}
+	if v := strings.TrimSpace(c.Query("until")); v != "" {
+		if t, err := time.Parse(time.RFC3339, v); err == nil {
+			f.Until = t
+		}
+	}
+	pageSize := 100
+	if v := strings.TrimSpace(c.Query("pageSize")); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			pageSize = n
+		}
+	}
+	f.Limit = pageSize
+	if v := strings.TrimSpace(c.Query("p")); v != "" {
+		if page, err := strconv.Atoi(v); err == nil && page > 1 {
+			f.Offset = (page - 1) * pageSize
+		}
+	}
+	return f
+}
+
+// toAuditRow maps a stored audit.Record to the operator wire row.
+func toAuditRow(r audit.Record) auditRow {
+	return auditRow{
+		Seq:        r.Seq,
+		Time:       r.Time.UTC().Format(time.RFC3339Nano),
+		Org:        r.Actor.Org,
+		Sub:        r.Actor.Sub,
+		Email:      r.Actor.Email,
+		Action:     r.Action,
+		Resource:   r.Resource.Type,
+		ResourceID: r.Resource.ID,
+		Method:     r.Method,
+		Path:       r.Path,
+		Result:     r.Outcome.Result,
+		Status:     r.Outcome.Status,
+		Reason:     r.Outcome.Reason,
+		SourceIP:   r.SourceIP,
+		UserAgent:  r.UserAgent,
+		RequestID:  r.RequestID,
+		IsAdmin:    r.Auth.IsAdmin,
+		Auth:       r.Auth.Method,
+		Hash:       r.Hash,
+		PrevHash:   r.PrevHash,
+	}
+}
+
+// auditFromIAM is the legacy federated view: when cloud has no local audit store,
+// forward the IAM get-records read verbatim (the prior behavior), so the endpoint
+// still surfaces IAM's own audit trail rather than an empty list.
+func (s *svc) auditFromIAM(c *zip.Ctx) error {
+	q := iamAuditQuery(c)
+	res, err := s.iam.getList(c.Context(), callerCreds(c), "/v1/iam/get-records", q)
+	if err != nil {
+		return fail(c, err.Error())
+	}
+	return okRaw(c, res.rows, res.total)
+}

clients/admin/audit_test.go

@@ -0,0 +1,238 @@
+package admin
+
+// Tests for the store-backed /v1/admin/audit + /v1/admin/audit/verify surface.
+// They wire admin against a REAL audit.Recorder (on-disk SQLite) seeded with
+// records, drive requests through the whole zip app, and assert the query
+// results, the integrity summary, and the global-admin gate.
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"testing"
+	"time"
+
+	fiber "github.com/gofiber/fiber/v3"
+	"github.com/hanzoai/cloud/audit"
+	"github.com/zap-proto/zip"
+	luxlog "github.com/luxfi/log"
+)
+
+// mountWithStore builds a zip app with admin's audit routes wired to a real audit
+// store, and returns the store + a request helper. Only the audit routes are
+// mounted here (the rest are covered by mount()); this keeps the store-backed
+// tests focused.
+func mountWithStore(t *testing.T) (*audit.Recorder, func(method, path string, hdr map[string]string) (*http.Response, []byte)) {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "audit.db")
+	rec, err := audit.Open(path, nil)
+	if err != nil {
+		t.Fatalf("audit.Open: %v", err)
+	}
+	t.Cleanup(func() { _ = rec.Close() })
+
+	app := zip.New(zip.Config{Logger: luxlog.New("test")})
+	s := &svc{adminOrg: "admin", auditStore: rec}
+	app.Get("/v1/admin/audit", s.guard(s.audit))
+	app.Get("/v1/admin/audit/verify", s.guard(s.auditVerify))
+	fa := app.Fiber()
+
+	do := func(method, p string, hdr map[string]string) (*http.Response, []byte) {
+		t.Helper()
+		req := httptest.NewRequest(method, p, nil)
+		for k, v := range hdr {
+			req.Header.Set(k, v)
+		}
+		resp, err := fa.Test(req, fiber.TestConfig{Timeout: 30 * time.Second})
+		if err != nil {
+			t.Fatalf("%s %s: %v", method, p, err)
+		}
+		b, _ := io.ReadAll(resp.Body)
+		return resp, b
+	}
+	return rec, do
+}
+
+func seedAudit(t *testing.T, rec *audit.Recorder, n int) {
+	t.Helper()
+	ctx := context.Background()
+	for i := 0; i < n; i++ {
+		_, err := rec.Append(ctx, audit.Record{
+			Time:     time.Now().UTC(),
+			Actor:    audit.Actor{Org: "admin", Sub: "z@hanzo.ai"},
+			Action:   "DELETE /v1/admin/orgs",
+			Resource: audit.Resource{Type: "org", ID: "acme"},
+			Auth:     audit.AuthContext{Method: "jwt", IsAdmin: true},
+			Outcome:  audit.Outcome{Result: "success", Status: 200},
+			Method:   "DELETE",
+			Path:     "/v1/admin/orgs/acme",
+		})
+		if err != nil {
+			t.Fatalf("seed %d: %v", i, err)
+		}
+	}
+}
+
+var globalAdmin = map[string]string{"X-User-IsAdmin": "true", "X-Org-Id": "admin", "X-User-Id": "z@hanzo.ai"}
+
+// TestAdminAudit_ReturnsRealRecords proves GET /v1/admin/audit returns the
+// store's records (newest-first) with an accurate total and an integrity summary.
+func TestAdminAudit_ReturnsRealRecords(t *testing.T) {
+	rec, do := mountWithStore(t)
+	seedAudit(t, rec, 5)
+
+	resp, body := do("GET", "/v1/admin/audit", globalAdmin)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("audit: got %d (body=%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Data []struct {
+			Seq    uint64 `json:"seq"`
+			Action string `json:"action"`
+			Hash   string `json:"hash"`
+			Result string `json:"result"`
+		} `json:"data"`
+		Data2     int `json:"data2"`
+		Integrity struct {
+			OK    bool   `json:"ok"`
+			Count uint64 `json:"count"`
+		} `json:"integrity"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v (body=%s)", err, body)
+	}
+	if env.Data2 != 5 || len(env.Data) != 5 {
+		t.Fatalf("got %d rows / total %d, want 5/5", len(env.Data), env.Data2)
+	}
+	if env.Data[0].Seq < env.Data[len(env.Data)-1].Seq {
+		t.Errorf("not newest-first: %d..%d", env.Data[0].Seq, env.Data[len(env.Data)-1].Seq)
+	}
+	if env.Data[0].Hash == "" {
+		t.Error("row has no hash — chain linkage not surfaced")
+	}
+	if !env.Integrity.OK || env.Integrity.Count != 5 {
+		t.Errorf("integrity summary = %+v, want ok/count=5", env.Integrity)
+	}
+}
+
+// TestAdminAudit_Filters proves the query filters (result) reach the store.
+func TestAdminAudit_Filters(t *testing.T) {
+	rec, do := mountWithStore(t)
+	ctx := context.Background()
+	// One deny among successes.
+	_, _ = rec.Append(ctx, audit.Record{Action: "POST /v1/admin/roles", Actor: audit.Actor{Org: "admin"}, Outcome: audit.Outcome{Result: "deny", Status: 403}})
+	seedAudit(t, rec, 3)
+
+	resp, body := do("GET", "/v1/admin/audit?result=deny", globalAdmin)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("got %d (body=%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Data  []map[string]any `json:"data"`
+		Data2 int              `json:"data2"`
+	}
+	_ = json.Unmarshal(body, &env)
+	if env.Data2 != 1 || len(env.Data) != 1 {
+		t.Fatalf("result=deny returned %d/%d, want 1/1", len(env.Data), env.Data2)
+	}
+	if env.Data[0]["result"] != "deny" {
+		t.Errorf("filtered row result = %v, want deny", env.Data[0]["result"])
+	}
+}
+
+// TestAdminAudit_VerifyEndpoint proves GET /v1/admin/audit/verify returns the
+// integrity result for the chain.
+func TestAdminAudit_VerifyEndpoint(t *testing.T) {
+	rec, do := mountWithStore(t)
+	seedAudit(t, rec, 8)
+
+	resp, body := do("GET", "/v1/admin/audit/verify", globalAdmin)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("verify: got %d (body=%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Data struct {
+			OK       bool   `json:"ok"`
+			Count    uint64 `json:"count"`
+			BrokenAt int64  `json:"brokenAt"`
+			HeadHash string `json:"headHash"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v (body=%s)", err, body)
+	}
+	if !env.Data.OK || env.Data.Count != 8 || env.Data.BrokenAt != -1 {
+		t.Errorf("verify result = %+v, want ok/count=8/brokenAt=-1", env.Data)
+	}
+	if env.Data.HeadHash == "" {
+		t.Error("verify returned no head hash")
+	}
+}
+
+// TestAdminAudit_DeniedWithoutGlobalAdmin proves BOTH audit endpoints fail-closed
+// 403 for a non-global-admin, and — critically — the store is NEVER read on a
+// denied request (the gate runs before the handler, so no records leak to an
+// unauthorized caller). We assert non-leakage by seeding records and confirming
+// the denied response body contains none of them.
+func TestAdminAudit_DeniedWithoutGlobalAdmin(t *testing.T) {
+	rec, do := mountWithStore(t)
+	seedAudit(t, rec, 3)
+
+	cases := []struct {
+		name string
+		hdr  map[string]string
+	}{
+		{"no identity", map[string]string{}},
+		{"tenant admin (org != adminOrg, no minted IsAdmin)", map[string]string{"X-Org-Id": "acme", "X-User-Id": "mallory"}},
+		{"forged-looking but non-admin", map[string]string{"X-Org-Id": "acme"}},
+	}
+	for _, ep := range []string{"/v1/admin/audit", "/v1/admin/audit/verify"} {
+		for _, tc := range cases {
+			resp, body := do("GET", ep, tc.hdr)
+			if resp.StatusCode != http.StatusForbidden {
+				t.Errorf("%s [%s]: got %d, want 403 (body=%s)", ep, tc.name, resp.StatusCode, body)
+			}
+			// No record content must appear in a denied response.
+			if len(body) > 0 && (contains(body, "DELETE /v1/admin/orgs") || contains(body, `"hash"`)) {
+				t.Errorf("%s [%s]: denied response leaked audit data: %s", ep, tc.name, body)
+			}
+		}
+	}
+}
+
+// TestAdminAudit_FallsBackToIAMWhenNoStore proves that when no local store is
+// configured (auditStore == nil), /v1/admin/audit still serves the federated IAM
+// view rather than erroring — preserving the prior capability. Covered by the
+// existing TestAudit_MapsRecords (IAM proxy path); here we assert the nil-store
+// verify endpoint reports "not configured" rather than panicking.
+func TestAdminAudit_VerifyWithoutStore(t *testing.T) {
+	app := zip.New(zip.Config{Logger: luxlog.New("test")})
+	s := &svc{adminOrg: "admin"} // no auditStore
+	app.Get("/v1/admin/audit/verify", s.guard(s.auditVerify))
+	req := httptest.NewRequest("GET", "/v1/admin/audit/verify", nil)
+	for k, v := range globalAdmin {
+		req.Header.Set(k, v)
+	}
+	resp, err := app.Fiber().Test(req, fiber.TestConfig{Timeout: 30 * time.Second})
+	if err != nil {
+		t.Fatalf("verify: %v", err)
+	}
+	body, _ := io.ReadAll(resp.Body)
+	// A well-formed error envelope, not a 500/panic.
+	if resp.StatusCode != http.StatusOK || !contains(body, "not configured") {
+		t.Errorf("nil-store verify = %d %s, want an ok-envelope 'not configured' error", resp.StatusCode, body)
+	}
+}
+
+func contains(b []byte, sub string) bool {
+	s := string(b)
+	for i := 0; i+len(sub) <= len(s); i++ {
+		if s[i:i+len(sub)] == sub {
+			return true
+		}
+	}
+	return false
+}

clients/admin/cockpit_test.go

@@ -0,0 +1,500 @@
+package admin
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	fiber "github.com/gofiber/fiber/v3"
+	"github.com/hanzoai/cloud/audit"
+)
+
+// ── rich stateful fakes for the customer-management surfaces ──────────────────
+
+// cockpitFakes bundles a stateful IAM + commerce fake and the mounted `do` helper,
+// exposing the recorded state (forbidden flips, deposits) tests assert on.
+type cockpitFakes struct {
+	iam      *httptest.Server
+	commerce *httptest.Server
+	svc      *svc
+	do       func(method, path string, hdr map[string]string, body string) (*http.Response, []byte)
+
+	mu          sync.Mutex
+	forbidden   map[string]bool  // "owner/name" -> forbidden (mutated by update-user)
+	updateCalls []string         // ids passed to update-user
+	deposits    []depositCapture // deposits commerce received
+	balances    map[string]int64 // org -> availableCents (mutated by deposit)
+}
+
+type depositCapture struct {
+	org    string
+	user   string
+	amount int64
+}
+
+// adminHdr is a validated global-admin identity (what SanitizeIdentity mints for
+// owner==AdminOrg) plus a replayable credential.
+func adminHdr() map[string]string {
+	return map[string]string{
+		"X-User-IsAdmin": "true", "X-Org-Id": "admin", "X-User-Id": "admin/z", "X-User-Email": "z@hanzo.ai",
+		"Authorization": "Bearer operator-jwt", "Cookie": "iam_access_token=operator-jwt",
+	}
+}
+
+// newCockpitFakes builds the stateful fleet: orgs acme + globex (owned by admin),
+// with users, balances, subscriptions, and a dated usage ledger — all relative to
+// `now` so the analytics windows are stable whenever the test runs.
+func newCockpitFakes(t *testing.T) *cockpitFakes {
+	t.Helper()
+	now := time.Now().UTC()
+	f := &cockpitFakes{
+		forbidden: map[string]bool{},
+		balances:  map[string]int64{"acme": 20000, "globex": 5000},
+	}
+	spend := map[string]int64{"acme": 1500, "globex": 300}
+
+	// Signup + usage dates relative to now so analytics windows include them.
+	acmeCreated := now.AddDate(0, 0, -45).Format(time.RFC3339)
+	globexCreated := now.AddDate(0, 0, -20).Format(time.RFC3339)
+	usage := map[string][]txn{
+		"acme": {
+			{ID: "t1", Type: "withdraw", Amount: 100, Currency: "usd", CreatedAt: now.AddDate(0, 0, -40).Format(time.RFC3339)},
+			{ID: "t2", Type: "withdraw", Amount: 200, Currency: "usd", CreatedAt: now.AddDate(0, 0, -5).Format(time.RFC3339)},
+			{ID: "t3", Type: "deposit", Amount: 20000, Currency: "usd", CreatedAt: now.AddDate(0, 0, -46).Format(time.RFC3339)},
+		},
+		"globex": {
+			{ID: "t4", Type: "withdraw", Amount: 400, Currency: "usd", CreatedAt: now.AddDate(0, 0, -3).Format(time.RFC3339)},
+		},
+	}
+	// users per org (owner/name): forbidden read live from f.forbidden.
+	type u struct{ owner, name, email, key string; admin bool }
+	users := map[string][]u{
+		"acme":   {{"acme", "anna", "anna@acme.test", "hk-anna-secret", true}, {"acme", "bob", "bob@acme.test", "", false}},
+		"globex": {{"globex", "gwen", "gwen@globex.test", "hk-gwen-secret", true}},
+	}
+
+	f.iam = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		q := r.URL.Query()
+		switch {
+		case strings.HasSuffix(r.URL.Path, "/get-organizations"):
+			fmt.Fprintf(w, `{"status":"ok","msg":"","data":[
+				{"owner":"admin","name":"acme","displayName":"Acme Inc","createdTime":%q},
+				{"owner":"admin","name":"globex","displayName":"Globex","createdTime":%q}
+			],"data2":2}`, acmeCreated, globexCreated)
+		case strings.HasSuffix(r.URL.Path, "/get-users"):
+			owner := q.Get("owner")
+			rows := []string{}
+			for _, us := range users[owner] {
+				f.mu.Lock()
+				forb := f.forbidden[us.owner+"/"+us.name]
+				f.mu.Unlock()
+				created := acmeCreated
+				if owner == "globex" {
+					created = globexCreated
+				}
+				rows = append(rows, fmt.Sprintf(`{"owner":%q,"name":%q,"email":%q,"isAdmin":%v,"isForbidden":%v,"accessKey":%q,"createdTime":%q,"lastSigninTime":%q}`,
+					us.owner, us.name, us.email, us.admin, forb, us.key, created, now.AddDate(0, 0, -2).Format(time.RFC3339)))
+			}
+			fmt.Fprintf(w, `{"status":"ok","msg":"","data":[%s],"data2":%d}`, strings.Join(rows, ","), len(rows))
+		case strings.HasSuffix(r.URL.Path, "/get-user"):
+			id := q.Get("id")
+			parts := strings.SplitN(id, "/", 2)
+			owner := ""
+			if len(parts) == 2 {
+				owner = parts[0]
+			}
+			for _, us := range users[owner] {
+				if us.owner+"/"+us.name == id {
+					f.mu.Lock()
+					forb := f.forbidden[id]
+					f.mu.Unlock()
+					// Full object incl. fields update-user must preserve.
+					fmt.Fprintf(w, `{"status":"ok","msg":"","data":{"owner":%q,"name":%q,"email":%q,"isAdmin":%v,"isForbidden":%v,"accessKey":%q,"displayName":"X","phone":"","type":"normal-user"}}`,
+						us.owner, us.name, us.email, us.admin, forb, us.key)
+					return
+				}
+			}
+			w.WriteHeader(404)
+			io.WriteString(w, `{"status":"error","msg":"not found"}`)
+		case strings.HasSuffix(r.URL.Path, "/update-user"):
+			id := q.Get("id")
+			body, _ := io.ReadAll(r.Body)
+			var obj map[string]any
+			_ = json.Unmarshal(body, &obj)
+			forb, _ := obj["isForbidden"].(bool)
+			f.mu.Lock()
+			f.forbidden[id] = forb
+			f.updateCalls = append(f.updateCalls, id)
+			f.mu.Unlock()
+			io.WriteString(w, `{"status":"ok","msg":"","data":"Affected"}`)
+		default:
+			w.WriteHeader(404)
+			io.WriteString(w, `{"status":"error","msg":"not found"}`)
+		}
+	}))
+
+	f.commerce = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		org := r.Header.Get("X-Org-Id")
+		q := r.URL.Query()
+		user := q.Get("user")
+		f.mu.Lock()
+		bal := f.balances[org]
+		f.mu.Unlock()
+		sp := int64(0)
+		if org != "" && user == org {
+			sp = spend[org]
+		} else {
+			bal = 0 // wrong subject/namespace → empty wallet (the live contract)
+		}
+		switch {
+		case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/deposit"):
+			var req struct {
+				User     string `json:"user"`
+				Amount   int64  `json:"amount"`
+				Currency string `json:"currency"`
+			}
+			body, _ := io.ReadAll(r.Body)
+			_ = json.Unmarshal(body, &req)
+			f.mu.Lock()
+			f.balances[org] += req.Amount
+			f.deposits = append(f.deposits, depositCapture{org: org, user: req.User, amount: req.Amount})
+			f.mu.Unlock()
+			w.WriteHeader(201)
+			fmt.Fprintf(w, `{"transactionId":"dep-%d","user":%q,"amount":%d,"currency":%q,"type":"deposit"}`, req.Amount, req.User, req.Amount, req.Currency)
+		case strings.HasSuffix(r.URL.Path, "/usage-rollup"):
+			fmt.Fprintf(w, `{"consumedCents":%d,"overageCents":0,"balance":{"balanceCents":%d,"availableCents":%d}}`, sp, bal, bal)
+		case strings.HasSuffix(r.URL.Path, "/balance"):
+			fmt.Fprintf(w, `{"user":%q,"currency":"usd","available":%d,"balance":%d}`, user, bal, bal)
+		case strings.HasSuffix(r.URL.Path, "/subscriptions"):
+			if org == "acme" && user == "acme" {
+				io.WriteString(w, `{"subscriptions":[{"status":"active","plan":{"name":"Pro","price":5000,"currency":"usd","interval":"month"}}]}`)
+			} else {
+				io.WriteString(w, `{"subscriptions":[]}`)
+			}
+		case strings.HasSuffix(r.URL.Path, "/transactions"):
+			// Commerce serves the ledger WRAPPED as { count, transactions:[...] }
+			// (the live contract) — the fake mirrors it so the decode is guarded
+			// against the real shape, not a bare array a mock would let pass.
+			rows := usage[org]
+			b, _ := json.Marshal(map[string]any{"count": len(rows), "transactions": rows})
+			w.Write(b)
+		default:
+			w.WriteHeader(404)
+			io.WriteString(w, `{"status":"error","msg":"not found"}`)
+		}
+	}))
+
+	_, s, fa := mountSvc(t, f.iam.URL, f.commerce.URL, "")
+	f.svc = s
+	f.do = func(method, path string, hdr map[string]string, body string) (*http.Response, []byte) {
+		t.Helper()
+		var rdr io.Reader
+		if body != "" {
+			rdr = strings.NewReader(body)
+		}
+		req := httptest.NewRequest(method, path, rdr)
+		if body != "" {
+			req.Header.Set("Content-Type", "application/json")
+		}
+		for k, v := range hdr {
+			req.Header.Set(k, v)
+		}
+		resp, err := fa.Test(req, fiber.TestConfig{Timeout: 30 * time.Second})
+		if err != nil {
+			t.Fatalf("%s %s: %v", method, path, err)
+		}
+		bb, _ := io.ReadAll(resp.Body)
+		return resp, bb
+	}
+	t.Cleanup(func() { f.iam.Close(); f.commerce.Close() })
+	return f
+}
+
+// TestCustomers_ListRealFleet proves the fleet customer list is real: every field
+// (owner email, plan, balance, spend, MRR, status, user count) comes from the live
+// IAM + commerce upstreams.
+func TestCustomers_ListRealFleet(t *testing.T) {
+	f := newCockpitFakes(t)
+	resp, body := f.do("GET", "/v1/admin/customers", adminHdr(), "")
+	if resp.StatusCode != 200 {
+		t.Fatalf("customers: %d (%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Data  []customerRow `json:"data"`
+		Data2 int           `json:"data2"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if env.Data2 != 2 || len(env.Data) != 2 {
+		t.Fatalf("want 2 customers, got %d (%+v)", len(env.Data), env.Data)
+	}
+	acme := env.Data[0] // sorted: acme, globex
+	if acme.Org != "acme" || acme.OwnerEmail != "anna@acme.test" || acme.Plan != "Pro" {
+		t.Errorf("acme identity wrong: %+v", acme)
+	}
+	if acme.BalanceCents != 20000 || acme.SpendCents != 1500 || acme.MRRCents != 5000 {
+		t.Errorf("acme money wrong: bal=%d spend=%d mrr=%d", acme.BalanceCents, acme.SpendCents, acme.MRRCents)
+	}
+	if acme.Users != 2 || acme.Status != "active" {
+		t.Errorf("acme users/status wrong: users=%d status=%s", acme.Users, acme.Status)
+	}
+}
+
+// TestCustomerDetail_RealAndNoSecretLeak proves the detail is real AND that the
+// hk- access key VALUE never appears in the response (presence only).
+func TestCustomerDetail_RealAndNoSecretLeak(t *testing.T) {
+	f := newCockpitFakes(t)
+	resp, body := f.do("GET", "/v1/admin/customers/acme", adminHdr(), "")
+	if resp.StatusCode != 200 {
+		t.Fatalf("detail: %d (%s)", resp.StatusCode, body)
+	}
+	if strings.Contains(string(body), "hk-anna-secret") {
+		t.Fatalf("SECRET LEAK: the access key value appears in the customer detail response")
+	}
+	var env struct {
+		Data customerDetail `json:"data"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	d := env.Data
+	if d.Org != "acme" || d.Plan != "Pro" || d.BalanceCents != 20000 || d.MRRCents != 5000 {
+		t.Errorf("detail money/plan wrong: %+v", d)
+	}
+	// anna has a key, bob does not → apiKeys count = 1.
+	if d.APIKeys != 1 {
+		t.Errorf("apiKeys = %d, want 1", d.APIKeys)
+	}
+	if len(d.Users) != 2 {
+		t.Fatalf("want 2 users, got %d", len(d.Users))
+	}
+	// The users carry hasApiKey (presence) but NO key value field exists in the type.
+	var anna *customerUser
+	for i := range d.Users {
+		if d.Users[i].Name == "anna" {
+			anna = &d.Users[i]
+		}
+	}
+	if anna == nil || !anna.HasAPIKey || !anna.IsAdmin {
+		t.Errorf("anna mapping wrong: %+v", anna)
+	}
+	if len(d.Transactions) == 0 {
+		t.Error("detail must include the real ledger transactions")
+	}
+}
+
+// TestGrantCredit_DepositLandsAndAudited proves grant credit is a REAL commerce
+// deposit (right org/subject) reflected in the balance AND recorded to the
+// tamper-evident audit trail with a before/after.
+func TestGrantCredit_DepositLandsAndAudited(t *testing.T) {
+	f := newCockpitFakes(t)
+	rec, err := audit.Open(":memory:", nil)
+	if err != nil {
+		t.Fatalf("audit open: %v", err)
+	}
+	defer rec.Close()
+	f.svc.auditStore = rec
+
+	resp, body := f.do("POST", "/v1/admin/customers/acme/credit", adminHdr(), `{"amountCents":5000,"reason":"support comp"}`)
+	if resp.StatusCode != 200 {
+		t.Fatalf("credit: %d (%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Status string `json:"status"`
+		Data   struct {
+			GrantedCents  int64  `json:"grantedCents"`
+			BalanceCents  int64  `json:"balanceCents"`
+			TransactionID string `json:"transactionId"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if env.Status != "ok" || env.Data.GrantedCents != 5000 {
+		t.Fatalf("grant envelope wrong: %+v", env)
+	}
+	// The balance reflects the grant (20000 + 5000).
+	if env.Data.BalanceCents != 25000 {
+		t.Errorf("balance after grant = %d, want 25000", env.Data.BalanceCents)
+	}
+	// Commerce received a deposit for the RIGHT org + subject (X-Org-Id=acme, user=acme).
+	f.mu.Lock()
+	deps := append([]depositCapture(nil), f.deposits...)
+	f.mu.Unlock()
+	if len(deps) != 1 || deps[0].org != "acme" || deps[0].user != "acme" || deps[0].amount != 5000 {
+		t.Fatalf("deposit not landed on the right subject: %+v", deps)
+	}
+	// The action is on the tamper-evident trail with a before/after balance.
+	rows, total, err := rec.Query(context.Background(), audit.Filter{Action: "admin.customer.credit"})
+	if err != nil || total < 1 || len(rows) < 1 {
+		t.Fatalf("credit not audited: total=%d err=%v", total, err)
+	}
+	r := rows[0]
+	if r.Actor.Org != "admin" || r.Outcome.Result != "success" || r.Resource.ID != "acme" {
+		t.Errorf("audit record wrong: %+v", r)
+	}
+	if !strings.Contains(string(r.Before), "balanceCents") || !strings.Contains(string(r.After), "grantedCents") {
+		t.Errorf("audit before/after missing: before=%s after=%s", r.Before, r.After)
+	}
+}
+
+// TestGrantCredit_Validation proves the guardrails (positive amount, cap, real org).
+func TestGrantCredit_Validation(t *testing.T) {
+	f := newCockpitFakes(t)
+	cases := []struct {
+		name, org, body string
+		wantStatus      int
+		wantErr         bool
+	}{
+		{"zero amount", "acme", `{"amountCents":0}`, 200, true},
+		{"negative", "acme", `{"amountCents":-100}`, 200, true},
+		{"over cap", "acme", `{"amountCents":999999999}`, 200, true},
+		{"unknown org", "nope", `{"amountCents":100}`, 404, true},
+	}
+	for _, tc := range cases {
+		resp, body := f.do("POST", "/v1/admin/customers/"+tc.org+"/credit", adminHdr(), tc.body)
+		if resp.StatusCode != tc.wantStatus {
+			t.Errorf("%s: status %d, want %d (%s)", tc.name, resp.StatusCode, tc.wantStatus, body)
+		}
+		if tc.wantErr && !strings.Contains(string(body), `"error"`) {
+			t.Errorf("%s: expected error envelope, got %s", tc.name, body)
+		}
+	}
+	// No deposit should have landed for any invalid grant.
+	f.mu.Lock()
+	n := len(f.deposits)
+	f.mu.Unlock()
+	if n != 0 {
+		t.Errorf("invalid grants must NOT deposit, but %d landed", n)
+	}
+}
+
+// TestSuspendReactivate_ForbidsUsersAndAudits proves suspend flips IAM isForbidden
+// on every org user (the real access lever) and is audited, and reactivate reverses
+// it — the customer's status reflects the change on a re-list.
+func TestSuspendReactivate_ForbidsUsersAndAudits(t *testing.T) {
+	f := newCockpitFakes(t)
+	rec, _ := audit.Open(":memory:", nil)
+	defer rec.Close()
+	f.svc.auditStore = rec
+
+	// Suspend acme.
+	resp, body := f.do("POST", "/v1/admin/customers/acme/suspend", adminHdr(), "")
+	if resp.StatusCode != 200 {
+		t.Fatalf("suspend: %d (%s)", resp.StatusCode, body)
+	}
+	// Both acme users were update-user'd to forbidden.
+	f.mu.Lock()
+	if !f.forbidden["acme/anna"] || !f.forbidden["acme/bob"] {
+		t.Errorf("suspend did not forbid both users: %+v", f.forbidden)
+	}
+	f.mu.Unlock()
+
+	// A re-list shows acme suspended (all users forbidden).
+	_, lb := f.do("GET", "/v1/admin/customers", adminHdr(), "")
+	var env struct{ Data []customerRow `json:"data"` }
+	_ = json.Unmarshal(lb, &env)
+	for _, c := range env.Data {
+		if c.Org == "acme" && c.Status != "suspended" {
+			t.Errorf("acme status = %q after suspend, want suspended", c.Status)
+		}
+	}
+	// Audited.
+	if _, total, _ := rec.Query(context.Background(), audit.Filter{Action: "admin.customer.suspend"}); total < 1 {
+		t.Errorf("suspend not audited")
+	}
+
+	// Reactivate reverses it.
+	if _, rb := f.do("POST", "/v1/admin/customers/acme/reactivate", adminHdr(), ""); !strings.Contains(string(rb), `"suspended":false`) {
+		t.Errorf("reactivate response wrong: %s", rb)
+	}
+	f.mu.Lock()
+	if f.forbidden["acme/anna"] || f.forbidden["acme/bob"] {
+		t.Errorf("reactivate did not clear forbidden: %+v", f.forbidden)
+	}
+	f.mu.Unlock()
+}
+
+// TestRevenue_RealAggregate proves the fleet revenue board: totals, paying-customer
+// count, ARPU, and the per-customer table are real commerce aggregates.
+func TestRevenue_RealAggregate(t *testing.T) {
+	f := newCockpitFakes(t)
+	resp, body := f.do("GET", "/v1/admin/revenue", adminHdr(), "")
+	if resp.StatusCode != 200 {
+		t.Fatalf("revenue: %d (%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Data revenueData `json:"data"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	d := env.Data
+	if d.TotalBalancesCents != 25000 { // acme 20000 + globex 5000
+		t.Errorf("total balances = %d, want 25000", d.TotalBalancesCents)
+	}
+	if d.TotalSpendCents != 1800 { // 1500 + 300
+		t.Errorf("total spend = %d, want 1800", d.TotalSpendCents)
+	}
+	if d.MRRCents != 5000 {
+		t.Errorf("MRR = %d, want 5000", d.MRRCents)
+	}
+	if d.PayingCustomers != 2 {
+		t.Errorf("paying customers = %d, want 2", d.PayingCustomers)
+	}
+	if d.ARPUCents != 900 { // 1800 / 2
+		t.Errorf("ARPU = %d, want 900", d.ARPUCents)
+	}
+	if len(d.PerCustomer) != 2 || d.PerCustomer[0].Org != "acme" { // sorted by spend desc
+		t.Errorf("per-customer table wrong: %+v", d.PerCustomer)
+	}
+	if len(d.SpendTrend) == 0 {
+		t.Error("revenue must include a real spend trend")
+	}
+}
+
+// TestAnalytics_HandlerRealWiring proves the analytics handler wires IAM signups +
+// commerce ledger into a REAL cohort/active/growth board, and flags computed=true.
+func TestAnalytics_HandlerRealWiring(t *testing.T) {
+	f := newCockpitFakes(t)
+	resp, body := f.do("GET", "/v1/admin/analytics?range=all", adminHdr(), "")
+	if resp.StatusCode != 200 {
+		t.Fatalf("analytics: %d (%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Data analyticsData `json:"data"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	d := env.Data
+	if d.TotalCustomers != 2 {
+		t.Errorf("total customers = %d, want 2", d.TotalCustomers)
+	}
+	// Real ledger present → retention/active/usage computed, growth always.
+	if !d.Computed["growth"] || !d.Computed["retention"] || !d.Computed["active"] {
+		t.Errorf("computed flags must be true with a real ledger: %+v", d.Computed)
+	}
+	// Both customers had recent usage → MAU covers them.
+	if d.MAU < 1 {
+		t.Errorf("MAU = %d, want >=1 (recent usage)", d.MAU)
+	}
+	// Retention grid has cohorts from the two signups.
+	if len(d.Retention.Cohorts) == 0 {
+		t.Error("retention grid must have cohorts from real signups")
+	}
+	// Top customer by usage present (acme 300c > globex 400c? globex 400 wins).
+	if len(d.TopCustomers) == 0 {
+		t.Error("top customers must be populated from real usage")
+	}
+}

clients/admin/commerce.go

@@ -0,0 +1,440 @@
+package admin
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+// commerceClient reads the commerce billing S2S surface (/v1/billing/*, /v1/costs)
+// for the money panels (spend, tokens, credits, COGS). Commerce runs as its own
+// deployment; these are HTTP calls authenticated with the admin-scoped
+// COMMERCE_SERVICE_TOKEN (a KMS-sourced secret already on the cloud env — never
+// hard-coded here). PER-ORG reads (balance/usage-rollup/subscriptions) resolve the
+// org's billing namespace from the TRUSTED X-Org-Id header — commerce's EdgeAuth
+// trusts it ONLY when the bearer is the service token — and key the wallet under the
+// bare org slug (`user`). The fleet-wide /v1/costs god-view is org-INDEPENDENT
+// (DigitalOcean + provider vendor bills) and sends NO org, so commerce falls back to
+// its own service namespace (COMMERCE_SERVICE_ORG) there. (An earlier revision sent
+// X-IAM-Org-Id, which commerce does NOT read — every per-org money panel read $0.)
+type commerceClient struct {
+	base  string // e.g. http://commerce.hanzo.svc.cluster.local:8001
+	token string // admin S2S bearer (secret; never logged)
+	http  *http.Client
+}
+
+func newCommerceClient(base, token string) *commerceClient {
+	return &commerceClient{
+		base:  strings.TrimRight(strings.TrimSpace(base), "/"),
+		token: strings.TrimSpace(token),
+		http:  &http.Client{Timeout: 15 * time.Second},
+	}
+}
+
+func (c *commerceClient) configured() bool { return c != nil && c.base != "" }
+
+// rollup is the org-scoped billing view commerce serves at /v1/billing/usage-rollup.
+// Cents are the canonical unit; consumedCents is the org's month-to-date spend.
+type rollup struct {
+	ConsumedCents int64 `json:"consumedCents"`
+	OverageCents  int64 `json:"overageCents"`
+	Balance       struct {
+		BalanceCents   int64 `json:"balanceCents"`
+		AvailableCents int64 `json:"availableCents"`
+	} `json:"balance"`
+}
+
+// usageRollup fetches the current-month rollup for one billing subject (an IAM
+// "org/user" identity) in org `org`. commerce keys usage per user; the operator
+// aggregates across an org's users when a full breakdown is needed. Returns a
+// zero rollup (not an error) when commerce is not configured so a partial deploy
+// degrades to honest zeros rather than a 5xx.
+func (c *commerceClient) usageRollup(ctx context.Context, org, user string) (rollup, error) {
+	var out rollup
+	if !c.configured() {
+		return out, nil
+	}
+	q := url.Values{"user": {user}}
+	body, err := c.get(ctx, "/v1/billing/usage-rollup", q, org)
+	if err != nil {
+		return out, err
+	}
+	if err := json.Unmarshal(body, &out); err != nil {
+		return out, fmt.Errorf("commerce rollup decode: %w", err)
+	}
+	return out, nil
+}
+
+// balanceAll is the org's prepaid credit balance across currencies (cents).
+// Sourced from /v1/billing/balance/all for the "Credits" tile.
+type balanceAll struct {
+	Balances map[string]struct {
+		Available int64 `json:"available"`
+		Balance   int64 `json:"balance"`
+	} `json:"balances"`
+}
+
+// creditsCents returns the org's available credit balance in USD cents. Zero
+// (not an error) when commerce is unconfigured.
+func (c *commerceClient) creditsCents(ctx context.Context, org, user string) (int64, error) {
+	if !c.configured() {
+		return 0, nil
+	}
+	q := url.Values{"user": {user}, "currency": {"usd"}}
+	body, err := c.get(ctx, "/v1/billing/balance", q, org)
+	if err != nil {
+		return 0, err
+	}
+	var b struct {
+		Available int64 `json:"available"`
+	}
+	if err := json.Unmarshal(body, &b); err != nil {
+		return 0, fmt.Errorf("commerce balance decode: %w", err)
+	}
+	return b.Available, nil
+}
+
+// subscriptionsWire is the /v1/billing/subscriptions list shape the MRR + plan
+// readers fold over. Only the fields we need are decoded (status + plan
+// name/price/interval); commerce emits plan.price in the currency's minor unit
+// (cents) per its wire.
+type subscriptionsWire struct {
+	Subscriptions []struct {
+		Status string `json:"status"`
+		Plan   struct {
+			Name     string `json:"name"`
+			ID       string `json:"id"`
+			Price    int64  `json:"price"`
+			Currency string `json:"currency"`
+			Interval string `json:"interval"`
+		} `json:"plan"`
+	} `json:"subscriptions"`
+}
+
+// subSummary is the plan + MRR view of an org's subscriptions in ONE read: the
+// active plan name (the customer's tier), the normalized monthly recurring cents,
+// and whether any subscription is active. "pay-as-you-go" is the honest default
+// for a metered customer with no active subscription (not a fabricated tier).
+type subSummary struct {
+	Plan   string // active plan name, else "pay-as-you-go"
+	MRR    int64  // monthly-normalized recurring cents from active subs
+	Active bool   // any active/trialing subscription present
+}
+
+// subscriptionSummary reads /v1/billing/subscriptions ONCE and derives both the
+// plan tier and the MRR contribution, so the customer list/detail and the revenue
+// board share a single upstream read (DRY). Only "active"/"trialing" subscriptions
+// count; canceled/past-due do not. An honest zero/"pay-as-you-go" (not an error)
+// when commerce is unconfigured, so a partial deploy degrades to honest values.
+func (c *commerceClient) subscriptionSummary(ctx context.Context, org, user string) (subSummary, error) {
+	sum := subSummary{Plan: "pay-as-you-go"}
+	if !c.configured() {
+		return sum, nil
+	}
+	q := url.Values{"user": {user}}
+	body, err := c.get(ctx, "/v1/billing/subscriptions", q, org)
+	if err != nil {
+		return sum, err
+	}
+	var w subscriptionsWire
+	if err := json.Unmarshal(body, &w); err != nil {
+		return sum, fmt.Errorf("commerce subscriptions decode: %w", err)
+	}
+	for _, s := range w.Subscriptions {
+		switch strings.ToLower(strings.TrimSpace(s.Status)) {
+		case "active", "trialing":
+			sum.MRR += monthlyNormalizedCents(s.Plan.Price, s.Plan.Interval)
+			sum.Active = true
+			if name := strings.TrimSpace(s.Plan.Name); name != "" && sum.Plan == "pay-as-you-go" {
+				sum.Plan = name
+			}
+		}
+	}
+	return sum, nil
+}
+
+// mrrCents returns the monthly-recurring-revenue contribution of org `org`'s
+// ACTIVE subscriptions (see subscriptionSummary). Kept as the narrow reader the
+// finance/revenue folds call; it delegates so there is ONE subscriptions decode.
+func (c *commerceClient) mrrCents(ctx context.Context, org, user string) (int64, error) {
+	sum, err := c.subscriptionSummary(ctx, org, user)
+	return sum.MRR, err
+}
+
+// monthlyNormalizedCents normalizes a plan price to a monthly figure by its
+// billing interval so annual and monthly plans are comparable in one MRR sum.
+func monthlyNormalizedCents(priceCents int64, interval string) int64 {
+	switch strings.ToLower(strings.TrimSpace(interval)) {
+	case "year", "yearly", "annual", "annually":
+		return priceCents / 12
+	case "week", "weekly":
+		return priceCents * 52 / 12
+	case "day", "daily":
+		return priceCents * 365 / 12
+	default: // month/monthly and anything unrecognized → treat as monthly
+		return priceCents
+	}
+}
+
+// vendorCost mirrors commerce's api/costs.VendorCost — one line of what WE pay a
+// vendor for a service in a period (COGS, USD cents). Decoded verbatim from
+// GET /v1/costs so the finance board renders the per-vendor breakdown without
+// re-deriving any cost cloud-side.
+type vendorCost struct {
+	Vendor      string `json:"vendor"`
+	Service     string `json:"service"`
+	AmountCents int64  `json:"amountCents"`
+	Source      string `json:"source"` // "actual" | "estimated"
+	Note        string `json:"note,omitempty"`
+}
+
+// costReport is the GET /v1/costs response: every vendor COGS line for a period
+// plus the total. TotalCents is the platform's whole COGS (DigitalOcean compute +
+// the LLM providers we resell) — the single figure the finance margin math folds.
+type costReport struct {
+	Period     string       `json:"period"`
+	Vendors    []vendorCost `json:"vendors"`
+	TotalCents int64        `json:"totalCents"`
+	Currency   string       `json:"currency"`
+}
+
+// costs reads commerce's vendor-COGS god-view (GET /v1/costs) for a period — the
+// SINGLE source of truth for what we pay every vendor. It authenticates with the
+// admin S2S service token (COMMERCE_SERVICE_TOKEN, no IAM user identity), which
+// commerce's requireCostsAdmin admits on its M2M path (Admin bit + empty Subject).
+//
+// This is a PLATFORM god-view, deliberately NOT per-org, so NO org selector is
+// sent: the DigitalOcean compute and OpenAI COGS lines are read from the vendor
+// billing APIs (global, org-independent) and the metered LLM estimates come from
+// commerce's own service namespace (COMMERCE_SERVICE_ORG) — which commerce resolves
+// from its service-token config, never from a request header. Returns a zero report
+// (not an error) when commerce is unconfigured so a partial deploy degrades to
+// honest zeros rather than a 5xx.
+func (c *commerceClient) costs(ctx context.Context, period string) (costReport, error) {
+	var out costReport
+	if !c.configured() {
+		return out, nil
+	}
+	q := url.Values{}
+	if period != "" {
+		q.Set("period", period)
+	}
+	// Empty org: /v1/costs is fleet-wide; commerce uses COMMERCE_SERVICE_ORG.
+	body, err := c.get(ctx, "/v1/costs", q, "")
+	if err != nil {
+		return out, err
+	}
+	if err := json.Unmarshal(body, &out); err != nil {
+		return out, fmt.Errorf("commerce costs decode: %w", err)
+	}
+	return out, nil
+}
+
+// depositResult is the /v1/billing/deposit 201 response — the transaction id of
+// the credit that landed. The operator surfaces it as the receipt of a grant.
+type depositResult struct {
+	TransactionID string `json:"transactionId"`
+	User          string `json:"user"`
+	Amount        int64  `json:"amount"`
+	Currency      string `json:"currency"`
+}
+
+// deposit grants credit to an org's wallet by creating a commerce Deposit
+// transaction (POST /v1/billing/deposit). This is the ONE money-in primitive the
+// admin credit action uses (refunds/comps/support) — it is symmetric with the
+// balance READ: the same X-Org-Id=<org> namespace + `user`=<org> subject the
+// creditsCents/usageRollup reads resolve, so a grant lands exactly where the
+// balance panel reads it. Authenticated with the admin S2S COMMERCE_SERVICE_TOKEN
+// (commerce's /billing admin group), which is the same credential the reads use.
+// Commerce's EdgeAuth additionally pins the body `user` to the X-Org-Id subject,
+// so the grant can never be mis-targeted to another org's wallet. amountCents must
+// be positive (a grant, never a silent debit) — the handler validates + caps it.
+func (c *commerceClient) deposit(ctx context.Context, org, user string, amountCents int64, currency, notes, tags string) (depositResult, error) {
+	var out depositResult
+	if !c.configured() {
+		return out, errUnconfigured
+	}
+	if currency == "" {
+		currency = "usd"
+	}
+	body, err := json.Marshal(map[string]any{
+		"user":     user,
+		"currency": currency,
+		"amount":   amountCents,
+		"notes":    notes,
+		"tags":     tags,
+	})
+	if err != nil {
+		return out, err
+	}
+	respBody, err := c.post(ctx, "/v1/billing/deposit", org, body)
+	if err != nil {
+		return out, err
+	}
+	if err := json.Unmarshal(respBody, &out); err != nil {
+		return out, fmt.Errorf("commerce deposit decode: %w", err)
+	}
+	return out, nil
+}
+
+// txn is one commerce ledger row (GET /v1/billing/transactions). Cents is the
+// canonical unit; Type is "deposit" (credit) or "withdraw" (usage/consumption).
+// CreatedAt is the RFC3339 event time the analytics fold buckets on.
+type txn struct {
+	ID        string `json:"id"`
+	Type      string `json:"type"`
+	Amount    int64  `json:"amount"`
+	Currency  string `json:"currency"`
+	Tags      string `json:"tags,omitempty"`
+	Notes     string `json:"notes,omitempty"`
+	CreatedAt string `json:"createdAt"`
+}
+
+// transactions reads an org's ledger (GET /v1/billing/transactions) for one
+// billing subject. The rows carry a real event timestamp + type, so the analytics
+// aggregator can derive signup-cohort retention, active-customer windows, churn,
+// and usage-over-time from actual consumption events — NOT a fabricated series.
+// `limit` bounds the read (the endpoint sorts newest-first). Returns an empty
+// slice (not an error) when commerce is unconfigured so a partial deploy degrades
+// to an honest empty history rather than a 5xx.
+func (c *commerceClient) transactions(ctx context.Context, org, user string, limit int) ([]txn, error) {
+	if !c.configured() {
+		return nil, nil
+	}
+	q := url.Values{"user": {user}}
+	if limit > 0 {
+		q.Set("limit", fmt.Sprintf("%d", limit))
+	}
+	body, err := c.get(ctx, "/v1/billing/transactions", q, org)
+	if err != nil {
+		return nil, err
+	}
+	// Commerce serves the ledger WRAPPED as { count, transactions:[...] } (verified
+	// live). Decode that shape; tolerate a bare array too so a contract change in
+	// either direction degrades gracefully rather than silently reading zero rows
+	// (which would make the analytics honest-empty despite real usage).
+	var wrap struct {
+		Transactions []txn `json:"transactions"`
+	}
+	if err := json.Unmarshal(body, &wrap); err == nil && wrap.Transactions != nil {
+		return wrap.Transactions, nil
+	}
+	var rows []txn
+	if err := json.Unmarshal(body, &rows); err != nil {
+		return nil, fmt.Errorf("commerce transactions decode: %w", err)
+	}
+	return rows, nil
+}
+
+// post performs one admin-authenticated commerce POST (JSON body) and returns the
+// raw response. It carries the SAME trust context as get: the admin S2S service
+// token as the bearer and X-Org-Id=<org> as the per-org namespace selector that
+// commerce's EdgeAuth trusts only after verifying the service token. A non-2xx is
+// an error (the caller surfaces it honestly + records the failed attempt in the
+// audit trail — a grant that did not land is never reported as success).
+func (c *commerceClient) post(ctx context.Context, path, org string, body []byte) ([]byte, error) {
+	u := c.base + path
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("Content-Type", "application/json")
+	if c.token != "" {
+		req.Header.Set("Authorization", "Bearer "+c.token)
+	}
+	if org != "" {
+		req.Header.Set("X-Org-Id", org)
+	}
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("commerce unreachable: %w", err)
+	}
+	defer resp.Body.Close()
+	respBody, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, fmt.Errorf("commerce status %d", resp.StatusCode)
+	}
+	return respBody, nil
+}
+
+// get performs one admin-authenticated commerce GET and returns the raw body.
+func (c *commerceClient) get(ctx context.Context, path string, q url.Values, org string) ([]byte, error) {
+	u := c.base + path
+	if enc := q.Encode(); enc != "" {
+		u += "?" + enc
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept", "application/json")
+	if c.token != "" {
+		req.Header.Set("Authorization", "Bearer "+c.token)
+	}
+	if org != "" {
+		// Commerce's EdgeAuth (middleware/edgeauth.go) trusts X-Org-Id ONLY after it
+		// verifies the bearer is the COMMERCE_SERVICE_TOKEN, then resolves the per-org
+		// billing namespace from it. This is the service-to-service org selector.
+		// X-IAM-Org-Id is NOT read by commerce — it silently resolved to the default
+		// (COMMERCE_SERVICE_ORG) namespace, so every real org's balance/spend read $0.
+		req.Header.Set("X-Org-Id", org)
+	}
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("commerce unreachable: %w", err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, fmt.Errorf("commerce status %d", resp.StatusCode)
+	}
+	return body, nil
+}
+
+// healthClient probes an upstream's /v1/o11y/health (or any health path) so the
+// overview can report System Health honestly. A non-2xx or unreachable upstream
+// is reported as not-ok — never masked.
+type healthClient struct {
+	url  string
+	http *http.Client
+}
+
+func newHealthClient(u string) *healthClient {
+	return &healthClient{url: strings.TrimSpace(u), http: &http.Client{Timeout: 8 * time.Second}}
+}
+
+func (h *healthClient) configured() bool { return h != nil && h.url != "" }
+
+// ok reports whether the o11y health endpoint answers 2xx.
+func (h *healthClient) ok(ctx context.Context) (bool, error) {
+	if !h.configured() {
+		return false, fmt.Errorf("o11y health not configured")
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, h.url, nil)
+	if err != nil {
+		return false, err
+	}
+	resp, err := h.http.Do(req)
+	if err != nil {
+		return false, fmt.Errorf("o11y unreachable: %w", err)
+	}
+	defer resp.Body.Close()
+	_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16))
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return false, fmt.Errorf("o11y health %d", resp.StatusCode)
+	}
+	return true, nil
+}

clients/admin/compute.go

@@ -0,0 +1,232 @@
+// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package admin
+
+// compute — GET /v1/admin/compute, the cross-tenant compute-analytics read the
+// operator's Bots and Machines boards (admin.hanzo.ai) group into an
+// org → app → project tree. It aggregates the operator-owned usage table
+// hanzo.compute_usage(org, app, project, kind, event, machine_id, size,
+// price_cents, ts) — the same warehouse (`datastore`, ClickHouse) the analytics
+// subsystem reads, over the SAME shared client (aiobject.DatastoreQuery), no
+// second connection. `kind` is an OPEN LowCardinality spectrum (bot | machine |
+// cluster | nodepool | container | function | …) — a bot is a machine running the
+// @hanzo/bot agent, a machine is raw compute visor opens — and each console lens
+// reuses this one endpoint with a different `?kind=` (Bots=bot, Machines=machine).
+//
+// GLOBAL-ADMIN ONLY (the s.guard wrap in admin.go), all-orgs by default; this is
+// an AGGREGATOR — admin holds no compute state, it only reads. Honest by
+// construction, exactly like the analytics events lens: no datastore connected, or
+// the events table not provisioned yet (the emitter is still being wired) → the
+// real empty list, NEVER a fabricated fleet. admin creates NO table (the datastore
+// stream owns hanzo.compute_usage). Money is USD cents end to end.
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	aiobject "github.com/hanzoai/ai/object"
+	"github.com/zap-proto/zip"
+)
+
+// computeTable is the operator-owned compute-usage warehouse table (named to match
+// the existing hanzo.cloud_usage convention; the visor/commerce emitter writes it).
+// admin only READS it (never creates it — mirrors how analytics treats hanzo.events).
+const computeTable = "hanzo.compute_usage"
+
+// terminalComputeEvents are the lifecycle events whose LATEST occurrence means a
+// machine is no longer running (mirrors the console foldEvents terminal set). A
+// CLOSED server-side constant — never user input — so rendering it into the argMax
+// check is injection-safe.
+var terminalComputeEvents = []string{
+	"stop", "stopped", "destroy", "destroyed", "terminate", "terminated",
+	"delete", "deleted", "off", "shutdown", "expire", "expired",
+}
+
+// computeLeaf is one (org, app, project, kind) rollup: distinct machines of that
+// kind, how many are currently active (latest event non-terminal), the billed
+// spend over the window, and the most recent event. The console folds these into
+// the org → app → project tree.
+type computeLeaf struct {
+	Org        string `json:"org"`
+	App        string `json:"app"`
+	Project    string `json:"project"`
+	Kind       string `json:"kind"`
+	Machines   int64  `json:"machines"`
+	Active     int64  `json:"active"`
+	SpendCents int64  `json:"spendCents"`
+	LastTs     string `json:"lastTs"`
+}
+
+// compute answers GET /v1/admin/compute. ?kind=<kind> and ?org= narrow the
+// aggregate; ?range=24h|7d|30d bounds it (default 30d). Global-admin only.
+func (s *svc) compute(c *zip.Ctx) error {
+	ctx := c.Context()
+	// Honest-empty when the warehouse is not connected or the usage table is not
+	// provisioned yet (the visor/commerce emitter is still being wired).
+	if !aiobject.DatastoreEnabled() || !computeTableExists(ctx) {
+		return okList(c, []computeLeaf{}, 0)
+	}
+
+	// `kind` is an OPEN LowCardinality spectrum (bot | machine | cluster | nodepool |
+	// container | function | …), matched as a PLAIN STRING — no enum assumption. Each
+	// console lens passes its own kind; empty = all kinds. Case-normalized to the
+	// warehouse's lower-case convention.
+	kind := strings.ToLower(strings.TrimSpace(c.Query("kind")))
+	sql, args := buildComputeQuery(c.Query("range"), kind, strings.TrimSpace(c.Query("org")))
+	rows, err := aiobject.DatastoreQuery(ctx, sql, args...)
+	if err != nil {
+		return fail(c, "compute query: "+err.Error())
+	}
+	leaves := computeLeavesFromRows(rows)
+	return okList(c, leaves, len(leaves))
+}
+
+// buildComputeQuery assembles the two-level roll-up (pure, so it is unit-tested).
+// The inner query resolves each machine's LATEST lifecycle state (argMax event by
+// ts) + its billed spend; the outer counts machines, counts the still-active ones,
+// and sums spend per (org, app, project, kind). `kind` is a PLAIN STRING over an
+// open LowCardinality spectrum (no enum assumption; any non-empty value filters) and
+// the terminal set is a constant, so nothing user-derived is interpolated — org,
+// kind, and the time bound are all POSITIONAL parameters.
+func buildComputeQuery(rangeLabel, kind, org string) (string, []any) {
+	where := "ts >= ?"
+	args := []any{chTS(computeSince(rangeLabel))}
+	if kind != "" {
+		where += " AND kind = ?"
+		args = append(args, kind)
+	}
+	if org != "" {
+		where += " AND org = ?"
+		args = append(args, org)
+	}
+	sql := "SELECT org, app, project, kind, " +
+		"count() AS machines, countIf(active) AS active, sum(spend) AS spend_cents, max(last_ts) AS last_ts " +
+		"FROM (SELECT org, app, project, kind, machine_id, " +
+		"sum(price_cents) AS spend, max(ts) AS last_ts, " +
+		"argMax(event, ts) NOT IN (" + terminalComputeSQL() + ") AS active " +
+		"FROM " + computeTable + " WHERE " + where + " " +
+		"GROUP BY org, app, project, kind, machine_id) " +
+		"GROUP BY org, app, project, kind ORDER BY spend_cents DESC"
+	return sql, args
+}
+
+// computeLeavesFromRows maps the DatastoreQuery rows onto []computeLeaf (pure).
+func computeLeavesFromRows(rows []map[string]any) []computeLeaf {
+	leaves := make([]computeLeaf, 0, len(rows))
+	for _, r := range rows {
+		leaves = append(leaves, computeLeaf{
+			Org:        chStr(r["org"]),
+			App:        chStr(r["app"]),
+			Project:    chStr(r["project"]),
+			Kind:       chStr(r["kind"]),
+			Machines:   chInt64(r["machines"]),
+			Active:     chInt64(r["active"]),
+			SpendCents: chInt64(r["spend_cents"]),
+			LastTs:     chTime(r["last_ts"]),
+		})
+	}
+	return leaves
+}
+
+// computeTableExists probes for the operator-owned events table. Any error → false
+// (honest "not available yet"), mirroring analytics.tableExists.
+func computeTableExists(ctx context.Context) bool {
+	rows, err := aiobject.DatastoreQuery(ctx, "EXISTS TABLE "+computeTable)
+	if err != nil || len(rows) == 0 {
+		return false
+	}
+	for _, v := range rows[0] {
+		return chInt64(v) == 1
+	}
+	return false
+}
+
+// computeSince maps the ?range enum to a lower time bound (default 30d).
+func computeSince(rangeLabel string) time.Time {
+	now := time.Now().UTC()
+	switch strings.TrimSpace(rangeLabel) {
+	case "24h":
+		return now.Add(-24 * time.Hour)
+	case "7d":
+		return now.Add(-7 * 24 * time.Hour)
+	default:
+		return now.Add(-30 * 24 * time.Hour)
+	}
+}
+
+// terminalComputeSQL renders the terminal-event set as a ClickHouse string list.
+func terminalComputeSQL() string {
+	quoted := make([]string, len(terminalComputeEvents))
+	for i, e := range terminalComputeEvents {
+		quoted[i] = "'" + e + "'"
+	}
+	return strings.Join(quoted, ",")
+}
+
+// chTS formats a time as a ClickHouse DateTime literal (UTC), bound as a string arg.
+func chTS(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05") }
+
+// ── map[string]any coercers (the DatastoreQuery row shape) ───────────────────
+//
+// The ClickHouse driver decodes each column to its native Go type (uint64 for
+// count()/sum(UInt*), time.Time for DateTime, string for String); these accept
+// those natives so a driver/transport change can't crash a read.
+
+func chInt64(v any) int64 {
+	switch n := v.(type) {
+	case int:
+		return int64(n)
+	case int64:
+		return n
+	case int32:
+		return int64(n)
+	case uint:
+		return int64(n)
+	case uint64:
+		return int64(n)
+	case uint32:
+		return int64(n)
+	case uint16:
+		return int64(n)
+	case uint8:
+		return int64(n)
+	case float64:
+		return int64(n)
+	case float32:
+		return int64(n)
+	default:
+		return 0
+	}
+}
+
+func chStr(v any) string {
+	if s, ok := v.(string); ok {
+		return s
+	}
+	return ""
+}
+
+// chTime coerces a ClickHouse DateTime (time.Time) to an RFC3339 UTC string.
+func chTime(v any) string {
+	switch t := v.(type) {
+	case time.Time:
+		return t.UTC().Format(time.RFC3339)
+	case string:
+		return t
+	default:
+		return ""
+	}
+}

clients/admin/compute_test.go

@@ -0,0 +1,143 @@
+// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package admin
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+// TestBuildComputeQuery_Filters proves the WHERE clause binds the range, and adds
+// kind + org as POSITIONAL params only when supplied — never interpolated.
+func TestBuildComputeQuery_Filters(t *testing.T) {
+	// Unfiltered: time bound only.
+	sql, args := buildComputeQuery("30d", "", "")
+	if len(args) != 1 {
+		t.Fatalf("unfiltered args = %d, want 1 (time bound)", len(args))
+	}
+	// Reads the real table the datastore stream + visor emitter write.
+	if !strings.Contains(sql, "FROM hanzo.compute_usage") {
+		t.Errorf("query must read hanzo.compute_usage; got %q", sql)
+	}
+	if !strings.Contains(sql, "GROUP BY org, app, project, kind") {
+		t.Errorf("query must group by (org, app, project, kind); got %q", sql)
+	}
+	if strings.Contains(sql, "AND kind = ?") || strings.Contains(sql, "AND org = ?") {
+		t.Errorf("unfiltered query must not add kind/org predicates; got %q", sql)
+	}
+
+	// kind=bot + org: two extra bound params, in order.
+	sql, args = buildComputeQuery("7d", "bot", "acme")
+	if len(args) != 3 {
+		t.Fatalf("kind+org args = %d, want 3", len(args))
+	}
+	if args[1] != "bot" || args[2] != "acme" {
+		t.Errorf("args = %v, want [<ts> bot acme]", args)
+	}
+	if !strings.Contains(sql, "AND kind = ?") || !strings.Contains(sql, "AND org = ?") {
+		t.Errorf("filtered query must bind kind + org; got %q", sql)
+	}
+
+	// OPEN SPECTRUM: an arbitrary kind (not bot/machine) filters too — no enum
+	// whitelist. Future Clusters/Functions lenses reuse this endpoint unchanged.
+	sql, args = buildComputeQuery("30d", "cluster", "")
+	if len(args) != 2 || args[1] != "cluster" {
+		t.Fatalf("kind=cluster args = %v, want [<ts> cluster]", args)
+	}
+	if !strings.Contains(sql, "AND kind = ?") {
+		t.Errorf("an arbitrary kind must still bind the kind predicate; got %q", sql)
+	}
+}
+
+// TestComputeSince maps the range enum to a lower time bound (default 30d).
+func TestComputeSince(t *testing.T) {
+	now := time.Now().UTC()
+	cases := map[string]time.Duration{
+		"24h": 24 * time.Hour,
+		"7d":  7 * 24 * time.Hour,
+		"30d": 30 * 24 * time.Hour,
+		"":    30 * 24 * time.Hour, // default
+		"xyz": 30 * 24 * time.Hour, // unknown → default
+	}
+	for label, want := range cases {
+		got := now.Sub(computeSince(label))
+		if d := got - want; d < -2*time.Second || d > 2*time.Second {
+			t.Errorf("computeSince(%q) lookback = %v, want ≈%v", label, got, want)
+		}
+	}
+}
+
+// TestTerminalComputeSQL renders the terminal set as a quoted CH list.
+func TestTerminalComputeSQL(t *testing.T) {
+	got := terminalComputeSQL()
+	for _, e := range []string{"'stop'", "'destroy'", "'terminated'", "'shutdown'"} {
+		if !strings.Contains(got, e) {
+			t.Errorf("terminal list missing %s; got %q", e, got)
+		}
+	}
+	if strings.Contains(got, "'start'") || strings.Contains(got, "'provision'") {
+		t.Errorf("terminal list must NOT include running states; got %q", got)
+	}
+}
+
+// TestComputeLeavesFromRows maps the driver's native row types (uint64 counts,
+// time.Time DateTime, string dims) onto typed leaves — the honest-empty and the
+// real-row paths both.
+func TestComputeLeavesFromRows(t *testing.T) {
+	if got := computeLeavesFromRows(nil); len(got) != 0 {
+		t.Fatalf("nil rows → %d leaves, want 0", len(got))
+	}
+	ts := time.Date(2026, 7, 1, 12, 0, 0, 0, time.UTC)
+	rows := []map[string]any{
+		{
+			"org": "acme", "app": "web", "project": "prod", "kind": "machine",
+			"machines": uint64(4), "active": uint64(3), "spend_cents": uint64(1200), "last_ts": ts,
+		},
+	}
+	got := computeLeavesFromRows(rows)
+	if len(got) != 1 {
+		t.Fatalf("rows → %d leaves, want 1", len(got))
+	}
+	l := got[0]
+	if l.Org != "acme" || l.App != "web" || l.Project != "prod" || l.Kind != "machine" {
+		t.Errorf("dims wrong: %+v", l)
+	}
+	if l.Machines != 4 || l.Active != 3 || l.SpendCents != 1200 {
+		t.Errorf("counts wrong: %+v", l)
+	}
+	if l.LastTs != "2026-07-01T12:00:00Z" {
+		t.Errorf("lastTs = %q, want 2026-07-01T12:00:00Z", l.LastTs)
+	}
+}
+
+// TestComputeCoercers proves the map coercers accept the driver natives + degrade.
+func TestComputeCoercers(t *testing.T) {
+	if chInt64(uint64(7)) != 7 || chInt64(int64(7)) != 7 || chInt64(float64(7)) != 7 {
+		t.Error("chInt64 must accept uint64/int64/float64")
+	}
+	if chInt64("nope") != 0 || chInt64(nil) != 0 {
+		t.Error("chInt64 must degrade non-numerics to 0")
+	}
+	if chStr("x") != "x" || chStr(42) != "" {
+		t.Error("chStr must pass strings, degrade others to empty")
+	}
+	if chTime(time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) != "2026-01-02T03:04:05Z" {
+		t.Error("chTime must format time.Time as RFC3339 UTC")
+	}
+	if chTime(123) != "" {
+		t.Error("chTime must degrade non-time to empty")
+	}
+}

clients/admin/customers.go

@@ -0,0 +1,526 @@
+package admin
+
+// The CUSTOMER management surface (/v1/admin/customers*) — the operator cockpit's
+// core: the live fleet customer list (incl. new self-serve signups), one-customer
+// detail, and the audited management ACTIONS (grant credit, suspend, reactivate).
+//
+// It aggregates the SAME real upstreams the rest of admin reads — IAM for the org
+// directory + user/owner/status, commerce for balance/spend/plan/ledger — and adds
+// the two write levers an operator needs to run the paid cloud:
+//
+//   - GRANT CREDIT is a real commerce Deposit (refunds/comps/support) landing in
+//     the org's own wallet, symmetric with the balance read.
+//   - SUSPEND / REACTIVATE flips IAM `isForbidden` on the org's users. That is the
+//     platform's REAL access lever: IAM refuses a forbidden user at login AND at
+//     token issuance (object/check.go + object/token_oauth.go), so a suspended
+//     customer cannot sign in or mint a fresh token — no new enforcement path is
+//     invented, and it is fully reversible.
+//
+// SECURITY. Every route is mounted behind s.guard (global-admin only, fail-closed)
+// exactly like the read surface. The write actions REPLAY THE CALLER'S OWN global-
+// admin credential to IAM (no service credential added — IAM re-checks
+// IsGlobalAdmin, so admin can never mutate a boundary the caller couldn't already
+// cross), and each is recorded to cloud's tamper-evident audit trail with a
+// redacted BEFORE/AFTER (the AU "before/after on config-affecting change"), on top
+// of the uniform request record the audit middleware already writes for every
+// /v1/admin/* mutation. No customer card data is ever read or exposed here.
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/hanzoai/cloud/audit"
+	"github.com/zap-proto/zip"
+)
+
+// ── wire shapes (operator contract) ──────────────────────────────────────────
+
+// customerRow is one row in GET /v1/admin/customers — a fleet customer at a glance.
+type customerRow struct {
+	Org          string `json:"org"`
+	Display      string `json:"display"`
+	OwnerEmail   string `json:"ownerEmail"`
+	Plan         string `json:"plan"`
+	Status       string `json:"status"` // "active" | "suspended"
+	Users        int    `json:"users"`
+	BalanceCents int64  `json:"balanceCents"`
+	SpendCents   int64  `json:"spendCents"`
+	MRRCents     int64  `json:"mrrCents"`
+	Created      string `json:"created"`
+	LastActive   string `json:"lastActive"`
+}
+
+// customerUser is one member in the customer detail (no secrets — the AccessKey
+// PRESENCE is surfaced as hasApiKey, never the key itself).
+type customerUser struct {
+	Name       string `json:"name"`
+	Email      string `json:"email"`
+	IsAdmin    bool   `json:"isAdmin"`
+	Forbidden  bool   `json:"forbidden"`
+	HasAPIKey  bool   `json:"hasApiKey"`
+	LastSignin string `json:"lastSignin"`
+	Created    string `json:"created"`
+}
+
+// customerTxn is one ledger row in the detail's top-up/usage history.
+type customerTxn struct {
+	ID       string `json:"id"`
+	Type     string `json:"type"` // "deposit" (credit) | "withdraw" (usage)
+	Cents    int64  `json:"cents"`
+	Currency string `json:"currency"`
+	Notes    string `json:"notes,omitempty"`
+	Time     string `json:"time"`
+}
+
+// customerDetail is GET /v1/admin/customers/:org.
+type customerDetail struct {
+	Org          string         `json:"org"`
+	Display      string         `json:"display"`
+	OwnerEmail   string         `json:"ownerEmail"`
+	Plan         string         `json:"plan"`
+	Status       string         `json:"status"`
+	Created      string         `json:"created"`
+	BalanceCents int64          `json:"balanceCents"`
+	SpendCents   int64          `json:"spendCents"`
+	MRRCents     int64          `json:"mrrCents"`
+	APIKeys      int            `json:"apiKeys"`
+	Users        []customerUser `json:"users"`
+	Transactions []customerTxn  `json:"transactions"`
+}
+
+// ── GET /v1/admin/customers — the fleet customer list ────────────────────────
+
+// maxCustomerConcurrency bounds the per-org enrichment fan-out so a large fleet
+// does not open one upstream connection per org at once. Admin is low-QPS; 8 keeps
+// latency low without hammering IAM/commerce.
+const maxCustomerConcurrency = 8
+
+func (s *svc) customers(c *zip.Ctx) error {
+	ctx := c.Context()
+	cr := callerCreds(c)
+	orgs, err := s.listOrgs(ctx, cr)
+	if err != nil {
+		return fail(c, err.Error())
+	}
+
+	rows := make([]customerRow, len(orgs))
+	sem := make(chan struct{}, maxCustomerConcurrency)
+	var wg sync.WaitGroup
+	for i, o := range orgs {
+		wg.Add(1)
+		sem <- struct{}{}
+		go func(i int, o iamOrg) {
+			defer wg.Done()
+			defer func() { <-sem }()
+			rows[i] = s.enrichCustomer(ctx, cr, o)
+		}(i, o)
+	}
+	wg.Wait()
+
+	sort.Slice(rows, func(i, j int) bool { return rows[i].Org < rows[j].Org })
+	return okList(c, rows, len(rows))
+}
+
+// enrichCustomer folds one org's real IAM + commerce reads into a customer row.
+// Each read is best-effort: an upstream miss degrades that field to its honest
+// zero/empty (never a fabricated value), so one flaky org never fails the fleet.
+func (s *svc) enrichCustomer(ctx context.Context, cr creds, o iamOrg) customerRow {
+	subj := orgSubject(o.Name)
+	users, _ := s.orgUsers(ctx, cr, o.Name)
+	spend, credits := s.orgMoney(ctx, o.Name)
+	sub, _ := s.commerce.subscriptionSummary(ctx, o.Name, subj)
+
+	return customerRow{
+		Org:          o.Name,
+		Display:      display(o.DisplayName, o.Name),
+		OwnerEmail:   ownerEmail(users),
+		Plan:         sub.Plan,
+		Status:       statusOf(users),
+		Users:        len(users),
+		BalanceCents: credits,
+		SpendCents:   spend,
+		MRRCents:     sub.MRR,
+		Created:      o.CreatedTime,
+		LastActive:   lastActiveOf(users),
+	}
+}
+
+// ── GET /v1/admin/customers/:org — one customer's detail ─────────────────────
+
+func (s *svc) customerDetail(c *zip.Ctx) error {
+	ctx := c.Context()
+	cr := callerCreds(c)
+	org := customerOrgParam(c)
+	if org == "" {
+		return fail(c, "org is required")
+	}
+
+	o, err := s.findOrg(ctx, cr, org)
+	if err != nil {
+		return fail(c, err.Error())
+	}
+	if o == nil {
+		return c.JSON(404, map[string]any{"status": "error", "msg": "customer not found", "data": nil})
+	}
+
+	subj := orgSubject(org)
+	users, _ := s.orgUsers(ctx, cr, org)
+	spend, credits := s.orgMoney(ctx, org)
+	sub, _ := s.commerce.subscriptionSummary(ctx, org, subj)
+	txns, _ := s.commerce.transactions(ctx, org, subj, 50)
+
+	rows := make([]customerUser, 0, len(users))
+	apiKeys := 0
+	for _, u := range users {
+		hasKey := strings.TrimSpace(u.AccessKey) != ""
+		if hasKey {
+			apiKeys++
+		}
+		rows = append(rows, customerUser{
+			Name:       u.Name,
+			Email:      u.Email,
+			IsAdmin:    u.IsAdmin,
+			Forbidden:  u.IsForbidden,
+			HasAPIKey:  hasKey,
+			LastSignin: u.LastSigninTime,
+			Created:    u.CreatedTime,
+		})
+	}
+
+	ledger := make([]customerTxn, 0, len(txns))
+	for _, t := range txns {
+		ledger = append(ledger, customerTxn{
+			ID:       t.ID,
+			Type:     t.Type,
+			Cents:    t.Amount,
+			Currency: t.Currency,
+			Notes:    t.Notes,
+			Time:     t.CreatedAt,
+		})
+	}
+
+	return ok(c, customerDetail{
+		Org:          org,
+		Display:      display(o.DisplayName, org),
+		OwnerEmail:   ownerEmail(users),
+		Plan:         sub.Plan,
+		Status:       statusOf(users),
+		Created:      o.CreatedTime,
+		BalanceCents: credits,
+		SpendCents:   spend,
+		MRRCents:     sub.MRR,
+		APIKeys:      apiKeys,
+		Users:        rows,
+		Transactions: ledger,
+	})
+}
+
+// ── POST /v1/admin/customers/:org/credit — grant credit ──────────────────────
+
+// creditRequest is the grant body. AmountCents is the credit to add (positive
+// only — a grant, never a silent debit). Reason is the operator's justification,
+// recorded in the audit trail's before/after (refund / comp / support).
+type creditRequest struct {
+	AmountCents int64  `json:"amountCents"`
+	Currency    string `json:"currency"`
+	Reason      string `json:"reason"`
+}
+
+// maxGrantCents caps a single grant at $100,000 — a guardrail against a fat-finger
+// operator credit, not a policy limit. A larger comp is deliberate + should be
+// deliberate (two grants), and the cap keeps a typo from minting a fortune.
+const maxGrantCents int64 = 100 * 100 * 1000
+
+func (s *svc) grantCredit(c *zip.Ctx) error {
+	ctx := c.Context()
+	cr := callerCreds(c)
+	org := customerOrgParam(c)
+	if org == "" {
+		return fail(c, "org is required")
+	}
+
+	var req creditRequest
+	if err := c.Bind(&req); err != nil {
+		return fail(c, "invalid request body")
+	}
+	if req.AmountCents <= 0 {
+		return fail(c, "amountCents must be positive")
+	}
+	if req.AmountCents > maxGrantCents {
+		return fail(c, fmt.Sprintf("amountCents exceeds the %d-cent per-grant cap", maxGrantCents))
+	}
+	currency := strings.ToLower(strings.TrimSpace(req.Currency))
+	if currency == "" {
+		currency = "usd"
+	}
+
+	// Validate the target is a REAL org (never mint an orphan wallet on a typo).
+	o, err := s.findOrg(ctx, cr, org)
+	if err != nil {
+		return fail(c, err.Error())
+	}
+	if o == nil {
+		return c.JSON(404, map[string]any{"status": "error", "msg": "customer not found", "data": nil})
+	}
+
+	subj := orgSubject(org)
+	before, _ := s.commerce.creditsCents(ctx, org, subj)
+
+	notes := grantNote(c, req.Reason)
+	res, derr := s.commerce.deposit(ctx, org, subj, req.AmountCents, currency, notes, "admin-grant")
+	if derr != nil {
+		// The grant did not land — record the FAILED attempt (accountability), then
+		// surface the error. Never report a grant that failed as success.
+		s.emitAudit(c, "admin.customer.credit", "credit", org,
+			map[string]any{"balanceCents": before},
+			map[string]any{"amountCents": req.AmountCents, "currency": currency, "reason": req.Reason, "error": derr.Error()},
+			audit.Outcome{Result: "error", Status: 200, Reason: "grant failed"})
+		return fail(c, "grant failed: "+derr.Error())
+	}
+
+	after, _ := s.commerce.creditsCents(ctx, org, subj)
+	s.emitAudit(c, "admin.customer.credit", "credit", org,
+		map[string]any{"balanceCents": before},
+		map[string]any{"balanceCents": after, "grantedCents": req.AmountCents, "currency": currency, "reason": req.Reason, "transactionId": res.TransactionID},
+		audit.Outcome{Result: "success", Status: 200})
+
+	return ok(c, map[string]any{
+		"org":           org,
+		"grantedCents":  req.AmountCents,
+		"currency":      currency,
+		"balanceCents":  after,
+		"transactionId": res.TransactionID,
+	})
+}
+
+// ── POST /v1/admin/customers/:org/{suspend,reactivate} — access control ──────
+
+func (s *svc) suspendCustomer(c *zip.Ctx) error  { return s.setForbidden(c, true) }
+func (s *svc) reactivateCustomer(c *zip.Ctx) error { return s.setForbidden(c, false) }
+
+// setForbidden flips IAM `isForbidden` on every member of the org — suspend
+// (forbidden=true) cuts login + token issuance; reactivate restores it. Each
+// user's FULL object is read, the one field flipped, and written back (update-user
+// replaces the row), replaying the caller's global-admin credential so IAM
+// authorizes it. Best-effort per user with an aggregated result: a partial failure
+// is reported honestly (affected vs failed), never masked as a clean success. The
+// action is recorded with a redacted before/after user tally.
+func (s *svc) setForbidden(c *zip.Ctx, forbidden bool) error {
+	ctx := c.Context()
+	cr := callerCreds(c)
+	org := customerOrgParam(c)
+	if org == "" {
+		return fail(c, "org is required")
+	}
+
+	o, err := s.findOrg(ctx, cr, org)
+	if err != nil {
+		return fail(c, err.Error())
+	}
+	if o == nil {
+		return c.JSON(404, map[string]any{"status": "error", "msg": "customer not found", "data": nil})
+	}
+
+	users, err := s.orgUsers(ctx, cr, org)
+	if err != nil {
+		return fail(c, err.Error())
+	}
+
+	beforeForbidden := 0
+	for _, u := range users {
+		if u.IsForbidden {
+			beforeForbidden++
+		}
+	}
+
+	var affected, failed []string
+	for _, u := range users {
+		id := u.Owner + "/" + u.Name
+		full, gerr := s.iam.getUserRaw(ctx, cr, id)
+		if gerr != nil {
+			failed = append(failed, u.Name)
+			continue
+		}
+		full["isForbidden"] = forbidden
+		if uerr := s.iam.updateUserRaw(ctx, cr, id, full); uerr != nil {
+			failed = append(failed, u.Name)
+			continue
+		}
+		affected = append(affected, u.Name)
+	}
+
+	action := "admin.customer.suspend"
+	if !forbidden {
+		action = "admin.customer.reactivate"
+	}
+	result := "success"
+	reason := ""
+	if len(failed) > 0 {
+		result = "error"
+		reason = fmt.Sprintf("%d user(s) not updated", len(failed))
+	}
+	s.emitAudit(c, action, "customer", org,
+		map[string]any{"suspended": beforeForbidden == len(users) && len(users) > 0, "forbiddenUsers": beforeForbidden, "totalUsers": len(users)},
+		map[string]any{"suspended": forbidden, "affected": affected, "failed": failed},
+		audit.Outcome{Result: result, Status: 200, Reason: reason})
+
+	return ok(c, map[string]any{
+		"org":       org,
+		"suspended": forbidden,
+		"affected":  affected,
+		"failed":    failed,
+	})
+}
+
+// ── aggregation + derivation helpers ─────────────────────────────────────────
+
+// orgUsers reads an org's members (a bounded page) as the typed subset the
+// customer surface folds over. It is the ONE IAM read that yields the user count,
+// the owner email, the suspend status, and the API-key presence — so a customer
+// row costs a single get-users call, not four.
+func (s *svc) orgUsers(ctx context.Context, cr creds, org string) ([]iamUser, error) {
+	q := url.Values{}
+	q.Set("owner", org)
+	q.Set("p", "1")
+	q.Set("pageSize", "200")
+	res, err := s.iam.getList(ctx, cr, "/v1/iam/get-users", q)
+	if err != nil {
+		return nil, err
+	}
+	var raw []iamUser
+	if len(res.rows) > 0 {
+		if err := json.Unmarshal(res.rows, &raw); err != nil {
+			return nil, fmt.Errorf("users decode: %w", err)
+		}
+	}
+	return raw, nil
+}
+
+// findOrg returns the IAM org by slug (nil, nil when it does not exist) so a
+// management action can validate its target before acting — never credit or
+// suspend an org that isn't real.
+func (s *svc) findOrg(ctx context.Context, cr creds, org string) (*iamOrg, error) {
+	orgs, err := s.listOrgs(ctx, cr)
+	if err != nil {
+		return nil, err
+	}
+	for i := range orgs {
+		if orgs[i].Name == org {
+			return &orgs[i], nil
+		}
+	}
+	return nil, nil
+}
+
+// ownerEmail picks the org's admin user's email (the account owner), falling back
+// to the first user with an email. Empty when no user carries one.
+func ownerEmail(users []iamUser) string {
+	for _, u := range users {
+		if u.IsAdmin && strings.TrimSpace(u.Email) != "" {
+			return u.Email
+		}
+	}
+	for _, u := range users {
+		if strings.TrimSpace(u.Email) != "" {
+			return u.Email
+		}
+	}
+	return ""
+}
+
+// statusOf derives the suspend status: an org is "suspended" only when it has at
+// least one user and EVERY user is forbidden (a partial forbid is still "active" —
+// the operator sees the per-user state in the detail). Honest by construction.
+func statusOf(users []iamUser) string {
+	if len(users) == 0 {
+		return "active"
+	}
+	for _, u := range users {
+		if !u.IsForbidden {
+			return "active"
+		}
+	}
+	return "suspended"
+}
+
+// lastActiveOf returns the most recent user sign-in across the org (RFC3339), the
+// best "last active" signal available from IAM. Empty when no user has signed in.
+func lastActiveOf(users []iamUser) string {
+	last := ""
+	for _, u := range users {
+		if u.LastSigninTime > last {
+			last = u.LastSigninTime
+		}
+	}
+	return last
+}
+
+// customerOrgParam reads + trims the :org path param.
+func customerOrgParam(c *zip.Ctx) string { return strings.TrimSpace(c.Param("org")) }
+
+// grantNote composes the deposit note from the operator's reason (bounded), so the
+// commerce ledger row itself carries the justification alongside the audit trail.
+func grantNote(c *zip.Ctx, reason string) string {
+	r := strings.TrimSpace(reason)
+	if len(r) > 200 {
+		r = r[:200]
+	}
+	by := strings.TrimSpace(c.UserEmail())
+	if by == "" {
+		by = strings.TrimSpace(c.User())
+	}
+	if r == "" {
+		r = "operator credit"
+	}
+	if by != "" {
+		return fmt.Sprintf("Admin grant by %s: %s", by, r)
+	}
+	return "Admin grant: " + r
+}
+
+// emitAudit writes ONE compliance record for a management action to cloud's
+// tamper-evident trail: who (the validated global admin from the sanitized
+// identity — the gate already proved it), what (action + resource), the redacted
+// before/after, and the outcome. This is the "before/after on a config-affecting
+// change" the request-level middleware record cannot carry (it never reads bodies).
+// Best-effort: the audit MIDDLEWARE is the AU-5 fail-closed authority for the
+// request; a failure here is logged loud, never silent, and never double-fails the
+// response. A nil store (unconfigured deployment) is a no-op, like the middleware.
+func (s *svc) emitAudit(c *zip.Ctx, action, resType, resID string, before, after any, outcome audit.Outcome) {
+	if s.auditStore == nil {
+		return
+	}
+	rec := audit.Record{
+		Actor:     audit.Actor{Org: strings.TrimSpace(c.Org()), Sub: strings.TrimSpace(c.User()), Email: strings.TrimSpace(c.UserEmail())},
+		Action:    action,
+		Resource:  audit.Resource{Type: resType, ID: resID},
+		Auth:      audit.AuthContext{Method: "jwt", IsAdmin: c.IsAdmin()},
+		Outcome:   outcome,
+		UserAgent: c.Header("User-Agent"),
+		RequestID: c.RequestID(),
+		Method:    c.Method(),
+		Path:      c.Path(),
+		Before:    audit.Redact(mustJSON(before)),
+		After:     audit.Redact(mustJSON(after)),
+	}
+	if _, err := s.auditStore.Append(c.Context(), rec); err != nil {
+		c.Log().Error("admin: audit emit failed (request-level record still applies)",
+			"action", action, "resource", resType, "id", resID, "err", err)
+	}
+}
+
+// mustJSON marshals v to raw JSON for the audit before/after, returning an empty
+// object on the (unexpected) marshal error rather than panicking — a metadata
+// diff must never crash a money/access action.
+func mustJSON(v any) json.RawMessage {
+	b, err := json.Marshal(v)
+	if err != nil {
+		return json.RawMessage("{}")
+	}
+	return b
+}

clients/admin/digitalocean.go

@@ -0,0 +1,178 @@
+package admin
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// doClient reads DigitalOcean's billing API for the finance dashboard's cost
+// side. DO is our PRIMARY venue (a ~$40k promotional credit); this client turns
+// the customer balance + billing history into the cents the finance aggregator
+// folds into gross margin and runway.
+//
+// Auth is a single personal-access token, DO_API_TOKEN, sourced from a KMSSecret
+// on the cloud env — NEVER hard-coded (kms.hanzo.ai is the only secret store).
+// When the token is unset the client is UNCONFIGURED and every read reports the
+// honest not-configured state; the finance endpoint then returns
+// cost.digitalocean = {configured:false} rather than a fabricated number.
+//
+// DIGITALOCEAN SIGN CONVENTION (authoritative, from DO's public OpenAPI spec):
+// GET /v2/customers/my/balance returns three DECIMAL-DOLLAR STRINGS —
+//   - account_balance:      most-recent billing balance, accounts-receivable sign.
+//     POSITIVE = the customer OWES DO; NEGATIVE = the customer
+//     holds CREDIT (DO owes us). Our promo credit shows as a
+//     NEGATIVE account_balance, so credit-remaining = -account_balance.
+//   - month_to_date_usage:  spend in the current billing period (positive dollars).
+//   - month_to_date_balance = account_balance + month_to_date_usage.
+//
+// We convert dollars→cents once at the edge and work in int64 cents everywhere after.
+type doClient struct {
+	base  string // DO API base; https://api.digitalocean.com in prod
+	token string // DO_API_TOKEN (secret; never logged)
+	http  *http.Client
+}
+
+// doAPIBase is DigitalOcean's public API host. Overridable in tests via
+// newDOClientWithBase so a fake server can stand in.
+const doAPIBase = "https://api.digitalocean.com"
+
+func newDOClient(token string) *doClient {
+	return newDOClientWithBase(doAPIBase, token)
+}
+
+func newDOClientWithBase(base, token string) *doClient {
+	return &doClient{
+		base:  strings.TrimRight(strings.TrimSpace(base), "/"),
+		token: strings.TrimSpace(token),
+		http:  &http.Client{Timeout: 15 * time.Second},
+	}
+}
+
+// configured reports whether a DO token is present. Unconfigured → the finance
+// endpoint returns cost.digitalocean = {configured:false}, never a fake balance.
+func (c *doClient) configured() bool { return c != nil && c.token != "" }
+
+// doBalance is the decoded /v2/customers/my/balance response. Dollars are parsed
+// into cents at decode time so no float dollars leak past this boundary.
+type doBalance struct {
+	// AccountBalanceCents mirrors DO's account_balance (accounts-receivable sign:
+	// positive = owed to DO, negative = credit we hold).
+	AccountBalanceCents     int64
+	MonthToDateBalanceCents int64
+	MonthToDateUsageCents   int64
+	GeneratedAt             string
+}
+
+// doBalanceWire is the raw DO JSON (all money fields are decimal-dollar strings).
+type doBalanceWire struct {
+	MonthToDateBalance string `json:"month_to_date_balance"`
+	AccountBalance     string `json:"account_balance"`
+	MonthToDateUsage   string `json:"month_to_date_usage"`
+	GeneratedAt        string `json:"generated_at"`
+}
+
+// balance fetches the customer balance and converts every dollar string to cents.
+func (c *doClient) balance(ctx context.Context) (doBalance, error) {
+	var out doBalance
+	if !c.configured() {
+		return out, fmt.Errorf("DO_API_TOKEN not configured")
+	}
+	body, err := c.get(ctx, "/v2/customers/my/balance")
+	if err != nil {
+		return out, err
+	}
+	var w doBalanceWire
+	if err := json.Unmarshal(body, &w); err != nil {
+		return out, fmt.Errorf("do balance decode: %w", err)
+	}
+	out = doBalance{
+		AccountBalanceCents:     dollarsToCents(w.AccountBalance),
+		MonthToDateBalanceCents: dollarsToCents(w.MonthToDateBalance),
+		MonthToDateUsageCents:   dollarsToCents(w.MonthToDateUsage),
+		GeneratedAt:             strings.TrimSpace(w.GeneratedAt),
+	}
+	return out, nil
+}
+
+// doHistoryEntry is one row of the billing history (used to build the burn-down
+// timeseries). amount is a decimal-dollar string in DO's wire.
+type doHistoryEntry struct {
+	Description string `json:"description"`
+	AmountCents int64  `json:"-"`
+	Amount      string `json:"amount"`
+	Date        string `json:"date"`
+	Type        string `json:"type"`
+	InvoiceID   string `json:"invoice_id"`
+}
+
+// history fetches recent billing history (Invoice/Credit/Payment entries). Used
+// only to render the credit burn-down series; a failure here is non-fatal (the
+// finance endpoint still returns the balance-derived tiles with an empty series).
+func (c *doClient) history(ctx context.Context, perPage int) ([]doHistoryEntry, error) {
+	if !c.configured() {
+		return nil, fmt.Errorf("DO_API_TOKEN not configured")
+	}
+	if perPage <= 0 {
+		perPage = 50
+	}
+	body, err := c.get(ctx, "/v2/customers/my/billing_history?per_page="+strconv.Itoa(perPage))
+	if err != nil {
+		return nil, err
+	}
+	var w struct {
+		BillingHistory []doHistoryEntry `json:"billing_history"`
+	}
+	if err := json.Unmarshal(body, &w); err != nil {
+		return nil, fmt.Errorf("do billing_history decode: %w", err)
+	}
+	for i := range w.BillingHistory {
+		w.BillingHistory[i].AmountCents = dollarsToCents(w.BillingHistory[i].Amount)
+	}
+	return w.BillingHistory, nil
+}
+
+// get performs one token-authenticated DO GET and returns the raw body.
+func (c *doClient) get(ctx context.Context, path string) ([]byte, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("Authorization", "Bearer "+c.token)
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("digitalocean unreachable: %w", err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, fmt.Errorf("digitalocean status %d", resp.StatusCode)
+	}
+	return body, nil
+}
+
+// dollarsToCents parses a DO decimal-dollar string ("23.44", "-40000.00") into
+// integer cents, rounding to the nearest cent. A blank/invalid string is 0 —
+// DO always sends a value, so this only guards against a malformed field, and a
+// zero there is the honest fallback (never a fabricated amount).
+func dollarsToCents(s string) int64 {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return 0
+	}
+	f, err := strconv.ParseFloat(s, 64)
+	if err != nil {
+		return 0
+	}
+	return int64(math.Round(f * 100))
+}

clients/admin/finance.go

@@ -0,0 +1,331 @@
+package admin
+
+import (
+	"context"
+	"errors"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/zap-proto/zip"
+)
+
+// errUnconfigured marks an upstream that is not wired on this deployment (no DO
+// token / no commerce URL). srcOf reports it as a not-ok source so the console
+// shows the honest not-configured state rather than a fabricated read.
+var errUnconfigured = errors.New("not configured")
+
+// errPartialRevenue marks a revenue read that succeeded at the org-list level but
+// had one or more per-org failures — the fleet total is real but PARTIAL. srcOf
+// reports it as a not-ok source so the console shows a degraded state rather than
+// presenting an under-count as authoritative.
+var errPartialRevenue = errors.New("partial: one or more org revenue reads failed")
+
+// ── /v1/admin/finance — SaaS business/finance dashboard (FinanceData) ─────────
+//
+// The profitability panel the Hanzo Admin Console renders on admin.hanzo.ai: what
+// we pay every vendor (COGS), what we earn, the resulting gross margin, how fast
+// we're burning the DigitalOcean promo credit, and the runway that credit + burn
+// imply. It is GLOBAL-ADMIN ONLY (s.guard) — financial data is Hanzo-internal and
+// must never reach a customer or tenant-admin.
+//
+// Like the rest of admin it FABRICATES NOTHING and it OWNS NO cost logic. COGS is
+// the SINGLE source of truth in commerce (GET /v1/costs: DigitalOcean compute +
+// the LLM providers we resell) — cloud CONSUMES it, never re-reads a vendor's
+// billing API to derive a cost, so the margin uses the whole multi-vendor COGS.
+// Revenue + MRR come from commerce billing (honest zeros when unreachable). The
+// one direct vendor read that remains is the DigitalOcean promo-CREDIT balance +
+// burn-down history — an ORTHOGONAL treasury view (how long the credit lasts), NOT
+// a COGS: commerce tracks what we SPEND with DO (the compute line), not our prepaid
+// credit balance, so it can't provide it. The derived margin/runway math is a pure
+// function (computeFinance) with a unit test proving the numbers and every
+// unconfigured path.
+
+// financeData is the full /v1/admin/finance aggregate (FinanceData).
+type financeData struct {
+	Cost        financeCost    `json:"cost"`
+	Revenue     financeRevenue `json:"revenue"`
+	Derived     financeDerived `json:"derived"`
+	GeneratedAt string         `json:"generatedAt"`
+	Sources     []sourceStatus `json:"sources"`
+}
+
+// financeCost is the platform COGS view — what WE pay our vendors. Its authority
+// is commerce GET /v1/costs (the SINGLE vendor-COGS source of truth): TotalCents is
+// the whole-platform COGS the margin math folds, and Vendors is the per-vendor
+// breakdown (DigitalOcean compute + each LLM provider we resell) the console
+// renders as a donut. Configured is false (and every number 0) when commerce
+// /v1/costs is unreachable — the console then shows the honest not-configured state.
+//
+// DigitalOcean here is an ORTHOGONAL treasury view (promo-credit remaining + the
+// burn-down series), NOT part of COGS: its month-to-date spend is NO LONGER the
+// margin cost (TotalCents is) — it feeds only the runway projection. Commerce owns
+// the DO compute COGS line; this is our prepaid-credit balance, which commerce
+// does not track, so it stays a direct DO account read.
+type financeCost struct {
+	Configured bool         `json:"configured"`
+	Error      string       `json:"error,omitempty"`
+	Period     string       `json:"period"`
+	TotalCents int64        `json:"totalCents"`
+	Vendors    []vendorCost `json:"vendors"`
+
+	DigitalOcean doCost `json:"digitalocean"`
+}
+
+// doCost is the DigitalOcean credit + spend view. When Configured is false every
+// number is zero and the console renders the honest "connect DO_API_TOKEN" state.
+type doCost struct {
+	Configured            bool             `json:"configured"`
+	Error                 string           `json:"error,omitempty"`
+	CreditRemainingCents  int64            `json:"creditRemainingCents"`
+	MonthToDateSpendCents int64            `json:"monthToDateSpendCents"`
+	AvgDailyBurnCents     int64            `json:"avgDailyBurnCents"`
+	AccountBalanceCents   int64            `json:"accountBalanceCents"`
+	GeneratedAt           string           `json:"generatedAt,omitempty"`
+	History               []doHistoryPoint `json:"history"`
+}
+
+// doHistoryPoint is one credit burn-down series point (usage charge over time).
+type doHistoryPoint struct {
+	Date        string `json:"date"`
+	AmountCents int64  `json:"amountCents"`
+	Type        string `json:"type"`
+	Description string `json:"description"`
+}
+
+// financeRevenue is the commerce revenue view (all money in USD cents).
+type financeRevenue struct {
+	Configured           bool  `json:"configured"`
+	TotalRevenueCents    int64 `json:"totalRevenueCents"`
+	MRRCents             int64 `json:"mrrCents"`
+	CreditsConsumedCents int64 `json:"creditsConsumedCents"`
+}
+
+// financeDerived is the pure profitability math. Runway is a pointer so it can be
+// null (no honest runway when burn is zero or DO is unconfigured).
+type financeDerived struct {
+	GrossMarginCents int64    `json:"grossMarginCents"`
+	GrossMarginPct   float64  `json:"grossMarginPct"`
+	RunwayDays       *float64 `json:"runwayDays"`
+	Profitable       bool     `json:"profitable"`
+}
+
+// financeInput is the raw material computeFinance folds into financeData. The
+// handler fills cost from the commerce COGS read (+ the DO-credit treasury view)
+// and revenue from commerce billing; the pure function does the math so the
+// derivation is unit-testable in isolation.
+type financeInput struct {
+	cost        financeCost
+	revenue     financeRevenue
+	generatedAt string
+	sources     []sourceStatus
+}
+
+// computeFinance is the PURE derivation: given the multi-vendor COGS view and the
+// commerce revenue view, it computes gross margin, margin %, runway, and
+// profitability. No I/O, no clock, no globals — everything it needs is in
+// financeInput, which is exactly why the finance math can be tested without any
+// network.
+//
+//	grossMarginCents = revenue - COGS(total, all vendors)
+//	grossMarginPct   = grossMargin / revenue * 100          (0 when revenue is 0)
+//	runwayDays       = DO creditRemaining / DO avgDailyBurn  (nil when burn 0 or DO off)
+//	profitable       = revenue > COGS
+func computeFinance(in financeInput) financeData {
+	cost := in.cost.TotalCents
+	rev := in.revenue.TotalRevenueCents
+
+	margin := rev - cost
+	var marginPct float64
+	if rev > 0 {
+		marginPct = (float64(margin) / float64(rev)) * 100
+	}
+
+	// Runway is the DO promo-credit treasury projection (orthogonal to COGS): how
+	// many days the remaining credit lasts at the current DO burn. Nil when DO is
+	// off or burn is 0 — never a fabricated infinity.
+	do := in.cost.DigitalOcean
+	var runway *float64
+	if do.Configured && do.AvgDailyBurnCents > 0 {
+		d := float64(do.CreditRemainingCents) / float64(do.AvgDailyBurnCents)
+		runway = &d
+	}
+
+	return financeData{
+		Cost:    in.cost,
+		Revenue: in.revenue,
+		Derived: financeDerived{
+			GrossMarginCents: margin,
+			GrossMarginPct:   marginPct,
+			RunwayDays:       runway,
+			Profitable:       rev > cost,
+		},
+		GeneratedAt: in.generatedAt,
+		Sources:     in.sources,
+	}
+}
+
+// finance answers GET /v1/admin/finance. It reads the multi-vendor COGS from
+// commerce /v1/costs, the DO promo-credit/burn-down treasury view, and the fleet
+// commerce revenue, then hands them to computeFinance. Global-admin only (mounted
+// under s.guard); no principal / tenant-admin / forged header → 403 before this
+// handler ever runs.
+func (s *svc) finance(c *zip.Ctx) error {
+	ctx := c.Context()
+	cr := callerCreds(c)
+	now := time.Now().UTC().Format(time.RFC3339)
+	period := time.Now().UTC().Format("2006-01")
+
+	var sources []sourceStatus
+
+	// ── COGS: commerce /v1/costs (the single vendor-COGS source of truth) ──
+	// cloud CONSUMES the multi-vendor breakdown (DigitalOcean compute + the LLM
+	// providers we resell) — it does NOT re-derive any vendor cost. TotalCents is
+	// the margin cost. Honest not-configured when commerce is unreachable.
+	cost := financeCost{Period: period}
+	if s.commerce.configured() {
+		report, err := s.commerce.costs(ctx, period)
+		if err != nil {
+			cost.Error = err.Error()
+			sources = append(sources, srcOf("commerce-costs", err, 0, now))
+		} else {
+			cost.Configured = true
+			cost.TotalCents = report.TotalCents
+			cost.Vendors = report.Vendors
+			if report.Period != "" {
+				cost.Period = report.Period
+			}
+			sources = append(sources, srcOf("commerce-costs", nil, len(report.Vendors), now))
+		}
+	} else {
+		cost.Error = "commerce /v1/costs not configured"
+		sources = append(sources, srcOf("commerce-costs", errUnconfigured, 0, now))
+	}
+	if cost.Vendors == nil {
+		cost.Vendors = []vendorCost{}
+	}
+
+	// ── DigitalOcean promo-credit / runway (orthogonal treasury view) ──
+	// The one direct vendor read that remains: our DO prepaid-credit balance +
+	// burn-down history, which commerce does not track. Its MTD spend feeds ONLY
+	// the runway projection — it is NOT the margin cost (that is cost.TotalCents).
+	do := doCost{Configured: s.do.configured()}
+	if !s.do.configured() {
+		do.Error = "DO_API_TOKEN not configured"
+		sources = append(sources, srcOf("digitalocean", errUnconfigured, 0, now))
+	} else {
+		bal, err := s.do.balance(ctx)
+		if err != nil {
+			do.Error = err.Error()
+			sources = append(sources, srcOf("digitalocean", err, 0, now))
+		} else {
+			// creditRemaining = -account_balance clamped at 0 (negative account
+			// balance = credit we hold; a positive balance means we owe DO → 0 credit).
+			credit := -bal.AccountBalanceCents
+			if credit < 0 {
+				credit = 0
+			}
+			do.CreditRemainingCents = credit
+			do.MonthToDateSpendCents = bal.MonthToDateUsageCents
+			do.AccountBalanceCents = bal.AccountBalanceCents
+			do.GeneratedAt = bal.GeneratedAt
+			do.AvgDailyBurnCents = avgDailyBurnCents(bal.MonthToDateUsageCents, time.Now().UTC())
+			do.History = s.doHistory(ctx)
+			sources = append(sources, srcOf("digitalocean", nil, 1, now))
+		}
+	}
+	if do.History == nil {
+		do.History = []doHistoryPoint{}
+	}
+	cost.DigitalOcean = do
+
+	// ── Revenue: commerce (fleet-wide) ────────────────────────────────────
+	// Configured means the revenue source was actually READ, not merely wired: on a
+	// transient IAM/commerce failure it stays FALSE so computeFinance and the console
+	// never fabricate a negative margin / red "burning" alarm from a fake zero.
+	rev := financeRevenue{}
+	if !s.commerce.configured() {
+		sources = append(sources, srcOf("commerce", errUnconfigured, 0, now))
+	} else if orgs, orgErr := s.listOrgs(ctx, cr); orgErr != nil {
+		// The revenue source is unreadable → honest not-configured, never a zero
+		// that would flip the margin negative on an upstream hiccup.
+		sources = append(sources, srcOf("commerce", orgErr, 0, now))
+	} else {
+		var totalRev, mrr int64
+		partial := false
+		for _, o := range orgs {
+			subj := orgSubject(o.Name)
+			if r, e := s.commerce.usageRollup(ctx, o.Name, subj); e == nil {
+				totalRev += r.ConsumedCents
+			} else {
+				partial = true
+			}
+			if m, e := s.commerce.mrrCents(ctx, o.Name, subj); e == nil {
+				mrr += m
+			} else {
+				partial = true
+			}
+		}
+		// Realized revenue = what customers consumed (metered spend). Credits
+		// consumed mirrors that same figure at the fleet level.
+		rev.Configured = true
+		rev.TotalRevenueCents = totalRev
+		rev.CreditsConsumedCents = totalRev
+		rev.MRRCents = mrr
+		// A per-org read failure means the fleet total is PARTIAL — mark the source
+		// not-ok so the console shows a degraded state, never presents an under-count
+		// as authoritative.
+		if partial {
+			sources = append(sources, srcOf("commerce", errPartialRevenue, len(orgs), now))
+		} else {
+			sources = append(sources, srcOf("commerce", nil, len(orgs), now))
+		}
+	}
+
+	return ok(c, computeFinance(financeInput{
+		cost:        cost,
+		revenue:     rev,
+		generatedAt: now,
+		sources:     sources,
+	}))
+}
+
+// doHistory reads DO billing history into the burn-down series (best-effort:
+// a failure yields an empty series, never a fabricated trend). Only usage-side
+// entries (Invoice/charges) shape the burn-down; the series stays honest-empty
+// when history is unavailable.
+func (s *svc) doHistory(ctx context.Context) []doHistoryPoint {
+	entries, err := s.do.history(ctx, 60)
+	if err != nil {
+		return []doHistoryPoint{}
+	}
+	pts := make([]doHistoryPoint, 0, len(entries))
+	for _, e := range entries {
+		pts = append(pts, doHistoryPoint{
+			Date:        e.Date,
+			AmountCents: e.AmountCents,
+			Type:        e.Type,
+			Description: e.Description,
+		})
+	}
+	return pts
+}
+
+// avgDailyBurnCents derives the average daily DO burn from month-to-date usage:
+// month-to-date spend divided by the number of elapsed days in the current month
+// (at least 1, so day 1 doesn't divide by zero). This is the honest run-rate the
+// runway projection uses — a real read (MTD usage) over real elapsed time, never
+// an invented rate.
+func avgDailyBurnCents(monthToDateSpendCents int64, now time.Time) int64 {
+	day := now.Day()
+	if day < 1 {
+		day = 1
+	}
+	return monthToDateSpendCents / int64(day)
+}
+
+// doTokenFromEnv reads the DigitalOcean token from the environment. Sourced from
+// a KMSSecret on the cloud deployment (DO_API_TOKEN) — never hard-coded.
+func doTokenFromEnv() string {
+	return strings.TrimSpace(os.Getenv("DO_API_TOKEN"))
+}

clients/admin/finance_test.go

@@ -0,0 +1,431 @@
+package admin
+
+import (
+	"encoding/json"
+	"io"
+	"math"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+// TestComputeFinance_Math is the PURE derivation proof: given a fixed multi-vendor
+// COGS view and commerce revenue view, gross margin, margin %, runway, and
+// profitability are exactly the arithmetic the dashboard promises — no I/O. The
+// margin cost is the COGS total (all vendors); runway is the DO-credit projection.
+func TestComputeFinance_Math(t *testing.T) {
+	// COGS: $30k total across vendors. DO treasury: $40k credit, $10k left, burning
+	// $1k/day. Revenue: $35k realized.
+	in := financeInput{
+		cost: financeCost{
+			Configured: true,
+			TotalCents: 3_000_000, // $30,000 COGS across all vendors (the margin cost)
+			DigitalOcean: doCost{
+				Configured:           true,
+				CreditRemainingCents: 1_000_000, // $10,000 promo credit remaining
+				AvgDailyBurnCents:    100_000,   // $1,000/day DO burn (runway input)
+			},
+		},
+		revenue: financeRevenue{
+			Configured:        true,
+			TotalRevenueCents: 3_500_000, // $35,000 revenue
+			MRRCents:          500_000,   // $5,000 MRR
+		},
+	}
+	got := computeFinance(in)
+
+	// margin = 35,000 - 30,000 = $5,000
+	if got.Derived.GrossMarginCents != 500_000 {
+		t.Errorf("grossMarginCents = %d, want 500000 ($5,000)", got.Derived.GrossMarginCents)
+	}
+	// marginPct = 5,000 / 35,000 * 100 = 14.2857…%
+	if math.Abs(got.Derived.GrossMarginPct-14.285714) > 0.0001 {
+		t.Errorf("grossMarginPct = %f, want ≈14.2857", got.Derived.GrossMarginPct)
+	}
+	// runway = 10,000 / 1,000 = 10 days
+	if got.Derived.RunwayDays == nil || math.Abs(*got.Derived.RunwayDays-10) > 1e-9 {
+		t.Errorf("runwayDays = %v, want 10", got.Derived.RunwayDays)
+	}
+	// revenue (35k) > cost (30k) → profitable this month.
+	if !got.Derived.Profitable {
+		t.Error("profitable must be true when revenue > cost")
+	}
+}
+
+// TestComputeFinance_BurningFasterThanEarning proves the red state: cost exceeds
+// revenue → negative margin, not profitable, runway still finite.
+func TestComputeFinance_BurningFasterThanEarning(t *testing.T) {
+	in := financeInput{
+		cost: financeCost{
+			Configured: true,
+			TotalCents: 4_000_000, // $40,000 COGS (the margin cost)
+			DigitalOcean: doCost{
+				Configured:           true,
+				CreditRemainingCents: 2_000_000, // $20,000 left
+				AvgDailyBurnCents:    200_000,   // $2,000/day
+			},
+		},
+		revenue: financeRevenue{Configured: true, TotalRevenueCents: 1_000_000}, // $10,000
+	}
+	got := computeFinance(in)
+	if got.Derived.GrossMarginCents != -3_000_000 { // 10k - 40k = -30k
+		t.Errorf("grossMarginCents = %d, want -3000000", got.Derived.GrossMarginCents)
+	}
+	if got.Derived.Profitable {
+		t.Error("must NOT be profitable when cost > revenue")
+	}
+	// runway = 20,000 / 2,000 = 10 days
+	if got.Derived.RunwayDays == nil || math.Abs(*got.Derived.RunwayDays-10) > 1e-9 {
+		t.Errorf("runwayDays = %v, want 10", got.Derived.RunwayDays)
+	}
+}
+
+// TestComputeFinance_HonestUnconfigured proves the DO-off path: no fabricated
+// credit/burn, runway is NULL (not zero), margin is just revenue (cost 0), and
+// margin % is 0 when revenue is 0.
+func TestComputeFinance_HonestUnconfigured(t *testing.T) {
+	in := financeInput{
+		cost:    financeCost{Configured: false, DigitalOcean: doCost{Configured: false}}, // commerce + DO both off
+		revenue: financeRevenue{Configured: false},
+	}
+	got := computeFinance(in)
+	if got.Cost.DigitalOcean.Configured {
+		t.Error("DO must report configured:false when the token is unset")
+	}
+	if got.Cost.DigitalOcean.CreditRemainingCents != 0 || got.Cost.DigitalOcean.AvgDailyBurnCents != 0 {
+		t.Error("unconfigured DO must not fabricate credit/burn")
+	}
+	// runway is null (nil) — no honest runway without a burn rate.
+	if got.Derived.RunwayDays != nil {
+		t.Errorf("runwayDays must be nil when DO is unconfigured, got %v", *got.Derived.RunwayDays)
+	}
+	if got.Derived.GrossMarginPct != 0 {
+		t.Errorf("grossMarginPct must be 0 when revenue is 0, got %f", got.Derived.GrossMarginPct)
+	}
+	// revenue 0 is not > cost 0 → not profitable.
+	if got.Derived.Profitable {
+		t.Error("zero revenue and zero cost is not profitable")
+	}
+}
+
+// TestComputeFinance_ZeroBurnNullRunway proves runway is null when DO is
+// configured but burn is zero (no division by zero, no fabricated infinity).
+func TestComputeFinance_ZeroBurnNullRunway(t *testing.T) {
+	in := financeInput{
+		cost:    financeCost{Configured: true, TotalCents: 0, DigitalOcean: doCost{Configured: true, CreditRemainingCents: 4_000_000, AvgDailyBurnCents: 0}},
+		revenue: financeRevenue{Configured: true, TotalRevenueCents: 100_000},
+	}
+	got := computeFinance(in)
+	if got.Derived.RunwayDays != nil {
+		t.Errorf("runwayDays must be nil when burn is 0, got %v", *got.Derived.RunwayDays)
+	}
+}
+
+// TestAvgDailyBurn_ElapsedDays proves the run-rate is MTD spend over elapsed
+// days (≥1), a real read over real time — never an invented rate.
+func TestAvgDailyBurn_ElapsedDays(t *testing.T) {
+	// $3,000 MTD on the 10th → $300/day.
+	got := avgDailyBurnCents(300_000, time.Date(2026, 7, 10, 12, 0, 0, 0, time.UTC))
+	if got != 30_000 {
+		t.Errorf("avgDailyBurn = %d, want 30000 ($300/day)", got)
+	}
+	// Day 1 must not divide by zero.
+	if avgDailyBurnCents(50_000, time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)) != 50_000 {
+		t.Error("day-1 burn must be the full MTD spend (divide by 1)")
+	}
+}
+
+// TestDollarsToCents proves the DO decimal-dollar → cents parsing, including the
+// negative (credit) case and the blank fallback.
+func TestDollarsToCents(t *testing.T) {
+	cases := []struct {
+		in   string
+		want int64
+	}{
+		{"23.44", 2344},
+		{"-40000.00", -4_000_000}, // promo credit held (negative account_balance)
+		{"12.23", 1223},
+		{"0", 0},
+		{"", 0},
+		{"  5.5 ", 550},
+		{"garbage", 0},
+	}
+	for _, c := range cases {
+		if got := dollarsToCents(c.in); got != c.want {
+			t.Errorf("dollarsToCents(%q) = %d, want %d", c.in, got, c.want)
+		}
+	}
+}
+
+// TestMonthlyNormalizedCents proves annual/monthly normalization for MRR.
+func TestMonthlyNormalizedCents(t *testing.T) {
+	if got := monthlyNormalizedCents(12_000, "year"); got != 1_000 {
+		t.Errorf("yearly $120 → monthly = %d, want 1000", got)
+	}
+	if got := monthlyNormalizedCents(2_000, "month"); got != 2_000 {
+		t.Errorf("monthly must pass through, got %d", got)
+	}
+	if got := monthlyNormalizedCents(2_000, ""); got != 2_000 {
+		t.Errorf("unknown interval must be treated as monthly, got %d", got)
+	}
+}
+
+// newFakeDO serves the DO billing API with fixed decimal-dollar strings so the
+// finance aggregation is deterministic. account_balance is NEGATIVE (credit held).
+func newFakeDO() *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		switch {
+		case strings.HasSuffix(r.URL.Path, "/customers/my/balance"):
+			// $10,000 credit remaining (account_balance = -10000.00), $3,000 MTD usage.
+			io.WriteString(w, `{"month_to_date_balance":"-7000.00","account_balance":"-10000.00","month_to_date_usage":"3000.00","generated_at":"2026-07-15T00:00:00Z"}`)
+		case strings.HasSuffix(r.URL.Path, "/customers/my/billing_history"):
+			io.WriteString(w, `{"billing_history":[
+				{"description":"Invoice for June 2026","amount":"2800.00","date":"2026-06-01T00:00:00Z","type":"Invoice","invoice_id":"1"},
+				{"description":"Promo credit","amount":"-40000.00","date":"2026-05-01T00:00:00Z","type":"Credit","invoice_id":""}
+			],"meta":{"total":2}}`)
+		default:
+			w.WriteHeader(404)
+		}
+	}))
+}
+
+// TestFinance_RealAggregation drives GET /v1/admin/finance against fake DO +
+// commerce and proves the whole pipe: DO credit/spend/burn derived with the
+// right sign, commerce revenue + MRR summed fleet-wide, and the derived margin/
+// runway from computeFinance — all in one envelope.
+func TestFinance_RealAggregation(t *testing.T) {
+	iam := newFakeIAM()
+	defer iam.server.Close()
+	commerce := newFakeCommerceFinance()
+	defer commerce.Close()
+	do := newFakeDO()
+	defer do.Close()
+
+	doReq, s, _ := mountSvc(t, iam.server.URL, commerce.URL, "")
+	s.do = newDOClientWithBase(do.URL, "test-do-token") // configured DO client
+	admin := map[string]string{
+		"X-User-IsAdmin": "true", "X-Org-Id": "admin",
+		"Authorization": "Bearer operator-jwt", "Cookie": "iam_access_token=operator-jwt",
+	}
+	resp, body := doReq("GET", "/v1/admin/finance", admin)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("finance: got %d (body=%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Status string      `json:"status"`
+		Data   financeData `json:"data"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if env.Status != "ok" {
+		t.Fatalf("finance status = %q, want ok", env.Status)
+	}
+	d := env.Data
+
+	// ── COGS: commerce /v1/costs — DO compute $3,000 + OpenAI $500 = $3,500 total,
+	// the multi-vendor breakdown that is now the margin cost (not the DO MTD spend).
+	if !d.Cost.Configured {
+		t.Fatal("commerce COGS must be configured in this test")
+	}
+	if d.Cost.TotalCents != 350_000 {
+		t.Errorf("cost.totalCents = %d, want 350000 ($3,500 multi-vendor COGS)", d.Cost.TotalCents)
+	}
+	if len(d.Cost.Vendors) != 2 {
+		t.Fatalf("cost.vendors must carry 2 lines (DO + OpenAI), got %d", len(d.Cost.Vendors))
+	}
+	if d.Cost.Vendors[0].Vendor == "" || d.Cost.Vendors[0].AmountCents == 0 {
+		t.Errorf("vendor line must carry a vendor + amount, got %+v", d.Cost.Vendors[0])
+	}
+
+	// ── DO treasury: credit = -account_balance = $10,000; MTD usage $3,000 (runway
+	// input, NOT the margin cost); burn-down history preserved.
+	if !d.Cost.DigitalOcean.Configured {
+		t.Fatal("DO must be configured in this test")
+	}
+	if d.Cost.DigitalOcean.CreditRemainingCents != 1_000_000 {
+		t.Errorf("creditRemaining = %d, want 1000000 ($10,000 = -account_balance)", d.Cost.DigitalOcean.CreditRemainingCents)
+	}
+	if d.Cost.DigitalOcean.MonthToDateSpendCents != 300_000 {
+		t.Errorf("monthToDateSpend = %d, want 300000 ($3,000)", d.Cost.DigitalOcean.MonthToDateSpendCents)
+	}
+	if d.Cost.DigitalOcean.AccountBalanceCents != -1_000_000 {
+		t.Errorf("accountBalance = %d, want -1000000 (negative = credit held)", d.Cost.DigitalOcean.AccountBalanceCents)
+	}
+	if len(d.Cost.DigitalOcean.History) != 2 {
+		t.Errorf("history must carry 2 entries, got %d", len(d.Cost.DigitalOcean.History))
+	}
+
+	// ── Commerce revenue: 2 orgs × $150 consumed = $300; MRR 2 × $50 = $100.
+	if !d.Revenue.Configured {
+		t.Fatal("commerce must be configured in this test")
+	}
+	if d.Revenue.TotalRevenueCents != 30_000 {
+		t.Errorf("totalRevenue = %d, want 30000 (2 orgs × $150)", d.Revenue.TotalRevenueCents)
+	}
+	if d.Revenue.MRRCents != 10_000 {
+		t.Errorf("MRR = %d, want 10000 (2 orgs × $50/mo active sub)", d.Revenue.MRRCents)
+	}
+
+	// ── Derived: margin = revenue 30,000 - COGS 350,000 = -320,000 (COGS > revenue).
+	if d.Derived.GrossMarginCents != -320_000 {
+		t.Errorf("grossMargin = %d, want -320000 (revenue 30k - COGS 350k)", d.Derived.GrossMarginCents)
+	}
+	if d.Derived.Profitable {
+		t.Error("not profitable: revenue $300 < COGS $3,500")
+	}
+	// runway present because DO configured + burn > 0.
+	if d.Derived.RunwayDays == nil {
+		t.Error("runwayDays must be present when DO burn > 0")
+	}
+	// Every source reported (digitalocean + commerce both ok).
+	src := map[string]sourceStatus{}
+	for _, x := range d.Sources {
+		src[x.Name] = x
+	}
+	if !src["digitalocean"].OK {
+		t.Errorf("digitalocean source must be ok: %+v", src["digitalocean"])
+	}
+	if !src["commerce"].OK {
+		t.Errorf("commerce source must be ok: %+v", src["commerce"])
+	}
+	if !src["commerce-costs"].OK {
+		t.Errorf("commerce-costs source must be ok: %+v", src["commerce-costs"])
+	}
+}
+
+// TestFinance_HonestUnconfiguredDO proves the ONE thing the user must provide:
+// with no DO_API_TOKEN the endpoint returns cost.digitalocean = {configured:false},
+// zero credit/burn, null runway — the honest state, never a fabricated $40k.
+func TestFinance_HonestUnconfiguredDO(t *testing.T) {
+	iam := newFakeIAM()
+	defer iam.server.Close()
+	commerce := newFakeCommerceFinance()
+	defer commerce.Close()
+
+	doReq, _, _ := mountSvc(t, iam.server.URL, commerce.URL, "") // s.do already has empty token → unconfigured
+	admin := map[string]string{"X-User-IsAdmin": "true", "X-Org-Id": "admin"}
+
+	resp, body := doReq("GET", "/v1/admin/finance", admin)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("finance: got %d (body=%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Data financeData `json:"data"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	d := env.Data
+	// COGS still flows from commerce even with the DO treasury read off — the DO
+	// compute COGS line belongs to commerce /v1/costs, decoupled from our DO credit
+	// read, so a missing DO_API_TOKEN never blanks the margin.
+	if !d.Cost.Configured || d.Cost.TotalCents == 0 || len(d.Cost.Vendors) == 0 {
+		t.Errorf("commerce COGS must remain configured with vendors when DO treasury is off: %+v", d.Cost)
+	}
+	if d.Cost.DigitalOcean.Configured {
+		t.Error("DO must report configured:false with no token")
+	}
+	if d.Cost.DigitalOcean.CreditRemainingCents != 0 || d.Cost.DigitalOcean.AvgDailyBurnCents != 0 {
+		t.Error("unconfigured DO must not fabricate credit/burn")
+	}
+	if d.Cost.DigitalOcean.Error == "" {
+		t.Error("unconfigured DO must carry an honest error string")
+	}
+	if d.Derived.RunwayDays != nil {
+		t.Errorf("runway must be null when DO is unconfigured, got %v", *d.Derived.RunwayDays)
+	}
+	// History must be an empty array (renders as an empty chart), never nil/fabricated.
+	if d.Cost.DigitalOcean.History == nil {
+		t.Error("history must be [] (empty array), not null")
+	}
+	// Commerce still reports its real revenue even with DO off.
+	if d.Revenue.TotalRevenueCents != 30_000 {
+		t.Errorf("commerce revenue must still be real with DO off, got %d", d.Revenue.TotalRevenueCents)
+	}
+	// The digitalocean source must be present and NOT ok (honest not-configured).
+	var doSrc *sourceStatus
+	for i := range d.Sources {
+		if d.Sources[i].Name == "digitalocean" {
+			doSrc = &d.Sources[i]
+		}
+	}
+	if doSrc == nil || doSrc.OK {
+		t.Errorf("digitalocean source must be present and not-ok when unconfigured: %+v", doSrc)
+	}
+}
+
+// TestFinance_RevenueSourceDown_NoFabrication proves the anti-fabrication property
+// (RED MED-1): when the revenue source (IAM listOrgs) is unreadable but commerce
+// COGS is fine, revenue reports configured:false (never a fake zero), so the board
+// cannot render a fabricated negative margin / "burning" alarm. COGS flows on.
+func TestFinance_RevenueSourceDown_NoFabrication(t *testing.T) {
+	commerce := newFakeCommerceFinance()
+	defer commerce.Close()
+
+	// IAM points nowhere reachable → listOrgs errors; commerce /v1/costs still 200s.
+	doReq, _, _ := mountSvc(t, "http://127.0.0.1:0", commerce.URL, "")
+	admin := map[string]string{"X-User-IsAdmin": "true", "X-Org-Id": "admin"}
+
+	resp, body := doReq("GET", "/v1/admin/finance", admin)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("finance: got %d (body=%s)", resp.StatusCode, body)
+	}
+	var env struct {
+		Data financeData `json:"data"`
+	}
+	if err := json.Unmarshal(body, &env); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	d := env.Data
+	// Revenue source unreadable → honest not-configured, NOT a fabricated zero.
+	if d.Revenue.Configured {
+		t.Error("revenue must report configured:false when the IAM org list is unreadable")
+	}
+	if d.Revenue.TotalRevenueCents != 0 {
+		t.Errorf("unreadable revenue must be 0, got %d", d.Revenue.TotalRevenueCents)
+	}
+	// COGS is independent — still configured from commerce /v1/costs.
+	if !d.Cost.Configured || d.Cost.TotalCents == 0 {
+		t.Errorf("COGS must remain configured when the revenue source is down: %+v", d.Cost)
+	}
+	// The commerce (revenue) source is present and NOT ok — honest degraded state.
+	var revSrc *sourceStatus
+	for i := range d.Sources {
+		if d.Sources[i].Name == "commerce" {
+			revSrc = &d.Sources[i]
+		}
+	}
+	if revSrc == nil || revSrc.OK {
+		t.Errorf("commerce revenue source must be present and not-ok when unreadable: %+v", revSrc)
+	}
+}
+
+// newFakeCommerceFinance serves the vendor-COGS god-view (/v1/costs) plus
+// usage-rollup ($150 consumed) and subscriptions (one active $50/mo sub) so the
+// finance COGS + revenue + MRR aggregation is deterministic.
+func newFakeCommerceFinance() *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		switch {
+		case strings.HasSuffix(r.URL.Path, "/costs"):
+			// The vendor-COGS god-view: DO compute $3,000 + OpenAI $500 = $3,500 total.
+			io.WriteString(w, `{"period":"2026-07","vendors":[
+				{"vendor":"digitalocean","service":"compute","amountCents":300000,"source":"actual","currency":"usd"},
+				{"vendor":"openai","service":"llm-inference","amountCents":50000,"source":"actual","currency":"usd"}
+			],"totalCents":350000,"currency":"usd"}`)
+		case strings.HasSuffix(r.URL.Path, "/usage-rollup"):
+			io.WriteString(w, `{"consumedCents":15000,"overageCents":0,"balance":{"balanceCents":0,"availableCents":0}}`)
+		case strings.HasSuffix(r.URL.Path, "/subscriptions"):
+			io.WriteString(w, `{"subscriptions":[
+				{"status":"active","plan":{"price":5000,"currency":"usd","interval":"month"}},
+				{"status":"canceled","plan":{"price":9900,"currency":"usd","interval":"month"}}
+			]}`)
+		default:
+			w.WriteHeader(404)
+		}
+	}))
+}

clients/admin/iam.go

@@ -0,0 +1,231 @@
+package admin
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// iamClient reads the IAM management surface (/v1/iam/get-*) on behalf of a
+// verified global-admin caller. IAM runs as its own deployment (not fused into
+// this binary — see subsystems.go), so these are HTTP calls, not Go method
+// dispatch. Every call REPLAYS THE CALLER'S OWN credential (session cookie +
+// Authorization), so IAM authorizes the read as the same principal the gateway
+// already validated as a global admin. admin adds NO service credential of
+// its own here: it never widens what the caller could read directly, and IAM's
+// own IsGlobalAdmin gate stays the second line of defense.
+type iamClient struct {
+	base string // e.g. http://iam.hanzo.svc.cluster.local:8000
+	http *http.Client
+}
+
+func newIAMClient(base string) *iamClient {
+	return &iamClient{
+		base: strings.TrimRight(strings.TrimSpace(base), "/"),
+		http: &http.Client{Timeout: 15 * time.Second},
+	}
+}
+
+func (c *iamClient) configured() bool { return c != nil && c.base != "" }
+
+// creds is the caller's replayed authorization context: the raw Cookie header
+// and Authorization bearer captured off the inbound request. IAM authenticates
+// exactly as it does for the browser (credentials: 'include').
+type creds struct {
+	cookie string
+	auth   string
+}
+
+// envelope is the uniform /v1 response shape every /v1/iam handler returns.
+// data is the payload; data2 the list total (paginated reads).
+type envelope struct {
+	Status string          `json:"status"`
+	Msg    string          `json:"msg"`
+	Data   json.RawMessage `json:"data"`
+	Data2  json.RawMessage `json:"data2"`
+}
+
+// listResult is a decoded paginated read: the raw rows and the backend total.
+type listResult struct {
+	rows  json.RawMessage
+	total int
+}
+
+// getList calls an IAM get-* endpoint and returns the raw data array + data2
+// total. A non-ok envelope is an error (surfaced honestly to the operator).
+func (c *iamClient) getList(ctx context.Context, cr creds, path string, q url.Values) (listResult, error) {
+	env, err := c.get(ctx, cr, path, q)
+	if err != nil {
+		return listResult{}, err
+	}
+	total := envTotal(env.Data2, env.Data)
+	return listResult{rows: env.Data, total: total}, nil
+}
+
+// get performs one authenticated GET and decodes the /v1 envelope.
+func (c *iamClient) get(ctx context.Context, cr creds, path string, q url.Values) (envelope, error) {
+	if !c.configured() {
+		return envelope{}, fmt.Errorf("iam endpoint not configured")
+	}
+	u := c.base + path
+	if enc := q.Encode(); enc != "" {
+		u += "?" + enc
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
+	if err != nil {
+		return envelope{}, err
+	}
+	req.Header.Set("Accept", "application/json")
+	if cr.cookie != "" {
+		req.Header.Set("Cookie", cr.cookie)
+	}
+	if cr.auth != "" {
+		req.Header.Set("Authorization", cr.auth)
+	}
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return envelope{}, fmt.Errorf("iam unreachable: %w", err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
+	if err != nil {
+		return envelope{}, err
+	}
+	if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
+		return envelope{}, fmt.Errorf("iam denied (%d)", resp.StatusCode)
+	}
+	var env envelope
+	if err := json.Unmarshal(body, &env); err != nil {
+		return envelope{}, fmt.Errorf("iam non-envelope response (%d)", resp.StatusCode)
+	}
+	if env.Status != "ok" {
+		msg := env.Msg
+		if msg == "" {
+			msg = fmt.Sprintf("iam status %d", resp.StatusCode)
+		}
+		return envelope{}, fmt.Errorf("iam: %s", msg)
+	}
+	return env, nil
+}
+
+// getUserRaw fetches ONE user as its FULL wire object (GET /v1/iam/get-user?id=
+// owner/name), preserving every field. The suspend/reactivate action reads the
+// whole object, flips isForbidden, and writes it back — update-user REPLACES the
+// row, so operating on the full object (not a typed subset) is what keeps every
+// other field intact. Replays the caller's own credential, so IAM authorizes the
+// read as the same validated global admin.
+func (c *iamClient) getUserRaw(ctx context.Context, cr creds, id string) (map[string]any, error) {
+	q := url.Values{"id": {id}}
+	env, err := c.get(ctx, cr, "/v1/iam/get-user", q)
+	if err != nil {
+		return nil, err
+	}
+	var user map[string]any
+	if err := json.Unmarshal(env.Data, &user); err != nil {
+		return nil, fmt.Errorf("iam get-user decode: %w", err)
+	}
+	if user == nil {
+		return nil, fmt.Errorf("iam get-user %q: empty", id)
+	}
+	return user, nil
+}
+
+// updateUserRaw writes a full user object back (POST /v1/iam/update-user?id=
+// owner/name). The caller's replayed credential is a VALIDATED global admin, whom
+// IAM's CheckPermissionForUpdateUser admits to set privileged fields (isForbidden)
+// on any user — a tenant/org-admin is refused by IAM itself, so this can never be
+// abused to suspend across a boundary the caller couldn't already cross. admin
+// adds no service credential of its own; IAM re-checks IsGlobalAdmin.
+func (c *iamClient) updateUserRaw(ctx context.Context, cr creds, id string, user map[string]any) error {
+	q := url.Values{"id": {id}}
+	body, err := json.Marshal(user)
+	if err != nil {
+		return err
+	}
+	_, err = c.post(ctx, cr, "/v1/iam/update-user", q, body)
+	return err
+}
+
+// post performs one authenticated POST (JSON body) replaying the caller's cookie +
+// bearer, and decodes the /v1 envelope. A non-ok envelope (or an IAM 401/403) is
+// an error the mutation surfaces honestly + records as a failed audited attempt.
+func (c *iamClient) post(ctx context.Context, cr creds, path string, q url.Values, body []byte) (envelope, error) {
+	if !c.configured() {
+		return envelope{}, fmt.Errorf("iam endpoint not configured")
+	}
+	u := c.base + path
+	if enc := q.Encode(); enc != "" {
+		u += "?" + enc
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, bytes.NewReader(body))
+	if err != nil {
+		return envelope{}, err
+	}
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("Content-Type", "application/json")
+	if cr.cookie != "" {
+		req.Header.Set("Cookie", cr.cookie)
+	}
+	if cr.auth != "" {
+		req.Header.Set("Authorization", cr.auth)
+	}
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return envelope{}, fmt.Errorf("iam unreachable: %w", err)
+	}
+	defer resp.Body.Close()
+	respBody, err := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
+	if err != nil {
+		return envelope{}, err
+	}
+	if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
+		return envelope{}, fmt.Errorf("iam denied (%d)", resp.StatusCode)
+	}
+	var env envelope
+	if err := json.Unmarshal(respBody, &env); err != nil {
+		return envelope{}, fmt.Errorf("iam non-envelope response (%d)", resp.StatusCode)
+	}
+	if env.Status != "ok" {
+		msg := env.Msg
+		if msg == "" {
+			msg = fmt.Sprintf("iam status %d", resp.StatusCode)
+		}
+		return envelope{}, fmt.Errorf("iam: %s", msg)
+	}
+	return env, nil
+}
+
+// envTotal reads data2 as the list total when present, else counts data rows.
+func envTotal(data2, data json.RawMessage) int {
+	if n, ok := asInt(data2); ok {
+		return n
+	}
+	var rows []json.RawMessage
+	if json.Unmarshal(data, &rows) == nil {
+		return len(rows)
+	}
+	return 0
+}
+
+// asInt decodes a JSON number (data2 may arrive as a bare int).
+func asInt(raw json.RawMessage) (int, bool) {
+	t := strings.TrimSpace(string(raw))
+	if t == "" || t == "null" {
+		return 0, false
+	}
+	if n, err := strconv.Atoi(t); err == nil {
+		return n, true
+	}
+	var f float64
+	if json.Unmarshal(raw, &f) == nil {
+		return int(f), true
+	}
+	return 0, false
+}

clients/admin/o11y.go

@@ -0,0 +1,424 @@
+// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package admin
+
+// o11y — GET /v1/admin/o11y, the GLOBAL fleet-wide observability read that powers
+// the operator's o11y board on admin.hanzo.ai. It is the un-org-scoped twin of the
+// per-org console o11y: the same signals, aggregated across EVERY tenant, over the
+// ONE hanzoai/datastore (ClickHouse) — the same warehouse + shared client
+// (aiobject.DatastoreQuery) the analytics/compute lenses already use, no second
+// connection.
+//
+// Signals, each from its canonical table in the one datastore:
+//   - LLM usage  → hanzo.cloud_usage         : requests, tokens, cost, errors, top orgs, top models
+//   - Traces     → signoz_traces.distributed_signoz_index_v3 : request count, latency p50/p95/p99,
+//                                                              error rate, top services
+//   - Logs       → signoz_logs.distributed_logs_v2 : fleet log volume + volume-over-time
+//   - LLM gens   → langfuse.observations      : generations + cost (fleet-wide; honest-empty today)
+//
+// GLOBAL-ADMIN ONLY (the s.guard wrap in admin.go): the gateway strips a client
+// X-Org-Id and re-mints from the JWT owner, and this handler applies NO org filter,
+// so it is the ONE place a fleet operator crosses tenants — a non-admin bearer is
+// refused 403 before a single row is read. Fail-closed.
+//
+// Honest by construction, exactly like compute/analytics: no datastore connected →
+// the real empty aggregate, never a fabricated fleet. admin READS only; it owns and
+// creates NO table (the ZAP-fed collector + the ai-owned cloud_usage ledger own the
+// data). Money is USD cents end to end; latency is milliseconds; time bounds are
+// POSITIONAL parameters (never interpolated), and the bucket interval is a
+// server-side constant — injection-safe.
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	aiobject "github.com/hanzoai/ai/object"
+	"github.com/zap-proto/zip"
+)
+
+// Fully-qualified datastore tables. admin only READS these — the ZAP collector
+// (signoz_*), the ai ledger (hanzo.cloud_usage), and Langfuse own their writes.
+const (
+	o11yUsageTable   = "hanzo.cloud_usage"
+	o11yTraceTable   = "signoz_traces.distributed_signoz_index_v3"
+	o11yLogTable     = "signoz_logs.distributed_logs_v2"
+	o11yLangfuseObs  = "langfuse.observations"
+	o11yTopN         = 10
+	o11yServiceLimit = 12
+)
+
+// o11yGlobal is the whole fleet o11y board payload.
+type o11yGlobal struct {
+	Range       string          `json:"range"`
+	Start       string          `json:"start"`
+	End         string          `json:"end"`
+	Totals      o11yTotals      `json:"totals"`
+	Series      []o11ySeries    `json:"series"`
+	LogSeries   []o11yLogPoint  `json:"logSeries"`
+	TopOrgs     []o11yOrgStat   `json:"topOrgs"`
+	TopModels   []o11yModelStat `json:"topModels"`
+	TopServices []o11ySvcStat   `json:"topServices"`
+	LLM         o11yLLM         `json:"llm"`
+}
+
+// o11yTotals is the fleet KPI band. LLM half from cloud_usage; RED half from traces;
+// volume from logs. Every field is a real aggregate or an honest zero.
+type o11yTotals struct {
+	// LLM usage (hanzo.cloud_usage), all orgs.
+	Requests         int64 `json:"requests"`
+	Tokens           int64 `json:"tokens"`
+	PromptTokens     int64 `json:"promptTokens"`
+	CompletionTokens int64 `json:"completionTokens"`
+	CostCents        int64 `json:"costCents"`
+	Errors           int64 `json:"errors"`
+	Orgs             int64 `json:"orgs"`
+	Models           int64 `json:"models"`
+	// Traces (signoz_index_v3), all services.
+	TraceCount     int64   `json:"traceCount"`
+	LatencyP50Ms   float64 `json:"latencyP50Ms"`
+	LatencyP95Ms   float64 `json:"latencyP95Ms"`
+	LatencyP99Ms   float64 `json:"latencyP99Ms"`
+	TraceErrorRate float64 `json:"traceErrorRate"` // percent (0..100)
+	Services       int64   `json:"services"`
+	// Logs (distributed_logs_v2), fleet volume over the window.
+	LogVolume int64 `json:"logVolume"`
+}
+
+// o11ySeries is one usage time bucket (fleet-wide).
+type o11ySeries struct {
+	Ts        string `json:"ts"`
+	Requests  int64  `json:"requests"`
+	Tokens    int64  `json:"tokens"`
+	CostCents int64  `json:"costCents"`
+	Errors    int64  `json:"errors"`
+}
+
+// o11yLogPoint is one log-volume time bucket (fleet-wide).
+type o11yLogPoint struct {
+	Ts    string `json:"ts"`
+	Count int64  `json:"count"`
+}
+
+// o11yOrgStat is one row of the top-orgs-by-usage leaderboard.
+type o11yOrgStat struct {
+	Org       string `json:"org"`
+	Requests  int64  `json:"requests"`
+	Tokens    int64  `json:"tokens"`
+	CostCents int64  `json:"costCents"`
+}
+
+// o11yModelStat is one row of the top-models leaderboard.
+type o11yModelStat struct {
+	Model     string `json:"model"`
+	Requests  int64  `json:"requests"`
+	Tokens    int64  `json:"tokens"`
+	CostCents int64  `json:"costCents"`
+}
+
+// o11ySvcStat is one row of the top-services (by trace volume) leaderboard.
+type o11ySvcStat struct {
+	Service      string  `json:"service"`
+	Requests     int64   `json:"requests"`
+	ErrorRate    float64 `json:"errorRate"` // percent (0..100)
+	LatencyP95Ms float64 `json:"latencyP95Ms"`
+}
+
+// o11yLLM is the fleet-wide Langfuse generation rollup (near-empty today → honest).
+type o11yLLM struct {
+	Generations int64   `json:"generations"`
+	CostUsd     float64 `json:"costUsd"`
+}
+
+// o11y answers GET /v1/admin/o11y. ?range=24h|7d|30d bounds the window (default 30d).
+// GLOBAL-ADMIN ONLY (s.guard). Every signal degrades independently: a table that is
+// absent or errors contributes its zero-value, never a failure — the fleet board
+// always renders what the datastore actually holds.
+func (s *svc) o11y(c *zip.Ctx) error {
+	ctx := c.Context()
+	rangeLabel := o11yRange(c.Query("range"))
+	since := computeSince(rangeLabel)
+	payload := o11yGlobal{
+		Range:       rangeLabel,
+		Start:       since.Format(time.RFC3339),
+		End:         time.Now().UTC().Format(time.RFC3339),
+		Series:      []o11ySeries{},
+		LogSeries:   []o11yLogPoint{},
+		TopOrgs:     []o11yOrgStat{},
+		TopModels:   []o11yModelStat{},
+		TopServices: []o11ySvcStat{},
+	}
+
+	// Honest-empty when the warehouse is not connected: the board renders its zero
+	// state, never a fabricated fleet.
+	if !aiobject.DatastoreEnabled() {
+		return ok(c, payload)
+	}
+
+	sinceTS := chTS(since)         // DateTime literal — cloud_usage.timestamp, traces.timestamp
+	sinceNanos := since.UnixNano() // UInt64 nanos — logs.timestamp
+	interval := o11yBucket(rangeLabel)
+
+	// LLM usage totals (all orgs).
+	if rows, err := aiobject.DatastoreQuery(ctx, o11yUsageTotalsSQL(), sinceTS); err == nil {
+		fillUsageTotals(&payload.Totals, firstRowOr(rows))
+	}
+	// Trace RED metrics (all services).
+	if rows, err := aiobject.DatastoreQuery(ctx, o11yTraceTotalsSQL(), sinceTS); err == nil {
+		fillTraceTotals(&payload.Totals, firstRowOr(rows))
+	}
+	// Fleet log volume.
+	if rows, err := aiobject.DatastoreQuery(ctx, o11yLogVolumeSQL(), sinceNanos); err == nil {
+		payload.Totals.LogVolume = chInt64(firstRowOr(rows)["c"])
+	}
+	// Usage time-series (fleet).
+	if rows, err := aiobject.DatastoreQuery(ctx, o11yUsageSeriesSQL(interval), sinceTS); err == nil {
+		payload.Series = usageSeriesFromRows(rows)
+	}
+	// Log-volume time-series (fleet).
+	if rows, err := aiobject.DatastoreQuery(ctx, o11yLogSeriesSQL(interval), sinceNanos); err == nil {
+		payload.LogSeries = logSeriesFromRows(rows)
+	}
+	// Top orgs by usage.
+	if rows, err := aiobject.DatastoreQuery(ctx, o11yTopOrgsSQL(), sinceTS); err == nil {
+		payload.TopOrgs = topOrgsFromRows(rows)
+	}
+	// Top models by usage.
+	if rows, err := aiobject.DatastoreQuery(ctx, o11yTopModelsSQL(), sinceTS); err == nil {
+		payload.TopModels = topModelsFromRows(rows)
+	}
+	// Top services by trace volume.
+	if rows, err := aiobject.DatastoreQuery(ctx, o11yTopServicesSQL(), sinceTS); err == nil {
+		payload.TopServices = topServicesFromRows(rows)
+	}
+	// Fleet LLM generations (Langfuse) — best-effort; near-empty today.
+	if rows, err := aiobject.DatastoreQuery(ctx, o11yLLMSQL(), sinceTS); err == nil {
+		r := firstRowOr(rows)
+		payload.LLM = o11yLLM{Generations: chInt64(r["gens"]), CostUsd: chFloat64(r["cost"])}
+	}
+
+	return ok(c, payload)
+}
+
+// ── pure SQL builders (static SQL + one positional time bound; unit-tested) ──
+
+func o11yUsageTotalsSQL() string {
+	return "SELECT count() AS requests, sum(total_tokens) AS tokens, " +
+		"sum(prompt_tokens) AS prompt_tokens, sum(completion_tokens) AS completion_tokens, " +
+		"sum(cost_cents) AS cost_cents, countIf(status = 'error') AS errors, " +
+		"uniqExact(organization) AS orgs, uniqExact(model) AS models " +
+		"FROM " + o11yUsageTable + " WHERE timestamp >= ?"
+}
+
+func o11yTraceTotalsSQL() string {
+	return "SELECT count() AS traces, " +
+		"round(quantile(0.5)(durationNano) / 1e6, 2) AS p50, " +
+		"round(quantile(0.95)(durationNano) / 1e6, 2) AS p95, " +
+		"round(quantile(0.99)(durationNano) / 1e6, 2) AS p99, " +
+		"round(100 * countIf(has_error) / greatest(count(), 1), 3) AS err_rate, " +
+		"uniqExact(serviceName) AS services " +
+		"FROM " + o11yTraceTable + " WHERE timestamp >= ?"
+}
+
+func o11yLogVolumeSQL() string {
+	return "SELECT count() AS c FROM " + o11yLogTable + " WHERE timestamp >= ?"
+}
+
+func o11yUsageSeriesSQL(interval string) string {
+	return "SELECT toStartOfInterval(timestamp, INTERVAL " + interval + ") AS ts, " +
+		"count() AS requests, sum(total_tokens) AS tokens, sum(cost_cents) AS cost_cents, " +
+		"countIf(status = 'error') AS errors " +
+		"FROM " + o11yUsageTable + " WHERE timestamp >= ? GROUP BY ts ORDER BY ts"
+}
+
+func o11yLogSeriesSQL(interval string) string {
+	return "SELECT toStartOfInterval(toDateTime(timestamp / 1000000000), INTERVAL " + interval + ") AS ts, " +
+		"count() AS c FROM " + o11yLogTable + " WHERE timestamp >= ? GROUP BY ts ORDER BY ts"
+}
+
+func o11yTopOrgsSQL() string {
+	return "SELECT organization AS org, count() AS requests, sum(total_tokens) AS tokens, " +
+		"sum(cost_cents) AS cost_cents FROM " + o11yUsageTable +
+		" WHERE timestamp >= ? GROUP BY org ORDER BY requests DESC LIMIT " + strconv.Itoa(o11yTopN)
+}
+
+func o11yTopModelsSQL() string {
+	return "SELECT model, count() AS requests, sum(total_tokens) AS tokens, " +
+		"sum(cost_cents) AS cost_cents FROM " + o11yUsageTable +
+		" WHERE timestamp >= ? AND model != '' GROUP BY model ORDER BY requests DESC LIMIT " + strconv.Itoa(o11yTopN)
+}
+
+func o11yTopServicesSQL() string {
+	return "SELECT serviceName AS service, count() AS requests, " +
+		"round(100 * countIf(has_error) / greatest(count(), 1), 3) AS error_rate, " +
+		"round(quantile(0.95)(durationNano) / 1e6, 2) AS p95 " +
+		"FROM " + o11yTraceTable + " WHERE timestamp >= ? AND serviceName != '' " +
+		"GROUP BY service ORDER BY requests DESC LIMIT " + strconv.Itoa(o11yServiceLimit)
+}
+
+func o11yLLMSQL() string {
+	return "SELECT count() AS gens, toFloat64(sum(total_cost)) AS cost FROM " + o11yLangfuseObs +
+		" WHERE type = 'GENERATION' AND start_time >= ?"
+}
+
+// ── pure row parsers (unit-tested) ──
+
+func fillUsageTotals(t *o11yTotals, r map[string]any) {
+	t.Requests = chInt64(r["requests"])
+	t.Tokens = chInt64(r["tokens"])
+	t.PromptTokens = chInt64(r["prompt_tokens"])
+	t.CompletionTokens = chInt64(r["completion_tokens"])
+	t.CostCents = chInt64(r["cost_cents"])
+	t.Errors = chInt64(r["errors"])
+	t.Orgs = chInt64(r["orgs"])
+	t.Models = chInt64(r["models"])
+}
+
+func fillTraceTotals(t *o11yTotals, r map[string]any) {
+	t.TraceCount = chInt64(r["traces"])
+	t.LatencyP50Ms = chFloat64(r["p50"])
+	t.LatencyP95Ms = chFloat64(r["p95"])
+	t.LatencyP99Ms = chFloat64(r["p99"])
+	t.TraceErrorRate = chFloat64(r["err_rate"])
+	t.Services = chInt64(r["services"])
+}
+
+func usageSeriesFromRows(rows []map[string]any) []o11ySeries {
+	out := make([]o11ySeries, 0, len(rows))
+	for _, r := range rows {
+		out = append(out, o11ySeries{
+			Ts:        chTime(r["ts"]),
+			Requests:  chInt64(r["requests"]),
+			Tokens:    chInt64(r["tokens"]),
+			CostCents: chInt64(r["cost_cents"]),
+			Errors:    chInt64(r["errors"]),
+		})
+	}
+	return out
+}
+
+func logSeriesFromRows(rows []map[string]any) []o11yLogPoint {
+	out := make([]o11yLogPoint, 0, len(rows))
+	for _, r := range rows {
+		out = append(out, o11yLogPoint{Ts: chTime(r["ts"]), Count: chInt64(r["c"])})
+	}
+	return out
+}
+
+func topOrgsFromRows(rows []map[string]any) []o11yOrgStat {
+	out := make([]o11yOrgStat, 0, len(rows))
+	for _, r := range rows {
+		out = append(out, o11yOrgStat{
+			Org:       chStr(r["org"]),
+			Requests:  chInt64(r["requests"]),
+			Tokens:    chInt64(r["tokens"]),
+			CostCents: chInt64(r["cost_cents"]),
+		})
+	}
+	return out
+}
+
+func topModelsFromRows(rows []map[string]any) []o11yModelStat {
+	out := make([]o11yModelStat, 0, len(rows))
+	for _, r := range rows {
+		out = append(out, o11yModelStat{
+			Model:     chStr(r["model"]),
+			Requests:  chInt64(r["requests"]),
+			Tokens:    chInt64(r["tokens"]),
+			CostCents: chInt64(r["cost_cents"]),
+		})
+	}
+	return out
+}
+
+func topServicesFromRows(rows []map[string]any) []o11ySvcStat {
+	out := make([]o11ySvcStat, 0, len(rows))
+	for _, r := range rows {
+		out = append(out, o11ySvcStat{
+			Service:      chStr(r["service"]),
+			Requests:     chInt64(r["requests"]),
+			ErrorRate:    chFloat64(r["error_rate"]),
+			LatencyP95Ms: chFloat64(r["p95"]),
+		})
+	}
+	return out
+}
+
+// ── small pure helpers ──
+
+// o11yRange normalizes the ?range enum (default 30d).
+func o11yRange(v string) string {
+	switch strings.TrimSpace(v) {
+	case "24h":
+		return "24h"
+	case "7d":
+		return "7d"
+	default:
+		return "30d"
+	}
+}
+
+// o11yBucket maps the range to a fixed ClickHouse interval clause (a server-side
+// CONSTANT — never user input — so it is safe to render into the SQL). ~24-30
+// buckets across the window keeps the charts legible.
+func o11yBucket(rangeLabel string) string {
+	switch rangeLabel {
+	case "24h":
+		return "1 HOUR"
+	case "7d":
+		return "6 HOUR"
+	default:
+		return "1 DAY"
+	}
+}
+
+// firstRowOr returns the first row or an empty map (never nil), so a parser reads
+// honest zeros from an empty result instead of panicking.
+func firstRowOr(rows []map[string]any) map[string]any {
+	if len(rows) == 0 {
+		return map[string]any{}
+	}
+	return rows[0]
+}
+
+// chFloat64 coerces a ClickHouse numeric cell to float64 (the round()/quantile()
+// columns land as float64; a Decimal serialized to string is parsed). The twin of
+// chInt64 for the latency/error-rate/cost fields. Non-numeric → 0 (honest zero).
+func chFloat64(v any) float64 {
+	switch n := v.(type) {
+	case float64:
+		return n
+	case float32:
+		return float64(n)
+	case int:
+		return float64(n)
+	case int64:
+		return float64(n)
+	case int32:
+		return float64(n)
+	case uint64:
+		return float64(n)
+	case uint32:
+		return float64(n)
+	case string:
+		f, err := strconv.ParseFloat(strings.TrimSpace(n), 64)
+		if err != nil {
+			return 0
+		}
+		return f
+	default:
+		return 0
+	}
+}

clients/admin/o11y_test.go

@@ -0,0 +1,179 @@
+// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package admin
+
+import (
+	"strings"
+	"testing"
+)
+
+// TestO11yRange normalizes the enum and defaults to 30d.
+func TestO11yRange(t *testing.T) {
+	for in, want := range map[string]string{"24h": "24h", "7d": "7d", "30d": "30d", "": "30d", "bogus": "30d", " 7d ": "7d"} {
+		if got := o11yRange(in); got != want {
+			t.Errorf("o11yRange(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+// TestO11yBucket maps each range to a fixed, injection-safe interval constant.
+func TestO11yBucket(t *testing.T) {
+	for r, want := range map[string]string{"24h": "1 HOUR", "7d": "6 HOUR", "30d": "1 DAY"} {
+		if got := o11yBucket(r); got != want {
+			t.Errorf("o11yBucket(%q) = %q, want %q", r, got, want)
+		}
+	}
+}
+
+// TestO11ySQL_ReadsCanonicalTables proves every fleet query reads the ONE
+// datastore's canonical table, binds the time bound as a POSITIONAL param (one
+// `?`), and never interpolates user input. The bucket interval is the only
+// rendered value and it is a server-side constant.
+func TestO11ySQL_ReadsCanonicalTables(t *testing.T) {
+	cases := []struct {
+		name, sql, table string
+		wantQMarks       int
+	}{
+		{"usageTotals", o11yUsageTotalsSQL(), "hanzo.cloud_usage", 1},
+		{"traceTotals", o11yTraceTotalsSQL(), "signoz_traces.distributed_signoz_index_v3", 1},
+		{"logVolume", o11yLogVolumeSQL(), "signoz_logs.distributed_logs_v2", 1},
+		{"usageSeries", o11yUsageSeriesSQL("1 HOUR"), "hanzo.cloud_usage", 1},
+		{"logSeries", o11yLogSeriesSQL("1 HOUR"), "signoz_logs.distributed_logs_v2", 1},
+		{"topOrgs", o11yTopOrgsSQL(), "hanzo.cloud_usage", 1},
+		{"topModels", o11yTopModelsSQL(), "hanzo.cloud_usage", 1},
+		{"topServices", o11yTopServicesSQL(), "signoz_traces.distributed_signoz_index_v3", 1},
+		{"llm", o11yLLMSQL(), "langfuse.observations", 1},
+	}
+	for _, c := range cases {
+		if !strings.Contains(c.sql, "FROM "+c.table) {
+			t.Errorf("%s must read %s; got %q", c.name, c.table, c.sql)
+		}
+		if n := strings.Count(c.sql, "?"); n != c.wantQMarks {
+			t.Errorf("%s: %d bind params, want %d (time bound only) — no interpolation; got %q", c.name, n, c.wantQMarks, c.sql)
+		}
+	}
+}
+
+// TestO11ySeriesSQL_IntervalBound proves the (constant) bucket interval is
+// rendered into the series queries and grouped/ordered by the bucket.
+func TestO11ySeriesSQL_IntervalBound(t *testing.T) {
+	for _, iv := range []string{"1 HOUR", "6 HOUR", "1 DAY"} {
+		u := o11yUsageSeriesSQL(iv)
+		if !strings.Contains(u, "INTERVAL "+iv) || !strings.Contains(u, "GROUP BY ts ORDER BY ts") {
+			t.Errorf("usage series must bucket by INTERVAL %s; got %q", iv, u)
+		}
+		l := o11yLogSeriesSQL(iv)
+		if !strings.Contains(l, "INTERVAL "+iv) {
+			t.Errorf("log series must bucket by INTERVAL %s; got %q", iv, l)
+		}
+	}
+}
+
+// TestO11yTop_LimitAndOrder proves the leaderboards bound + order the result.
+func TestO11yTop_LimitAndOrder(t *testing.T) {
+	if !strings.Contains(o11yTopOrgsSQL(), "ORDER BY requests DESC LIMIT 10") {
+		t.Errorf("topOrgs must order by requests desc, limit %d", o11yTopN)
+	}
+	if !strings.Contains(o11yTopServicesSQL(), "LIMIT 12") {
+		t.Errorf("topServices must limit %d", o11yServiceLimit)
+	}
+	// The LLM lens is scoped to generations only (not spans/events).
+	if !strings.Contains(o11yLLMSQL(), "type = 'GENERATION'") {
+		t.Errorf("llm lens must scope to GENERATION observations; got %q", o11yLLMSQL())
+	}
+}
+
+// TestFillUsageTotals reads the ClickHouse row into the KPI band across the
+// numeric variants the driver returns (uint64/int64/float64), honest zeros on
+// an empty row.
+func TestFillUsageTotals(t *testing.T) {
+	var empty o11yTotals
+	fillUsageTotals(&empty, map[string]any{})
+	if empty.Requests != 0 || empty.Tokens != 0 || empty.Orgs != 0 {
+		t.Fatalf("empty row must yield honest zeros; got %+v", empty)
+	}
+	var got o11yTotals
+	fillUsageTotals(&got, map[string]any{
+		"requests": uint64(274), "tokens": uint64(102597), "prompt_tokens": uint64(60000),
+		"completion_tokens": uint64(42597), "cost_cents": uint64(216), "errors": uint64(41),
+		"orgs": uint64(3), "models": uint64(42),
+	})
+	if got.Requests != 274 || got.Tokens != 102597 || got.CostCents != 216 || got.Orgs != 3 || got.Models != 42 || got.Errors != 41 {
+		t.Fatalf("usage totals mis-parsed: %+v", got)
+	}
+}
+
+// TestFillTraceTotals maps the RED metrics, including the float latency/error-rate
+// columns (round()/quantile() land as float64; a Decimal-as-string is parsed).
+func TestFillTraceTotals(t *testing.T) {
+	var got o11yTotals
+	fillTraceTotals(&got, map[string]any{
+		"traces": uint64(4044354), "p50": float64(12.5), "p95": float64(340.2),
+		"p99": "901.7", "err_rate": float64(1.25), "services": uint64(8),
+	})
+	if got.TraceCount != 4044354 || got.LatencyP50Ms != 12.5 || got.LatencyP95Ms != 340.2 {
+		t.Fatalf("trace latency mis-parsed: %+v", got)
+	}
+	if got.LatencyP99Ms != 901.7 { // string→float64 path
+		t.Errorf("p99 string→float64 = %v, want 901.7", got.LatencyP99Ms)
+	}
+	if got.TraceErrorRate != 1.25 || got.Services != 8 {
+		t.Errorf("trace error/services mis-parsed: %+v", got)
+	}
+}
+
+// TestTopParsers map ClickHouse rows into the leaderboard view-models and preserve
+// order (the SQL already ORDER BYs; the parser must not reorder or drop rows).
+func TestTopParsers(t *testing.T) {
+	orgs := topOrgsFromRows([]map[string]any{
+		{"org": "hanzo", "requests": uint64(154), "tokens": uint64(38966), "cost_cents": uint64(114)},
+		{"org": "maxpower", "requests": uint64(118), "tokens": uint64(61550), "cost_cents": uint64(101)},
+	})
+	if len(orgs) != 2 || orgs[0].Org != "hanzo" || orgs[1].Org != "maxpower" || orgs[0].Requests != 154 {
+		t.Fatalf("top orgs mis-parsed/reordered: %+v", orgs)
+	}
+	svcs := topServicesFromRows([]map[string]any{
+		{"service": "ingress", "requests": uint64(308125), "error_rate": float64(45.56), "p95": float64(38.5)},
+		{"service": "gateway", "requests": uint64(1112), "error_rate": float64(0), "p95": float64(8.3)},
+	})
+	if len(svcs) != 2 || svcs[0].Service != "ingress" || svcs[0].ErrorRate != 45.56 || svcs[1].LatencyP95Ms != 8.3 {
+		t.Fatalf("top services mis-parsed: %+v", svcs)
+	}
+	// Empty input → empty (non-nil) slice, never a panic.
+	if got := usageSeriesFromRows(nil); got == nil || len(got) != 0 {
+		t.Errorf("nil rows must yield empty slice, got %v", got)
+	}
+}
+
+// TestChFloat64 covers the numeric coercions the float accessor must survive.
+func TestChFloat64(t *testing.T) {
+	cases := map[string]struct {
+		in   any
+		want float64
+	}{
+		"float64": {float64(3.14), 3.14},
+		"float32": {float32(2.5), 2.5},
+		"int64":   {int64(7), 7},
+		"uint64":  {uint64(9), 9},
+		"string":  {"12.5", 12.5},
+		"badstr":  {"nope", 0},
+		"nil":     {nil, 0},
+	}
+	for name, c := range cases {
+		if got := chFloat64(c.in); got != c.want {
+			t.Errorf("chFloat64(%s=%v) = %v, want %v", name, c.in, got, c.want)
+		}
+	}
+}

clients/admin/revenue.go

@@ -0,0 +1,154 @@
+package admin
+
+// Fleet REVENUE aggregate (/v1/admin/revenue) — the operator's money board: total
+// prepaid balances held, total realized spend, MRR, a per-customer revenue table,
+// ARPU, and a real spend trend. Global-admin only (s.guard).
+//
+// This is ORTHOGONAL to /v1/admin/finance: finance is the COGS/margin god-view
+// (what WE pay vendors, gross margin, DO-credit runway); revenue is the CUSTOMER
+// money view (what each customer holds/spends/subscribes). Both read commerce, but
+// answer different questions — one is "are we profitable", the other is "who are
+// our paying customers and what do they pay". Every number is a real commerce read;
+// an unreachable org degrades to honest zero (never a fabricated figure), and a
+// partial fleet read marks its source degraded rather than presenting an undercount
+// as authoritative.
+
+import (
+	"context"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/zap-proto/zip"
+)
+
+// revenueCustomer is one row of the per-customer revenue table.
+type revenueCustomer struct {
+	Org          string `json:"org"`
+	Display      string `json:"display"`
+	Plan         string `json:"plan"`
+	BalanceCents int64  `json:"balanceCents"`
+	SpendCents   int64  `json:"spendCents"`
+	MRRCents     int64  `json:"mrrCents"`
+}
+
+// revenueData is the whole GET /v1/admin/revenue payload.
+type revenueData struct {
+	TotalBalancesCents int64             `json:"totalBalancesCents"`
+	TotalSpendCents    int64             `json:"totalSpendCents"`
+	MRRCents           int64             `json:"mrrCents"`
+	Customers          int               `json:"customers"`
+	PayingCustomers    int               `json:"payingCustomers"`
+	ARPUCents          int64             `json:"arpuCents"`
+	PerCustomer        []revenueCustomer `json:"perCustomer"`
+	SpendTrend         []seriesPoint     `json:"spendTrend"`
+	GeneratedAt        string            `json:"generatedAt"`
+	Sources            []sourceStatus    `json:"sources"`
+}
+
+func (s *svc) revenue(c *zip.Ctx) error {
+	ctx := c.Context()
+	cr := callerCreds(c)
+	now := time.Now().UTC()
+
+	orgs, err := s.listOrgs(ctx, cr)
+	if err != nil {
+		return fail(c, err.Error())
+	}
+
+	// Per-org money, fanned out concurrently (balance + spend + plan/MRR).
+	rows := make([]revenueCustomer, len(orgs))
+	oks := make([]bool, len(orgs))
+	sem := make(chan struct{}, maxCustomerConcurrency)
+	var wg sync.WaitGroup
+	for i, o := range orgs {
+		wg.Add(1)
+		sem <- struct{}{}
+		go func(i int, o iamOrg) {
+			defer wg.Done()
+			defer func() { <-sem }()
+			rows[i], oks[i] = s.revenueOf(ctx, o)
+		}(i, o)
+	}
+	wg.Wait()
+
+	var totalBal, totalSpend, mrr int64
+	paying := 0
+	partial := false
+	for i, r := range rows {
+		totalBal += r.BalanceCents
+		totalSpend += r.SpendCents
+		mrr += r.MRRCents
+		if r.SpendCents > 0 || r.MRRCents > 0 {
+			paying++
+		}
+		if !oks[i] {
+			partial = true
+		}
+	}
+	arpu := int64(0)
+	if paying > 0 {
+		arpu = totalSpend / int64(paying)
+	}
+
+	// Real 30-day spend trend from the usage ledger (honest empty when no usage).
+	acts, ledgerOK := s.fleetActivity(ctx, orgs)
+	trend := spendSeries(acts, now.AddDate(0, 0, -30), now, "day")
+
+	// Highest-revenue customers first.
+	sort.Slice(rows, func(i, j int) bool {
+		if rows[i].SpendCents != rows[j].SpendCents {
+			return rows[i].SpendCents > rows[j].SpendCents
+		}
+		return rows[i].BalanceCents > rows[j].BalanceCents
+	})
+
+	nowStr := now.Format(time.RFC3339)
+	sources := []sourceStatus{srcOf("iam", nil, len(orgs), nowStr)}
+	if partial {
+		sources = append(sources, srcOf("commerce", errPartialRevenue, len(orgs), nowStr))
+	} else {
+		sources = append(sources, srcOf("commerce", nil, len(orgs), nowStr))
+	}
+	if !ledgerOK {
+		sources = append(sources, srcOf("commerce-ledger", errPartialRevenue, 0, nowStr))
+	}
+
+	return ok(c, revenueData{
+		TotalBalancesCents: totalBal,
+		TotalSpendCents:    totalSpend,
+		MRRCents:           mrr,
+		Customers:          len(orgs),
+		PayingCustomers:    paying,
+		ARPUCents:          arpu,
+		PerCustomer:        rows,
+		SpendTrend:         trend,
+		GeneratedAt:        nowStr,
+		Sources:            sources,
+	})
+}
+
+// revenueOf reads one org's money view (balance + spend + plan/MRR). Returns
+// (row, ok): ok is false when the spend OR balance read failed, so the caller can
+// mark the fleet total PARTIAL rather than presenting an undercount as complete.
+func (s *svc) revenueOf(ctx context.Context, o iamOrg) (revenueCustomer, bool) {
+	subj := orgSubject(o.Name)
+	row := revenueCustomer{Org: o.Name, Display: display(o.DisplayName, o.Name), Plan: "pay-as-you-go"}
+	ok := true
+
+	if r, err := s.commerce.usageRollup(ctx, o.Name, subj); err == nil {
+		row.SpendCents = r.ConsumedCents
+	} else {
+		ok = false
+	}
+	if credits, err := s.commerce.creditsCents(ctx, o.Name, subj); err == nil {
+		row.BalanceCents = credits
+	} else {
+		ok = false
+	}
+	if sub, err := s.commerce.subscriptionSummary(ctx, o.Name, subj); err == nil {
+		row.MRRCents = sub.MRR
+		row.Plan = sub.Plan
+	}
+	return row, ok
+}

clients/admin/types.go

@@ -0,0 +1,131 @@
+package admin
+
+// Response shapes for /v1/admin/*. Each mirrors the operator's api.ts contract
+// (admin/apps/operator/src/lib/api.ts) field-for-field — the JSON tags ARE the
+// contract, so the operator's TypeScript types decode these one-to-one.
+
+// adminMe is the operator identity (AdminMe / GET /v1/admin/me).
+type adminMe struct {
+	Owner         string `json:"owner"`
+	Name          string `json:"name"`
+	Email         string `json:"email"`
+	DisplayName   string `json:"displayName"`
+	IsGlobalAdmin bool   `json:"isGlobalAdmin"`
+}
+
+// sourceStatus is the freshness of one upstream the aggregator pulls from
+// (SourceStatus / overview.sources[]).
+type sourceStatus struct {
+	Name  string `json:"name"`
+	OK    bool   `json:"ok"`
+	Rows  int    `json:"rows"`
+	Error string `json:"error"`
+	At    string `json:"at"`
+}
+
+// overviewData is the fleet overview tiles (OverviewData / GET /v1/admin/overview).
+type overviewData struct {
+	Orgs           int            `json:"orgs"`
+	Users          int            `json:"users"`
+	Products       int            `json:"products"`
+	ActiveProducts int            `json:"activeProducts"`
+	Drift          int            `json:"drift"`
+	SpendCents30d  int64          `json:"spendCents30d"`
+	Tokens30d      int64          `json:"tokens30d"`
+	CreditsCents   int64          `json:"creditsCents"`
+	LastSync       string         `json:"lastSync"`
+	Sources        []sourceStatus `json:"sources"`
+}
+
+// orgRow is one tenant row (OrgRow / GET /v1/admin/orgs).
+type orgRow struct {
+	Org          string `json:"org"`
+	Display      string `json:"display"`
+	Users        int    `json:"users"`
+	Products     int    `json:"products"`
+	SpendCents   int64  `json:"spendCents"`
+	CreditsCents int64  `json:"creditsCents"`
+	Tokens       int64  `json:"tokens"`
+	Created      string `json:"created"`
+}
+
+// operatorUser is one user in the cross-org directory (OperatorUser / GET
+// /v1/admin/users).
+type operatorUser struct {
+	Owner         string `json:"owner"`
+	Name          string `json:"name"`
+	Email         string `json:"email"`
+	DisplayName   string `json:"displayName"`
+	IsAdmin       bool   `json:"isAdmin"`
+	IsGlobalAdmin bool   `json:"isGlobalAdmin"`
+	Tag           string `json:"tag"`
+	Created       string `json:"created"`
+	LastSignin    string `json:"lastSignin"`
+	Forbidden     bool   `json:"forbidden"`
+}
+
+// usage roll-up (UsageData / GET /v1/admin/usage).
+type usageTotals struct {
+	SpendCents int64 `json:"spendCents"`
+	Tokens     int64 `json:"tokens"`
+	Requests   int64 `json:"requests"`
+}
+
+type usagePoint struct {
+	Date       string `json:"date"`
+	SpendCents int64  `json:"spendCents"`
+	Tokens     int64  `json:"tokens"`
+	Requests   int64  `json:"requests"`
+}
+
+type usageByProduct struct {
+	Product    string `json:"product"`
+	SpendCents int64  `json:"spendCents"`
+	Tokens     int64  `json:"tokens"`
+}
+
+type usageData struct {
+	Totals    usageTotals      `json:"totals"`
+	Series    []usagePoint     `json:"series"`
+	ByProduct []usageByProduct `json:"byProduct"`
+}
+
+// productRow is one product/workload row (ProductRow / GET /v1/admin/products).
+type productRow struct {
+	Name        string `json:"name"`
+	Kind        string `json:"kind"`
+	Org         string `json:"org"`
+	Cluster     string `json:"cluster"`
+	DeclaredTag string `json:"declaredTag"`
+	RunningTag  string `json:"runningTag"`
+	Health      string `json:"health"`
+	Drift       bool   `json:"drift"`
+	Updated     string `json:"updated"`
+}
+
+// ── IAM wire shapes (the subset admin decodes from get-* payloads) ─────────
+
+// iamOrg is the IAM Organization subset the aggregators fold over.
+type iamOrg struct {
+	Owner       string `json:"owner"`
+	Name        string `json:"name"`
+	DisplayName string `json:"displayName"`
+	CreatedTime string `json:"createdTime"`
+}
+
+// iamUser is the IAM User subset mapped into OperatorUser. AccessKey is decoded
+// ONLY to derive API-key PRESENCE (hasApiKey) for the customer detail — its VALUE
+// is never surfaced in any admin response (the hk- key is a credential, not a
+// display field), so no secret leaves this binary.
+type iamUser struct {
+	Owner          string `json:"owner"`
+	Name           string `json:"name"`
+	Email          string `json:"email"`
+	DisplayName    string `json:"displayName"`
+	Tag            string `json:"tag"`
+	CreatedTime    string `json:"createdTime"`
+	LastSigninTime string `json:"lastSigninTime"`
+	IsAdmin        bool   `json:"isAdmin"`
+	IsForbidden    bool   `json:"isForbidden"`
+	AccessKey      string `json:"accessKey"`
+}

clients/agents/agents.go

@@ -0,0 +1,989 @@
+// Package agents mounts the Hanzo Cloud /v1/agents surface: per-org autonomous
+// agent definitions and their runs. An agent is a model + a system prompt
+// (instructions) + a set of tool names; running one executes a real chat
+// completion through the in-process AI client (the SAME gateway path the rest
+// of the console uses) and records the run. Tenant isolation is the
+// gateway-minted X-Org-Id (HIP-0026) enforced as the org column on every
+// query, so one tenant can never read, run, or delete another's agents.
+//
+// Surface (all org-scoped; console2's AgentsModule reads {agents:[...]}):
+//
+//	GET    /v1/agents               list agents for the org      -> {agents:[...]}
+//	POST   /v1/agents               create an agent              -> Agent
+//	GET    /v1/agents/:ref          agent detail + recent runs   -> AgentDetail
+//	PATCH  /v1/agents/:ref          update an agent              -> Agent
+//	DELETE /v1/agents/:ref          delete an agent (+ its runs)
+//	POST   /v1/agents/:ref/run      run the agent {input}        -> RunResult
+//	GET    /v1/agents/:ref/runs     run history                  -> {runs:[...]}
+//
+// :ref is either the agent's public id (the `agent_...` handle create and list
+// return) OR its org-unique name — resolved by Store.Resolve, so a created agent
+// is immediately gettable and runnable by whatever create/list handed back.
+//
+// The store is SQLite in deps.DataDir (Base/SQLite-only). It holds definitions
+// and run I/O only — never a secret; tool credentials live in KMS by reference.
+package agents
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/hanzoai/cloud"
+	"github.com/hanzoai/cloud/clients/principal"
+	"github.com/hanzoai/cloud/types"
+	"github.com/hanzoai/commerce/metering"
+	luxlog "github.com/luxfi/log"
+	"github.com/zap-proto/zip"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// agentTracer emits the per-run/per-step agent spans (shipped over ZAP to
+// o11y). A run is one root span; each step nests an LLM GenAI client span.
+var agentTracer = otel.Tracer("hanzo.ai/cloud/agents")
+
+// nameRE constrains an agent's org-unique name at the create boundary — the one
+// place a name is written. Path addressing (Store.Resolve, parameterized) accepts
+// the name OR the `agent_...` id, so it needs no separate path validation.
+var nameRE = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$`)
+
+const (
+	maxInstructions = 32 * 1024 // system prompt cap
+	maxInput        = 128 * 1024
+	// maxRef bounds the free-text bot-lifecycle references (compute machine id,
+	// service-account id). They are opaque identifiers, not documents — a
+	// generous 256 keeps a client from bloating the per-org SQLite with a
+	// multi-megabyte "id".
+	maxRef = 256
+
+	// agentFeeEnvPrefix is the operator knob for the flat per-run fee. The
+	// effective fee is cloud.ResourceFeeCents(agentFeeEnvPrefix, meterKind): a
+	// global CLOUD_AGENT_FEE_CENTS override wins over the $1.00 default; set it
+	// to 0 to make agent runs free (and therefore un-gated). This is a per-RUN
+	// fee — the honest, policy-set unit an agent run bills. Token-based pricing
+	// is intentionally NOT used here: the in-process AIClient returns only the
+	// completion content (types.ChatResponse{Content}), no token counts, so
+	// charging per-token would be fabricated. Duration is recorded on the run.
+	agentFeeEnvPrefix = "CLOUD_AGENT_FEE_CENTS"
+	// meterKind is the commerce "provider"/attribution label for agent spend —
+	// the task's product:"agent". One value so every agent run (HTTP or
+	// scheduled) is attributed identically.
+	meterKind = "agent"
+	// schedulerActor is the Actor recorded on a scheduled run that has no IAM
+	// service account bound. Real service-account identity (the keystone) rides
+	// in Agent.ServiceAccountID when present.
+	schedulerActor = "scheduler"
+	// maxLongRunningPerOrg caps an org's scheduler footprint: how many scheduled
+	// long-running agents it may create. Each scheduled agent adds recurring
+	// gate+run+debit load to the shared store, so a per-org bound stops one
+	// tenant from self-amplifying the once-a-minute scan. Overridable by ops via
+	// CLOUD_AGENT_MAX_LONG_RUNNING.
+	maxLongRunningPerOrg = 100
+	longRunningCapEnv    = "CLOUD_AGENT_MAX_LONG_RUNNING"
+)
+
+type svc struct {
+	store *Store
+	ai    types.AIClient
+	log   luxlog.Logger
+	// bill is the shared per-org gate+meter (reuses deps.Metering, the ONE
+	// commerce client — the same object ml/provisioning use). Nil/!Enabled()
+	// makes Gate allow and Meter a no-op, so an unconfigured deployment runs
+	// agents without billing rather than failing closed on a missing ledger.
+	bill *cloud.ResourceMeter
+	// sched is the long-running-agent scheduler; nil until started, stopped on
+	// Shutdown. It shares svc so it runs agents through the SAME runAgent path.
+	sched *scheduler
+	// bus is the in-process fan-out behind the live session/event stream (SSE +
+	// ZAP). Set in Mount; nil-safe (a direct-construct unit test skips fan-out).
+	bus *bus
+	// tasks is the seam to the hanzoai/tasks durable-execution engine that control
+	// commands forward to for task-backed sessions. Defaults to the disabled
+	// controller (record-only) until a live tasks client is wired in Mount.
+	tasks TaskController
+}
+
+var mounted *svc
+
+// ---- HTTP response shapes (the published contract) ----
+
+type agentView struct {
+	ID               string   `json:"id"`
+	Name             string   `json:"name"`
+	Model            string   `json:"model"`
+	Description      string   `json:"description,omitempty"`
+	Tools            []string `json:"tools"`
+	Status           string   `json:"status"`
+	ExecutionMode    string   `json:"executionMode"`
+	Schedule         string   `json:"schedule,omitempty"`
+	ComputeRef       string   `json:"computeRef,omitempty"`
+	ServiceAccountID string   `json:"serviceAccountId,omitempty"`
+	Runs             int      `json:"runs"`
+	CreatedAt        string   `json:"createdAt"`
+	UpdatedAt        string   `json:"updatedAt"`
+}
+
+type agentDetail struct {
+	agentView
+	Instructions string    `json:"instructions"`
+	RecentRuns   []runView `json:"recentRuns"`
+}
+
+type runView struct {
+	ID         string `json:"id"`
+	Status     string `json:"status"`
+	Model      string `json:"model"`
+	Input      string `json:"input"`
+	Output     string `json:"output,omitempty"`
+	Error      string `json:"error,omitempty"`
+	DurationMs int64  `json:"durationMs"`
+	CreatedAt  string `json:"createdAt"`
+}
+
+// ---- overview shapes (console2 Agents dashboard: metrics + activity) ----
+//
+// These mirror the console's normalizers EXACTLY (console2 src/lib/api/agents.ts:
+// normalizeMetrics reads {series:[{key,points:[{t,v}]}], resource:{...}};
+// normalizeActivity reads {activity:[{id,kind,agent,message,at}]}). Every number
+// is derived from real agent_runs rows — never a fabricated trend. A metric this
+// store cannot source (CPU/mem/storage/cost metering) is emitted as JSON null so
+// the shape is honest and the UI renders "—".
+
+type seriesPoint struct {
+	T string `json:"t"` // bucket start, RFC3339 UTC
+	V int    `json:"v"` // real invocation count in the bucket
+}
+
+type seriesLine struct {
+	Key    string        `json:"key"` // agent name
+	Points []seriesPoint `json:"points"`
+}
+
+// resourceUsage is the Resource Usage panel rollup. This store holds agent
+// definitions and run I/O only — it does NOT meter CPU/memory/storage/cost — so
+// every field is nil, marshalling to explicit JSON null (honest "no data", not 0).
+type resourceUsage struct {
+	CPUVcpuHours   *float64 `json:"cpuVcpuHours"`
+	MemGbHours     *float64 `json:"memGbHours"`
+	StorageIoBytes *float64 `json:"storageIoBytes"`
+	CostCents      *float64 `json:"costCents"`
+}
+
+type metricsView struct {
+	Range    string        `json:"range"`  // echoes the requested window (24H|7D|30D)
+	Series   []seriesLine  `json:"series"` // per-agent invocation histogram (real)
+	Resource resourceUsage `json:"resource"`
+}
+
+type activityView struct {
+	ID      string `json:"id"`
+	Kind    string `json:"kind"`  // invoked|failed|created|updated (from real events)
+	Agent   string `json:"agent"` // agent name
+	Message string `json:"message,omitempty"`
+	At      string `json:"at"` // RFC3339 UTC
+}
+
+func rfc3339(unix int64) string {
+	if unix == 0 {
+		return ""
+	}
+	return time.Unix(unix, 0).UTC().Format(time.RFC3339)
+}
+
+func toView(a Agent, runs int) agentView {
+	return agentView{
+		ID: a.ID, Name: a.Name, Model: a.Model, Description: a.Description,
+		Tools: nonNil(a.Tools), Status: a.Status,
+		ExecutionMode: a.ExecutionMode, Schedule: a.Schedule,
+		ComputeRef: a.ComputeRef, ServiceAccountID: a.ServiceAccountID,
+		Runs:      runs,
+		CreatedAt: rfc3339(a.CreatedAt), UpdatedAt: rfc3339(a.UpdatedAt),
+	}
+}
+
+func toRunView(r Run) runView {
+	return runView{
+		ID: r.ID, Status: r.Status, Model: r.Model, Input: r.Input, Output: r.Output,
+		Error: r.Error, DurationMs: r.DurationMs, CreatedAt: rfc3339(r.CreatedAt),
+	}
+}
+
+func nonNil(xs []string) []string {
+	if xs == nil {
+		return []string{}
+	}
+	return xs
+}
+
+// Mount wires the agents surface onto app per HIP-0106.
+func Mount(app *zip.App, deps cloud.Deps) error {
+	if app == nil {
+		return fmt.Errorf("agents.Mount: nil zip.App")
+	}
+	log := deps.Logger
+	if log == nil {
+		return fmt.Errorf("agents.Mount: nil deps.Logger")
+	}
+	log = log.New("subsystem", "agents")
+	if deps.DataDir == "" {
+		return fmt.Errorf("agents.Mount: empty DataDir")
+	}
+	if err := os.MkdirAll(deps.DataDir, 0o755); err != nil {
+		return fmt.Errorf("agents.Mount: data dir: %w", err)
+	}
+	store, err := openStore(filepath.Join(deps.DataDir, "agents.db"))
+	if err != nil {
+		return fmt.Errorf("agents.Mount: open store: %w", err)
+	}
+	// deps.AI may be nil when no gateway is configured; run() degrades honestly.
+	s := &svc{
+		store: store,
+		ai:    deps.AI,
+		log:   log,
+		bill:  cloud.NewResourceMeter(deps, meterKind),
+		bus:   newBus(),
+		// TASKS PLUG-IN POINT: durable execution rides hanzoai/tasks, not a
+		// bespoke engine. Default is record-only; wiring client.Dial(TASKS_URL)
+		// from github.com/hanzoai/tasks/pkg/sdk/client here makes control forward
+		// to the engine's Signal/Cancel API (see sessions_tasks.go).
+		tasks: disabledTaskController{},
+	}
+	mounted = s
+
+	app.Get("/v1/agents", s.list)
+	app.Post("/v1/agents", s.create)
+	// Static org-wide surfaces MUST register before the :ref wildcard: Fiber
+	// matches routes in registration order, so a bare `/v1/agents/:ref` would
+	// otherwise capture "metrics"/"activity"/"sessions" as a ref and 404 them
+	// (Red route audit). Registering the literals first makes them win.
+	app.Get("/v1/agents/metrics", s.metrics)
+	app.Get("/v1/agents/activity", s.activity)
+	// Live agent-session control plane: /v1/agents/sessions[/...]. Registered
+	// before :name for the same registration-order reason (and internally the
+	// static /stream precedes /:id).
+	s.mountSessions(app)
+	app.Get("/v1/agents/:ref", s.get)
+	app.Patch("/v1/agents/:ref", s.update)
+	app.Delete("/v1/agents/:ref", s.del)
+	app.Post("/v1/agents/:ref/run", s.run)
+	app.Get("/v1/agents/:ref/runs", s.runs)
+
+	// Long-running scheduler: invokes each long-running agent's run on its cron
+	// cadence through the SAME runAgent path as the HTTP handler (one run path,
+	// one gate, one meter). Only started when inference is wired — with no AI a
+	// scheduled run could never execute, so there is nothing to schedule.
+	if s.ai != nil {
+		s.sched = newScheduler(s, log)
+		s.sched.start()
+	}
+
+	log.Info("agents mounted", "ai", s.ai != nil, "billing", s.bill.Enabled(),
+		"scheduler", s.sched != nil, "brand", deps.Brand)
+	return nil
+}
+
+func init() {
+	cloud.RegisterWithShutdown("agents", 127, func(app any, deps cloud.Deps) error {
+		a, ok := app.(*zip.App)
+		if !ok {
+			return fmt.Errorf("agents.Mount: app is %T, want *zip.App", app)
+		}
+		return Mount(a, deps)
+	}, func(ctx context.Context) error {
+		// Graceful teardown: stop the scheduler (drain in-flight runs) and close
+		// the store. Bounded by the caller's shutdown deadline so a stuck run
+		// can't hang SIGTERM.
+		return Shutdown(ctx)
+	})
+}
+
+// ---- handlers ----
+
+type createReq struct {
+	Name             string   `json:"name"`
+	Model            string   `json:"model"`
+	Instructions     string   `json:"instructions"`
+	Description      string   `json:"description"`
+	Tools            []string `json:"tools"`
+	ExecutionMode    string   `json:"executionMode"`
+	Schedule         string   `json:"schedule"`
+	ComputeRef       string   `json:"computeRef"`
+	ServiceAccountID string   `json:"serviceAccountId"`
+}
+
+func (s *svc) create(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	var body createReq
+	if err := c.Bind(&body); err != nil {
+		return err
+	}
+	name := strings.TrimSpace(body.Name)
+	if name == "" {
+		return zip.ErrBadRequest("name is required")
+	}
+	if !nameRE.MatchString(name) {
+		return zip.ErrBadRequest("name must match ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
+	}
+	model := strings.TrimSpace(body.Model)
+	if model == "" {
+		return zip.ErrBadRequest("model is required")
+	}
+	if len(body.Instructions) > maxInstructions {
+		return zip.ErrBadRequest("instructions too large")
+	}
+	mode, schedule, err := validateLifecycle(body.ExecutionMode, body.Schedule)
+	if err != nil {
+		return err
+	}
+	computeRef, err := validateRef("computeRef", body.ComputeRef)
+	if err != nil {
+		return err
+	}
+	serviceAccountID, err := validateRef("serviceAccountId", body.ServiceAccountID)
+	if err != nil {
+		return err
+	}
+	// Cap the org's scheduler footprint (Red LOW-1): a tenant cannot create an
+	// unbounded number of scheduled agents that each add recurring load to the
+	// shared store. Only counts when this create is itself long-running.
+	if mode == ModeLongRunning {
+		n, err := s.store.CountLongRunning(c.Context(), org)
+		if err != nil {
+			return zip.Errorf(http.StatusInternalServerError, "count: %v", err)
+		}
+		if n >= longRunningCap() {
+			return zip.Errorf(http.StatusConflict,
+				"long-running agent limit reached for this org (max %d)", longRunningCap())
+		}
+	}
+	id, err := genID("agent")
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "rng: %v", err)
+	}
+	now := time.Now().Unix()
+	a := Agent{
+		ID: id, Org: org, Name: name, Model: model, Instructions: body.Instructions,
+		Description: strings.TrimSpace(body.Description), Tools: cleanList(body.Tools),
+		Status: "ready", ExecutionMode: mode, Schedule: schedule,
+		ComputeRef: computeRef, ServiceAccountID: serviceAccountID,
+		CreatedAt: now, UpdatedAt: now,
+	}
+	if err := s.store.Create(c.Context(), a); err != nil {
+		if err == errConflict {
+			return zip.ErrConflict("agent already exists in this org")
+		}
+		return zip.Errorf(http.StatusInternalServerError, "persist: %v", err)
+	}
+	return c.JSON(http.StatusCreated, toView(a, 0))
+}
+
+func (s *svc) list(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	rows, err := s.store.List(c.Context(), org)
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "list: %v", err)
+	}
+	out := make([]agentView, 0, len(rows))
+	for _, a := range rows {
+		n, err := s.store.CountRuns(c.Context(), org, a.Name)
+		if err != nil {
+			return zip.Errorf(http.StatusInternalServerError, "runs: %v", err)
+		}
+		out = append(out, toView(a, n))
+	}
+	return c.JSON(http.StatusOK, map[string]any{"agents": out})
+}
+
+func (s *svc) get(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	a, err := s.store.Resolve(c.Context(), org, refParam(c))
+	if err == errNotFound {
+		return zip.ErrNotFound("agent not found")
+	}
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "get: %v", err)
+	}
+	runs, err := s.store.ListRuns(c.Context(), org, a.Name, 20)
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "runs: %v", err)
+	}
+	rv := make([]runView, 0, len(runs))
+	for _, r := range runs {
+		rv = append(rv, toRunView(r))
+	}
+	return c.JSON(http.StatusOK, agentDetail{
+		agentView: toView(a, len(runs)), Instructions: a.Instructions, RecentRuns: rv,
+	})
+}
+
+type updateReq struct {
+	Model            *string   `json:"model"`
+	Instructions     *string   `json:"instructions"`
+	Description      *string   `json:"description"`
+	Tools            *[]string `json:"tools"`
+	ExecutionMode    *string   `json:"executionMode"`
+	Schedule         *string   `json:"schedule"`
+	ComputeRef       *string   `json:"computeRef"`
+	ServiceAccountID *string   `json:"serviceAccountId"`
+}
+
+func (s *svc) update(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	a, err := s.store.Resolve(c.Context(), org, refParam(c))
+	if err == errNotFound {
+		return zip.ErrNotFound("agent not found")
+	}
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "get: %v", err)
+	}
+	var body updateReq
+	if err := c.Bind(&body); err != nil {
+		return err
+	}
+	if body.Model != nil {
+		m := strings.TrimSpace(*body.Model)
+		if m == "" {
+			return zip.ErrBadRequest("model cannot be empty")
+		}
+		a.Model = m
+	}
+	if body.Instructions != nil {
+		if len(*body.Instructions) > maxInstructions {
+			return zip.ErrBadRequest("instructions too large")
+		}
+		a.Instructions = *body.Instructions
+	}
+	if body.Description != nil {
+		a.Description = strings.TrimSpace(*body.Description)
+	}
+	if body.Tools != nil {
+		a.Tools = cleanList(*body.Tools)
+	}
+	if body.ComputeRef != nil {
+		if a.ComputeRef, err = validateRef("computeRef", *body.ComputeRef); err != nil {
+			return err
+		}
+	}
+	if body.ServiceAccountID != nil {
+		if a.ServiceAccountID, err = validateRef("serviceAccountId", *body.ServiceAccountID); err != nil {
+			return err
+		}
+	}
+	// Re-validate the lifecycle from the RESULTING mode+schedule so a partial
+	// update can't leave a long-running agent without a valid cron (which the
+	// scheduler would then skip forever). Absent fields keep the stored value.
+	wasLongRunning := a.ExecutionMode == ModeLongRunning
+	mode, schedule := a.ExecutionMode, a.Schedule
+	if body.ExecutionMode != nil {
+		mode = *body.ExecutionMode
+	}
+	if body.Schedule != nil {
+		schedule = *body.Schedule
+	}
+	if a.ExecutionMode, a.Schedule, err = validateLifecycle(mode, schedule); err != nil {
+		return err
+	}
+	// Enforce the per-org scheduler cap on a TRANSITION into long-running, so a
+	// tenant can't sidestep the create-time cap by making N one-shot agents and
+	// PATCHing them to long-running (Red LOW-1 follow-up). Only counts when the
+	// agent was NOT already long-running (a no-op re-save of an existing
+	// long-running agent must not 409 against its own row).
+	if a.ExecutionMode == ModeLongRunning && !wasLongRunning {
+		n, cerr := s.store.CountLongRunning(c.Context(), org)
+		if cerr != nil {
+			return zip.Errorf(http.StatusInternalServerError, "count: %v", cerr)
+		}
+		if n >= longRunningCap() {
+			return zip.Errorf(http.StatusConflict,
+				"long-running agent limit reached for this org (max %d)", longRunningCap())
+		}
+	}
+	a.UpdatedAt = time.Now().Unix()
+	if err := s.store.Update(c.Context(), a); err != nil {
+		if err == errNotFound {
+			return zip.ErrNotFound("agent not found")
+		}
+		return zip.Errorf(http.StatusInternalServerError, "update: %v", err)
+	}
+	n, _ := s.store.CountRuns(c.Context(), org, a.Name)
+	return c.JSON(http.StatusOK, toView(a, n))
+}
+
+func (s *svc) del(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	// Resolve id-or-name first, then delete by the canonical name (agent_runs
+	// cascades on agent_name). Deleting by a raw id would never match the store's
+	// name key and silently 404 a real agent.
+	a, err := s.store.Resolve(c.Context(), org, refParam(c))
+	if err == errNotFound {
+		return zip.ErrNotFound("agent not found")
+	}
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "resolve: %v", err)
+	}
+	deleted, err := s.store.Delete(c.Context(), org, a.Name)
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "delete: %v", err)
+	}
+	if !deleted {
+		return zip.ErrNotFound("agent not found")
+	}
+	return c.NoContent(http.StatusNoContent)
+}
+
+type runReq struct {
+	Input string `json:"input"`
+}
+
+// run executes the agent: it composes the agent's instructions with the caller
+// input and runs a real chat completion via the in-process AI client, then
+// records the run. Every returned run reflects an execution that actually
+// happened — an inference failure is recorded and returned as an error run, not
+// hidden and not fabricated.
+func (s *svc) run(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	// tenant() above already required a VALIDATED principal (principal.Tenant
+	// returns ok only when c.User() — set solely from a JWT SanitizeIdentity
+	// verified — is non-empty), so every path here, run included, is closed to the
+	// no-bearer direct-to-pod forge path. This explicit re-assertion is a local,
+	// money-path invariant: a run MOVES MONEY (debits the org's commerce ledger),
+	// so the debit's principal requirement is stated where the money moves and
+	// never silently depends on tenant()'s internals (Red MEDIUM-2). Same guard the
+	// s3 / provisioning subsystems use.
+	if strings.TrimSpace(c.User()) == "" {
+		return zip.ErrForbidden("a validated principal is required to run an agent")
+	}
+	a, err := s.store.Resolve(c.Context(), org, refParam(c))
+	if err == errNotFound {
+		return zip.ErrNotFound("agent not found")
+	}
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "get: %v", err)
+	}
+	var body runReq
+	if err := c.Bind(&body); err != nil {
+		return err
+	}
+	if len(body.Input) > maxInput {
+		return zip.ErrBadRequest("input too large")
+	}
+	if s.ai == nil {
+		return zip.Errorf(http.StatusServiceUnavailable, "inference is not configured on this deployment")
+	}
+
+	// Pre-authorize the caller's org balance BEFORE any inference (fail-closed).
+	// The actor is the validated principal (org/sub) when present, else the bare
+	// org — recorded on the debit for attribution. Gating here means an unfunded
+	// org gets 402 and NO free inference; an unreachable commerce gets 503.
+	actor := billingActor(org, c.User())
+	r, gateErr := s.runAgent(c.Context(), a, body.Input, actor, c.RequestID(), cloud.ClientIP(c))
+	if gateErr != nil {
+		return cloud.DenyResource(c, gateErr)
+	}
+	if r.Status != "ok" {
+		// The run is recorded; surface the upstream failure honestly.
+		return c.JSON(http.StatusBadGateway, toRunView(r))
+	}
+	return c.JSON(http.StatusOK, toRunView(r))
+}
+
+// runAgent is the ONE run path — shared by the HTTP handler and the scheduler.
+// It (1) pre-authorizes the AGENT's OWN org balance (fail-closed) so no unfunded
+// tenant ever gets free inference, (2) executes one real completion, (3) records
+// the run regardless of outcome (the history is real), and (4) debits the run
+// fee to the agent's org ONLY on success. A non-nil error is a BALANCE-GATE
+// denial (out-of-funds / commerce-unknown) that the caller renders (402/503) —
+// it means no run happened. A run that executed but the model failed returns a
+// recorded error-status Run and a nil error.
+func (s *svc) runAgent(ctx context.Context, a Agent, input, actor, requestID, clientIP string) (Run, error) {
+	// Root span per run — the whole trace (balance gate → step → LLM call)
+	// nests under it, shipped over ZAP to o11y (SigNoz).
+	ctx, span := agentTracer.Start(ctx, "agent.run "+a.Name, trace.WithSpanKind(trace.SpanKindInternal))
+	defer span.End()
+	span.SetAttributes(
+		attribute.String("hanzo.agent.name", a.Name),
+		attribute.String("hanzo.agent.org", a.Org),
+		attribute.String("gen_ai.request.model", a.Model),
+	)
+
+	fee := cloud.ResourceFeeCents(agentFeeEnvPrefix, meterKind)
+	// Gate the AGENT's own org — never a caller default, never another tenant.
+	// fee<=0 or unconfigured billing makes this a no-op (allows).
+	if err := s.bill.Gate(ctx, a.Org, meterKind, fee); err != nil {
+		span.RecordError(err)
+		span.SetStatus(codes.Error, "balance gate denied")
+		return Run{}, err
+	}
+
+	r := executeRun(ctx, s.ai, a.Org, a, input)
+	span.SetAttributes(
+		attribute.String("hanzo.agent.run_id", r.ID),
+		attribute.String("hanzo.agent.run_status", r.Status),
+		attribute.Int64("hanzo.agent.duration_ms", r.DurationMs),
+	)
+	if r.Status == "error" {
+		span.SetStatus(codes.Error, r.Error)
+	}
+	if err := s.store.InsertRun(ctx, r); err != nil {
+		s.log.Warn("record run failed", "org", a.Org, "agent", a.Name, "err", err)
+	}
+
+	// Make the run visible in the live session registry as a ROOT session (the
+	// same registry the @hanzo/dev outer-agent + subagent flows use). Best-effort:
+	// it NEVER fails the run — the run and its billing already happened. DRY: this
+	// is the ONE run path (HTTP + scheduler), so every run becomes a session here.
+	s.openRunSession(ctx, a, r, actor)
+
+	// Bill only a successful run (mirrors the edge gate: failed work is not
+	// charged). Rich attribution: product=agent (Provider), the agent's model,
+	// and the actor for the audit trail. Fire-and-forget on a background context.
+	if r.Status == "ok" {
+		s.bill.MeterUsage(a.Org, meterKind, metering.Usage{
+			AmountCents: fee,
+			Model:       a.Model,
+			Actor:       actor,
+			RequestID:   requestID,
+			ClientIP:    clientIP,
+		})
+	}
+	return r, nil
+}
+
+// executeRun composes the agent's instructions with the caller input, runs one
+// real chat completion through the AI client, and returns the resulting Run —
+// status "ok" with output, or "error" with the upstream failure. Pure of HTTP
+// and persistence so it is directly testable; the caller records + responds.
+func executeRun(ctx context.Context, ai types.AIClient, org string, a Agent, input string) Run {
+	// Child step span; the AI client opens its own GenAI span nested under this.
+	ctx, span := agentTracer.Start(ctx, "agent.step", trace.WithSpanKind(trace.SpanKindInternal))
+	defer span.End()
+	span.SetAttributes(attribute.String("gen_ai.request.model", a.Model))
+
+	prompt := a.Instructions
+	if in := strings.TrimSpace(input); in != "" {
+		if prompt != "" {
+			prompt += "\n\n"
+		}
+		prompt += in
+	}
+	start := time.Now()
+	resp, aiErr := ai.ChatCompletion(ctx, &types.ChatRequest{Model: a.Model, Prompt: prompt})
+	dur := time.Since(start).Milliseconds()
+	id, _ := genID("run")
+	r := Run{
+		ID: id, Org: org, AgentName: a.Name, Model: a.Model, Input: input,
+		DurationMs: dur, CreatedAt: time.Now().Unix(),
+	}
+	if aiErr != nil {
+		span.RecordError(aiErr)
+		span.SetStatus(codes.Error, "agent step failed")
+		r.Status = "error"
+		r.Error = aiErr.Error()
+	} else {
+		r.Status = "ok"
+		if resp != nil {
+			r.Output = resp.Content
+		}
+	}
+	return r
+}
+
+func (s *svc) runs(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	a, err := s.store.Resolve(c.Context(), org, refParam(c))
+	if err == errNotFound {
+		return zip.ErrNotFound("agent not found")
+	}
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "resolve: %v", err)
+	}
+	limit := 50
+	if q := strings.TrimSpace(c.Query("limit")); q != "" {
+		if n, err := strconv.Atoi(q); err == nil {
+			limit = n
+		}
+	}
+	runs, err := s.store.ListRuns(c.Context(), org, a.Name, limit)
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "runs: %v", err)
+	}
+	out := make([]runView, 0, len(runs))
+	for _, r := range runs {
+		out = append(out, toRunView(r))
+	}
+	return c.JSON(http.StatusOK, map[string]any{"runs": out})
+}
+
+// metrics serves the invocations-over-time histogram for the org's Agents
+// dashboard. Every point is a REAL count of recorded runs in that time bucket —
+// one series line per agent that ran in the window. The Resource Usage rollup is
+// all-null because this store meters no CPU/memory/storage/cost; the console
+// renders those as "—" rather than a fabricated figure. No runs => empty series
+// (an honest "not connected / no activity yet"), never a synthesized trend.
+func (s *svc) metrics(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	rng, buckets, step := metricsWindow(c.Query("range"))
+	now := time.Now()
+	start := now.Add(-time.Duration(buckets) * step) // last bucket ends at now
+	runs, err := s.store.RunsSince(c.Context(), org, start.Unix(), 10000)
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "metrics: %v", err)
+	}
+	// Bucket real runs per agent. counts[agent][i] = invocations in bucket i.
+	counts := map[string][]int{}
+	var order []string
+	for _, r := range runs {
+		idx := int(time.Unix(r.CreatedAt, 0).Sub(start) / step)
+		if idx < 0 {
+			idx = 0
+		}
+		if idx >= buckets {
+			idx = buckets - 1
+		}
+		if _, seen := counts[r.AgentName]; !seen {
+			counts[r.AgentName] = make([]int, buckets)
+			order = append(order, r.AgentName)
+		}
+		counts[r.AgentName][idx]++
+	}
+	sort.Strings(order) // deterministic series order
+	series := make([]seriesLine, 0, len(order))
+	for _, name := range order {
+		pts := make([]seriesPoint, buckets)
+		for i := 0; i < buckets; i++ {
+			pts[i] = seriesPoint{
+				T: start.Add(time.Duration(i) * step).UTC().Format(time.RFC3339),
+				V: counts[name][i],
+			}
+		}
+		series = append(series, seriesLine{Key: name, Points: pts})
+	}
+	return c.JSON(http.StatusOK, metricsView{Range: rng, Series: series, Resource: resourceUsage{}})
+}
+
+// metricsWindow maps a console range token to (canonical token, bucket count,
+// bucket width). Unknown/empty defaults to 30D. Each range yields >=4 buckets so
+// the console's trendPct has real halves to compare.
+func metricsWindow(raw string) (rng string, buckets int, step time.Duration) {
+	switch strings.ToUpper(strings.TrimSpace(raw)) {
+	case "24H":
+		return "24H", 24, time.Hour
+	case "7D":
+		return "7D", 7, 24 * time.Hour
+	default:
+		return "30D", 30, 24 * time.Hour
+	}
+}
+
+// activity serves the org-wide recent-activity feed. Events are REAL: each
+// recorded run is an invoked (ok) or failed (error) event; each agent's own
+// create/update timestamps are created/updated events. Merged, newest first,
+// capped. Nothing is invented — an org with no agents and no runs gets [].
+func (s *svc) activity(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	const limit = 50
+	runs, err := s.store.RunsSince(c.Context(), org, 0, 200)
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "activity runs: %v", err)
+	}
+	rows, err := s.store.List(c.Context(), org)
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "activity agents: %v", err)
+	}
+	evs := make([]activityView, 0, len(runs)+2*len(rows))
+	for _, r := range runs {
+		kind, msg := "invoked", "Invoked "+r.Model
+		if r.Status == "error" {
+			kind, msg = "failed", trimMsg(r.Error)
+		}
+		evs = append(evs, activityView{ID: r.ID, Kind: kind, Agent: r.AgentName, Message: msg, At: rfc3339(r.CreatedAt)})
+	}
+	for _, a := range rows {
+		evs = append(evs, activityView{ID: a.ID + ":created", Kind: "created", Agent: a.Name, Message: "Agent created", At: rfc3339(a.CreatedAt)})
+		if a.UpdatedAt > a.CreatedAt {
+			evs = append(evs, activityView{ID: a.ID + ":updated", Kind: "updated", Agent: a.Name, Message: "Configuration updated", At: rfc3339(a.UpdatedAt)})
+		}
+	}
+	// Newest first. rfc3339 is UTC ("Z"), so lexical order == chronological.
+	sort.SliceStable(evs, func(i, j int) bool { return evs[i].At > evs[j].At })
+	if len(evs) > limit {
+		evs = evs[:limit]
+	}
+	return c.JSON(http.StatusOK, map[string]any{"activity": evs})
+}
+
+// trimMsg bounds an error string for the activity feed without hiding it.
+func trimMsg(s string) string {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return "Run failed"
+	}
+	if len(s) > 200 {
+		return s[:200]
+	}
+	return s
+}
+
+// ---- helpers ----
+
+// refParam is the URL path segment addressing an agent: its public id or its
+// org-unique name. Store.Resolve accepts either — see the package doc.
+func refParam(c *zip.Ctx) string { return strings.TrimSpace(c.Param("ref")) }
+
+// tenant resolves the org — the tenant isolation KEY. It uses c.Org() EXACTLY
+// as SanitizeIdentity minted it from the validated IAM owner claim (HIP-0026):
+// never lowercased/stripped/truncated. Normalizing would collapse distinct
+// owners into one bucket (Red HIGH-1). Reject only empty or pathologically
+// long. No magic "admin" bucket — a global admin operating on per-org data
+// carries an explicit org, so an empty org is a true 403.
+func tenant(c *zip.Ctx) (string, bool) { return principal.Tenant(c) }
+
+// validateLifecycle normalizes and validates the execution mode + schedule.
+// Empty mode defaults to one-shot. A long-running agent MUST carry a schedule
+// that parses as a 5-field cron (else the scheduler would silently never fire
+// it); a one-shot agent's schedule is cleared (it is meaningless without the
+// scheduler). Returns the normalized (mode, schedule) or a 400.
+func validateLifecycle(mode, schedule string) (string, string, error) {
+	mode = strings.TrimSpace(mode)
+	if mode == "" {
+		mode = ModeOneShot
+	}
+	schedule = strings.TrimSpace(schedule)
+	switch mode {
+	case ModeOneShot:
+		return ModeOneShot, "", nil // schedule is meaningless one-shot; drop it.
+	case ModeLongRunning:
+		if schedule == "" {
+			return "", "", zip.ErrBadRequest("a long-running agent requires a 'schedule' (5-field cron)")
+		}
+		if _, err := parseCron(schedule); err != nil {
+			return "", "", zip.ErrBadRequest("invalid 'schedule': " + err.Error())
+		}
+		return ModeLongRunning, schedule, nil
+	default:
+		return "", "", zip.ErrBadRequest("executionMode must be 'one-shot' or 'long-running'")
+	}
+}
+
+// longRunningCap resolves the per-org scheduled-agent limit from the operator
+// env override, falling back to the default. A non-positive/invalid override is
+// ignored so a typo can never remove the cap.
+func longRunningCap() int {
+	if v := strings.TrimSpace(os.Getenv(longRunningCapEnv)); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			return n
+		}
+	}
+	return maxLongRunningPerOrg
+}
+
+// validateRef bounds an opaque lifecycle reference (compute id / service-account
+// id). Returns the trimmed value or a 400 when it exceeds maxRef.
+func validateRef(field, v string) (string, error) {
+	v = strings.TrimSpace(v)
+	if len(v) > maxRef {
+		return "", zip.ErrBadRequest(field + " too long")
+	}
+	return v, nil
+}
+
+// billingActor is the "org/sub" identity recorded on a debit for the audit
+// trail. It never selects which balance is gated — that is always the org — but
+// attributes the spend to a principal. Falls back to the bare org when no
+// validated user subject is present (e.g. a service-token caller).
+func billingActor(org, sub string) string {
+	sub = strings.TrimSpace(sub)
+	if org != "" && sub != "" {
+		return org + "/" + sub
+	}
+	if sub != "" {
+		return sub
+	}
+	return org
+}
+
+func cleanList(xs []string) []string {
+	seen := map[string]bool{}
+	var out []string
+	for _, x := range xs {
+		x = strings.TrimSpace(x)
+		if x == "" || len(x) > 128 || seen[x] {
+			continue
+		}
+		seen[x] = true
+		out = append(out, x)
+		if len(out) >= 64 {
+			break
+		}
+	}
+	return out
+}
+
+func genID(prefix string) (string, error) {
+	var b [16]byte
+	if _, err := rand.Read(b[:]); err != nil {
+		return "", err
+	}
+	return prefix + "_" + hex.EncodeToString(b[:]), nil
+}
+
+// Shutdown stops the scheduler (draining in-flight runs, bounded by ctx) and
+// closes the agents store. Idempotent — safe to call when nothing is mounted.
+func Shutdown(ctx context.Context) error {
+	if mounted == nil {
+		return nil
+	}
+	if mounted.sched != nil {
+		mounted.sched.stop(ctx)
+	}
+	// Close the live-stream bus so every open SSE/ZAP subscriber's loop returns
+	// and its handler unblocks within the shutdown deadline.
+	if mounted.bus != nil {
+		mounted.bus.close()
+	}
+	var err error
+	if mounted.store != nil {
+		err = mounted.store.Close()
+	}
+	mounted = nil
+	return err
+}

clients/agents/agents_test.go

@@ -0,0 +1,187 @@
+package agents
+
+import (
+	"context"
+	"errors"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/hanzoai/cloud/types"
+)
+
+func testStore(t *testing.T) *Store {
+	t.Helper()
+	s, err := openStore(filepath.Join(t.TempDir(), "agents.db"))
+	if err != nil {
+		t.Fatalf("openStore: %v", err)
+	}
+	t.Cleanup(func() { _ = s.Close() })
+	return s
+}
+
+func mk(org, name string) Agent {
+	now := time.Now().Unix()
+	return Agent{
+		ID: org + "-" + name + "-id", Org: org, Name: name, Model: "gpt-4o-mini",
+		Instructions: "You are " + name, Description: "d", Tools: []string{"http"},
+		Status: "ready", CreatedAt: now, UpdatedAt: now,
+	}
+}
+
+// fakeAI is a deterministic AIClient for exercising executeRun without a real
+// gateway — proves the compose + record contract, not the model.
+type fakeAI struct {
+	gotModel  string
+	gotPrompt string
+	content   string
+	err       error
+}
+
+func (f *fakeAI) ChatCompletion(_ context.Context, req *types.ChatRequest) (*types.ChatResponse, error) {
+	f.gotModel = req.Model
+	f.gotPrompt = req.Prompt
+	if f.err != nil {
+		return nil, f.err
+	}
+	return &types.ChatResponse{Content: f.content}, nil
+}
+
+func TestCreateGetListDelete(t *testing.T) {
+	s := testStore(t)
+	ctx := context.Background()
+	if err := s.Create(ctx, mk("maxpower", "helper")); err != nil {
+		t.Fatalf("create: %v", err)
+	}
+	if err := s.Create(ctx, mk("maxpower", "helper")); err != errConflict {
+		t.Fatalf("duplicate create should conflict, got %v", err)
+	}
+	a, err := s.Get(ctx, "maxpower", "helper")
+	if err != nil || a.Model != "gpt-4o-mini" {
+		t.Fatalf("get: %v model=%q", err, a.Model)
+	}
+	list, _ := s.List(ctx, "maxpower")
+	if len(list) != 1 {
+		t.Fatalf("want 1 agent, got %d", len(list))
+	}
+	deleted, err := s.Delete(ctx, "maxpower", "helper")
+	if err != nil || !deleted {
+		t.Fatalf("delete: %v deleted=%v", err, deleted)
+	}
+	if _, err := s.Get(ctx, "maxpower", "helper"); err != errNotFound {
+		t.Fatalf("want notfound after delete, got %v", err)
+	}
+}
+
+// TestResolveByIdOrName proves the id/name split that made a created agent
+// un-gettable is closed: Store.Resolve returns the SAME agent whether addressed
+// by its public id (the handle create/list return) or its org-unique name, and
+// stays fail-closed for another tenant's ref and unknown refs.
+func TestResolveByIdOrName(t *testing.T) {
+	s := testStore(t)
+	ctx := context.Background()
+	a := mk("maxpower", "helper") // ID = "maxpower-helper-id", Name = "helper"
+	if err := s.Create(ctx, a); err != nil {
+		t.Fatalf("create: %v", err)
+	}
+
+	byID, err := s.Resolve(ctx, "maxpower", a.ID)
+	if err != nil || byID.Name != "helper" || byID.ID != a.ID {
+		t.Fatalf("resolve by id must return the agent, got %+v err=%v", byID, err)
+	}
+	byName, err := s.Resolve(ctx, "maxpower", "helper")
+	if err != nil || byName.ID != a.ID {
+		t.Fatalf("resolve by name must return the SAME agent, got %+v err=%v", byName, err)
+	}
+	if byID.ID != byName.ID {
+		t.Fatalf("id and name must resolve to the same row: %q vs %q", byID.ID, byName.ID)
+	}
+
+	// Cross-org: maxpower's id/name must be invisible to acme (fail-closed).
+	if _, err := s.Resolve(ctx, "acme", a.ID); err != errNotFound {
+		t.Fatalf("cross-org resolve by id must be errNotFound, got %v", err)
+	}
+	if _, err := s.Resolve(ctx, "acme", "helper"); err != errNotFound {
+		t.Fatalf("cross-org resolve by name must be errNotFound, got %v", err)
+	}
+	// Unknown ref.
+	if _, err := s.Resolve(ctx, "maxpower", "agent_deadbeef"); err != errNotFound {
+		t.Fatalf("unknown ref must be errNotFound, got %v", err)
+	}
+}
+
+// TestTenantIsolation: one org cannot read, run-log, or delete another's agents.
+func TestTenantIsolation(t *testing.T) {
+	s := testStore(t)
+	ctx := context.Background()
+	if err := s.Create(ctx, mk("maxpower", "shared")); err != nil {
+		t.Fatalf("seed maxpower: %v", err)
+	}
+	if err := s.Create(ctx, mk("acme", "shared")); err != nil {
+		t.Fatalf("seed acme: %v", err)
+	}
+	// Record a run for maxpower's agent only.
+	if err := s.InsertRun(ctx, Run{ID: "r1", Org: "maxpower", AgentName: "shared", Status: "ok", CreatedAt: time.Now().Unix()}); err != nil {
+		t.Fatalf("insert run: %v", err)
+	}
+
+	mpRuns, _ := s.ListRuns(ctx, "maxpower", "shared", 50)
+	if len(mpRuns) != 1 {
+		t.Fatalf("maxpower should have 1 run, got %d", len(mpRuns))
+	}
+	acRuns, _ := s.ListRuns(ctx, "acme", "shared", 50)
+	if len(acRuns) != 0 {
+		t.Fatalf("acme must NOT see maxpower's runs, got %d", len(acRuns))
+	}
+	if n, _ := s.CountRuns(ctx, "acme", "shared"); n != 0 {
+		t.Fatalf("acme run count must be 0, got %d", n)
+	}
+
+	// acme deleting "shared" must not remove maxpower's agent or its run log.
+	if _, err := s.Delete(ctx, "acme", "shared"); err != nil {
+		t.Fatalf("acme delete own: %v", err)
+	}
+	if _, err := s.Get(ctx, "maxpower", "shared"); err != nil {
+		t.Fatalf("maxpower agent must survive acme delete: %v", err)
+	}
+	if n, _ := s.CountRuns(ctx, "maxpower", "shared"); n != 1 {
+		t.Fatalf("maxpower run log must survive acme delete, got %d", n)
+	}
+}
+
+func TestExecuteRunOK(t *testing.T) {
+	ai := &fakeAI{content: "hi there"}
+	a := mk("maxpower", "greeter")
+	a.Instructions = "You are a greeter."
+	r := executeRun(context.Background(), ai, "maxpower", a, "say hi")
+
+	if r.Status != "ok" {
+		t.Fatalf("want ok, got %q err=%q", r.Status, r.Error)
+	}
+	if r.Output != "hi there" {
+		t.Fatalf("output should be the model content, got %q", r.Output)
+	}
+	if ai.gotModel != "gpt-4o-mini" {
+		t.Fatalf("run must use the agent's model, got %q", ai.gotModel)
+	}
+	if ai.gotPrompt != "You are a greeter.\n\nsay hi" {
+		t.Fatalf("prompt must compose instructions + input, got %q", ai.gotPrompt)
+	}
+	if r.Org != "maxpower" || r.AgentName != "greeter" {
+		t.Fatalf("run must be scoped to the org+agent, got %+v", r)
+	}
+}
+
+func TestExecuteRunRecordsError(t *testing.T) {
+	ai := &fakeAI{err: errors.New("model unavailable")}
+	r := executeRun(context.Background(), ai, "maxpower", mk("maxpower", "x"), "in")
+	if r.Status != "error" {
+		t.Fatalf("want error status, got %q", r.Status)
+	}
+	if r.Error != "model unavailable" {
+		t.Fatalf("error must be recorded honestly, got %q", r.Error)
+	}
+	if r.Output != "" {
+		t.Fatalf("failed run must not fabricate output, got %q", r.Output)
+	}
+}

clients/agents/billing_test.go

@@ -0,0 +1,281 @@
+package agents
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/hanzoai/cloud"
+	"github.com/hanzoai/cloud/types"
+	"github.com/hanzoai/commerce/metering"
+	luxlog "github.com/luxfi/log"
+	"github.com/zap-proto/zip"
+)
+
+// errTest is the model-failure the "failed run is not billed" case injects.
+var errTest = errors.New("model unavailable")
+
+// billServer is a minimal commerce double: it returns a fixed balance and
+// records the X-Org-Id header (the tenant the debit lands on) + the usage body
+// of every debit. X-Org-Id is the header commerce's service-token auth reads
+// (metering >= v0.1.2), so a wrong tenant here would prove a cross-tenant leak.
+type billServer struct {
+	available int64
+
+	mu        sync.Mutex
+	usageOrg  string
+	usageBody []byte
+	usages    int32
+	balances  int32
+}
+
+func (b *billServer) start(t *testing.T) string {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc("/v1/billing/balance", func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&b.balances, 1)
+		_ = json.NewEncoder(w).Encode(map[string]any{"available": b.available})
+	})
+	mux.HandleFunc("/v1/billing/usage", func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&b.usages, 1)
+		body, _ := io.ReadAll(r.Body)
+		b.mu.Lock()
+		b.usageOrg, b.usageBody = r.Header.Get("X-Org-Id"), body
+		b.mu.Unlock()
+		w.WriteHeader(http.StatusOK)
+		_, _ = io.WriteString(w, `{"transactionId":"tx_1","type":"usage"}`)
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv.URL
+}
+
+func (b *billServer) debits() int32 { return atomic.LoadInt32(&b.usages) }
+func (b *billServer) lastDebit() (string, []byte) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.usageOrg, b.usageBody
+}
+
+// waitForDebit polls a condition briefly — debits are recorded on a detached
+// goroutine, so the assertion must wait for the async write.
+func waitForDebit(cond func() bool) bool {
+	for i := 0; i < 200; i++ {
+		if cond() {
+			return true
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	return cond()
+}
+
+// mountBilled mounts the agents surface with a REAL metering client pointed at
+// the fake commerce (default org "hanzo", so every "acme is billed" assertion
+// proves the per-call org override scopes the ledger to the CALLER). No
+// scheduler is started here (deps.AI is set, but these tests exercise the HTTP
+// run path; scheduler tests drive tick() directly).
+func mountBilled(t *testing.T, commerceURL string, ai types.AIClient) *zip.App {
+	t.Helper()
+	m, err := metering.New(metering.Config{BaseURL: commerceURL, Token: "svc-tok", Org: "hanzo"})
+	if err != nil {
+		t.Fatalf("metering.New: %v", err)
+	}
+	app := zip.New(zip.Config{Logger: luxlog.New("test")})
+	deps := cloud.Deps{Logger: luxlog.New("test"), DataDir: t.TempDir(), AI: ai, Metering: m}
+	if err := Mount(app, deps); err != nil {
+		t.Fatalf("Mount: %v", err)
+	}
+	t.Cleanup(func() { _ = Shutdown(context.Background()) })
+	return app
+}
+
+// TestRunGatesUnfundedOrg: a run for an org with a non-positive balance is
+// refused 402 and NO usage is recorded and (fail-closed) no inference output is
+// returned — an unfunded tenant gets no free agent run.
+func TestRunGatesUnfundedOrg(t *testing.T) {
+	bs := &billServer{available: 0}
+	app := mountBilled(t, bs.start(t), &fakeAI{content: "should not run"})
+
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "a", "model": "gpt-4o-mini", "instructions": "x"}); code != http.StatusCreated {
+		t.Fatalf("create want 201, got %d", code)
+	}
+	code, body := do(t, app, http.MethodPost, "/v1/agents/a/run", "acme", map[string]any{"input": "hi"})
+	if code != http.StatusPaymentRequired {
+		t.Fatalf("unfunded run want 402, got %d (%s)", code, body)
+	}
+	if bs.debits() != 0 {
+		t.Fatalf("a refused run must not debit, got %d", bs.debits())
+	}
+}
+
+// TestRunGatesUnderfundedOrg: an org with a POSITIVE balance that is still less
+// than the run fee is refused 402 — the gate enforces available >= fee, not
+// merely available > 0, so a 1-cent balance can't authorize a $1 run and take
+// the ledger negative (Red MEDIUM-1). Default fee is $1.00 (100c).
+func TestRunGatesUnderfundedOrg(t *testing.T) {
+	bs := &billServer{available: 1} // 1 cent, fee is 100 cents
+	app := mountBilled(t, bs.start(t), &fakeAI{content: "should not run"})
+
+	do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "a", "model": "m", "instructions": "x"})
+	code, body := do(t, app, http.MethodPost, "/v1/agents/a/run", "acme", map[string]any{"input": "hi"})
+	if code != http.StatusPaymentRequired {
+		t.Fatalf("underfunded (1c < 100c fee) run want 402, got %d (%s)", code, body)
+	}
+	if bs.debits() != 0 {
+		t.Fatalf("a gate-refused run must not debit, got %d", bs.debits())
+	}
+}
+
+// TestRunDebitsCallerOrg: a funded run returns the output AND debits the CALLER
+// org (acme, never the client default 'hanzo'), with product=agent + the agent's
+// model on the usage transaction.
+func TestRunDebitsCallerOrg(t *testing.T) {
+	bs := &billServer{available: 100000}
+	app := mountBilled(t, bs.start(t), &fakeAI{content: "the answer"})
+
+	do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "a", "model": "gpt-4o-mini", "instructions": "x"})
+	code, body := do(t, app, http.MethodPost, "/v1/agents/a/run", "acme", map[string]any{"input": "hi"})
+	if code != http.StatusOK {
+		t.Fatalf("funded run want 200, got %d (%s)", code, body)
+	}
+	if !waitForDebit(func() bool { return bs.debits() == 1 }) {
+		t.Fatalf("a successful run must debit once, got %d", bs.debits())
+	}
+	org, ubody := bs.lastDebit()
+	if org != "acme" {
+		t.Fatalf("debited org %q, want caller %q (never default 'hanzo')", org, "acme")
+	}
+	var u struct {
+		User     string `json:"user"`
+		Amount   int64  `json:"amount"`
+		Model    string `json:"model"`
+		Provider string `json:"provider"`
+		Actor    string `json:"actor"`
+	}
+	_ = json.Unmarshal(ubody, &u)
+	if u.User != "acme" {
+		t.Fatalf("debit user = %q, want caller org %q", u.User, "acme")
+	}
+	if u.Amount != cloud.DefaultResourceFeeCents {
+		t.Fatalf("debit amount = %d, want default fee %d", u.Amount, cloud.DefaultResourceFeeCents)
+	}
+	if u.Provider != meterKind {
+		t.Fatalf("debit provider = %q, want %q (product:agent)", u.Provider, meterKind)
+	}
+	if u.Model != "gpt-4o-mini" {
+		t.Fatalf("debit model = %q, want the agent's model", u.Model)
+	}
+	if u.Actor == "" {
+		t.Fatalf("debit must carry an actor for the audit trail")
+	}
+}
+
+// TestRunByReturnedIDMetersOnce: running an agent addressed by the id create
+// returned debits the caller org EXACTLY ONCE with product=agent — the run path
+// meters identically whether the agent is addressed by id or by name.
+func TestRunByReturnedIDMetersOnce(t *testing.T) {
+	bs := &billServer{available: 100000}
+	app := mountBilled(t, bs.start(t), &fakeAI{content: "the answer"})
+
+	_, body := do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "a", "model": "gpt-4o-mini", "instructions": "x"})
+	var created agentView
+	if err := json.Unmarshal(body, &created); err != nil || created.ID == "" {
+		t.Fatalf("create must return an id, got %s (err %v)", body, err)
+	}
+
+	code, rbody := do(t, app, http.MethodPost, "/v1/agents/"+created.ID+"/run", "acme", map[string]any{"input": "hi"})
+	if code != http.StatusOK {
+		t.Fatalf("run by returned id want 200, got %d (%s)", code, rbody)
+	}
+	if !waitForDebit(func() bool { return bs.debits() == 1 }) {
+		t.Fatalf("a run by id must debit exactly once, got %d", bs.debits())
+	}
+	org, ubody := bs.lastDebit()
+	if org != "acme" {
+		t.Fatalf("debited org %q, want caller %q", org, "acme")
+	}
+	var u struct {
+		Provider string `json:"provider"`
+		Model    string `json:"model"`
+	}
+	_ = json.Unmarshal(ubody, &u)
+	if u.Provider != meterKind || u.Model != "gpt-4o-mini" {
+		t.Fatalf("debit must be product=agent for the agent's model, got provider=%q model=%q", u.Provider, u.Model)
+	}
+}
+
+// TestFailedRunNotBilled: when the model errors, the run is recorded as an error
+// but NOT billed — failed work is never charged (mirrors the edge gate).
+func TestFailedRunNotBilled(t *testing.T) {
+	bs := &billServer{available: 100000}
+	app := mountBilled(t, bs.start(t), &fakeAI{err: errTest})
+
+	do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "a", "model": "m", "instructions": "x"})
+	code, _ := do(t, app, http.MethodPost, "/v1/agents/a/run", "acme", map[string]any{"input": "hi"})
+	if code != http.StatusBadGateway {
+		t.Fatalf("errored run want 502, got %d", code)
+	}
+	// Give any (erroneous) async debit a chance to land, then assert none did.
+	if waitForDebit(func() bool { return bs.debits() > 0 }) {
+		t.Fatalf("a failed run must NOT be billed, got %d debits", bs.debits())
+	}
+}
+
+// TestRunRequiresValidatedPrincipal: a run with only a client X-Org-Id (no
+// validated X-User-Id — the direct-to-pod no-bearer path) is refused 403 and
+// NEVER debits. A money-moving action can't ride an unauthenticated, forgeable
+// org header (Red MEDIUM-2). Read/create still work on the org header alone.
+func TestRunRequiresValidatedPrincipal(t *testing.T) {
+	bs := &billServer{available: 100000}
+	app := mountBilled(t, bs.start(t), &fakeAI{content: "must not run"})
+
+	// create is allowed with X-User-Id (via do()).
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "a", "model": "m", "instructions": "x"}); code != http.StatusCreated {
+		t.Fatalf("create want 201, got %d", code)
+	}
+	// A raw run request carrying ONLY X-Org-Id (no X-User-Id) must be 403.
+	req := httptest.NewRequest(http.MethodPost, "/v1/agents/a/run", nil)
+	req.Header.Set("X-Org-Id", "acme") // forged/unvalidated org, no principal
+	resp, err := app.Fiber().Test(req)
+	if err != nil {
+		t.Fatalf("Test: %v", err)
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode != http.StatusForbidden {
+		t.Fatalf("run without a validated principal want 403, got %d", resp.StatusCode)
+	}
+	if waitForDebit(func() bool { return bs.debits() > 0 }) {
+		t.Fatalf("an unauthenticated run must never debit, got %d", bs.debits())
+	}
+}
+
+// TestRunAgentGateFailClosedOnUnreachableCommerce: when commerce cannot be
+// reached, the gate denies (fail-closed) and no run executes — runAgent returns
+// the gate error and the fake AI is never called.
+func TestRunAgentGateFailClosedOnUnreachableCommerce(t *testing.T) {
+	// Point at a dead URL so Authorize errors (unknown balance -> fail-closed).
+	m, _ := metering.New(metering.Config{BaseURL: "http://127.0.0.1:1", Token: "t", Org: "hanzo", Timeout: 200 * time.Millisecond})
+	ai := &fakeAI{content: "must not run"}
+	s := &svc{store: testStore(t), ai: ai, log: luxlog.New("test"), bill: cloud.NewResourceMeter(cloud.Deps{Metering: m, Logger: luxlog.New("test")}, meterKind)}
+	a := mk("acme", "x")
+	_, gateErr := s.runAgent(context.Background(), a, "hi", "acme", "", "")
+	if gateErr == nil {
+		t.Fatal("unreachable commerce must fail closed (non-nil gate error)")
+	}
+	if ai.gotPrompt != "" {
+		t.Fatalf("no inference must run when the gate denies, got prompt %q", ai.gotPrompt)
+	}
+}

clients/agents/cron.go

@@ -0,0 +1,150 @@
+package agents
+
+// A minimal, dependency-free 5-field cron matcher — the ONE schedule grammar
+// long-running agents use. It is deliberately tiny (no seconds field, no
+// @-macros, no timezones beyond UTC) because the scheduler ticks once a minute
+// and only needs to answer "does this expression fire at this minute?".
+//
+// Grammar (standard 5-field, all times UTC):
+//
+//	minute hour day-of-month month day-of-week
+//	 0-59   0-23    1-31      1-12     0-6 (Sun=0)
+//
+// Each field is a comma list of terms; a term is "*", a number, a range "a-b",
+// or a step "*/n" or "a-b/n". Day-of-month and day-of-week combine with OR when
+// BOTH are restricted (Vixie-cron semantics), else AND — matching what operators
+// expect from "0 9 * * 1" (09:00 on Mondays).
+//
+// Hand-rolled instead of pulling a cron module: it is ~one screen, fully
+// unit-tested, and keeps the dependency surface minimal (no new module in a
+// binary that mounts every subsystem).
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// schedule is a parsed cron expression: one bitset per field (bit i set => the
+// field matches value i). domRestricted/dowRestricted record whether the
+// day-of-month / day-of-week field was anything other than "*", which selects
+// the OR-vs-AND combination rule.
+type schedule struct {
+	min, hour, dom, mon, dow uint64
+	domRestricted            bool
+	dowRestricted            bool
+}
+
+// fieldRange bounds each cron field (inclusive).
+type fieldRange struct{ min, max int }
+
+var cronRanges = [5]fieldRange{
+	{0, 59}, // minute
+	{0, 23}, // hour
+	{1, 31}, // day of month
+	{1, 12}, // month
+	{0, 6},  // day of week (Sunday=0)
+}
+
+// parseCron parses a 5-field cron expression or returns an error describing the
+// first malformed field. Whitespace between fields is collapsed.
+func parseCron(expr string) (schedule, error) {
+	fields := strings.Fields(strings.TrimSpace(expr))
+	if len(fields) != 5 {
+		return schedule{}, fmt.Errorf("cron: want 5 fields, got %d in %q", len(fields), expr)
+	}
+	var s schedule
+	dst := []*uint64{&s.min, &s.hour, &s.dom, &s.mon, &s.dow}
+	for i, f := range fields {
+		bits, err := parseField(f, cronRanges[i])
+		if err != nil {
+			return schedule{}, fmt.Errorf("cron field %d (%q): %w", i+1, f, err)
+		}
+		*dst[i] = bits
+	}
+	s.domRestricted = fields[2] != "*"
+	s.dowRestricted = fields[4] != "*"
+	return s, nil
+}
+
+// parseField parses one comma-separated cron field into a bitset over r.
+func parseField(f string, r fieldRange) (uint64, error) {
+	if f == "" {
+		return 0, fmt.Errorf("empty field")
+	}
+	var bits uint64
+	for _, term := range strings.Split(f, ",") {
+		tb, err := parseTerm(term, r)
+		if err != nil {
+			return 0, err
+		}
+		bits |= tb
+	}
+	return bits, nil
+}
+
+// parseTerm parses a single term: "*", "n", "a-b", "*/n", or "a-b/n".
+func parseTerm(term string, r fieldRange) (uint64, error) {
+	step := 1
+	if i := strings.IndexByte(term, '/'); i >= 0 {
+		n, err := strconv.Atoi(term[i+1:])
+		if err != nil || n <= 0 {
+			return 0, fmt.Errorf("bad step %q", term)
+		}
+		step = n
+		term = term[:i]
+	}
+
+	lo, hi := r.min, r.max
+	switch {
+	case term == "*":
+		// full range with the parsed step.
+	case strings.IndexByte(term, '-') > 0:
+		i := strings.IndexByte(term, '-')
+		a, err1 := strconv.Atoi(term[:i])
+		b, err2 := strconv.Atoi(term[i+1:])
+		if err1 != nil || err2 != nil {
+			return 0, fmt.Errorf("bad range %q", term)
+		}
+		lo, hi = a, b
+	default:
+		n, err := strconv.Atoi(term)
+		if err != nil {
+			return 0, fmt.Errorf("bad number %q", term)
+		}
+		lo, hi = n, n
+	}
+
+	if lo < r.min || hi > r.max || lo > hi {
+		return 0, fmt.Errorf("value out of range [%d,%d]", r.min, r.max)
+	}
+	var bits uint64
+	for v := lo; v <= hi; v += step {
+		bits |= 1 << uint(v)
+	}
+	return bits, nil
+}
+
+// matches reports whether the schedule fires at t (evaluated in UTC, minute
+// granularity). The day-of-month / day-of-week combination follows Vixie cron:
+// when BOTH are restricted the day matches if EITHER matches (OR); otherwise the
+// unrestricted field is a wildcard and the restricted one is ANDed.
+func (s schedule) matches(t time.Time) bool {
+	t = t.UTC()
+	if s.min&(1<<uint(t.Minute())) == 0 {
+		return false
+	}
+	if s.hour&(1<<uint(t.Hour())) == 0 {
+		return false
+	}
+	if s.mon&(1<<uint(int(t.Month()))) == 0 {
+		return false
+	}
+	domHit := s.dom&(1<<uint(t.Day())) != 0
+	dowHit := s.dow&(1<<uint(int(t.Weekday()))) != 0
+	if s.domRestricted && s.dowRestricted {
+		return domHit || dowHit
+	}
+	return domHit && dowHit
+}

clients/agents/cron_test.go

@@ -0,0 +1,105 @@
+package agents
+
+import (
+	"testing"
+	"time"
+)
+
+func TestParseCronErrors(t *testing.T) {
+	bad := []string{
+		"",             // empty
+		"* * * *",      // 4 fields
+		"* * * * * *",  // 6 fields
+		"60 * * * *",   // minute out of range
+		"* 24 * * *",   // hour out of range
+		"* * 0 * *",    // dom below range
+		"* * 32 * *",   // dom above range
+		"* * * 13 *",   // month above range
+		"* * * * 7",    // dow above range
+		"*/0 * * * *",  // zero step
+		"5-1 * * * *",  // inverted range
+		"abc * * * *",  // non-numeric
+		"1,,2 * * * *", // empty term
+	}
+	for _, expr := range bad {
+		if _, err := parseCron(expr); err == nil {
+			t.Errorf("parseCron(%q) = nil error, want error", expr)
+		}
+	}
+}
+
+func TestParseCronValid(t *testing.T) {
+	for _, expr := range []string{
+		"* * * * *", "*/5 * * * *", "0 9 * * 1", "0 0 1 * *",
+		"0,30 * * * *", "0-15 * * * *", "0 9-17/2 * * 1-5", "0 0 * * 0",
+	} {
+		if _, err := parseCron(expr); err != nil {
+			t.Errorf("parseCron(%q) unexpected error: %v", expr, err)
+		}
+	}
+}
+
+func at(t *testing.T, s string) time.Time {
+	t.Helper()
+	tm, err := time.Parse("2006-01-02 15:04 MST", s+" UTC")
+	if err != nil {
+		t.Fatalf("bad test time %q: %v", s, err)
+	}
+	return tm
+}
+
+func TestCronMatches(t *testing.T) {
+	cases := []struct {
+		expr string
+		when string // "YYYY-MM-DD HH:MM"
+		want bool
+	}{
+		{"* * * * *", "2026-07-01 12:34", true},
+		{"*/5 * * * *", "2026-07-01 12:35", true},
+		{"*/5 * * * *", "2026-07-01 12:36", false},
+		{"0 9 * * *", "2026-07-01 09:00", true},
+		{"0 9 * * *", "2026-07-01 09:01", false},
+		{"0 9 * * *", "2026-07-01 10:00", false},
+		// 2026-07-06 is a Monday; 0 9 * * 1 fires 09:00 Mondays.
+		{"0 9 * * 1", "2026-07-06 09:00", true},
+		{"0 9 * * 1", "2026-07-07 09:00", false}, // Tuesday
+		{"0-15 * * * *", "2026-07-01 12:15", true},
+		{"0-15 * * * *", "2026-07-01 12:16", false},
+		{"0 0 1 * *", "2026-08-01 00:00", true},  // first of month
+		{"0 0 1 * *", "2026-08-02 00:00", false}, // second of month
+		{"0 9-17/2 * * *", "2026-07-01 09:00", true},
+		{"0 9-17/2 * * *", "2026-07-01 11:00", true},
+		{"0 9-17/2 * * *", "2026-07-01 10:00", false}, // 10 not in 9,11,13,15,17
+	}
+	for _, c := range cases {
+		s, err := parseCron(c.expr)
+		if err != nil {
+			t.Fatalf("parseCron(%q): %v", c.expr, err)
+		}
+		if got := s.matches(at(t, c.when)); got != c.want {
+			t.Errorf("%q matches %q = %v, want %v", c.expr, c.when, got, c.want)
+		}
+	}
+}
+
+// TestCronDOMDOWOrSemantics: when BOTH day-of-month and day-of-week are
+// restricted, Vixie cron fires if EITHER matches. "0 0 13 * 5" fires on the
+// 13th OR on any Friday.
+func TestCronDOMDOWOrSemantics(t *testing.T) {
+	s, err := parseCron("0 0 13 * 5") // 5 = Friday
+	if err != nil {
+		t.Fatalf("parse: %v", err)
+	}
+	// 2026-07-13 is a Monday -> matches via DOM (the 13th).
+	if !s.matches(at(t, "2026-07-13 00:00")) {
+		t.Error("should fire on the 13th regardless of weekday")
+	}
+	// 2026-07-03 is a Friday -> matches via DOW.
+	if !s.matches(at(t, "2026-07-03 00:00")) {
+		t.Error("should fire on a Friday regardless of day-of-month")
+	}
+	// 2026-07-06 is a Monday, not the 13th -> no match.
+	if s.matches(at(t, "2026-07-06 00:00")) {
+		t.Error("must NOT fire on a non-13th non-Friday")
+	}
+}

clients/agents/http_test.go

@@ -0,0 +1,312 @@
+package agents
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/hanzoai/cloud"
+	"github.com/hanzoai/cloud/types"
+	luxlog "github.com/luxfi/log"
+	"github.com/zap-proto/zip"
+)
+
+// mountApp mounts the agents surface with a deterministic fake AI so run() is
+// exercised end-to-end over HTTP without a real gateway. Pass a nil interface
+// to exercise the no-inference fail-closed path.
+func mountApp(t *testing.T, ai types.AIClient) *zip.App {
+	t.Helper()
+	app := zip.New(zip.Config{Logger: luxlog.New("test")})
+	if err := Mount(app, cloud.Deps{Logger: luxlog.New("test"), DataDir: t.TempDir(), AI: ai}); err != nil {
+		t.Fatalf("Mount: %v", err)
+	}
+	// Mount starts the scheduler goroutine when AI is non-nil and sets the global
+	// `mounted` singleton; tear both down at test end so the loop goroutine can't
+	// leak and clobber a later test's singleton (Red re-review LOW).
+	t.Cleanup(func() { _ = Shutdown(context.Background()) })
+	return app
+}
+
+func do(t *testing.T, app *zip.App, method, path, org string, body any) (int, []byte) {
+	t.Helper()
+	var r io.Reader
+	if body != nil {
+		b, _ := json.Marshal(body)
+		r = bytes.NewReader(b)
+	}
+	req := httptest.NewRequest(method, path, r)
+	if body != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	if org != "" {
+		req.Header.Set("X-Org-Id", org)
+		// A validated principal: the run path (money-moving) requires a non-empty
+		// c.User() (X-User-Id). SanitizeIdentity sets this only from a verified
+		// JWT; the test app has no sanitizer, so we inject it directly, exactly as
+		// the gateway would. Empty org => no user (the anonymous 403 path).
+		req.Header.Set("X-User-Id", "u-"+org)
+	}
+	resp, err := app.Fiber().Test(req)
+	if err != nil {
+		t.Fatalf("Test %s %s: %v", method, path, err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	b, _ := io.ReadAll(resp.Body)
+	return resp.StatusCode, b
+}
+
+func TestHTTPGateIsolationAndRun(t *testing.T) {
+	app := mountApp(t, &fakeAI{content: "the answer"})
+
+	if code, _ := do(t, app, http.MethodGet, "/v1/agents", "", nil); code != http.StatusForbidden {
+		t.Fatalf("no-org list want 403, got %d", code)
+	}
+
+	// maxpower creates an agent (model required).
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "maxpower",
+		map[string]any{"name": "helper", "model": "gpt-4o-mini", "instructions": "be terse"}); code != http.StatusCreated {
+		t.Fatalf("create want 201, got %d", code)
+	}
+	// model is required — creating without one is a 400.
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "maxpower",
+		map[string]any{"name": "nomodel"}); code != http.StatusBadRequest {
+		t.Fatalf("create without model want 400, got %d", code)
+	}
+
+	// List shape is {agents:[...]}.
+	code, body := do(t, app, http.MethodGet, "/v1/agents", "maxpower", nil)
+	var listed struct {
+		Agents []agentView `json:"agents"`
+	}
+	_ = json.Unmarshal(body, &listed)
+	if code != http.StatusOK || len(listed.Agents) != 1 || listed.Agents[0].Name != "helper" {
+		t.Fatalf("maxpower should see [helper], got %d %+v", code, listed.Agents)
+	}
+
+	// run executes via the (fake) AI and returns a real recorded run.
+	code, body = do(t, app, http.MethodPost, "/v1/agents/helper/run", "maxpower", map[string]any{"input": "hi"})
+	if code != http.StatusOK {
+		t.Fatalf("run want 200, got %d (%s)", code, body)
+	}
+	var rv runView
+	_ = json.Unmarshal(body, &rv)
+	if rv.Status != "ok" || rv.Output != "the answer" {
+		t.Fatalf("run should return the model output, got %+v", rv)
+	}
+
+	// The run was recorded and is org-scoped.
+	code, body = do(t, app, http.MethodGet, "/v1/agents/helper/runs", "maxpower", nil)
+	if code != http.StatusOK || !bytes.Contains(body, []byte("the answer")) {
+		t.Fatalf("runs history want the recorded run, got %d %s", code, body)
+	}
+
+	// acme cannot see, run, or read runs for maxpower's agent.
+	code, body = do(t, app, http.MethodGet, "/v1/agents", "acme", nil)
+	_ = json.Unmarshal(body, &listed)
+	if code != http.StatusOK || len(listed.Agents) != 0 {
+		t.Fatalf("acme must see zero agents, got %d %+v", code, listed.Agents)
+	}
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents/helper/run", "acme", map[string]any{"input": "hi"}); code != http.StatusNotFound {
+		t.Fatalf("acme run on maxpower agent want 404, got %d", code)
+	}
+}
+
+// TestHTTPCreateThenGetRunByReturnedID reproduces Dave's exact flow and proves
+// the id/name disconnect is fixed: create returns an id, and GETting AND running
+// that agent BY THE RETURNED ID (not just the name) both resolve the SAME agent.
+// Before the fix, get/run keyed the path only against the name column, so the id
+// create handed back 404'd — a created agent was not runnable.
+func TestHTTPCreateThenGetRunByReturnedID(t *testing.T) {
+	app := mountApp(t, &fakeAI{content: "the answer"})
+
+	// Create — capture the id the API returns (exactly what a client keeps).
+	code, body := do(t, app, http.MethodPost, "/v1/agents", "maxpower",
+		map[string]any{"name": "verify-run", "model": "gpt-4o-mini", "instructions": "be terse"})
+	if code != http.StatusCreated {
+		t.Fatalf("create want 201, got %d (%s)", code, body)
+	}
+	var created agentView
+	if err := json.Unmarshal(body, &created); err != nil {
+		t.Fatalf("create shape: %v (%s)", err, body)
+	}
+	if created.ID == "" || created.Name != "verify-run" {
+		t.Fatalf("create must return id+name, got %+v", created)
+	}
+	id := created.ID
+
+	// GET by the RETURNED ID must be 200 and the same agent (was 404 pre-fix).
+	code, body = do(t, app, http.MethodGet, "/v1/agents/"+id, "maxpower", nil)
+	if code != http.StatusOK {
+		t.Fatalf("GET by returned id want 200, got %d (%s)", code, body)
+	}
+	var got agentDetail
+	_ = json.Unmarshal(body, &got)
+	if got.ID != id || got.Name != "verify-run" {
+		t.Fatalf("GET by id resolved the wrong agent, got %+v", got.agentView)
+	}
+
+	// GET by NAME must resolve the SAME agent (both identifiers work).
+	code, body = do(t, app, http.MethodGet, "/v1/agents/verify-run", "maxpower", nil)
+	if code != http.StatusOK {
+		t.Fatalf("GET by name want 200, got %d (%s)", code, body)
+	}
+	var byName agentDetail
+	_ = json.Unmarshal(body, &byName)
+	if byName.ID != id {
+		t.Fatalf("GET by name must be the SAME agent as by id: %q vs %q", byName.ID, id)
+	}
+
+	// RUN by the RETURNED ID must execute and return real output (was 404 pre-fix).
+	code, body = do(t, app, http.MethodPost, "/v1/agents/"+id+"/run", "maxpower", map[string]any{"input": "hi"})
+	if code != http.StatusOK {
+		t.Fatalf("run by returned id want 200, got %d (%s)", code, body)
+	}
+	var rv runView
+	_ = json.Unmarshal(body, &rv)
+	if rv.Status != "ok" || rv.Output != "the answer" {
+		t.Fatalf("run by id must return the model output, got %+v", rv)
+	}
+
+	// The run recorded under the agent is visible via runs-by-id AND runs-by-name.
+	code, body = do(t, app, http.MethodGet, "/v1/agents/"+id+"/runs", "maxpower", nil)
+	if code != http.StatusOK || !bytes.Contains(body, []byte("the answer")) {
+		t.Fatalf("runs by id want the recorded run, got %d %s", code, body)
+	}
+	code, body = do(t, app, http.MethodGet, "/v1/agents/verify-run/runs", "maxpower", nil)
+	if code != http.StatusOK || !bytes.Contains(body, []byte("the answer")) {
+		t.Fatalf("runs by name want the same recorded run, got %d %s", code, body)
+	}
+
+	// Cross-org fail-closed: acme cannot GET or run maxpower's agent BY ITS ID.
+	if c2, _ := do(t, app, http.MethodGet, "/v1/agents/"+id, "acme", nil); c2 != http.StatusNotFound {
+		t.Fatalf("acme GET maxpower agent by id want 404, got %d", c2)
+	}
+	if c2, _ := do(t, app, http.MethodPost, "/v1/agents/"+id+"/run", "acme", map[string]any{"input": "x"}); c2 != http.StatusNotFound {
+		t.Fatalf("acme run maxpower agent by id want 404, got %d", c2)
+	}
+}
+
+// TestHTTPMetricsAndActivityNotShadowed proves /v1/agents/metrics and
+// /v1/agents/activity resolve to their own handlers (not captured by the :name
+// wildcard) and that every number is derived from REAL recorded runs.
+func TestHTTPMetricsAndActivityNotShadowed(t *testing.T) {
+	app := mountApp(t, &fakeAI{content: "ok"})
+
+	// Both org-wide surfaces require a tenant, like every other route.
+	if code, _ := do(t, app, http.MethodGet, "/v1/agents/metrics", "", nil); code != http.StatusForbidden {
+		t.Fatalf("no-org metrics want 403, got %d", code)
+	}
+	if code, _ := do(t, app, http.MethodGet, "/v1/agents/activity", "", nil); code != http.StatusForbidden {
+		t.Fatalf("no-org activity want 403, got %d", code)
+	}
+
+	// Empty org: honest empty shapes, NOT a 404 (proves no wildcard shadowing)
+	// and NOT a fabricated trend.
+	code, body := do(t, app, http.MethodGet, "/v1/agents/metrics?range=7D", "maxpower", nil)
+	if code != http.StatusOK {
+		t.Fatalf("metrics want 200 (not shadowed 404), got %d (%s)", code, body)
+	}
+	var m struct {
+		Range    string       `json:"range"`
+		Series   []seriesLine `json:"series"`
+		Resource struct {
+			CPUVcpuHours   *float64 `json:"cpuVcpuHours"`
+			MemGbHours     *float64 `json:"memGbHours"`
+			StorageIoBytes *float64 `json:"storageIoBytes"`
+			CostCents      *float64 `json:"costCents"`
+		} `json:"resource"`
+	}
+	if err := json.Unmarshal(body, &m); err != nil {
+		t.Fatalf("metrics shape: %v (%s)", err, body)
+	}
+	if m.Range != "7D" || len(m.Series) != 0 {
+		t.Fatalf("empty-org metrics want range=7D, no series, got %+v", m)
+	}
+	if m.Resource.CPUVcpuHours != nil || m.Resource.CostCents != nil {
+		t.Fatalf("resource metering is unsourced — must be null, got %+v", m.Resource)
+	}
+
+	code, body = do(t, app, http.MethodGet, "/v1/agents/activity", "maxpower", nil)
+	if code != http.StatusOK {
+		t.Fatalf("activity want 200 (not shadowed 404), got %d (%s)", code, body)
+	}
+	var empty struct {
+		Activity []activityView `json:"activity"`
+	}
+	_ = json.Unmarshal(body, &empty)
+	if len(empty.Activity) != 0 {
+		t.Fatalf("empty-org activity want [], got %+v", empty.Activity)
+	}
+
+	// Seed a real agent + a real run, then the surfaces must reflect exactly it.
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "maxpower",
+		map[string]any{"name": "helper", "model": "gpt-4o-mini"}); code != http.StatusCreated {
+		t.Fatalf("create want 201, got %d", code)
+	}
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents/helper/run", "maxpower", map[string]any{"input": "hi"}); code != http.StatusOK {
+		t.Fatalf("run want 200, got %d", code)
+	}
+
+	// Metrics now carry a real invocation series for "helper" summing to 1.
+	_, body = do(t, app, http.MethodGet, "/v1/agents/metrics?range=24H", "maxpower", nil)
+	_ = json.Unmarshal(body, &m)
+	if len(m.Series) != 1 || m.Series[0].Key != "helper" {
+		t.Fatalf("metrics want one series for helper, got %+v", m.Series)
+	}
+	total := 0
+	for _, p := range m.Series[0].Points {
+		total += p.V
+	}
+	if total != 1 {
+		t.Fatalf("real invocation total want 1, got %d", total)
+	}
+
+	// Activity now carries the real invoked event + the created event, newest first.
+	_, body = do(t, app, http.MethodGet, "/v1/agents/activity", "maxpower", nil)
+	var feed struct {
+		Activity []activityView `json:"activity"`
+	}
+	_ = json.Unmarshal(body, &feed)
+	var invoked, created bool
+	for _, e := range feed.Activity {
+		if e.Agent != "helper" {
+			t.Fatalf("activity must be scoped to helper, got %+v", e)
+		}
+		switch e.Kind {
+		case "invoked":
+			invoked = true
+		case "created":
+			created = true
+		}
+	}
+	if !invoked || !created {
+		t.Fatalf("activity want a real invoked + created event, got %+v", feed.Activity)
+	}
+
+	// Cross-org isolation: acme sees none of maxpower's metrics/activity.
+	_, body = do(t, app, http.MethodGet, "/v1/agents/metrics?range=24H", "acme", nil)
+	_ = json.Unmarshal(body, &m)
+	if len(m.Series) != 0 {
+		t.Fatalf("acme must see zero series, got %+v", m.Series)
+	}
+	_, body = do(t, app, http.MethodGet, "/v1/agents/activity", "acme", nil)
+	_ = json.Unmarshal(body, &feed)
+	if len(feed.Activity) != 0 {
+		t.Fatalf("acme must see zero activity, got %+v", feed.Activity)
+	}
+}
+
+// TestHTTPRunWithoutAIFailsClosed: when no AI client is wired, run 503s and
+// never fabricates output.
+func TestHTTPRunWithoutAIFailsClosed(t *testing.T) {
+	app := mountApp(t, nil)
+	do(t, app, http.MethodPost, "/v1/agents", "maxpower",
+		map[string]any{"name": "a", "model": "m", "instructions": "x"})
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents/a/run", "maxpower", map[string]any{"input": "hi"}); code != http.StatusServiceUnavailable {
+		t.Fatalf("run without AI want 503, got %d", code)
+	}
+}

clients/agents/lifecycle_http_test.go

@@ -0,0 +1,153 @@
+package agents
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+	"testing"
+)
+
+// TestCreateRejectsOversizedRefs: computeRef/serviceAccountId are opaque ids,
+// bounded at the boundary — a multi-KB "id" must be a 400, not persisted.
+func TestCreateRejectsOversizedRefs(t *testing.T) {
+	app := mountApp(t, &fakeAI{content: "x"})
+	huge := strings.Repeat("a", maxRef+1)
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "big", "model": "m", "computeRef": huge}); code != http.StatusBadRequest {
+		t.Fatalf("oversized computeRef want 400, got %d", code)
+	}
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "big2", "model": "m", "serviceAccountId": huge}); code != http.StatusBadRequest {
+		t.Fatalf("oversized serviceAccountId want 400, got %d", code)
+	}
+}
+
+// TestCreateLongRunningRequiresValidCron: a long-running agent must carry a
+// parseable cron; missing/invalid schedule is a 400. A valid one is 201 and the
+// mode+schedule round-trip in the view.
+func TestCreateLongRunningValidation(t *testing.T) {
+	app := mountApp(t, &fakeAI{content: "x"})
+
+	// long-running without a schedule -> 400.
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "a", "model": "m", "executionMode": "long-running"}); code != http.StatusBadRequest {
+		t.Fatalf("long-running w/o schedule want 400, got %d", code)
+	}
+	// long-running with a bad cron -> 400.
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "b", "model": "m", "executionMode": "long-running", "schedule": "not a cron"}); code != http.StatusBadRequest {
+		t.Fatalf("long-running w/ bad cron want 400, got %d", code)
+	}
+	// unknown mode -> 400.
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "c", "model": "m", "executionMode": "daemon"}); code != http.StatusBadRequest {
+		t.Fatalf("unknown mode want 400, got %d", code)
+	}
+	// valid long-running -> 201, fields echoed.
+	code, body := do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "cron", "model": "m", "executionMode": "long-running",
+			"schedule": "*/5 * * * *", "computeRef": "vm-1", "serviceAccountId": "acme-cron"})
+	if code != http.StatusCreated {
+		t.Fatalf("valid long-running want 201, got %d (%s)", code, body)
+	}
+	var v agentView
+	_ = json.Unmarshal(body, &v)
+	if v.ExecutionMode != "long-running" || v.Schedule != "*/5 * * * *" ||
+		v.ComputeRef != "vm-1" || v.ServiceAccountID != "acme-cron" {
+		t.Fatalf("lifecycle fields not echoed in view: %+v", v)
+	}
+}
+
+// TestCreateOneShotDropsSchedule: a one-shot agent's schedule is meaningless and
+// dropped, so the view carries no schedule and the scheduler will never pick it.
+func TestCreateOneShotDropsSchedule(t *testing.T) {
+	app := mountApp(t, &fakeAI{content: "x"})
+	code, body := do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "one", "model": "m", "schedule": "* * * * *"})
+	if code != http.StatusCreated {
+		t.Fatalf("create want 201, got %d", code)
+	}
+	var v agentView
+	_ = json.Unmarshal(body, &v)
+	if v.ExecutionMode != "one-shot" || v.Schedule != "" {
+		t.Fatalf("one-shot must drop schedule, got mode=%q schedule=%q", v.ExecutionMode, v.Schedule)
+	}
+}
+
+// TestLongRunningPerOrgCap: an org cannot create more than the configured number
+// of scheduled long-running agents (Red LOW-1). One-shot agents don't count.
+func TestLongRunningPerOrgCap(t *testing.T) {
+	t.Setenv(longRunningCapEnv, "2")
+	app := mountApp(t, &fakeAI{content: "x"})
+	mk := func(name string) map[string]any {
+		return map[string]any{"name": name, "model": "m", "executionMode": "long-running", "schedule": "* * * * *"}
+	}
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "acme", mk("a")); code != http.StatusCreated {
+		t.Fatalf("1st long-running want 201, got %d", code)
+	}
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "acme", mk("b")); code != http.StatusCreated {
+		t.Fatalf("2nd long-running want 201, got %d", code)
+	}
+	// 3rd exceeds the cap of 2 -> 409.
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "acme", mk("c")); code != http.StatusConflict {
+		t.Fatalf("3rd long-running want 409 (cap), got %d", code)
+	}
+	// A one-shot agent is unaffected by the cap.
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "one", "model": "m"}); code != http.StatusCreated {
+		t.Fatalf("one-shot must not be capped, got %d", code)
+	}
+	// A DIFFERENT org has its own budget.
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "beta", mk("a")); code != http.StatusCreated {
+		t.Fatalf("other org's 1st long-running want 201, got %d", code)
+	}
+}
+
+// TestLongRunningCapNotBypassedByPatch: the cap can't be dodged by creating
+// one-shot agents (uncapped) then PATCHing them to long-running. Transition into
+// long-running is capped too; re-saving an already-long-running agent is not.
+func TestLongRunningCapNotBypassedByPatch(t *testing.T) {
+	t.Setenv(longRunningCapEnv, "1")
+	app := mountApp(t, &fakeAI{content: "x"})
+
+	// Fill the cap with one long-running agent.
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "lr", "model": "m", "executionMode": "long-running", "schedule": "* * * * *"}); code != http.StatusCreated {
+		t.Fatalf("seed long-running want 201, got %d", code)
+	}
+	// Create a one-shot agent (uncapped), then try to PATCH it to long-running.
+	do(t, app, http.MethodPost, "/v1/agents", "acme", map[string]any{"name": "sneaky", "model": "m"})
+	if code, _ := do(t, app, http.MethodPatch, "/v1/agents/sneaky", "acme",
+		map[string]any{"executionMode": "long-running", "schedule": "* * * * *"}); code != http.StatusConflict {
+		t.Fatalf("PATCH one-shot->long-running over cap want 409, got %d", code)
+	}
+	// Re-saving the EXISTING long-running agent (no transition) must NOT 409.
+	if code, _ := do(t, app, http.MethodPatch, "/v1/agents/lr", "acme",
+		map[string]any{"schedule": "*/2 * * * *"}); code != http.StatusOK {
+		t.Fatalf("no-op re-save of own long-running agent want 200, got %d", code)
+	}
+}
+
+// TestPatchToLongRunningValidates: PATCHing an agent to long-running without a
+// schedule is rejected; supplying a valid schedule in the same PATCH succeeds.
+func TestPatchToLongRunningValidates(t *testing.T) {
+	app := mountApp(t, &fakeAI{content: "x"})
+	do(t, app, http.MethodPost, "/v1/agents", "acme", map[string]any{"name": "a", "model": "m"})
+
+	// flip to long-running with no schedule -> 400.
+	if code, _ := do(t, app, http.MethodPatch, "/v1/agents/a", "acme",
+		map[string]any{"executionMode": "long-running"}); code != http.StatusBadRequest {
+		t.Fatalf("patch to long-running w/o schedule want 400, got %d", code)
+	}
+	// flip with a schedule -> 200.
+	code, body := do(t, app, http.MethodPatch, "/v1/agents/a", "acme",
+		map[string]any{"executionMode": "long-running", "schedule": "0 * * * *"})
+	if code != http.StatusOK {
+		t.Fatalf("patch to long-running w/ schedule want 200, got %d (%s)", code, body)
+	}
+	var v agentView
+	_ = json.Unmarshal(body, &v)
+	if v.ExecutionMode != "long-running" || v.Schedule != "0 * * * *" {
+		t.Fatalf("patch did not apply lifecycle: %+v", v)
+	}
+}

clients/agents/onbehalf.go

@@ -0,0 +1,57 @@
+package agents
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/hanzoai/cloud/clients/principal"
+)
+
+// RunOnBehalf runs agent `ref` for `org` ON BEHALF OF `userSub`, IN-PROCESS —
+// no gateway hop, no Cloudflare/IPv6 exposure. It is the clean in-process twin of
+// the HTTP s.run handler: the CALLER (e.g. the Slack integrations bridge) has
+// ALREADY authenticated org+userSub server-side, so this entry takes them
+// DIRECTLY and never reads an HTTP principal / JWT / zip.Ctx. It resolves the
+// agent org-scoped, runs it through the SAME runAgent → executeRun → meter path
+// as s.run (one run path: one balance gate, one debit, one recorded run, one live
+// session), and bills billingActor(org, userSub) against ORG's ledger.
+//
+// ISOLATION: org is the ONLY tenant key. Store.Resolve is org-scoped, so a caller
+// for org A can never resolve, run, or bill against org B's agent — exactly the
+// property the HTTP handler relies on the gateway-minted X-Org-Id for.
+//
+// A non-nil error means NO run happened: not mounted, invalid org, oversized
+// input, inference not configured, agent-not-found (errNotFound), or a
+// balance-gate denial (out-of-funds / commerce-unknown). A run that executed but
+// whose model failed returns a recorded error-status Run and a nil error.
+func RunOnBehalf(ctx context.Context, org, userSub, ref, input string) (Run, error) {
+	if mounted == nil {
+		return Run{}, fmt.Errorf("agents: not mounted")
+	}
+	return mounted.runOnBehalf(ctx, org, userSub, ref, input)
+}
+
+func (s *svc) runOnBehalf(ctx context.Context, org, userSub, ref, input string) (Run, error) {
+	org = strings.TrimSpace(org)
+	if org == "" || len(org) > principal.MaxOrgLen {
+		return Run{}, fmt.Errorf("agents: invalid org")
+	}
+	if len(input) > maxInput {
+		return Run{}, fmt.Errorf("agents: input too large")
+	}
+	if s.ai == nil {
+		return Run{}, fmt.Errorf("agents: inference is not configured on this deployment")
+	}
+	a, err := s.store.Resolve(ctx, org, strings.TrimSpace(ref))
+	if err != nil {
+		return Run{}, err // errNotFound or a real DB error — caller replies generically
+	}
+	// The actor attributes the spend to the acting principal (org/userSub) for the
+	// audit trail; the BALANCE gated + debited is always a.Org (== org), never the
+	// caller. Synthetic request id: in-process, there is no HTTP X-Request-Id; the
+	// client IP is empty (no socket).
+	actor := billingActor(org, userSub)
+	reqID, _ := genID("obh")
+	return s.runAgent(ctx, a, input, actor, reqID, "")
+}

clients/agents/onbehalf_test.go

@@ -0,0 +1,117 @@
+package agents
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"testing"
+)
+
+// TestRunOnBehalfBillsActor proves the in-process on-behalf-of path runs the
+// agent through the SAME meter path as the HTTP handler and bills the AGENT's org
+// with the actor "org/userSub" — the identity the Slack bridge passes for a linked
+// user. This is the deliverable's "RunOnBehalf bills the right actor" bar.
+func TestRunOnBehalfBillsActor(t *testing.T) {
+	bs := &billServer{available: 100000}
+	app := mountBilled(t, bs.start(t), &fakeAI{content: "on-behalf answer"})
+	_ = app
+
+	// Create the agent the bridge will address by ref (name "hanzo").
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "hanzo", "model": "gpt-4o-mini", "instructions": "x"}); code != http.StatusCreated {
+		t.Fatalf("create want 201, got %d", code)
+	}
+
+	run, err := RunOnBehalf(context.Background(), "acme", "U-slack-123", "hanzo", "hi from slack")
+	if err != nil {
+		t.Fatalf("RunOnBehalf: %v", err)
+	}
+	if run.Status != "ok" || run.Output != "on-behalf answer" {
+		t.Fatalf("run must succeed with the model output, got %+v", run)
+	}
+	if !waitForDebit(func() bool { return bs.debits() == 1 }) {
+		t.Fatalf("an on-behalf run must debit exactly once, got %d", bs.debits())
+	}
+	org, ubody := bs.lastDebit()
+	if org != "acme" {
+		t.Fatalf("debited org %q, want the agent's org 'acme' (never the caller default)", org)
+	}
+	var u struct {
+		Actor    string `json:"actor"`
+		Provider string `json:"provider"`
+		Model    string `json:"model"`
+	}
+	_ = json.Unmarshal(ubody, &u)
+	if u.Actor != "acme/U-slack-123" {
+		t.Fatalf("actor = %q, want billingActor(org,userSub)=%q", u.Actor, "acme/U-slack-123")
+	}
+	if u.Provider != meterKind || u.Model != "gpt-4o-mini" {
+		t.Fatalf("debit must be product=agent for the agent's model, got provider=%q model=%q", u.Provider, u.Model)
+	}
+}
+
+// TestRunOnBehalfOrgScoped proves the in-process path is tenant-isolated: a caller
+// for org A can never resolve/run/bill org B's agent — resolution is org-scoped, so
+// a cross-org ref is errNotFound and NOTHING is billed.
+func TestRunOnBehalfOrgScoped(t *testing.T) {
+	bs := &billServer{available: 100000}
+	app := mountBilled(t, bs.start(t), &fakeAI{content: "should not run"})
+	_ = app
+
+	// "secret" exists only in globex.
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "globex",
+		map[string]any{"name": "secret", "model": "m", "instructions": "y"}); code != http.StatusCreated {
+		t.Fatalf("create want 201, got %d", code)
+	}
+	// acme attempts to run globex's agent by ref.
+	if _, err := RunOnBehalf(context.Background(), "acme", "u", "secret", "hi"); err == nil {
+		t.Fatal("acme must NOT resolve globex's agent (org-scoped) — cross-tenant run")
+	}
+	// No inference, no debit.
+	if waitForDebit(func() bool { return bs.debits() > 0 }) {
+		t.Fatalf("a cross-org run must never debit, got %d", bs.debits())
+	}
+}
+
+// TestRunOnBehalfGatesUnfunded proves the on-behalf path shares the balance gate:
+// an unfunded org gets NO free inference (fail-closed) and the fake AI is never
+// called.
+func TestRunOnBehalfGatesUnfunded(t *testing.T) {
+	bs := &billServer{available: 0}
+	ai := &fakeAI{content: "must not run"}
+	app := mountBilled(t, bs.start(t), ai)
+	_ = app
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "a", "model": "m", "instructions": "x"}); code != http.StatusCreated {
+		t.Fatalf("create want 201, got %d", code)
+	}
+	if _, err := RunOnBehalf(context.Background(), "acme", "u", "a", "hi"); err == nil {
+		t.Fatal("unfunded on-behalf run must fail closed (balance-gate denial)")
+	}
+	if ai.gotPrompt != "" {
+		t.Fatalf("no inference must run when the gate denies, got prompt %q", ai.gotPrompt)
+	}
+	if bs.debits() != 0 {
+		t.Fatalf("a gate-refused run must not debit, got %d", bs.debits())
+	}
+}
+
+// TestRunOnBehalfNotMounted proves the package-level entry fails closed when the
+// agents subsystem is not mounted (no panic, an honest error).
+func TestRunOnBehalfNotMounted(t *testing.T) {
+	_ = Shutdown(context.Background())
+	if _, err := RunOnBehalf(context.Background(), "acme", "u", "hanzo", "hi"); err == nil {
+		t.Fatal("RunOnBehalf must fail when agents is not mounted")
+	}
+}
+
+// TestRunOnBehalfInvalidOrg proves an empty/oversized org is refused before any
+// store/inference — the org is a tenant key and must be bounded.
+func TestRunOnBehalfInvalidOrg(t *testing.T) {
+	bs := &billServer{available: 100000}
+	app := mountBilled(t, bs.start(t), &fakeAI{content: "x"})
+	_ = app
+	if _, err := RunOnBehalf(context.Background(), "", "u", "hanzo", "hi"); err == nil {
+		t.Fatal("empty org must be refused")
+	}
+}

clients/agents/scheduler.go

@@ -0,0 +1,290 @@
+package agents
+
+// The long-running-agent scheduler: it invokes each long-running agent's run on
+// its cron cadence, through the SAME svc.runAgent path the HTTP handler uses —
+// so a scheduled run is gated (fail-closed on the agent's OWN org balance),
+// executed, recorded, and billed identically to an interactive one. There is no
+// self-HTTP call: the endpoint's BEHAVIOR is the contract, and calling runAgent
+// directly keeps ONE run path (no duplicated gate/meter, no re-crossing the
+// identity boundary with a synthetic token).
+//
+// Cadence: one ticker fires every minute (cron's finest granularity). On each
+// tick it loads the long-running work set and, for every agent whose schedule
+// matches the current minute, launches a run — subject to two safety controls:
+//
+//   - Concurrency cap: at most maxConcurrentPerAgent in-flight runs per agent.
+//     A slow model must never let cron stack unbounded goroutines for one agent.
+//   - Exponential backoff: after a failed run (gate denial OR model error) an
+//     agent is skipped for a growing number of ticks (1,2,4,… up to a cap), so a
+//     persistently-failing or unfunded agent stops hammering commerce/the model.
+//     A success resets the backoff.
+//
+// All state is in-memory and keyed by org/name: the scheduler is a per-process
+// singleton owned by the mounted svc, torn down on Shutdown.
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"time"
+
+	luxlog "github.com/luxfi/log"
+)
+
+const (
+	// tickInterval is cron's resolution. Aligned to the top of each minute so a
+	// "* * * * *" agent fires once per minute, not on process-start phase.
+	tickInterval = time.Minute
+	// maxConcurrentPerAgent caps in-flight runs for ONE agent. A cron agent is
+	// expected to complete within its period; 1 means "never overlap a run with
+	// itself" (the safe default for periodic work). >1 would allow catch-up.
+	maxConcurrentPerAgent = 1
+	// maxBackoffTicks caps the exponential skip so a failing agent still retries
+	// roughly hourly rather than backing off forever.
+	maxBackoffTicks = 60
+	// runTimeout bounds a single scheduled run so one stuck completion cannot pin
+	// a concurrency slot indefinitely.
+	runTimeout = 10 * time.Minute
+)
+
+// agentState is the per-agent runtime bookkeeping the scheduler keeps between
+// ticks: how many runs are in flight, and the backoff countdown after failures.
+type agentState struct {
+	inFlight   int
+	failstreak int // consecutive failures; drives the backoff window.
+	skipRemain int // ticks still to skip before the next attempt.
+	parsed     schedule
+	parsedExpr string // the expression `parsed` was compiled from (recompile on change).
+}
+
+type scheduler struct {
+	svc    *svc
+	log    luxlog.Logger
+	cancel context.CancelFunc // cancels the loop + all in-flight run contexts.
+
+	mu     sync.Mutex
+	states map[string]*agentState // key: org + "\x00" + name
+	wg     sync.WaitGroup         // tracks in-flight run goroutines for clean shutdown.
+
+	// now is time.Now, overridable in tests for deterministic cron evaluation.
+	now func() time.Time
+	// tick, when non-nil, replaces the internal ticker so tests drive cadence.
+	tickC <-chan time.Time
+}
+
+func newScheduler(s *svc, log luxlog.Logger) *scheduler {
+	return &scheduler{
+		svc:    s,
+		log:    log.New("component", "scheduler"),
+		states: map[string]*agentState{},
+		now:    time.Now,
+	}
+}
+
+// start launches the scheduler loop in its own goroutine. Cancelled by stop().
+func (sc *scheduler) start() {
+	ctx, cancel := context.WithCancel(context.Background())
+	sc.cancel = cancel
+	sc.wg.Add(1)
+	go sc.loop(ctx)
+}
+
+// stop halts the scheduler and waits for in-flight runs to drain BEFORE the
+// caller closes the store — otherwise a run could InsertRun into a closed DB.
+//
+// Cancelling the loop context also cancels every in-flight run's derived
+// context, so a run whose AIClient honors ctx returns promptly. The drain wait
+// is bounded by the caller's shutdown ctx: if a run ignores cancellation and
+// runs long (up to runTimeout), stop returns at the deadline rather than hanging
+// SIGTERM. Idempotent.
+func (sc *scheduler) stop(ctx context.Context) {
+	if sc.cancel == nil {
+		return
+	}
+	sc.cancel()
+	sc.cancel = nil
+
+	done := make(chan struct{})
+	go func() { sc.wg.Wait(); close(done) }()
+	select {
+	case <-done: // clean drain
+	case <-ctx.Done():
+		sc.log.Warn("scheduler drain timed out; in-flight runs may not have recorded",
+			"err", ctx.Err())
+	}
+}
+
+// loop is the cadence driver. It uses the injected tick channel in tests, else a
+// real minute ticker. Each tick evaluates the whole long-running work set.
+func (sc *scheduler) loop(ctx context.Context) {
+	defer sc.wg.Done()
+	tickC := sc.tickC
+	if tickC == nil {
+		t := time.NewTicker(tickInterval)
+		defer t.Stop()
+		tickC = t.C
+	}
+	sc.log.Info("scheduler started", "interval", tickInterval)
+	for {
+		select {
+		case <-ctx.Done():
+			sc.log.Info("scheduler stopped")
+			return
+		case <-tickC:
+			sc.tick(ctx, sc.now())
+		}
+	}
+}
+
+// tick evaluates every long-running agent against the wall-clock minute now and
+// launches the ones that are due, are not backed off, and have a free
+// concurrency slot. It is separated from loop() so tests can invoke it directly.
+func (sc *scheduler) tick(ctx context.Context, now time.Time) {
+	agents, err := sc.svc.store.ListLongRunning(ctx)
+	if err != nil {
+		sc.log.Warn("scheduler: list long-running failed", "err", err)
+		return
+	}
+	live := make(map[string]bool, len(agents))
+	for _, a := range agents {
+		key := stateKey(a.Org, a.Name)
+		live[key] = true
+		if sc.due(a, key, now) {
+			sc.launch(ctx, a, key)
+		}
+	}
+	sc.pruneDeleted(live)
+}
+
+// due decides, under a SINGLE lock acquisition, whether agent a should run this
+// tick: it (re)compiles the cron on change, decrements a live backoff window,
+// checks the cron against now and the per-agent concurrency slot, and — when it
+// returns true — has already reserved the slot (inFlight++). All shared state
+// (parsed cron, backoff, inFlight) is touched only while holding sc.mu, so there
+// is no data race with the completion goroutine in launch().
+func (sc *scheduler) due(a Agent, key string, now time.Time) bool {
+	sc.mu.Lock()
+	defer sc.mu.Unlock()
+	st := sc.stateForLocked(key)
+
+	// Recompile the cron only when the expression changed (edits via PATCH).
+	if st.parsedExpr != a.Schedule {
+		p, err := parseCron(a.Schedule)
+		if err != nil {
+			// Stored schedule is invalid (create/update validate, but a
+			// hand-edited DB could carry garbage). Skip, don't crash.
+			sc.log.Warn("scheduler: bad stored schedule, skipping",
+				"org", a.Org, "agent", a.Name, "schedule", a.Schedule, "err", err)
+			return false
+		}
+		st.parsed, st.parsedExpr = p, a.Schedule
+	}
+
+	if st.skipRemain > 0 { // in a backoff window — consume one tick.
+		st.skipRemain--
+		return false
+	}
+	if !st.parsed.matches(now) || st.inFlight >= maxConcurrentPerAgent {
+		return false
+	}
+	st.inFlight++ // reserve the slot before launching.
+	return true
+}
+
+// launch runs one scheduled invocation in its own goroutine, updating the
+// agent's backoff/concurrency state on completion. The run is empty-input (a
+// scheduled agent acts on its own instructions) and attributed to its service
+// account when bound, else the synthetic scheduler actor.
+func (sc *scheduler) launch(ctx context.Context, a Agent, key string) {
+	sc.wg.Add(1)
+	go func() {
+		defer sc.wg.Done()
+		runCtx, cancel := context.WithTimeout(ctx, runTimeout)
+		defer cancel()
+
+		// Scheduled runs carry no HTTP request/IP; requestID/clientIP are empty.
+		r, gateErr := sc.svc.runAgent(runCtx, a, "", scheduledActor(a), "", "")
+
+		ok := gateErr == nil && r.Status == "ok"
+		sc.mu.Lock()
+		st := sc.stateForLocked(key)
+		st.inFlight--
+		if ok {
+			st.failstreakReset()
+		} else {
+			st.failstreakBump()
+		}
+		remain := st.skipRemain
+		streak := st.failstreak
+		sc.mu.Unlock()
+
+		switch {
+		case gateErr != nil:
+			sc.log.Warn("scheduled run gated (not executed)",
+				"org", a.Org, "agent", a.Name, "err", gateErr, "failstreak", streak, "backoffTicks", remain)
+		case r.Status != "ok":
+			sc.log.Warn("scheduled run errored",
+				"org", a.Org, "agent", a.Name, "err", r.Error, "failstreak", streak, "backoffTicks", remain)
+		default:
+			sc.log.Info("scheduled run ok", "org", a.Org, "agent", a.Name, "durationMs", r.DurationMs)
+		}
+	}()
+}
+
+// stateForLocked returns (creating if needed) the runtime state for an agent
+// key. The CALLER MUST hold sc.mu — every read/write of agentState fields is
+// serialized by that one lock, so the scheduler has no data race between a tick
+// deciding to run and a completion goroutine updating backoff/inFlight.
+func (sc *scheduler) stateForLocked(key string) *agentState {
+	st := sc.states[key]
+	if st == nil {
+		st = &agentState{}
+		sc.states[key] = st
+	}
+	return st
+}
+
+// pruneDeleted drops runtime state for agents that no longer appear in the work
+// set (deleted or switched to one-shot), but keeps any with a run still in
+// flight so its completion bookkeeping lands on live state.
+func (sc *scheduler) pruneDeleted(live map[string]bool) {
+	sc.mu.Lock()
+	defer sc.mu.Unlock()
+	for key, st := range sc.states {
+		if !live[key] && st.inFlight == 0 {
+			delete(sc.states, key)
+		}
+	}
+}
+
+// failstreakReset clears the failure streak and backoff after a success.
+func (st *agentState) failstreakReset() { st.failstreak, st.skipRemain = 0, 0 }
+
+// failstreakBump grows the failure streak and sets the next backoff window to
+// 2^(streak-1) ticks, capped — 1,2,4,8,… minutes between retries.
+func (st *agentState) failstreakBump() {
+	st.failstreak++
+	skip := 1 << uint(min(st.failstreak-1, 30)) // guard the shift; 2^30 >> cap.
+	if skip > maxBackoffTicks {
+		skip = maxBackoffTicks
+	}
+	st.skipRemain = skip
+}
+
+// stateKey namespaces runtime state by org+name. The NUL separator can never
+// appear in either (nameRE + org validation forbid it), so keys are injective.
+func stateKey(org, name string) string { return org + "\x00" + name }
+
+// scheduledActor is the audit-trail Actor for a scheduled run. It is ALWAYS
+// prefixed "scheduler" so a scheduled run can never masquerade as a validated
+// interactive principal (org/sub). When the agent carries a service-account id
+// it is appended as an UNVERIFIED hint (Red LOW-2): the id is client-supplied on
+// create and not yet checked against IAM (that is the service-account keystone),
+// so it must be clearly non-authoritative, not the bare "principal". Once IAM
+// agent service accounts land, this becomes a verified identity.
+func scheduledActor(a Agent) string {
+	if sa := strings.TrimSpace(a.ServiceAccountID); sa != "" {
+		return schedulerActor + ":" + a.Org + "/" + a.Name + " (sa:" + sa + " unverified)"
+	}
+	return schedulerActor + ":" + a.Org + "/" + a.Name
+}

clients/agents/scheduler_test.go

@@ -0,0 +1,311 @@
+package agents
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/hanzoai/cloud"
+	"github.com/hanzoai/cloud/types"
+	"github.com/hanzoai/commerce/metering"
+	luxlog "github.com/luxfi/log"
+)
+
+// countingAI records how many completions ran and can be told to fail, so
+// scheduler tests can assert on run count + drive backoff deterministically.
+type countingAI struct {
+	mu    sync.Mutex
+	calls int32
+	fail  bool
+	err   error
+	block chan struct{} // when non-nil, ChatCompletion blocks until closed.
+}
+
+func (c *countingAI) ChatCompletion(_ context.Context, _ *types.ChatRequest) (*types.ChatResponse, error) {
+	atomic.AddInt32(&c.calls, 1)
+	if c.block != nil {
+		<-c.block
+	}
+	c.mu.Lock()
+	fail, err := c.fail, c.err
+	c.mu.Unlock()
+	if fail {
+		if err == nil {
+			err = errTest
+		}
+		return nil, err
+	}
+	return &types.ChatResponse{Content: "done"}, nil
+}
+
+func (c *countingAI) count() int32 { return atomic.LoadInt32(&c.calls) }
+
+// schedSvc builds an svc + scheduler with NO billing (gate allows) and the given
+// AI, seeded with the supplied agents. Returns the scheduler for direct tick().
+func schedSvc(t *testing.T, ai types.AIClient, seed ...Agent) *scheduler {
+	t.Helper()
+	s := &svc{store: testStore(t), ai: ai, log: luxlog.New("test")}
+	for _, a := range seed {
+		if err := s.store.Create(context.Background(), a); err != nil {
+			t.Fatalf("seed %s/%s: %v", a.Org, a.Name, err)
+		}
+	}
+	sc := newScheduler(s, luxlog.New("test"))
+	return sc
+}
+
+func longRunning(org, name, cron string) Agent {
+	a := mk(org, name)
+	a.ExecutionMode, a.Schedule = ModeLongRunning, cron
+	return a
+}
+
+// waitFor polls a condition briefly (async run goroutines).
+func waitFor(cond func() bool) bool {
+	for i := 0; i < 200; i++ {
+		if cond() {
+			return true
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	return cond()
+}
+
+// TestSchedulerFiresDueAgent: a tick at a minute the cron matches launches one
+// run; a tick at a non-matching minute launches none.
+func TestSchedulerFiresDueAgent(t *testing.T) {
+	ai := &countingAI{}
+	sc := schedSvc(t, ai, longRunning("acme", "cron", "*/5 * * * *"))
+	ctx := context.Background()
+
+	sc.tick(ctx, at(t, "2026-07-01 12:36")) // 36 not multiple of 5 -> no fire
+	time.Sleep(20 * time.Millisecond)
+	if ai.count() != 0 {
+		t.Fatalf("non-matching minute must not fire, got %d", ai.count())
+	}
+
+	sc.tick(ctx, at(t, "2026-07-01 12:35")) // matches */5
+	if !waitFor(func() bool { return ai.count() == 1 }) {
+		t.Fatalf("matching minute must fire once, got %d", ai.count())
+	}
+}
+
+// TestSchedulerRecordsRun: a scheduled run is persisted to the run history, just
+// like an HTTP run — the scheduler shares runAgent.
+func TestSchedulerRecordsRun(t *testing.T) {
+	ai := &countingAI{}
+	sc := schedSvc(t, ai, longRunning("acme", "cron", "* * * * *"))
+	ctx := context.Background()
+	sc.tick(ctx, at(t, "2026-07-01 12:00"))
+	if !waitFor(func() bool {
+		runs, _ := sc.svc.store.ListRuns(ctx, "acme", "cron", 10)
+		return len(runs) == 1 && runs[0].Status == "ok"
+	}) {
+		runs, _ := sc.svc.store.ListRuns(ctx, "acme", "cron", 10)
+		t.Fatalf("scheduled run not recorded: %+v", runs)
+	}
+}
+
+// TestSchedulerBackoffOnFailure: after a failed run the agent is skipped for a
+// growing number of ticks, so a broken agent stops hammering. The first failing
+// tick fires; the immediately-following matching tick is skipped (backoff=1).
+func TestSchedulerBackoffOnFailure(t *testing.T) {
+	ai := &countingAI{fail: true}
+	sc := schedSvc(t, ai, longRunning("acme", "cron", "* * * * *"))
+	ctx := context.Background()
+
+	sc.tick(ctx, at(t, "2026-07-01 12:00"))
+	if !waitFor(func() bool { return ai.count() == 1 }) {
+		t.Fatalf("first tick should attempt the run, got %d", ai.count())
+	}
+	// Wait for the failure to register the backoff window.
+	if !waitFor(func() bool {
+		sc.mu.Lock()
+		defer sc.mu.Unlock()
+		st := sc.states[stateKey("acme", "cron")]
+		return st != nil && st.failstreak == 1 && st.skipRemain == 1
+	}) {
+		t.Fatal("failure should set failstreak=1, skipRemain=1")
+	}
+	// Next matching tick is consumed by backoff -> no new run.
+	sc.tick(ctx, at(t, "2026-07-01 12:01"))
+	time.Sleep(20 * time.Millisecond)
+	if ai.count() != 1 {
+		t.Fatalf("backoff tick must not fire, got %d", ai.count())
+	}
+	// The tick after that (skip exhausted) fires again.
+	sc.tick(ctx, at(t, "2026-07-01 12:02"))
+	if !waitFor(func() bool { return ai.count() == 2 }) {
+		t.Fatalf("post-backoff tick should fire, got %d", ai.count())
+	}
+}
+
+// TestSchedulerConcurrencyCap: a slow run holds the single per-agent slot, so a
+// second matching tick while it is in flight does NOT start a second run.
+func TestSchedulerConcurrencyCap(t *testing.T) {
+	ai := &countingAI{block: make(chan struct{})}
+	sc := schedSvc(t, ai, longRunning("acme", "cron", "* * * * *"))
+	ctx := context.Background()
+
+	sc.tick(ctx, at(t, "2026-07-01 12:00")) // starts run #1, which blocks
+	if !waitFor(func() bool { return ai.count() == 1 }) {
+		t.Fatalf("first run should start, got %d", ai.count())
+	}
+	sc.tick(ctx, at(t, "2026-07-01 12:01")) // slot busy -> no second run
+	time.Sleep(20 * time.Millisecond)
+	if ai.count() != 1 {
+		t.Fatalf("concurrency cap breached: %d runs in flight", ai.count())
+	}
+	close(ai.block) // let run #1 finish
+	if !waitFor(func() bool {
+		sc.mu.Lock()
+		defer sc.mu.Unlock()
+		st := sc.states[stateKey("acme", "cron")]
+		return st != nil && st.inFlight == 0
+	}) {
+		t.Fatal("in-flight count should drain to 0 after completion")
+	}
+}
+
+// TestSchedulerBillsScheduledRun: a scheduled tick goes through the SAME gate +
+// meter as an HTTP run — a funded agent's scheduled run debits its OWN org via
+// commerce (product=agent), proving the billing path is live on the cron path,
+// not just the HTTP handler (Red INFO-2).
+func TestSchedulerBillsScheduledRun(t *testing.T) {
+	bs := &billServer{available: 100000}
+	m, err := metering.New(metering.Config{BaseURL: bs.start(t), Token: "svc-tok", Org: "hanzo"})
+	if err != nil {
+		t.Fatalf("metering.New: %v", err)
+	}
+	s := &svc{
+		store: testStore(t),
+		ai:    &countingAI{},
+		log:   luxlog.New("test"),
+		bill:  cloud.NewResourceMeter(cloud.Deps{Metering: m, Logger: luxlog.New("test")}, meterKind),
+	}
+	if err := s.store.Create(context.Background(), longRunning("acme", "cron", "* * * * *")); err != nil {
+		t.Fatalf("seed: %v", err)
+	}
+	sc := newScheduler(s, luxlog.New("test"))
+
+	sc.tick(context.Background(), at(t, "2026-07-01 12:00"))
+	if !waitForDebit(func() bool { return bs.debits() == 1 }) {
+		t.Fatalf("a scheduled run on a funded org must debit once, got %d", bs.debits())
+	}
+	org, ubody := bs.lastDebit()
+	if org != "acme" {
+		t.Fatalf("scheduled debit org = %q, want the agent's own org acme", org)
+	}
+	var u struct {
+		User     string `json:"user"`
+		Provider string `json:"provider"`
+		Actor    string `json:"actor"`
+	}
+	_ = json.Unmarshal(ubody, &u)
+	if u.User != "acme" || u.Provider != meterKind {
+		t.Fatalf("scheduled debit user/provider = %q/%q, want acme/%s", u.User, u.Provider, meterKind)
+	}
+	// The actor MUST be the "scheduler:" namespace, never a bare "org/sub" that
+	// could be mistaken for a validated interactive principal (Red LOW-2).
+	if !strings.HasPrefix(u.Actor, schedulerActor+":") {
+		t.Fatalf("scheduled actor = %q, want a %q-prefixed (non-principal) actor", u.Actor, schedulerActor)
+	}
+}
+
+// TestSchedulerGatesUnfundedRun: a scheduled run on an unfunded org is gated
+// (fail-closed) so the model NEVER runs and nothing is debited — an unfunded
+// long-running agent can't burn free inference every minute.
+func TestSchedulerGatesUnfundedRun(t *testing.T) {
+	bs := &billServer{available: 0}
+	m, _ := metering.New(metering.Config{BaseURL: bs.start(t), Token: "t", Org: "hanzo"})
+	ai := &countingAI{}
+	s := &svc{
+		store: testStore(t),
+		ai:    ai,
+		log:   luxlog.New("test"),
+		bill:  cloud.NewResourceMeter(cloud.Deps{Metering: m, Logger: luxlog.New("test")}, meterKind),
+	}
+	if err := s.store.Create(context.Background(), longRunning("acme", "cron", "* * * * *")); err != nil {
+		t.Fatalf("seed: %v", err)
+	}
+	sc := newScheduler(s, luxlog.New("test"))
+
+	sc.tick(context.Background(), at(t, "2026-07-01 12:00"))
+	time.Sleep(40 * time.Millisecond)
+	if ai.count() != 0 {
+		t.Fatalf("unfunded scheduled run must NOT invoke the model, got %d", ai.count())
+	}
+	if bs.debits() != 0 {
+		t.Fatalf("unfunded scheduled run must not debit, got %d", bs.debits())
+	}
+}
+
+// TestSchedulerStopDrainsCleanly: stop() with an un-expired ctx cancels the loop
+// and waits for the (fast) in-flight run to finish before returning — the drain
+// path that lets Shutdown close the store safely.
+func TestSchedulerStopDrainsCleanly(t *testing.T) {
+	ai := &countingAI{}
+	sc := schedSvc(t, ai, longRunning("acme", "cron", "* * * * *"))
+	sc.start()
+	// Fire one run via a direct tick, then stop — stop must return after drain.
+	sc.tick(context.Background(), at(t, "2026-07-01 12:00"))
+	done := make(chan struct{})
+	go func() { sc.stop(context.Background()); close(done) }()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("stop() did not return — drain hung")
+	}
+	// After a clean stop, no run goroutine is left holding a slot.
+	sc.mu.Lock()
+	st := sc.states[stateKey("acme", "cron")]
+	inFlight := 0
+	if st != nil {
+		inFlight = st.inFlight
+	}
+	sc.mu.Unlock()
+	if inFlight != 0 {
+		t.Fatalf("after drain inFlight=%d, want 0", inFlight)
+	}
+}
+
+// TestSchedulerStopHonorsDeadline: a run that IGNORES cancellation (blocks) must
+// not hang stop() past the caller's deadline — stop returns at the ctx deadline
+// rather than waiting the full runTimeout.
+func TestSchedulerStopHonorsDeadline(t *testing.T) {
+	ai := &countingAI{block: make(chan struct{})}
+	sc := schedSvc(t, ai, longRunning("acme", "cron", "* * * * *"))
+	sc.start()
+	sc.tick(context.Background(), at(t, "2026-07-01 12:00"))
+	if !waitFor(func() bool { return ai.count() == 1 }) {
+		t.Fatal("run should have started")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+	sc.stop(ctx) // the run is blocked and ignores ctx; stop must return at deadline
+	if elapsed := time.Since(start); elapsed > 2*time.Second {
+		t.Fatalf("stop() waited %v — did not honor the 100ms deadline", elapsed)
+	}
+	// Release the stuck run and let it fully drain BEFORE the test's store
+	// cleanup, so the late InsertRun can't race a closed DB.
+	close(ai.block)
+	sc.wg.Wait()
+}
+
+// TestSchedulerOnlyLongRunning: a one-shot agent is never fired by the
+// scheduler even if its (dropped) schedule would have matched.
+func TestSchedulerOnlyLongRunning(t *testing.T) {
+	ai := &countingAI{}
+	one := mk("acme", "one") // one-shot default, no schedule
+	sc := schedSvc(t, ai, one)
+	sc.tick(context.Background(), at(t, "2026-07-01 12:00"))
+	time.Sleep(20 * time.Millisecond)
+	if ai.count() != 0 {
+		t.Fatalf("one-shot agent must never be scheduled, got %d", ai.count())
+	}
+}

clients/agents/sessions.go

@@ -0,0 +1,704 @@
+package agents
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/zap-proto/zip"
+)
+
+// This file mounts the LIVE agent-session control plane under /v1/agents/sessions
+// — the canonical registry every surface (the @hanzo/dev CLI outer agent,
+// hanzo.bot, console, chat, app) hangs off. It is the VIEW + control + ZAP-stream
+// layer over durable execution; the durable run itself is a hanzoai/tasks
+// workflow (see sessions_tasks.go), never a bespoke scheduler here.
+//
+//	POST   /v1/agents/sessions              register a session (opt parentSessionId) -> Session
+//	GET    /v1/agents/sessions              list live sessions (filter root/parent/status) -> {sessions:[...]}
+//	GET    /v1/agents/sessions/stream       SSE feed of session+event updates (rides ZAP)
+//	GET    /v1/agents/sessions/:id          detail + direct children + recent events -> SessionDetail
+//	PATCH  /v1/agents/sessions/:id          update status/title -> Session
+//	GET    /v1/agents/sessions/:id/tree     the full subagent-flow graph -> TreeNode
+//	POST   /v1/agents/sessions/:id/events   append an event (message/tool-call/spawn/log) -> Event
+//	POST   /v1/agents/sessions/:id/{pause,resume,stop,message}  control command -> {command,event,forwarded}
+//
+// Every route is org-scoped through principal.Tenant (a validated principal AND
+// a non-empty org), so cross-tenant reads/writes/control are refused fail-closed.
+
+// Event kinds — the closed vocabulary of a session's ordered log.
+const (
+	KindMessage  = "message"
+	KindToolCall = "tool-call"
+	KindSpawn    = "spawn"
+	KindLog      = "log"
+	KindStatus   = "status"
+	KindControl  = "control"
+)
+
+// Control commands — the closed vocabulary of remote steering.
+const (
+	CmdPause   = "pause"
+	CmdResume  = "resume"
+	CmdStop    = "stop"
+	CmdMessage = "message"
+)
+
+const (
+	maxTitle        = 512
+	maxAgentLabel   = 128
+	maxActor        = 256
+	maxSessionID    = 128
+	maxWorkflowRef  = 256
+	maxEventPayload = 64 * 1024
+	maxControlMsg   = 16 * 1024
+	recentEvents    = 50
+	treeNodeCap     = 10000
+)
+
+func validKind(k string) bool {
+	switch k {
+	case KindMessage, KindToolCall, KindSpawn, KindLog, KindStatus, KindControl:
+		return true
+	}
+	return false
+}
+
+// ---- HTTP shapes (the published contract) ----
+
+type sessionView struct {
+	ID              string `json:"id"`
+	Agent           string `json:"agent"`
+	Actor           string `json:"actor,omitempty"`
+	Status          string `json:"status"`
+	ParentSessionID string `json:"parentSessionId,omitempty"`
+	RootSessionID   string `json:"rootSessionId"`
+	Title           string `json:"title,omitempty"`
+	TaskWorkflowID  string `json:"taskWorkflowId,omitempty"`
+	TaskRunID       string `json:"taskRunId,omitempty"`
+	Events          int    `json:"events"`
+	Children        int    `json:"children"`
+	StartedAt       string `json:"startedAt"`
+	EndedAt         string `json:"endedAt,omitempty"`
+	CreatedAt       string `json:"createdAt"`
+	UpdatedAt       string `json:"updatedAt"`
+}
+
+type eventView struct {
+	ID        string          `json:"id"`
+	SessionID string          `json:"sessionId"`
+	Seq       int64           `json:"seq"`
+	Kind      string          `json:"kind"`
+	Actor     string          `json:"actor,omitempty"`
+	Payload   json.RawMessage `json:"payload,omitempty"`
+	CreatedAt string          `json:"createdAt"`
+}
+
+type sessionDetail struct {
+	sessionView
+	Children     []sessionView `json:"childSessions"`
+	RecentEvents []eventView   `json:"recentEvents"`
+}
+
+// treeNode is one node of the subagent-flow graph: a session plus its children,
+// recursively. Node = {session, children:[...]} — the session's own Children int
+// is the direct fan-out count, the children array is the materialised subtree.
+type treeNode struct {
+	Session  sessionView `json:"session"`
+	Children []treeNode  `json:"children"`
+}
+
+func toSessionView(x Session, events, children int) sessionView {
+	return sessionView{
+		ID: x.ID, Agent: x.Agent, Actor: x.Actor, Status: x.Status,
+		ParentSessionID: x.ParentID, RootSessionID: x.RootID, Title: x.Title,
+		TaskWorkflowID: x.TaskWorkflowID, TaskRunID: x.TaskRunID,
+		Events: events, Children: children,
+		StartedAt: rfc3339(x.StartedAt), EndedAt: rfc3339(x.EndedAt),
+		CreatedAt: rfc3339(x.CreatedAt), UpdatedAt: rfc3339(x.UpdatedAt),
+	}
+}
+
+func toEventView(e Event) eventView {
+	var p json.RawMessage
+	if e.Payload != "" {
+		p = json.RawMessage(e.Payload)
+	}
+	return eventView{
+		ID: e.ID, SessionID: e.SessionID, Seq: e.Seq, Kind: e.Kind, Actor: e.Actor,
+		Payload: p, CreatedAt: rfc3339(e.CreatedAt),
+	}
+}
+
+// mountSessions registers the sessions routes. It MUST be called before the
+// /v1/agents/:name wildcard (Fiber matches in registration order, so a bare
+// :name would otherwise capture "sessions"). Within the block, the static
+// /stream route precedes the /:id param for the same reason.
+func (s *svc) mountSessions(app *zip.App) {
+	app.Post("/v1/agents/sessions", s.registerSession)
+	app.Get("/v1/agents/sessions", s.listSessions)
+	app.Get("/v1/agents/sessions/stream", s.sessionsStream)
+	app.Get("/v1/agents/sessions/:id", s.getSession)
+	app.Patch("/v1/agents/sessions/:id", s.patchSession)
+	app.Get("/v1/agents/sessions/:id/tree", s.sessionTree)
+	app.Post("/v1/agents/sessions/:id/events", s.appendSessionEvent)
+	app.Post("/v1/agents/sessions/:id/pause", s.pauseSession)
+	app.Post("/v1/agents/sessions/:id/resume", s.resumeSession)
+	app.Post("/v1/agents/sessions/:id/stop", s.stopSession)
+	app.Post("/v1/agents/sessions/:id/message", s.messageSession)
+}
+
+func idParam(c *zip.Ctx) string { return strings.TrimSpace(c.Param("id")) }
+
+// ---- register ----
+
+type registerReq struct {
+	Agent           string `json:"agent"`
+	Actor           string `json:"actor"`
+	Title           string `json:"title"`
+	Status          string `json:"status"`
+	ParentSessionID string `json:"parentSessionId"`
+	TaskWorkflowID  string `json:"taskWorkflowId"`
+	TaskRunID       string `json:"taskRunId"`
+}
+
+func (s *svc) registerSession(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	var body registerReq
+	if err := c.Bind(&body); err != nil {
+		return err
+	}
+	agent := strings.TrimSpace(body.Agent)
+	if agent == "" {
+		return zip.ErrBadRequest("agent is required")
+	}
+	if len(agent) > maxAgentLabel {
+		return zip.ErrBadRequest("agent too long")
+	}
+	if len(body.Title) > maxTitle {
+		return zip.ErrBadRequest("title too long")
+	}
+	status := strings.TrimSpace(body.Status)
+	if status == "" {
+		status = StatusRunning
+	}
+	if !validStatus(status) {
+		return zip.ErrBadRequest("status must be running|paused|done|error")
+	}
+	actor := strings.TrimSpace(body.Actor)
+	if actor == "" {
+		actor = billingActor(org, c.User())
+	}
+	if len(actor) > maxActor {
+		return zip.ErrBadRequest("actor too long")
+	}
+	if len(body.TaskWorkflowID) > maxWorkflowRef || len(body.TaskRunID) > maxWorkflowRef {
+		return zip.ErrBadRequest("task workflow/run reference too long")
+	}
+
+	id, err := genID("sess")
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "rng: %v", err)
+	}
+	now := time.Now().Unix()
+	x := Session{
+		ID: id, Org: org, Agent: agent, Actor: actor, Status: status,
+		Title:          strings.TrimSpace(body.Title),
+		TaskWorkflowID: strings.TrimSpace(body.TaskWorkflowID),
+		TaskRunID:      strings.TrimSpace(body.TaskRunID),
+		StartedAt:      now, CreatedAt: now, UpdatedAt: now,
+	}
+	if isTerminalStatus(status) {
+		x.EndedAt = now
+	}
+
+	// Subagent linkage. A parent MUST exist IN THE SAME ORG — the tree can never
+	// cross a tenant boundary. RootID is inherited from the parent (all nodes in
+	// one flow share it); a session with no parent is itself a root.
+	parent := strings.TrimSpace(body.ParentSessionID)
+	if parent != "" {
+		p, perr := s.store.GetSession(c.Context(), org, parent)
+		if perr == errSessionNotFound {
+			return zip.ErrBadRequest("parentSessionId not found in this org")
+		}
+		if perr != nil {
+			return zip.Errorf(http.StatusInternalServerError, "parent: %v", perr)
+		}
+		x.ParentID = p.ID
+		x.RootID = p.RootID
+	} else {
+		x.RootID = id
+	}
+
+	if err := s.store.CreateSession(c.Context(), x); err != nil {
+		if err == errParentNotFound {
+			return zip.ErrBadRequest("parentSessionId not found in this org")
+		}
+		return zip.Errorf(http.StatusInternalServerError, "persist: %v", err)
+	}
+	s.publishSession(x, 0, 0)
+	return c.JSON(http.StatusCreated, toSessionView(x, 0, 0))
+}
+
+// ---- list ----
+
+func (s *svc) listSessions(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	f := SessionFilter{
+		Root:   trimField(c.Query("root")),
+		Parent: trimField(c.Query("parent")),
+		Status: trimField(c.Query("status")),
+		Limit:  queryInt(c, "limit"),
+	}
+	if f.Status != "" && !validStatus(f.Status) {
+		return zip.ErrBadRequest("status must be running|paused|done|error")
+	}
+	rows, err := s.store.ListSessions(c.Context(), org, f)
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "list: %v", err)
+	}
+	out := make([]sessionView, 0, len(rows))
+	for _, x := range rows {
+		ev, _ := s.store.CountEvents(c.Context(), org, x.ID)
+		ch, _ := s.store.CountChildren(c.Context(), org, x.ID)
+		out = append(out, toSessionView(x, ev, ch))
+	}
+	return c.JSON(http.StatusOK, map[string]any{"sessions": out})
+}
+
+// ---- detail ----
+
+func (s *svc) getSession(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	id := idParam(c)
+	if len(id) > maxSessionID {
+		return zip.ErrNotFound("session not found")
+	}
+	x, err := s.store.GetSession(c.Context(), org, id)
+	if err == errSessionNotFound {
+		return zip.ErrNotFound("session not found")
+	}
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "get: %v", err)
+	}
+	kids, err := s.store.ListSessions(c.Context(), org, SessionFilter{Parent: id})
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "children: %v", err)
+	}
+	events, err := s.store.ListEvents(c.Context(), org, id, 0, recentEvents)
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "events: %v", err)
+	}
+	evCount, _ := s.store.CountEvents(c.Context(), org, id)
+	kidViews := make([]sessionView, 0, len(kids))
+	for _, k := range kids {
+		kc, _ := s.store.CountChildren(c.Context(), org, k.ID)
+		ke, _ := s.store.CountEvents(c.Context(), org, k.ID)
+		kidViews = append(kidViews, toSessionView(k, ke, kc))
+	}
+	evViews := make([]eventView, 0, len(events))
+	for _, e := range events {
+		evViews = append(evViews, toEventView(e))
+	}
+	return c.JSON(http.StatusOK, sessionDetail{
+		sessionView:  toSessionView(x, evCount, len(kids)),
+		Children:     kidViews,
+		RecentEvents: evViews,
+	})
+}
+
+// ---- tree ----
+
+func (s *svc) sessionTree(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	id := idParam(c)
+	if len(id) > maxSessionID {
+		return zip.ErrNotFound("session not found")
+	}
+	x, err := s.store.GetSession(c.Context(), org, id)
+	if err == errSessionNotFound {
+		return zip.ErrNotFound("session not found")
+	}
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "get: %v", err)
+	}
+	// One indexed query pulls the whole tree (same RootID); assemble in memory.
+	nodes, err := s.store.ListTree(c.Context(), org, x.RootID, treeNodeCap)
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "tree: %v", err)
+	}
+	counts, err := s.store.EventCountsByRoot(c.Context(), org, x.RootID)
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "counts: %v", err)
+	}
+	return c.JSON(http.StatusOK, buildSubtree(nodes, counts, id))
+}
+
+// buildSubtree assembles the flat tree rows into the node rooted at rootAtID.
+// children map is built once; each node's Children int (fan-out) comes from the
+// map, its Events from counts. A missing rootAtID yields an empty node (the
+// caller already verified the session exists, so this is defensive).
+func buildSubtree(nodes []Session, counts map[string]int, rootAtID string) treeNode {
+	childrenOf := map[string][]Session{}
+	byID := map[string]Session{}
+	for _, n := range nodes {
+		byID[n.ID] = n
+		childrenOf[n.ParentID] = append(childrenOf[n.ParentID], n)
+	}
+	var build func(x Session) treeNode
+	build = func(x Session) treeNode {
+		kids := childrenOf[x.ID]
+		node := treeNode{Session: toSessionView(x, counts[x.ID], len(kids))}
+		for _, k := range kids {
+			node.Children = append(node.Children, build(k))
+		}
+		return node
+	}
+	root, ok := byID[rootAtID]
+	if !ok {
+		return treeNode{}
+	}
+	return build(root)
+}
+
+// ---- patch (status/title, surface-owned truth) ----
+
+type patchSessionReq struct {
+	Status *string `json:"status"`
+	Title  *string `json:"title"`
+}
+
+func (s *svc) patchSession(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	id := idParam(c)
+	x, err := s.store.GetSession(c.Context(), org, id)
+	if err == errSessionNotFound {
+		return zip.ErrNotFound("session not found")
+	}
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "get: %v", err)
+	}
+	var body patchSessionReq
+	if err := c.Bind(&body); err != nil {
+		return err
+	}
+	if body.Status != nil {
+		ns := strings.TrimSpace(*body.Status)
+		if !validStatus(ns) {
+			return zip.ErrBadRequest("status must be running|paused|done|error")
+		}
+		// A finished session stays finished (truthful, monotonic terminal state):
+		// reopening a done/error run would fabricate liveness.
+		if isTerminalStatus(x.Status) && ns != x.Status {
+			return zip.Errorf(http.StatusConflict, "session is %s; cannot change status", x.Status)
+		}
+		x.Status = ns
+		if isTerminalStatus(ns) && x.EndedAt == 0 {
+			x.EndedAt = time.Now().Unix()
+		}
+	}
+	if body.Title != nil {
+		if len(*body.Title) > maxTitle {
+			return zip.ErrBadRequest("title too long")
+		}
+		x.Title = strings.TrimSpace(*body.Title)
+	}
+	x.UpdatedAt = time.Now().Unix()
+	if err := s.store.UpdateSession(c.Context(), x); err != nil {
+		if err == errSessionNotFound {
+			return zip.ErrNotFound("session not found")
+		}
+		return zip.Errorf(http.StatusInternalServerError, "update: %v", err)
+	}
+	ev, _ := s.store.CountEvents(c.Context(), org, id)
+	ch, _ := s.store.CountChildren(c.Context(), org, id)
+	s.publishSession(x, ev, ch)
+	return c.JSON(http.StatusOK, toSessionView(x, ev, ch))
+}
+
+// ---- append event ----
+
+type eventReq struct {
+	Kind    string          `json:"kind"`
+	Actor   string          `json:"actor"`
+	Payload json.RawMessage `json:"payload"`
+}
+
+func (s *svc) appendSessionEvent(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	id := idParam(c)
+	x, err := s.store.GetSession(c.Context(), org, id)
+	if err == errSessionNotFound {
+		return zip.ErrNotFound("session not found")
+	}
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "get: %v", err)
+	}
+	var body eventReq
+	if err := c.Bind(&body); err != nil {
+		return err
+	}
+	kind := strings.TrimSpace(body.Kind)
+	if !validKind(kind) {
+		return zip.ErrBadRequest("kind must be message|tool-call|spawn|log|status|control")
+	}
+	if len(body.Payload) > maxEventPayload {
+		return zip.ErrBadRequest("payload too large")
+	}
+	if len(body.Payload) > 0 && !json.Valid(body.Payload) {
+		return zip.ErrBadRequest("payload must be valid JSON")
+	}
+	actor := strings.TrimSpace(body.Actor)
+	if actor == "" {
+		actor = billingActor(org, c.User())
+	}
+	if len(actor) > maxActor {
+		return zip.ErrBadRequest("actor too long")
+	}
+	evID, err := genID("evt")
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "rng: %v", err)
+	}
+	e, err := s.store.AppendEvent(c.Context(), Event{
+		ID: evID, SessionID: id, Org: org, Kind: kind, Actor: actor,
+		Payload: string(body.Payload), CreatedAt: time.Now().Unix(),
+	})
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "append: %v", err)
+	}
+	s.publishEvent(org, x.RootID, e)
+	return c.JSON(http.StatusCreated, toEventView(e))
+}
+
+// ---- control (record intent + forward to the tasks engine when task-backed) ----
+
+type controlReq struct {
+	Message string          `json:"message"`
+	Payload json.RawMessage `json:"payload"`
+}
+
+type controlPayload struct {
+	Command string          `json:"command"`
+	Message string          `json:"message,omitempty"`
+	Payload json.RawMessage `json:"payload,omitempty"`
+}
+
+func (s *svc) pauseSession(c *zip.Ctx) error   { return s.control(c, CmdPause) }
+func (s *svc) resumeSession(c *zip.Ctx) error  { return s.control(c, CmdResume) }
+func (s *svc) stopSession(c *zip.Ctx) error    { return s.control(c, CmdStop) }
+func (s *svc) messageSession(c *zip.Ctx) error { return s.control(c, CmdMessage) }
+
+// control records a steering command as a durable control event (the intent the
+// running surface consumes) and, when the session is backed by a hanzoai/tasks
+// workflow AND a tasks backend is wired, forwards it to the engine's signal/
+// cancel API. Org/actor-authorized: principal.Tenant already requires a validated
+// principal AND same-org ownership of the session, so no other tenant can steer.
+func (s *svc) control(c *zip.Ctx, command string) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	id := idParam(c)
+	x, err := s.store.GetSession(c.Context(), org, id)
+	if err == errSessionNotFound {
+		return zip.ErrNotFound("session not found")
+	}
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "get: %v", err)
+	}
+	if isTerminalStatus(x.Status) {
+		return zip.Errorf(http.StatusConflict, "session is %s; cannot %s a finished session", x.Status, command)
+	}
+	// The control body is optional (pause/resume/stop often carry none); only
+	// parse when present so a bodyless command is not a 400.
+	var body controlReq
+	if len(c.Body()) > 0 {
+		if err := c.Bind(&body); err != nil {
+			return err
+		}
+	}
+	if len(body.Message) > maxControlMsg {
+		return zip.ErrBadRequest("message too long")
+	}
+	if len(body.Payload) > maxEventPayload {
+		return zip.ErrBadRequest("payload too large")
+	}
+	if len(body.Payload) > 0 && !json.Valid(body.Payload) {
+		return zip.ErrBadRequest("payload must be valid JSON")
+	}
+	if command == CmdMessage && strings.TrimSpace(body.Message) == "" && len(body.Payload) == 0 {
+		return zip.ErrBadRequest("message requires a 'message' or 'payload'")
+	}
+
+	actor := billingActor(org, c.User())
+	cp, _ := json.Marshal(controlPayload{Command: command, Message: body.Message, Payload: body.Payload})
+	evID, err := genID("evt")
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "rng: %v", err)
+	}
+	e, err := s.store.AppendEvent(c.Context(), Event{
+		ID: evID, SessionID: id, Org: org, Kind: KindControl, Actor: actor,
+		Payload: string(cp), CreatedAt: time.Now().Unix(),
+	})
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "record control: %v", err)
+	}
+	s.publishEvent(org, x.RootID, e)
+
+	// Forward to the durable-execution engine when this session is task-backed.
+	// The intent is ALREADY durably recorded above, so a forward failure is
+	// reported (502) without losing the command; a session with no workflow link
+	// or no wired backend is record-only (stream-consuming surfaces act on it).
+	forwarded := false
+	if x.TaskWorkflowID != "" && s.tasks != nil && s.tasks.Enabled() {
+		var ferr error
+		if command == CmdStop {
+			ferr = s.tasks.Cancel(c.Context(), x.TaskWorkflowID, x.TaskRunID, reasonOf(body.Message))
+		} else {
+			ferr = s.tasks.Signal(c.Context(), x.TaskWorkflowID, x.TaskRunID, command, signalPayload(body))
+		}
+		if ferr != nil {
+			return zip.Errorf(http.StatusBadGateway, "control recorded but tasks forward failed: %v", ferr)
+		}
+		forwarded = true
+	}
+	return c.JSON(http.StatusOK, map[string]any{
+		"command": command, "event": toEventView(e), "forwarded": forwarded,
+	})
+}
+
+func reasonOf(msg string) string {
+	if m := strings.TrimSpace(msg); m != "" {
+		return m
+	}
+	return "stopped via control plane"
+}
+
+func signalPayload(b controlReq) []byte {
+	if len(b.Payload) > 0 {
+		return b.Payload
+	}
+	if b.Message != "" {
+		m, _ := json.Marshal(b.Message)
+		return m
+	}
+	return nil
+}
+
+// ---- run integration (#5): a /v1/agents/:name/run opens a root session ----
+
+// openRunSession records a completed agent run as a ROOT session so every run is
+// visible in the same registry the @hanzo/dev outer-agent flows use. Best-effort:
+// a bookkeeping failure NEVER fails the run (the run + its billing already
+// happened). A cloud one-shot run is a synchronous completion, so the session is
+// born terminal with one log event; TaskWorkflowID is left empty because the run
+// is not (yet) a tasks workflow — when runs are promoted to hanzoai/tasks
+// ExecuteWorkflow, set TaskWorkflowID/TaskRunID here from the workflow handle.
+func (s *svc) openRunSession(ctx context.Context, a Agent, r Run, actor string) {
+	if s.store == nil {
+		return
+	}
+	status := StatusDone
+	if r.Status != "ok" {
+		status = StatusError
+	}
+	id, err := genID("sess")
+	if err != nil {
+		s.log.Warn("run session: rng", "err", err)
+		return
+	}
+	ts := r.CreatedAt
+	if ts == 0 {
+		ts = time.Now().Unix()
+	}
+	x := Session{
+		ID: id, Org: a.Org, Agent: a.Name, Actor: actor, Status: status,
+		RootID: id, Title: runTitle(r.Input),
+		StartedAt: ts, EndedAt: ts, CreatedAt: ts, UpdatedAt: ts,
+	}
+	if err := s.store.CreateSession(ctx, x); err != nil {
+		s.log.Warn("run session: create", "org", a.Org, "agent", a.Name, "err", err)
+		return
+	}
+	payload, _ := json.Marshal(map[string]any{
+		"runId": r.ID, "status": r.Status, "model": r.Model,
+		"durationMs": r.DurationMs, "error": r.Error,
+	})
+	evID, err := genID("evt")
+	if err != nil {
+		s.publishSession(x, 0, 0)
+		return
+	}
+	e, aerr := s.store.AppendEvent(ctx, Event{
+		ID: evID, SessionID: id, Org: a.Org, Kind: KindLog, Actor: actor,
+		Payload: string(payload), CreatedAt: ts,
+	})
+	s.publishSession(x, 1, 0)
+	if aerr == nil {
+		s.publishEvent(a.Org, x.RootID, e)
+	}
+}
+
+func runTitle(input string) string {
+	t := strings.TrimSpace(input)
+	if t == "" {
+		return "agent run"
+	}
+	if len(t) > 120 {
+		t = t[:120]
+	}
+	return t
+}
+
+// ---- stream publish helpers (nil-safe: a bus-less svc, e.g. a direct-construct
+// unit test, simply skips the live fan-out; the store is still the truth) ----
+
+func (s *svc) publishSession(x Session, events, children int) {
+	if s.bus == nil {
+		return
+	}
+	v := toSessionView(x, events, children)
+	s.bus.publish(streamUpdate{Org: x.Org, RootID: x.RootID, Type: "session", Session: &v})
+}
+
+func (s *svc) publishEvent(org, rootID string, e Event) {
+	if s.bus == nil {
+		return
+	}
+	v := toEventView(e)
+	s.bus.publish(streamUpdate{Org: org, RootID: rootID, Type: "event", Event: &v})
+}
+
+// ---- small query helpers ----
+
+func trimField(v string) string { return strings.TrimSpace(v) }
+
+func queryInt(c *zip.Ctx, name string) int {
+	if q := strings.TrimSpace(c.Query(name)); q != "" {
+		if n, err := strconv.Atoi(q); err == nil && n >= 0 {
+			return n
+		}
+	}
+	return 0
+}

clients/agents/sessions_store.go

@@ -0,0 +1,390 @@
+package agents
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+)
+
+// A live agent-session is a running invocation — a cloud agent run, a bot loop,
+// or a @hanzo/dev CLI run spawning subagents. The SUBAGENT TREE is sessions
+// linked by ParentID: the outer agent is the root (ParentID==""), each spawned
+// subagent is a child, and RootID is the tree key every node in one flow shares.
+// It is the first-class, streamable form of the blue/red/cto fan-out tree.
+//
+// A session is NOT foreign-keyed to an agents row: an external surface (the
+// @hanzo/dev CLI) registers a session whose Agent is just a label, not a cloud
+// Agent definition. Tenant isolation is the Org column, enforced on every query
+// exactly like agents/runs — one file (agents.db), tenancy is the org.
+type Session struct {
+	ID        string
+	Org       string
+	Agent     string // agent name / type label (need not be a cloud Agent row)
+	Actor     string // the principal that started it (validated user, or a bound SA)
+	Status    string // running|paused|done|error
+	ParentID  string // "" for a root (the outer agent)
+	RootID    string // the tree key; == ID for a root
+	Title     string
+	StartedAt int64
+	EndedAt   int64 // 0 until a terminal status is reached
+	CreatedAt int64
+	UpdatedAt int64
+
+	// TaskWorkflowID / TaskRunID link this session to the hanzoai/tasks durable
+	// workflow that actually EXECUTES it. This registry is the view/control/stream
+	// layer; durable execution (retries, resumability, scheduling) is owned by
+	// hanzoai/tasks — NOT by a bespoke scheduler here. A root session maps to a
+	// tasks workflow (ExecuteWorkflow); a subagent maps to a child workflow keyed
+	// by the same RootID. When these are set, control (pause/resume/stop/message)
+	// forwards to the tasks Signal/Cancel API (see svc.tasks). Empty = a surface
+	// that consumes control from the event stream instead (today's @hanzo/dev).
+	TaskWorkflowID string
+	TaskRunID      string
+}
+
+// Event is one entry in a session's ordered log: a model message, a tool call, a
+// subagent spawn, a free log line, a status change, or a control command the
+// running surface consumes. Seq is monotonic PER SESSION so a subscriber can
+// resume from its last-seen point; Org is denormalised so every read stays
+// org-scoped without a join back to the session row.
+type Event struct {
+	ID        string
+	SessionID string
+	Org       string
+	Seq       int64
+	Kind      string // message|tool-call|spawn|log|status|control
+	Actor     string
+	Payload   string // opaque JSON blob (validated well-formed, size-bounded)
+	CreatedAt int64
+}
+
+// Session status values. running/paused are live; done/error are terminal.
+const (
+	StatusRunning = "running"
+	StatusPaused  = "paused"
+	StatusDone    = "done"
+	StatusError   = "error"
+)
+
+func isTerminalStatus(s string) bool { return s == StatusDone || s == StatusError }
+
+func validStatus(s string) bool {
+	switch s {
+	case StatusRunning, StatusPaused, StatusDone, StatusError:
+		return true
+	}
+	return false
+}
+
+var (
+	errSessionNotFound = errors.New("agents: session not found")
+	errParentNotFound  = errors.New("agents: parent session not found")
+)
+
+// migrateSessions creates the session + event tables. Called from migrate() so
+// the ONE agents.db carries agents, runs, sessions and events — one store, one
+// tenancy column, no second DB handle. Idempotent (IF NOT EXISTS).
+func (s *Store) migrateSessions() error {
+	const ddl = `
+CREATE TABLE IF NOT EXISTS agent_sessions (
+  id               TEXT PRIMARY KEY,
+  org              TEXT NOT NULL,
+  agent            TEXT NOT NULL DEFAULT '',
+  actor            TEXT NOT NULL DEFAULT '',
+  status           TEXT NOT NULL DEFAULT 'running',
+  parent_id        TEXT NOT NULL DEFAULT '',
+  root_id          TEXT NOT NULL DEFAULT '',
+  title            TEXT NOT NULL DEFAULT '',
+  started_at       INTEGER NOT NULL,
+  ended_at         INTEGER NOT NULL DEFAULT 0,
+  created_at       INTEGER NOT NULL,
+  updated_at       INTEGER NOT NULL,
+  task_workflow_id TEXT NOT NULL DEFAULT '',
+  task_run_id      TEXT NOT NULL DEFAULT ''
+);
+CREATE INDEX IF NOT EXISTS ix_sessions_org_root ON agent_sessions(org, root_id, created_at);
+CREATE INDEX IF NOT EXISTS ix_sessions_org_parent ON agent_sessions(org, parent_id, created_at);
+CREATE INDEX IF NOT EXISTS ix_sessions_org_status ON agent_sessions(org, status, updated_at);
+
+CREATE TABLE IF NOT EXISTS agent_session_events (
+  id         TEXT PRIMARY KEY,
+  session_id TEXT NOT NULL,
+  org        TEXT NOT NULL,
+  seq        INTEGER NOT NULL,
+  kind       TEXT NOT NULL,
+  actor      TEXT NOT NULL DEFAULT '',
+  payload    TEXT NOT NULL DEFAULT '',
+  created_at INTEGER NOT NULL
+);
+CREATE UNIQUE INDEX IF NOT EXISTS ux_events_session_seq ON agent_session_events(session_id, seq);
+CREATE INDEX IF NOT EXISTS ix_events_org_session_seq ON agent_session_events(org, session_id, seq);
+`
+	if _, err := s.db.Exec(ddl); err != nil {
+		return fmt.Errorf("migrate sessions: %w", err)
+	}
+	return nil
+}
+
+const sessionCols = `id,org,agent,actor,status,parent_id,root_id,title,started_at,ended_at,created_at,updated_at,task_workflow_id,task_run_id`
+
+func scanSession(sc interface{ Scan(...any) error }) (Session, error) {
+	var x Session
+	err := sc.Scan(&x.ID, &x.Org, &x.Agent, &x.Actor, &x.Status, &x.ParentID, &x.RootID,
+		&x.Title, &x.StartedAt, &x.EndedAt, &x.CreatedAt, &x.UpdatedAt,
+		&x.TaskWorkflowID, &x.TaskRunID)
+	return x, err
+}
+
+// CreateSession inserts one session. When ParentID is set it MUST reference an
+// existing session IN THE SAME ORG — the caller resolves it via GetSession first
+// so a cross-tenant or dangling parent can never link a tree. RootID is derived
+// by the caller (parent's root, or self for a root); this method persists what it
+// is given after a final same-org sanity check on the parent.
+func (s *Store) CreateSession(ctx context.Context, x Session) error {
+	if x.ParentID != "" {
+		// Re-verify the parent under the SAME org inside the write path so a
+		// TOCTOU between the handler's lookup and here cannot smuggle a foreign
+		// or deleted parent into the tree (fail-closed).
+		var org string
+		err := s.db.QueryRowContext(ctx,
+			`SELECT org FROM agent_sessions WHERE id=? AND org=?`, x.ParentID, x.Org).Scan(&org)
+		if errors.Is(err, sql.ErrNoRows) {
+			return errParentNotFound
+		}
+		if err != nil {
+			return fmt.Errorf("verify parent: %w", err)
+		}
+	}
+	_, err := s.db.ExecContext(ctx,
+		`INSERT INTO agent_sessions (`+sessionCols+`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
+		x.ID, x.Org, x.Agent, x.Actor, x.Status, x.ParentID, x.RootID, x.Title,
+		x.StartedAt, x.EndedAt, x.CreatedAt, x.UpdatedAt, x.TaskWorkflowID, x.TaskRunID)
+	if err != nil {
+		return fmt.Errorf("insert session: %w", err)
+	}
+	return nil
+}
+
+// GetSession returns the (org,id) session or errSessionNotFound. The org is part
+// of the key so one tenant can never resolve another's session id.
+func (s *Store) GetSession(ctx context.Context, org, id string) (Session, error) {
+	row := s.db.QueryRowContext(ctx,
+		`SELECT `+sessionCols+` FROM agent_sessions WHERE org=? AND id=?`, org, id)
+	x, err := scanSession(row)
+	if errors.Is(err, sql.ErrNoRows) {
+		return Session{}, errSessionNotFound
+	}
+	if err != nil {
+		return Session{}, fmt.Errorf("get session: %w", err)
+	}
+	return x, nil
+}
+
+// SessionFilter selects a slice of an org's sessions. The fields are AND-ed; a
+// zero field is "any". Scope picks the structural axis:
+//   - Root set   -> every session in that tree (root_id == Root).
+//   - Parent set -> the direct children of Parent (parent_id == Parent).
+//   - neither    -> roots only (parent_id == ”), the outer-agent view.
+type SessionFilter struct {
+	Root   string
+	Parent string
+	Status string
+	Limit  int
+}
+
+// ListSessions returns an org's sessions per filter, newest first, capped.
+func (s *Store) ListSessions(ctx context.Context, org string, f SessionFilter) ([]Session, error) {
+	limit := f.Limit
+	if limit <= 0 || limit > 500 {
+		limit = 100
+	}
+	where := "org=?"
+	args := []any{org}
+	switch {
+	case f.Root != "":
+		where += " AND root_id=?"
+		args = append(args, f.Root)
+	case f.Parent != "":
+		where += " AND parent_id=?"
+		args = append(args, f.Parent)
+	default:
+		where += " AND parent_id=''"
+	}
+	if f.Status != "" {
+		where += " AND status=?"
+		args = append(args, f.Status)
+	}
+	args = append(args, limit)
+	rows, err := s.db.QueryContext(ctx,
+		`SELECT `+sessionCols+` FROM agent_sessions WHERE `+where+
+			` ORDER BY created_at DESC, id ASC LIMIT ?`, args...)
+	if err != nil {
+		return nil, fmt.Errorf("list sessions: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+	var out []Session
+	for rows.Next() {
+		x, err := scanSession(rows)
+		if err != nil {
+			return nil, fmt.Errorf("scan session: %w", err)
+		}
+		out = append(out, x)
+	}
+	return out, rows.Err()
+}
+
+// ListTree returns EVERY session in one org's tree (root_id == root), oldest
+// first so a caller can assemble parent→child in a single pass. Capped so a
+// pathological tree can't produce an unbounded response.
+func (s *Store) ListTree(ctx context.Context, org, root string, cap int) ([]Session, error) {
+	if cap <= 0 || cap > 10000 {
+		cap = 10000
+	}
+	rows, err := s.db.QueryContext(ctx,
+		`SELECT `+sessionCols+` FROM agent_sessions WHERE org=? AND root_id=?
+		 ORDER BY created_at ASC, id ASC LIMIT ?`, org, root, cap)
+	if err != nil {
+		return nil, fmt.Errorf("list tree: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+	var out []Session
+	for rows.Next() {
+		x, err := scanSession(rows)
+		if err != nil {
+			return nil, fmt.Errorf("scan session: %w", err)
+		}
+		out = append(out, x)
+	}
+	return out, rows.Err()
+}
+
+// UpdateSession persists status/title/ended_at for an existing (org,id) session.
+// Scoped by org so a cross-tenant id can never mutate another's session.
+func (s *Store) UpdateSession(ctx context.Context, x Session) error {
+	res, err := s.db.ExecContext(ctx,
+		`UPDATE agent_sessions SET status=?, title=?, ended_at=?, updated_at=?
+		 WHERE org=? AND id=?`,
+		x.Status, x.Title, x.EndedAt, x.UpdatedAt, x.Org, x.ID)
+	if err != nil {
+		return fmt.Errorf("update session: %w", err)
+	}
+	n, _ := res.RowsAffected()
+	if n == 0 {
+		return errSessionNotFound
+	}
+	return nil
+}
+
+// CountChildren returns how many DIRECT children a session has (its fan-out).
+func (s *Store) CountChildren(ctx context.Context, org, id string) (int, error) {
+	var n int
+	err := s.db.QueryRowContext(ctx,
+		`SELECT COUNT(*) FROM agent_sessions WHERE org=? AND parent_id=?`, org, id).Scan(&n)
+	if err != nil {
+		return 0, fmt.Errorf("count children: %w", err)
+	}
+	return n, nil
+}
+
+// AppendEvent inserts one event, allocating the next per-session Seq. The store
+// runs on a single connection (SetMaxOpenConns(1)) so the read-then-write of the
+// max seq is serialised; the UNIQUE(session_id,seq) index is the final backstop.
+// The session's updated_at is bumped in the SAME transaction so "last activity"
+// stays truthful. Returns the persisted event (with Seq/CreatedAt) for streaming.
+func (s *Store) AppendEvent(ctx context.Context, e Event) (Event, error) {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return Event{}, fmt.Errorf("begin: %w", err)
+	}
+	defer func() { _ = tx.Rollback() }()
+
+	var next int64
+	if err := tx.QueryRowContext(ctx,
+		`SELECT COALESCE(MAX(seq),0)+1 FROM agent_session_events WHERE session_id=?`,
+		e.SessionID).Scan(&next); err != nil {
+		return Event{}, fmt.Errorf("next seq: %w", err)
+	}
+	e.Seq = next
+	if _, err := tx.ExecContext(ctx,
+		`INSERT INTO agent_session_events (id,session_id,org,seq,kind,actor,payload,created_at)
+		 VALUES (?,?,?,?,?,?,?,?)`,
+		e.ID, e.SessionID, e.Org, e.Seq, e.Kind, e.Actor, e.Payload, e.CreatedAt); err != nil {
+		return Event{}, fmt.Errorf("insert event: %w", err)
+	}
+	if _, err := tx.ExecContext(ctx,
+		`UPDATE agent_sessions SET updated_at=? WHERE org=? AND id=?`,
+		e.CreatedAt, e.Org, e.SessionID); err != nil {
+		return Event{}, fmt.Errorf("bump session: %w", err)
+	}
+	if err := tx.Commit(); err != nil {
+		return Event{}, fmt.Errorf("commit: %w", err)
+	}
+	return e, nil
+}
+
+// ListEvents returns a session's events in Seq order (optionally only those with
+// Seq > since, so a subscriber resumes exactly where it dropped), capped.
+func (s *Store) ListEvents(ctx context.Context, org, sessionID string, since int64, limit int) ([]Event, error) {
+	if limit <= 0 || limit > 1000 {
+		limit = 200
+	}
+	rows, err := s.db.QueryContext(ctx,
+		`SELECT id,session_id,org,seq,kind,actor,payload,created_at
+		 FROM agent_session_events WHERE org=? AND session_id=? AND seq>?
+		 ORDER BY seq ASC LIMIT ?`, org, sessionID, since, limit)
+	if err != nil {
+		return nil, fmt.Errorf("list events: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+	var out []Event
+	for rows.Next() {
+		var e Event
+		if err := rows.Scan(&e.ID, &e.SessionID, &e.Org, &e.Seq, &e.Kind, &e.Actor,
+			&e.Payload, &e.CreatedAt); err != nil {
+			return nil, fmt.Errorf("scan event: %w", err)
+		}
+		out = append(out, e)
+	}
+	return out, rows.Err()
+}
+
+// EventCountsByRoot returns per-session event counts for EVERY session in one
+// org's tree (root_id == root) in a SINGLE grouped query — so materialising a
+// tree of N nodes with real per-node event counts costs one round trip, not N,
+// and never hits SQLite's bound-parameter limit (the join scopes by root_id, not
+// an IN list of ids).
+func (s *Store) EventCountsByRoot(ctx context.Context, org, root string) (map[string]int, error) {
+	rows, err := s.db.QueryContext(ctx,
+		`SELECT e.session_id, COUNT(*)
+		 FROM agent_session_events e
+		 JOIN agent_sessions s ON s.id = e.session_id AND s.org = e.org
+		 WHERE e.org=? AND s.root_id=?
+		 GROUP BY e.session_id`, org, root)
+	if err != nil {
+		return nil, fmt.Errorf("event counts by root: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+	out := map[string]int{}
+	for rows.Next() {
+		var id string
+		var n int
+		if err := rows.Scan(&id, &n); err != nil {
+			return nil, fmt.Errorf("scan count: %w", err)
+		}
+		out[id] = n
+	}
+	return out, rows.Err()
+}
+
+// CountEvents returns how many events a session has (the list rollup).
+func (s *Store) CountEvents(ctx context.Context, org, sessionID string) (int, error) {
+	var n int
+	err := s.db.QueryRowContext(ctx,
+		`SELECT COUNT(*) FROM agent_session_events WHERE org=? AND session_id=?`,
+		org, sessionID).Scan(&n)
+	if err != nil {
+		return 0, fmt.Errorf("count events: %w", err)
+	}
+	return n, nil
+}

clients/agents/sessions_stream.go

@@ -0,0 +1,203 @@
+package agents
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/zap-proto/zip"
+)
+
+// streamUpdate is one live update fanned out to subscribers: either a session
+// lifecycle change (register / status / control) or an appended event. Org and
+// RootID are carried so the bus can filter by tenant AND a subscriber can scope
+// to a single subagent tree (?root=). Exactly one of Session/Event is set.
+type streamUpdate struct {
+	Org     string       `json:"-"`
+	RootID  string       `json:"-"`
+	Type    string       `json:"-"` // "session" | "event"
+	Session *sessionView `json:"session,omitempty"`
+	Event   *eventView   `json:"event,omitempty"`
+}
+
+// bus is the in-process publish/subscribe fan-out under the sessions surface. It
+// is the SINGLE seam the live stream hangs off:
+//
+//   - Today: the SSE handler (GET /v1/agents/sessions/stream) subscribes and
+//     writes each update as an SSE frame. Because zip's SendStreamWriter streams
+//     THROUGH the ZAP machine transport natively (proven by zip stream_test
+//     TestListenZAP_Streams), that SSE endpoint IS the live ZAP stream — a ZAP
+//     subscriber gets frames as they flush, no per-handler transport code.
+//
+//   - ZAP HOOK POINT: a future direct ZAP push subscription (e.g. a browser
+//     /zap duplex that grows server-push, or a tasks-events → session-events
+//     indexer) attaches by calling subscribe(org) and forwarding updates. The
+//     publisher side (publish) does not change.
+//
+// Delivery is best-effort and non-blocking: a slow subscriber never stalls a
+// writer. On buffer overrun the subscriber is dropped (its channel closed) and
+// the client reconnects + re-fetches truth from the GET endpoints. The GET tree/
+// list/detail endpoints are the source of truth; the stream is a live hint.
+type bus struct {
+	mu     sync.Mutex
+	subs   map[int]*subscriber
+	nextID int
+	closed bool
+}
+
+type subscriber struct {
+	org string // tenant filter — a subscriber only ever receives its own org
+	ch  chan streamUpdate
+}
+
+const subBuffer = 256
+
+func newBus() *bus { return &bus{subs: map[int]*subscriber{}} }
+
+// subscribe registers a tenant-scoped subscriber and returns its channel plus a
+// cancel func. cancel is idempotent. After close(), subscribe returns a closed
+// channel so a late subscriber exits immediately.
+func (b *bus) subscribe(org string) (<-chan streamUpdate, func()) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.closed {
+		ch := make(chan streamUpdate)
+		close(ch)
+		return ch, func() {}
+	}
+	id := b.nextID
+	b.nextID++
+	s := &subscriber{org: org, ch: make(chan streamUpdate, subBuffer)}
+	b.subs[id] = s
+	var once sync.Once
+	cancel := func() {
+		once.Do(func() {
+			b.mu.Lock()
+			defer b.mu.Unlock()
+			if cur, ok := b.subs[id]; ok && cur == s {
+				delete(b.subs, id)
+				close(s.ch)
+			}
+		})
+	}
+	return s.ch, cancel
+}
+
+// publish fans an update out to every subscriber of the update's org. Non-
+// blocking: if a subscriber's buffer is full it is dropped (channel closed), so
+// one stuck dashboard can never back-pressure a session write.
+func (b *bus) publish(u streamUpdate) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.closed {
+		return
+	}
+	for id, s := range b.subs {
+		if s.org != u.Org {
+			continue
+		}
+		select {
+		case s.ch <- u:
+		default:
+			// Overrun: drop this laggard. It reconnects and re-syncs via GET.
+			delete(b.subs, id)
+			close(s.ch)
+		}
+	}
+}
+
+// close tears the bus down on Shutdown: every subscriber channel is closed so
+// its SSE loop returns and the handler unblocks within the shutdown deadline.
+func (b *bus) close() {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.closed {
+		return
+	}
+	b.closed = true
+	for id, s := range b.subs {
+		delete(b.subs, id)
+		close(s.ch)
+	}
+}
+
+// sessionsStream is GET /v1/agents/sessions/stream — a Server-Sent Events feed of
+// live session + event updates for the caller's org. Optional ?root=<id> scopes
+// the feed to one subagent tree. Org-scoped (fail-closed): a subscriber only ever
+// receives its own tenant's updates because the bus filters on org.
+//
+// This handler streams over BOTH the plain HTTP listener and the ZAP machine
+// transport with no transport-specific code (zip SendStreamWriter is transport-
+// agnostic). Everything the stream loop needs is captured BEFORE SendStreamWriter
+// so the loop never touches the request Ctx after the handler returns (fasthttp
+// recycles it) — client-gone is detected by a flush error, bounded by a 25s
+// heartbeat.
+func (s *svc) sessionsStream(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("X-Org-Id required")
+	}
+	// CLONE the root filter: c.Query returns a zero-copy view into the fasthttp
+	// request buffer, and the stream loop below OUTLIVES this handler (it runs
+	// under SendStreamWriter after the Ctx is recycled). tenant() already clones
+	// org for exactly this reason; root is retained past the request the same way,
+	// so it must be an owned copy or the filter races a reused buffer.
+	root := strings.Clone(trimField(c.Query("root")))
+
+	c.SetHeader("Content-Type", "text/event-stream")
+	c.SetHeader("Cache-Control", "no-cache")
+	c.SetHeader("Connection", "keep-alive")
+	c.SetHeader("X-Accel-Buffering", "no") // defeat proxy buffering of the stream
+
+	ch, cancel := s.bus.subscribe(org)
+	return c.SendStreamWriter(func(w *bufio.Writer) {
+		defer cancel()
+		// Initial comment flushes headers so the client's EventSource opens.
+		if _, err := w.WriteString(": stream open\n\n"); err != nil {
+			return
+		}
+		if err := w.Flush(); err != nil {
+			return
+		}
+		hb := time.NewTicker(25 * time.Second)
+		defer hb.Stop()
+		for {
+			select {
+			case <-hb.C:
+				if _, err := w.WriteString(": ping\n\n"); err != nil {
+					return
+				}
+				if err := w.Flush(); err != nil {
+					return
+				}
+			case u, open := <-ch:
+				if !open {
+					return // bus closed this sub (overrun or Shutdown)
+				}
+				if root != "" && u.RootID != root {
+					continue
+				}
+				if !writeSSE(w, u.Type, u) {
+					return // client gone
+				}
+			}
+		}
+	})
+}
+
+// writeSSE writes one SSE frame (event: <type>\ndata: <json>\n\n) and flushes.
+// Returns false on any write/flush error (client disconnected) so the caller
+// stops the loop.
+func writeSSE(w *bufio.Writer, event string, v any) bool {
+	b, err := json.Marshal(v)
+	if err != nil {
+		return true // skip a bad frame, keep the stream alive
+	}
+	if _, err := fmt.Fprintf(w, "event: %s\ndata: %s\n\n", event, b); err != nil {
+		return false
+	}
+	return w.Flush() == nil
+}

clients/agents/sessions_tasks.go

@@ -0,0 +1,58 @@
+package agents
+
+import (
+	"context"
+	"errors"
+)
+
+// TaskController is the seam to the hanzoai/tasks durable-execution engine — the
+// ONE canonical engine for durable/retriable/scheduled agent work. This sessions
+// surface is the REGISTRY + control + ZAP-stream VIEW layer; it deliberately owns
+// NO scheduler, ticker, or lease. When a session is backed by a tasks workflow
+// (Session.TaskWorkflowID set), a control command forwards through this seam to
+// the engine's signal/cancel API instead of only being recorded.
+//
+// The method set mirrors github.com/hanzoai/tasks/pkg/sdk/client.Client exactly,
+// so the live adapter is a thin wrapper (Signal→Client.SignalWorkflow,
+// Cancel→Client.CancelWorkflow) with no impedance mismatch:
+//
+//	SignalWorkflow(ctx, workflowID, runID, signalName string, arg any) error
+//	CancelWorkflow(ctx, workflowID, runID string) error
+//
+// TASKS PLUG-IN POINT. The live controller is wired in Mount from a dialed tasks
+// client (client.Dial(TASKS_URL)); until the hanzoai/tasks native engine lands
+// (today its workflow opcodes return 501 by design — "the shape is in place so
+// callers depend on the API while the engine lands behind it"), the default is
+// the disabled controller: control is still durably RECORDED as a session event
+// for stream-consuming surfaces, and the forward is a clean no-op.
+type TaskController interface {
+	// Signal forwards a cooperative control signal (pause/resume/message) to the
+	// durable workflow backing a session. name is the signal name; payload is the
+	// opaque signal argument (e.g. a steer message), nil when there is none.
+	Signal(ctx context.Context, workflowID, runID, name string, payload []byte) error
+	// Cancel gracefully cancels the durable workflow backing a session (a stop).
+	Cancel(ctx context.Context, workflowID, runID, reason string) error
+	// Enabled reports whether a real tasks backend is wired. When false the
+	// control endpoints record the intent and skip the forward (honest degrade,
+	// same pattern as deps.AI).
+	Enabled() bool
+}
+
+// errTasksNotConfigured is returned by the disabled controller's Signal/Cancel.
+// Control handlers never surface it as a failure — they check Enabled() first —
+// but it exists so a mis-wired direct call fails closed with a clear message.
+var errTasksNotConfigured = errors.New("agents: tasks durable-execution backend not configured")
+
+// disabledTaskController is the fail-safe default: no engine wired. It records
+// nothing and forwards nothing; the control endpoints persist the command as a
+// session event regardless, which is what today's stream-consuming surfaces
+// (the @hanzo/dev CLI outer agent) act on.
+type disabledTaskController struct{}
+
+func (disabledTaskController) Signal(context.Context, string, string, string, []byte) error {
+	return errTasksNotConfigured
+}
+func (disabledTaskController) Cancel(context.Context, string, string, string) error {
+	return errTasksNotConfigured
+}
+func (disabledTaskController) Enabled() bool { return false }

clients/agents/sessions_test.go

@@ -0,0 +1,560 @@
+package agents
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/zap-proto/zip"
+)
+
+// ---- store-level: tree linking, seq, tenant isolation ----
+
+func testSessionStore(t *testing.T) *Store {
+	t.Helper()
+	s, err := openStore(filepath.Join(t.TempDir(), "agents.db"))
+	if err != nil {
+		t.Fatalf("openStore: %v", err)
+	}
+	t.Cleanup(func() { _ = s.Close() })
+	return s
+}
+
+func mkSession(org, id, parent, root string) Session {
+	now := time.Now().Unix()
+	return Session{
+		ID: id, Org: org, Agent: "dev", Actor: "u", Status: StatusRunning,
+		ParentID: parent, RootID: root, StartedAt: now, CreatedAt: now, UpdatedAt: now,
+	}
+}
+
+func TestSessionTreeLinkingStore(t *testing.T) {
+	s := testSessionStore(t)
+	ctx := context.Background()
+	// root -> child -> grandchild, all one org.
+	if err := s.CreateSession(ctx, mkSession("acme", "root", "", "root")); err != nil {
+		t.Fatalf("root: %v", err)
+	}
+	if err := s.CreateSession(ctx, mkSession("acme", "child", "root", "root")); err != nil {
+		t.Fatalf("child: %v", err)
+	}
+	if err := s.CreateSession(ctx, mkSession("acme", "gchild", "child", "root")); err != nil {
+		t.Fatalf("gchild: %v", err)
+	}
+	tree, err := s.ListTree(ctx, "acme", "root", 0)
+	if err != nil || len(tree) != 3 {
+		t.Fatalf("tree want 3 nodes, got %d (%v)", len(tree), err)
+	}
+	// A parent that does not exist in the org is refused (no dangling tree).
+	if err := s.CreateSession(ctx, Session{ID: "x", Org: "acme", ParentID: "nope", RootID: "nope", StartedAt: 1, CreatedAt: 1, UpdatedAt: 1}); err != errParentNotFound {
+		t.Fatalf("dangling parent want errParentNotFound, got %v", err)
+	}
+	// A parent in ANOTHER org is refused (tree can't cross tenants).
+	if err := s.CreateSession(ctx, mkSession("evil", "e", "root", "root")); err != errParentNotFound {
+		t.Fatalf("cross-tenant parent want errParentNotFound, got %v", err)
+	}
+	nc, _ := s.CountChildren(ctx, "acme", "root")
+	if nc != 1 {
+		t.Fatalf("root direct children want 1, got %d", nc)
+	}
+}
+
+func TestSessionEventSeqAndCounts(t *testing.T) {
+	s := testSessionStore(t)
+	ctx := context.Background()
+	_ = s.CreateSession(ctx, mkSession("acme", "root", "", "root"))
+	_ = s.CreateSession(ctx, mkSession("acme", "child", "root", "root"))
+	for i := 0; i < 3; i++ {
+		e, err := s.AppendEvent(ctx, Event{ID: genIDMust(t), SessionID: "root", Org: "acme", Kind: KindLog, CreatedAt: time.Now().Unix()})
+		if err != nil {
+			t.Fatalf("append: %v", err)
+		}
+		if e.Seq != int64(i+1) {
+			t.Fatalf("seq want %d, got %d", i+1, e.Seq)
+		}
+	}
+	_, _ = s.AppendEvent(ctx, Event{ID: genIDMust(t), SessionID: "child", Org: "acme", Kind: KindSpawn, CreatedAt: time.Now().Unix()})
+	counts, err := s.EventCountsByRoot(ctx, "acme", "root")
+	if err != nil {
+		t.Fatalf("counts: %v", err)
+	}
+	if counts["root"] != 3 || counts["child"] != 1 {
+		t.Fatalf("event counts want root=3 child=1, got %+v", counts)
+	}
+	// Cross-tenant read sees nothing.
+	other, _ := s.EventCountsByRoot(ctx, "evil", "root")
+	if len(other) != 0 {
+		t.Fatalf("cross-tenant counts must be empty, got %+v", other)
+	}
+	if n, _ := s.CountEvents(ctx, "evil", "root"); n != 0 {
+		t.Fatalf("cross-tenant event count want 0, got %d", n)
+	}
+}
+
+// TestSessionEventSeqConcurrent proves the store's per-session Seq is gap-free
+// and duplicate-free under CONCURRENT appends — the exact race vector #4. The
+// store runs on a single connection (SetMaxOpenConns(1)) so the MAX(seq)+1
+// read-then-write is serialised; the UNIQUE(session_id,seq) index is the final
+// backstop. N goroutines append to the SAME session in parallel; the returned
+// seqs must be EXACTLY {1..N} (no gap = no lost write, no dupe = no double
+// allocation), and the persisted count must equal N. Run under -race.
+func TestSessionEventSeqConcurrent(t *testing.T) {
+	s := testSessionStore(t)
+	ctx := context.Background()
+	if err := s.CreateSession(ctx, mkSession("acme", "root", "", "root")); err != nil {
+		t.Fatalf("root: %v", err)
+	}
+	const n = 64
+	var wg sync.WaitGroup
+	seqs := make([]int64, n)
+	errs := make([]error, n)
+	wg.Add(n)
+	for i := 0; i < n; i++ {
+		go func(i int) {
+			defer wg.Done()
+			id, err := genID("evt")
+			if err != nil {
+				errs[i] = err
+				return
+			}
+			e, err := s.AppendEvent(ctx, Event{
+				ID: id, SessionID: "root", Org: "acme", Kind: KindLog,
+				CreatedAt: time.Now().Unix(),
+			})
+			if err != nil {
+				errs[i] = err
+				return
+			}
+			seqs[i] = e.Seq
+		}(i)
+	}
+	wg.Wait()
+	seen := map[int64]bool{}
+	for i := 0; i < n; i++ {
+		if errs[i] != nil {
+			t.Fatalf("append %d: %v", i, errs[i])
+		}
+		if seen[seqs[i]] {
+			t.Fatalf("duplicate seq %d — MAX+1 allocation raced", seqs[i])
+		}
+		seen[seqs[i]] = true
+	}
+	for want := int64(1); want <= n; want++ {
+		if !seen[want] {
+			t.Fatalf("gap: seq %d missing — a concurrent append was lost", want)
+		}
+	}
+	if got, _ := s.CountEvents(ctx, "acme", "root"); got != n {
+		t.Fatalf("persisted event count want %d, got %d", n, got)
+	}
+}
+
+func genIDMust(t *testing.T) string {
+	t.Helper()
+	id, err := genID("evt")
+	if err != nil {
+		t.Fatalf("genID: %v", err)
+	}
+	return id
+}
+
+// ---- HTTP: helpers ----
+
+// doNoUser sends X-Org-Id WITHOUT X-User-Id — the anonymous-forge path the
+// principal gate must refuse (no validated principal).
+func doNoUser(t *testing.T, app *zip.App, method, path, org string, body any) (int, []byte) {
+	t.Helper()
+	var r io.Reader
+	if body != nil {
+		b, _ := json.Marshal(body)
+		r = bytes.NewReader(b)
+	}
+	req := httptest.NewRequest(method, path, r)
+	if body != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	if org != "" {
+		req.Header.Set("X-Org-Id", org)
+	}
+	resp, err := app.Fiber().Test(req)
+	if err != nil {
+		t.Fatalf("Test %s %s: %v", method, path, err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	b, _ := io.ReadAll(resp.Body)
+	return resp.StatusCode, b
+}
+
+func mustJSON(t *testing.T, b []byte, v any) {
+	t.Helper()
+	if err := json.Unmarshal(b, v); err != nil {
+		t.Fatalf("unmarshal %s: %v", b, err)
+	}
+}
+
+// register is a helper that POSTs a session and returns its view.
+func register(t *testing.T, app *zip.App, org string, body map[string]any) sessionView {
+	t.Helper()
+	code, b := do(t, app, http.MethodPost, "/v1/agents/sessions", org, body)
+	if code != http.StatusCreated {
+		t.Fatalf("register want 201, got %d (%s)", code, b)
+	}
+	var v sessionView
+	mustJSON(t, b, &v)
+	return v
+}
+
+// ---- HTTP: tree, org-scope, precedence ----
+
+func TestSessionsHTTPTreeAndScope(t *testing.T) {
+	app := mountApp(t, &fakeAI{content: "x"})
+
+	// Route precedence: /v1/agents/sessions is NOT captured by /v1/agents/:name.
+	code, b := do(t, app, http.MethodGet, "/v1/agents/sessions", "acme", nil)
+	if code != http.StatusOK {
+		t.Fatalf("list sessions want 200 (not shadowed by :name), got %d (%s)", code, b)
+	}
+	var empty struct {
+		Sessions []sessionView `json:"sessions"`
+	}
+	mustJSON(t, b, &empty)
+	if len(empty.Sessions) != 0 {
+		t.Fatalf("fresh org want 0 sessions, got %d", len(empty.Sessions))
+	}
+
+	// Build a tree: root (outer @hanzo/dev run) -> two subagents -> one grandchild.
+	root := register(t, app, "acme", map[string]any{"agent": "hanzo-dev", "title": "outer run"})
+	if root.RootSessionID != root.ID || root.ParentSessionID != "" {
+		t.Fatalf("root must self-root with no parent, got %+v", root)
+	}
+	childA := register(t, app, "acme", map[string]any{"agent": "planner", "parentSessionId": root.ID})
+	childB := register(t, app, "acme", map[string]any{"agent": "coder", "parentSessionId": root.ID})
+	gchild := register(t, app, "acme", map[string]any{"agent": "tester", "parentSessionId": childA.ID})
+	for _, c := range []sessionView{childA, childB, gchild} {
+		if c.RootSessionID != root.ID {
+			t.Fatalf("subagent %s must inherit rootSessionId %s, got %s", c.Agent, root.ID, c.RootSessionID)
+		}
+	}
+	if gchild.ParentSessionID != childA.ID {
+		t.Fatalf("grandchild parent want %s, got %s", childA.ID, gchild.ParentSessionID)
+	}
+
+	// Default list = ROOTS only (the outer-agent view).
+	code, b = do(t, app, http.MethodGet, "/v1/agents/sessions", "acme", nil)
+	mustJSON(t, b, &empty)
+	if code != http.StatusOK || len(empty.Sessions) != 1 || empty.Sessions[0].ID != root.ID {
+		t.Fatalf("default list want [root], got %d %+v", code, empty.Sessions)
+	}
+	if empty.Sessions[0].Children != 2 {
+		t.Fatalf("root fan-out want 2, got %d", empty.Sessions[0].Children)
+	}
+
+	// The tree endpoint returns the full subagent-flow graph.
+	code, b = do(t, app, http.MethodGet, "/v1/agents/sessions/"+root.ID+"/tree", "acme", nil)
+	if code != http.StatusOK {
+		t.Fatalf("tree want 200, got %d (%s)", code, b)
+	}
+	var tree treeNode
+	mustJSON(t, b, &tree)
+	if tree.Session.ID != root.ID || len(tree.Children) != 2 {
+		t.Fatalf("tree root want 2 children, got %+v", tree)
+	}
+	// Find childA subtree and confirm the grandchild hangs off it.
+	var found bool
+	for _, ch := range tree.Children {
+		if ch.Session.ID == childA.ID {
+			if len(ch.Children) != 1 || ch.Children[0].Session.ID != gchild.ID {
+				t.Fatalf("childA must have grandchild, got %+v", ch)
+			}
+			found = true
+		}
+	}
+	if !found {
+		t.Fatalf("childA not found in tree")
+	}
+
+	// Cross-tenant: evil cannot see, read, tree, control, or parent-under acme's root.
+	code, b = do(t, app, http.MethodGet, "/v1/agents/sessions", "evil", nil)
+	mustJSON(t, b, &empty)
+	if len(empty.Sessions) != 0 {
+		t.Fatalf("evil must see 0 sessions, got %d", len(empty.Sessions))
+	}
+	if code, _ := do(t, app, http.MethodGet, "/v1/agents/sessions/"+root.ID, "evil", nil); code != http.StatusNotFound {
+		t.Fatalf("evil get acme session want 404, got %d", code)
+	}
+	if code, _ := do(t, app, http.MethodGet, "/v1/agents/sessions/"+root.ID+"/tree", "evil", nil); code != http.StatusNotFound {
+		t.Fatalf("evil tree acme session want 404, got %d", code)
+	}
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents/sessions/"+root.ID+"/stop", "evil", nil); code != http.StatusNotFound {
+		t.Fatalf("evil control acme session want 404, got %d", code)
+	}
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents/sessions/"+root.ID+"/events", "evil",
+		map[string]any{"kind": "log"}); code != http.StatusNotFound {
+		t.Fatalf("evil append to acme session want 404, got %d", code)
+	}
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents/sessions", "evil",
+		map[string]any{"agent": "x", "parentSessionId": root.ID}); code != http.StatusBadRequest {
+		t.Fatalf("evil parent-under acme root want 400, got %d", code)
+	}
+}
+
+// ---- HTTP: events + status ----
+
+func TestSessionsHTTPEventsAndStatus(t *testing.T) {
+	app := mountApp(t, &fakeAI{content: "x"})
+	root := register(t, app, "acme", map[string]any{"agent": "dev"})
+
+	// Append events: message, tool-call, spawn — seq is monotonic.
+	for i, k := range []string{KindMessage, KindToolCall, KindSpawn} {
+		code, b := do(t, app, http.MethodPost, "/v1/agents/sessions/"+root.ID+"/events", "acme",
+			map[string]any{"kind": k, "payload": map[string]any{"n": i}})
+		if code != http.StatusCreated {
+			t.Fatalf("append %s want 201, got %d (%s)", k, code, b)
+		}
+		var ev eventView
+		mustJSON(t, b, &ev)
+		if ev.Seq != int64(i+1) || ev.Kind != k {
+			t.Fatalf("event %s seq want %d, got %+v", k, i+1, ev)
+		}
+	}
+	// Bad kind + bad payload are rejected.
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents/sessions/"+root.ID+"/events", "acme",
+		map[string]any{"kind": "bogus"}); code != http.StatusBadRequest {
+		t.Fatalf("bad kind want 400, got %d", code)
+	}
+
+	// Detail shows recent events + event count.
+	code, b := do(t, app, http.MethodGet, "/v1/agents/sessions/"+root.ID, "acme", nil)
+	if code != http.StatusOK {
+		t.Fatalf("detail want 200, got %d (%s)", code, b)
+	}
+	var det sessionDetail
+	mustJSON(t, b, &det)
+	if det.Events != 3 || len(det.RecentEvents) != 3 {
+		t.Fatalf("detail want 3 events, got %d / %d", det.Events, len(det.RecentEvents))
+	}
+
+	// PATCH running -> done sets endedAt; then terminal is monotonic.
+	code, b = do(t, app, http.MethodPatch, "/v1/agents/sessions/"+root.ID, "acme",
+		map[string]any{"status": StatusDone})
+	if code != http.StatusOK {
+		t.Fatalf("patch done want 200, got %d (%s)", code, b)
+	}
+	var done sessionView
+	mustJSON(t, b, &done)
+	if done.Status != StatusDone || done.EndedAt == "" {
+		t.Fatalf("done must set endedAt, got %+v", done)
+	}
+	if code, _ := do(t, app, http.MethodPatch, "/v1/agents/sessions/"+root.ID, "acme",
+		map[string]any{"status": StatusRunning}); code != http.StatusConflict {
+		t.Fatalf("reopen finished session want 409, got %d", code)
+	}
+}
+
+// ---- HTTP: control authz + tasks forward ----
+
+// fakeTasks is an enabled TaskController capturing the last forwarded op.
+type fakeTasks struct {
+	mu       sync.Mutex
+	signals  []string
+	cancels  int
+	lastWF   string
+	failNext bool
+}
+
+func (f *fakeTasks) Signal(_ context.Context, wf, _ string, name string, _ []byte) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.failNext {
+		f.failNext = false
+		return context.DeadlineExceeded
+	}
+	f.signals = append(f.signals, name)
+	f.lastWF = wf
+	return nil
+}
+func (f *fakeTasks) Cancel(_ context.Context, wf, _, _ string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.cancels++
+	f.lastWF = wf
+	return nil
+}
+func (f *fakeTasks) Enabled() bool { return true }
+
+func TestSessionsControlAuthzAndForward(t *testing.T) {
+	app := mountApp(t, &fakeAI{content: "x"})
+	ft := &fakeTasks{}
+	mounted.tasks = ft // inject an enabled durable-execution backend for this test
+
+	// A task-backed session forwards control to the tasks engine.
+	backed := register(t, app, "acme", map[string]any{
+		"agent": "dev", "taskWorkflowId": "wf-123", "taskRunId": "run-1",
+	})
+	// pause -> Signal("pause")
+	code, b := do(t, app, http.MethodPost, "/v1/agents/sessions/"+backed.ID+"/pause", "acme", nil)
+	if code != http.StatusOK {
+		t.Fatalf("pause want 200, got %d (%s)", code, b)
+	}
+	var res struct {
+		Command   string    `json:"command"`
+		Event     eventView `json:"event"`
+		Forwarded bool      `json:"forwarded"`
+	}
+	mustJSON(t, b, &res)
+	if !res.Forwarded || res.Command != CmdPause || res.Event.Kind != KindControl {
+		t.Fatalf("pause must forward + record a control event, got %+v", res)
+	}
+	// message (steer) -> Signal("message")
+	do(t, app, http.MethodPost, "/v1/agents/sessions/"+backed.ID+"/message", "acme",
+		map[string]any{"message": "focus on the bug"})
+	// stop -> Cancel
+	code, _ = do(t, app, http.MethodPost, "/v1/agents/sessions/"+backed.ID+"/stop", "acme", nil)
+	if code != http.StatusOK {
+		t.Fatalf("stop want 200, got %d", code)
+	}
+	ft.mu.Lock()
+	gotSignals, gotCancels, gotWF := append([]string{}, ft.signals...), ft.cancels, ft.lastWF
+	ft.mu.Unlock()
+	if len(gotSignals) != 2 || gotSignals[0] != CmdPause || gotSignals[1] != CmdMessage {
+		t.Fatalf("want signals [pause,message], got %v", gotSignals)
+	}
+	if gotCancels != 1 || gotWF != "wf-123" {
+		t.Fatalf("want 1 cancel on wf-123, got cancels=%d wf=%s", gotCancels, gotWF)
+	}
+
+	// Control is recorded as an event even on a NON-task-backed session
+	// (forwarded=false) — stream-consuming surfaces act on it.
+	plain := register(t, app, "acme", map[string]any{"agent": "dev"})
+	code, b = do(t, app, http.MethodPost, "/v1/agents/sessions/"+plain.ID+"/pause", "acme", nil)
+	mustJSON(t, b, &res)
+	if code != http.StatusOK || res.Forwarded {
+		t.Fatalf("plain pause want 200 forwarded=false, got %d %+v", code, res)
+	}
+	// The control command landed in the event log.
+	_, b = do(t, app, http.MethodGet, "/v1/agents/sessions/"+plain.ID, "acme", nil)
+	var det sessionDetail
+	mustJSON(t, b, &det)
+	if det.Events != 1 || det.RecentEvents[0].Kind != KindControl {
+		t.Fatalf("control must be recorded as an event, got %+v", det.RecentEvents)
+	}
+
+	// A forward FAILURE is a 502 but the intent is still recorded.
+	ft.failNext = true
+	code, _ = do(t, app, http.MethodPost, "/v1/agents/sessions/"+backed.ID+"/resume", "acme", nil)
+	// backed was stopped above (running still — stop only records/cancels, status
+	// is surface-owned), so resume is allowed; the forward fails -> 502.
+	if code != http.StatusBadGateway {
+		t.Fatalf("forward failure want 502, got %d", code)
+	}
+
+	// AuthZ: X-Org-Id without a validated principal (no X-User-Id) is refused.
+	if code, _ := doNoUser(t, app, http.MethodPost, "/v1/agents/sessions/"+backed.ID+"/pause", "acme", nil); code != http.StatusForbidden {
+		t.Fatalf("control without validated principal want 403, got %d", code)
+	}
+	if code, _ := doNoUser(t, app, http.MethodPost, "/v1/agents/sessions", "acme",
+		map[string]any{"agent": "x"}); code != http.StatusForbidden {
+		t.Fatalf("register without validated principal want 403, got %d", code)
+	}
+
+	// Control on a finished session is refused (409).
+	fin := register(t, app, "acme", map[string]any{"agent": "dev", "status": StatusDone})
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents/sessions/"+fin.ID+"/pause", "acme", nil); code != http.StatusConflict {
+		t.Fatalf("control a finished session want 409, got %d", code)
+	}
+}
+
+// ---- run integration (#5): a run opens a root session ----
+
+func TestRunOpensRootSession(t *testing.T) {
+	app := mountApp(t, &fakeAI{content: "the answer"})
+	do(t, app, http.MethodPost, "/v1/agents", "acme",
+		map[string]any{"name": "helper", "model": "m", "instructions": "x"})
+	if code, _ := do(t, app, http.MethodPost, "/v1/agents/helper/run", "acme", map[string]any{"input": "hi"}); code != http.StatusOK {
+		t.Fatalf("run want 200")
+	}
+	// The run is now visible as a root session with a log event.
+	_, b := do(t, app, http.MethodGet, "/v1/agents/sessions", "acme", nil)
+	var lst struct {
+		Sessions []sessionView `json:"sessions"`
+	}
+	mustJSON(t, b, &lst)
+	if len(lst.Sessions) != 1 {
+		t.Fatalf("run should open 1 root session, got %d", len(lst.Sessions))
+	}
+	s := lst.Sessions[0]
+	if s.Agent != "helper" || s.Status != StatusDone || s.Events != 1 {
+		t.Fatalf("run session shape wrong: %+v", s)
+	}
+}
+
+// ---- bus (the ZAP stream seam) ----
+
+func TestBusFanoutOrgFilterAndOverrun(t *testing.T) {
+	b := newBus()
+	chA, cancelA := b.subscribe("acme")
+	chB, _ := b.subscribe("evil")
+	defer cancelA()
+
+	b.publish(streamUpdate{Org: "acme", RootID: "r", Type: "session"})
+	select {
+	case u := <-chA:
+		if u.Org != "acme" {
+			t.Fatalf("acme sub got wrong org %s", u.Org)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("acme sub got no update")
+	}
+	// evil must NOT receive acme's update (org filter).
+	select {
+	case <-chB:
+		t.Fatal("evil sub must not receive acme update")
+	default:
+	}
+
+	// Overrun: fill acme's buffer past capacity — the laggard is dropped (closed).
+	for i := 0; i < subBuffer+10; i++ {
+		b.publish(streamUpdate{Org: "acme", RootID: "r", Type: "event"})
+	}
+	// Drain until closed.
+	dropped := false
+	for i := 0; i < subBuffer+20; i++ {
+		if _, open := <-chA; !open {
+			dropped = true
+			break
+		}
+	}
+	if !dropped {
+		t.Fatal("overrun laggard must be dropped (channel closed)")
+	}
+
+	// close() unblocks remaining subscribers.
+	b.close()
+	if _, open := <-chB; open {
+		t.Fatal("close() must close evil sub channel")
+	}
+}
+
+// TestPublishReachesSubscriber proves a live registration fans out to a bus
+// subscriber — the exact path the SSE/ZAP stream handler consumes.
+func TestPublishReachesSubscriber(t *testing.T) {
+	app := mountApp(t, &fakeAI{content: "x"})
+	ch, cancel := mounted.bus.subscribe("acme")
+	defer cancel()
+	root := register(t, app, "acme", map[string]any{"agent": "dev"})
+	select {
+	case u := <-ch:
+		if u.Type != "session" || u.Session == nil || u.Session.ID != root.ID {
+			t.Fatalf("subscriber should receive the registered session, got %+v", u)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("subscriber received no update for a live registration")
+	}
+}

clients/agents/store.go

@@ -0,0 +1,494 @@
+package agents
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+
+	// github.com/hanzoai/sqlite is the ONE Hanzo SQLite driver: it registers
+	// the "sqlite" database/sql name under both build tags (cgo →
+	// mattn+SQLCipher, encrypted at rest; !cgo → pure-Go modernc). Importing
+	// modernc directly instead would double-register "sqlite" under CGO and
+	// panic at init. Blank import registers the driver.
+	_ "github.com/hanzoai/sqlite"
+)
+
+var (
+	errConflict = errors.New("agents: agent already exists")
+	errNotFound = errors.New("agents: agent not found")
+)
+
+// Agent is the org-scoped definition of an autonomous worker: a model, a system
+// prompt (instructions), and a set of tool names it may call. Tenant isolation
+// is the org column, enforced on every query. It never stores a secret — tool
+// credentials live in KMS and are referenced by name at run time.
+//
+// The bot-lifecycle fields promote an agent from a one-shot callable into a
+// long-running bot (per hanzo-agent-bot-architecture: "Bot = Agent + compute +
+// long-running"):
+//
+//   - ExecutionMode: "one-shot" (default; runs only when POSTed) or
+//     "long-running" (the scheduler invokes it on Schedule).
+//   - Schedule: a 5-field cron expression; required when long-running, ignored
+//     otherwise. The scheduler evaluates it once a minute.
+//   - ComputeRef: an optional visor machine id the bot is bound to. It is an
+//     opaque reference here; binding/lifecycle is owned elsewhere.
+//   - ServiceAccountID: an optional IAM agent service-account (<org>-<agent>).
+//     When set it is the Actor recorded on scheduled-run billing so an
+//     autonomous run is attributable to a principal, not just the org.
+type Agent struct {
+	ID               string
+	Org              string
+	Name             string
+	Model            string
+	Instructions     string
+	Description      string
+	Tools            []string
+	Status           string
+	ExecutionMode    string
+	Schedule         string
+	ComputeRef       string
+	ServiceAccountID string
+	CreatedAt        int64
+	UpdatedAt        int64
+}
+
+// Execution modes. One-shot agents run only on an explicit POST; long-running
+// agents are additionally invoked by the scheduler on their Schedule.
+const (
+	ModeOneShot     = "one-shot"
+	ModeLongRunning = "long-running"
+)
+
+// Run is one execution of an agent: the input, the produced output (or error),
+// which model served it, and how long it took. Real history — every row is a
+// call that actually happened.
+type Run struct {
+	ID         string
+	Org        string
+	AgentName  string
+	Status     string
+	Model      string
+	Input      string
+	Output     string
+	Error      string
+	DurationMs int64
+	CreatedAt  int64
+}
+
+// Store is the agents database. ONE SQLite file ({DataDir}/agents.db) holds
+// every org's records; tenancy is the org column.
+type Store struct {
+	db *sql.DB
+}
+
+func openStore(path string) (*Store, error) {
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		return nil, fmt.Errorf("open sqlite %q: %w", path, err)
+	}
+	db.SetMaxOpenConns(1)
+	for _, pragma := range []string{
+		"PRAGMA busy_timeout=5000",
+		"PRAGMA journal_mode=WAL",
+		"PRAGMA foreign_keys=ON",
+	} {
+		if _, err := db.Exec(pragma); err != nil {
+			_ = db.Close()
+			return nil, fmt.Errorf("pragma %q: %w", pragma, err)
+		}
+	}
+	s := &Store{db: db}
+	if err := s.migrate(); err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+	return s, nil
+}
+
+func (s *Store) migrate() error {
+	const ddl = `
+CREATE TABLE IF NOT EXISTS agents (
+  id                 TEXT PRIMARY KEY,
+  org                TEXT NOT NULL,
+  name               TEXT NOT NULL,
+  model              TEXT NOT NULL DEFAULT '',
+  instructions       TEXT NOT NULL DEFAULT '',
+  description        TEXT NOT NULL DEFAULT '',
+  tools              TEXT NOT NULL DEFAULT '[]',
+  status             TEXT NOT NULL DEFAULT 'ready',
+  execution_mode     TEXT NOT NULL DEFAULT 'one-shot',
+  schedule           TEXT NOT NULL DEFAULT '',
+  compute_ref        TEXT NOT NULL DEFAULT '',
+  service_account_id TEXT NOT NULL DEFAULT '',
+  created_at         INTEGER NOT NULL,
+  updated_at         INTEGER NOT NULL
+);
+CREATE UNIQUE INDEX IF NOT EXISTS ux_agents_org_name ON agents(org, name);
+CREATE INDEX IF NOT EXISTS ix_agents_org_updated ON agents(org, updated_at);
+
+CREATE TABLE IF NOT EXISTS agent_runs (
+  id          TEXT PRIMARY KEY,
+  org         TEXT NOT NULL,
+  agent_name  TEXT NOT NULL,
+  status      TEXT NOT NULL,
+  model       TEXT NOT NULL DEFAULT '',
+  input       TEXT NOT NULL DEFAULT '',
+  output      TEXT NOT NULL DEFAULT '',
+  error       TEXT NOT NULL DEFAULT '',
+  duration_ms INTEGER NOT NULL DEFAULT 0,
+  created_at  INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS ix_runs_org_agent_created ON agent_runs(org, agent_name, created_at);
+`
+	if _, err := s.db.Exec(ddl); err != nil {
+		return fmt.Errorf("migrate: %w", err)
+	}
+	// Forward, idempotent migration for databases created before the
+	// bot-lifecycle columns existed. Each ADD COLUMN is guarded by a live
+	// column-existence check (PRAGMA table_info), so re-running migrate() on an
+	// already-upgraded DB is a no-op and never errors — the DDL above handles
+	// fresh DBs, this handles pre-existing ones. It touches no storage-backend
+	// knob (driverName/DSN), so the SQLite-only storage lockdown is unaffected.
+	if err := s.addColumns("agents", map[string]string{
+		"execution_mode":     "TEXT NOT NULL DEFAULT 'one-shot'",
+		"schedule":           "TEXT NOT NULL DEFAULT ''",
+		"compute_ref":        "TEXT NOT NULL DEFAULT ''",
+		"service_account_id": "TEXT NOT NULL DEFAULT ''",
+	}); err != nil {
+		return err
+	}
+	// Partial index for the once-a-minute scheduler scan — created AFTER the
+	// lifecycle columns exist (a legacy DB gains them just above), so it selects
+	// only the (typically few) scheduled long-running agents instead of
+	// full-scanning every org's agents on the single shared SQLite connection.
+	if _, err := s.db.Exec(`CREATE INDEX IF NOT EXISTS ix_agents_scheduled
+		ON agents(org, name) WHERE execution_mode='long-running' AND schedule<>''`); err != nil {
+		return fmt.Errorf("migrate: scheduled index: %w", err)
+	}
+	// Live agent-session control-plane tables live in the SAME agents.db (one
+	// store, one tenancy column) — sessions/events are to runs what the subagent
+	// tree is to a single call.
+	if err := s.migrateSessions(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// addColumns adds each missing column to table, idempotently. A column already
+// present is skipped; a fresh install (all present from the CREATE) is a no-op.
+func (s *Store) addColumns(table string, cols map[string]string) error {
+	have, err := s.columns(table)
+	if err != nil {
+		return err
+	}
+	for name, def := range cols {
+		if have[name] {
+			continue
+		}
+		// name/def are package-internal literals, never user input — no
+		// injection surface. SQLite forbids parameterizing DDL identifiers.
+		if _, err := s.db.Exec(`ALTER TABLE ` + table + ` ADD COLUMN ` + name + ` ` + def); err != nil {
+			return fmt.Errorf("migrate: add %s.%s: %w", table, name, err)
+		}
+	}
+	return nil
+}
+
+// columns returns the set of column names on table via PRAGMA table_info.
+func (s *Store) columns(table string) (map[string]bool, error) {
+	rows, err := s.db.Query(`PRAGMA table_info(` + table + `)`)
+	if err != nil {
+		return nil, fmt.Errorf("migrate: table_info %s: %w", table, err)
+	}
+	defer func() { _ = rows.Close() }()
+	have := map[string]bool{}
+	for rows.Next() {
+		var (
+			cid, notnull, pk int
+			name, ctype      string
+			dflt             sql.NullString
+		)
+		if err := rows.Scan(&cid, &name, &ctype, &notnull, &dflt, &pk); err != nil {
+			return nil, fmt.Errorf("migrate: scan table_info: %w", err)
+		}
+		have[name] = true
+	}
+	return have, rows.Err()
+}
+
+func (s *Store) Close() error { return s.db.Close() }
+
+func encodeList(xs []string) string {
+	if len(xs) == 0 {
+		return "[]"
+	}
+	b, err := json.Marshal(xs)
+	if err != nil {
+		return "[]"
+	}
+	return string(b)
+}
+
+func decodeList(s string) []string {
+	if s == "" {
+		return nil
+	}
+	var xs []string
+	if err := json.Unmarshal([]byte(s), &xs); err != nil {
+		return nil
+	}
+	return xs
+}
+
+const agentCols = `id,org,name,model,instructions,description,tools,status,execution_mode,schedule,compute_ref,service_account_id,created_at,updated_at`
+
+func scanAgent(sc interface{ Scan(...any) error }) (Agent, error) {
+	var a Agent
+	var tools string
+	err := sc.Scan(&a.ID, &a.Org, &a.Name, &a.Model, &a.Instructions, &a.Description,
+		&tools, &a.Status, &a.ExecutionMode, &a.Schedule, &a.ComputeRef, &a.ServiceAccountID,
+		&a.CreatedAt, &a.UpdatedAt)
+	a.Tools = decodeList(tools)
+	return a, err
+}
+
+// normalizeMode is the lowest-layer fail-safe default: an empty execution_mode
+// is stored as one-shot so NO path (handler, scheduler, or a direct store call)
+// can persist an agent the scheduler would treat ambiguously. The HTTP handler
+// also defaults+validates, but this makes the invariant hold at the store.
+func normalizeMode(m string) string {
+	if strings.TrimSpace(m) == "" {
+		return ModeOneShot
+	}
+	return m
+}
+
+// Create inserts one agent. A UNIQUE(org,name) violation surfaces as errConflict.
+func (s *Store) Create(ctx context.Context, a Agent) error {
+	a.ExecutionMode = normalizeMode(a.ExecutionMode)
+	_, err := s.db.ExecContext(ctx,
+		`INSERT INTO agents (`+agentCols+`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
+		a.ID, a.Org, a.Name, a.Model, a.Instructions, a.Description,
+		encodeList(a.Tools), a.Status, a.ExecutionMode, a.Schedule, a.ComputeRef,
+		a.ServiceAccountID, a.CreatedAt, a.UpdatedAt)
+	if err != nil {
+		if strings.Contains(err.Error(), "UNIQUE constraint failed") {
+			return errConflict
+		}
+		return fmt.Errorf("insert agent: %w", err)
+	}
+	return nil
+}
+
+// Get returns the agent for the exact (org,name) or errNotFound. It is the
+// precise name primitive; path-addressed handlers use Resolve (id-or-name).
+func (s *Store) Get(ctx context.Context, org, name string) (Agent, error) {
+	row := s.db.QueryRowContext(ctx, `SELECT `+agentCols+` FROM agents WHERE org=? AND name=?`, org, name)
+	a, err := scanAgent(row)
+	if errors.Is(err, sql.ErrNoRows) {
+		return Agent{}, errNotFound
+	}
+	if err != nil {
+		return Agent{}, fmt.Errorf("get agent: %w", err)
+	}
+	return a, nil
+}
+
+// Resolve returns the agent identified by ref within org, matching either its
+// public id (the `agent_...` handle create and list hand back) OR its org-unique
+// name. This is the ONE lookup every path-addressed handler (get/update/delete/
+// run/runs) uses, so a just-created agent is immediately addressable by exactly
+// the identifier create/list returned — no id-vs-name split. If a ref somehow
+// equals one agent's id and another's name, the id match wins (the stable public
+// handle is authoritative). Tenancy is the org filter, so a ref belonging to
+// another tenant is errNotFound — fail-closed, never cross-org.
+func (s *Store) Resolve(ctx context.Context, org, ref string) (Agent, error) {
+	row := s.db.QueryRowContext(ctx,
+		`SELECT `+agentCols+` FROM agents WHERE org=? AND (id=? OR name=?)
+		 ORDER BY (id=?) DESC LIMIT 1`, org, ref, ref, ref)
+	a, err := scanAgent(row)
+	if errors.Is(err, sql.ErrNoRows) {
+		return Agent{}, errNotFound
+	}
+	if err != nil {
+		return Agent{}, fmt.Errorf("resolve agent: %w", err)
+	}
+	return a, nil
+}
+
+// List returns every agent for org, most-recently-updated first.
+func (s *Store) List(ctx context.Context, org string) ([]Agent, error) {
+	rows, err := s.db.QueryContext(ctx,
+		`SELECT `+agentCols+` FROM agents WHERE org=? ORDER BY updated_at DESC, name ASC`, org)
+	if err != nil {
+		return nil, fmt.Errorf("list agents: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+	var out []Agent
+	for rows.Next() {
+		a, err := scanAgent(rows)
+		if err != nil {
+			return nil, fmt.Errorf("scan agent: %w", err)
+		}
+		out = append(out, a)
+	}
+	return out, rows.Err()
+}
+
+// Update overwrites the mutable fields of an existing agent.
+func (s *Store) Update(ctx context.Context, a Agent) error {
+	a.ExecutionMode = normalizeMode(a.ExecutionMode)
+	res, err := s.db.ExecContext(ctx,
+		`UPDATE agents SET model=?,instructions=?,description=?,tools=?,status=?,
+		 execution_mode=?,schedule=?,compute_ref=?,service_account_id=?,updated_at=?
+		 WHERE org=? AND name=?`,
+		a.Model, a.Instructions, a.Description, encodeList(a.Tools), a.Status,
+		a.ExecutionMode, a.Schedule, a.ComputeRef, a.ServiceAccountID, a.UpdatedAt, a.Org, a.Name)
+	if err != nil {
+		return fmt.Errorf("update agent: %w", err)
+	}
+	n, _ := res.RowsAffected()
+	if n == 0 {
+		return errNotFound
+	}
+	return nil
+}
+
+// ListLongRunning returns every agent across ALL orgs whose execution_mode is
+// long-running and that carries a non-empty schedule — the scheduler's work
+// set. It is the ONE cross-org query in this store; the scheduler is a trusted
+// in-process subsystem (not a tenant request), and each returned agent carries
+// its own Org so every downstream action (run, gate, meter) stays scoped to the
+// agent's own tenant.
+func (s *Store) ListLongRunning(ctx context.Context) ([]Agent, error) {
+	rows, err := s.db.QueryContext(ctx,
+		`SELECT `+agentCols+` FROM agents
+		 WHERE execution_mode=? AND schedule<>'' ORDER BY org, name`, ModeLongRunning)
+	if err != nil {
+		return nil, fmt.Errorf("list long-running: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+	var out []Agent
+	for rows.Next() {
+		a, err := scanAgent(rows)
+		if err != nil {
+			return nil, fmt.Errorf("scan agent: %w", err)
+		}
+		out = append(out, a)
+	}
+	return out, rows.Err()
+}
+
+// CountLongRunning returns how many scheduled long-running agents an org has —
+// used to cap an org's scheduler footprint at create time.
+func (s *Store) CountLongRunning(ctx context.Context, org string) (int, error) {
+	var n int
+	err := s.db.QueryRowContext(ctx,
+		`SELECT COUNT(*) FROM agents WHERE org=? AND execution_mode=? AND schedule<>''`,
+		org, ModeLongRunning).Scan(&n)
+	if err != nil {
+		return 0, fmt.Errorf("count long-running: %w", err)
+	}
+	return n, nil
+}
+
+// Delete removes an agent and its run history. Reports whether a row went.
+func (s *Store) Delete(ctx context.Context, org, name string) (bool, error) {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return false, fmt.Errorf("begin: %w", err)
+	}
+	defer func() { _ = tx.Rollback() }()
+	res, err := tx.ExecContext(ctx, `DELETE FROM agents WHERE org=? AND name=?`, org, name)
+	if err != nil {
+		return false, fmt.Errorf("delete agent: %w", err)
+	}
+	if _, err := tx.ExecContext(ctx, `DELETE FROM agent_runs WHERE org=? AND agent_name=?`, org, name); err != nil {
+		return false, fmt.Errorf("delete runs: %w", err)
+	}
+	n, _ := res.RowsAffected()
+	if err := tx.Commit(); err != nil {
+		return false, fmt.Errorf("commit: %w", err)
+	}
+	return n > 0, nil
+}
+
+// InsertRun records one agent execution.
+func (s *Store) InsertRun(ctx context.Context, r Run) error {
+	_, err := s.db.ExecContext(ctx,
+		`INSERT INTO agent_runs (id,org,agent_name,status,model,input,output,error,duration_ms,created_at)
+		 VALUES (?,?,?,?,?,?,?,?,?,?)`,
+		r.ID, r.Org, r.AgentName, r.Status, r.Model, r.Input, r.Output, r.Error, r.DurationMs, r.CreatedAt)
+	if err != nil {
+		return fmt.Errorf("insert run: %w", err)
+	}
+	return nil
+}
+
+// ListRuns returns the run history for (org,agent), newest first, capped.
+func (s *Store) ListRuns(ctx context.Context, org, agent string, limit int) ([]Run, error) {
+	if limit <= 0 || limit > 200 {
+		limit = 50
+	}
+	rows, err := s.db.QueryContext(ctx,
+		`SELECT id,org,agent_name,status,model,input,output,error,duration_ms,created_at
+		 FROM agent_runs WHERE org=? AND agent_name=? ORDER BY created_at DESC LIMIT ?`, org, agent, limit)
+	if err != nil {
+		return nil, fmt.Errorf("list runs: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+	var out []Run
+	for rows.Next() {
+		var r Run
+		if err := rows.Scan(&r.ID, &r.Org, &r.AgentName, &r.Status, &r.Model, &r.Input,
+			&r.Output, &r.Error, &r.DurationMs, &r.CreatedAt); err != nil {
+			return nil, fmt.Errorf("scan run: %w", err)
+		}
+		out = append(out, r)
+	}
+	return out, rows.Err()
+}
+
+// RunsSince returns the org's runs across ALL agents with created_at >= since,
+// newest first, capped. It powers the org-wide surfaces: the recent-activity
+// feed (since=0 → the newest runs regardless of age) and the invocation
+// histogram (since=windowStart → every run in the window, order-independent for
+// bucketing). since<=0 means "no lower bound". Tenancy is the org column, so a
+// caller never sees another org's runs. Every row is a real recorded execution.
+func (s *Store) RunsSince(ctx context.Context, org string, since int64, limit int) ([]Run, error) {
+	if limit <= 0 || limit > 10000 {
+		limit = 200
+	}
+	rows, err := s.db.QueryContext(ctx,
+		`SELECT id,org,agent_name,status,model,input,output,error,duration_ms,created_at
+		 FROM agent_runs WHERE org=? AND created_at>=? ORDER BY created_at DESC LIMIT ?`, org, since, limit)
+	if err != nil {
+		return nil, fmt.Errorf("runs since: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+	var out []Run
+	for rows.Next() {
+		var r Run
+		if err := rows.Scan(&r.ID, &r.Org, &r.AgentName, &r.Status, &r.Model, &r.Input,
+			&r.Output, &r.Error, &r.DurationMs, &r.CreatedAt); err != nil {
+			return nil, fmt.Errorf("scan run: %w", err)
+		}
+		out = append(out, r)
+	}
+	return out, rows.Err()
+}
+
+// CountRuns returns how many runs an org's agent has (for the list rollup).
+func (s *Store) CountRuns(ctx context.Context, org, agent string) (int, error) {
+	var n int
+	err := s.db.QueryRowContext(ctx,
+		`SELECT COUNT(*) FROM agent_runs WHERE org=? AND agent_name=?`, org, agent).Scan(&n)
+	if err != nil {
+		return 0, fmt.Errorf("count runs: %w", err)
+	}
+	return n, nil
+}

clients/agents/store_lifecycle_test.go

@@ -0,0 +1,146 @@
+package agents
+
+import (
+	"context"
+	"database/sql"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// TestLifecycleFieldsRoundTrip: the four bot-lifecycle columns persist and read
+// back through Create/Get/Update.
+func TestLifecycleFieldsRoundTrip(t *testing.T) {
+	s := testStore(t)
+	ctx := context.Background()
+	a := mk("acme", "sweeper")
+	a.ExecutionMode = ModeLongRunning
+	a.Schedule = "*/5 * * * *"
+	a.ComputeRef = "vm-123"
+	a.ServiceAccountID = "acme-sweeper"
+	if err := s.Create(ctx, a); err != nil {
+		t.Fatalf("create: %v", err)
+	}
+	got, err := s.Get(ctx, "acme", "sweeper")
+	if err != nil {
+		t.Fatalf("get: %v", err)
+	}
+	if got.ExecutionMode != ModeLongRunning || got.Schedule != "*/5 * * * *" ||
+		got.ComputeRef != "vm-123" || got.ServiceAccountID != "acme-sweeper" {
+		t.Fatalf("lifecycle fields not persisted: %+v", got)
+	}
+
+	got.Schedule = "0 9 * * 1"
+	got.ComputeRef = "vm-456"
+	got.UpdatedAt = time.Now().Unix()
+	if err := s.Update(ctx, got); err != nil {
+		t.Fatalf("update: %v", err)
+	}
+	got2, _ := s.Get(ctx, "acme", "sweeper")
+	if got2.Schedule != "0 9 * * 1" || got2.ComputeRef != "vm-456" {
+		t.Fatalf("update did not persist lifecycle edits: %+v", got2)
+	}
+}
+
+// TestDefaultExecutionMode: a fresh agent created without a mode reads back as
+// one-shot (the DEFAULT that the DDL + migration guarantee), never empty.
+func TestDefaultExecutionMode(t *testing.T) {
+	s := testStore(t)
+	ctx := context.Background()
+	if err := s.Create(ctx, mk("acme", "plain")); err != nil {
+		t.Fatalf("create: %v", err)
+	}
+	got, _ := s.Get(ctx, "acme", "plain")
+	if got.ExecutionMode != ModeOneShot {
+		t.Fatalf("default execution_mode = %q, want %q", got.ExecutionMode, ModeOneShot)
+	}
+}
+
+// TestListLongRunning: returns only long-running agents WITH a schedule, across
+// orgs, and each carries its own org (the scheduler scopes actions per agent).
+func TestListLongRunning(t *testing.T) {
+	s := testStore(t)
+	ctx := context.Background()
+
+	oneShot := mk("acme", "oneshot") // default one-shot
+	lr := mk("acme", "cron")
+	lr.ExecutionMode, lr.Schedule = ModeLongRunning, "* * * * *"
+	lrNoSched := mk("beta", "cron")
+	lrNoSched.ExecutionMode, lrNoSched.Schedule = ModeLongRunning, "" // no schedule -> excluded
+	lrOther := mk("beta", "nightly")
+	lrOther.ExecutionMode, lrOther.Schedule = ModeLongRunning, "0 0 * * *"
+
+	for _, a := range []Agent{oneShot, lr, lrNoSched, lrOther} {
+		if err := s.Create(ctx, a); err != nil {
+			t.Fatalf("seed %s/%s: %v", a.Org, a.Name, err)
+		}
+	}
+
+	got, err := s.ListLongRunning(ctx)
+	if err != nil {
+		t.Fatalf("list long-running: %v", err)
+	}
+	if len(got) != 2 {
+		t.Fatalf("want 2 scheduled agents (acme/cron, beta/nightly), got %d: %+v", len(got), got)
+	}
+	seen := map[string]string{}
+	for _, a := range got {
+		seen[a.Org+"/"+a.Name] = a.Schedule
+	}
+	if seen["acme/cron"] != "* * * * *" || seen["beta/nightly"] != "0 0 * * *" {
+		t.Fatalf("wrong scheduled set: %v", seen)
+	}
+	if _, bad := seen["beta/cron"]; bad {
+		t.Fatalf("long-running agent WITHOUT a schedule must be excluded")
+	}
+}
+
+// TestMigrationIdempotentOnLegacyDB: a DB created with the PRE-lifecycle schema
+// (no new columns) is migrated forward on open, existing rows survive with the
+// column defaults, and re-opening (re-running migrate) is a clean no-op.
+func TestMigrationIdempotentOnLegacyDB(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "legacy.db")
+
+	// Hand-build the legacy schema + a legacy row, exactly as the pre-lifecycle
+	// migrate() would have, then close.
+	legacy, err := sql.Open("sqlite", path)
+	if err != nil {
+		t.Fatalf("open legacy: %v", err)
+	}
+	const legacyDDL = `
+CREATE TABLE agents (
+  id TEXT PRIMARY KEY, org TEXT NOT NULL, name TEXT NOT NULL,
+  model TEXT NOT NULL DEFAULT '', instructions TEXT NOT NULL DEFAULT '',
+  description TEXT NOT NULL DEFAULT '', tools TEXT NOT NULL DEFAULT '[]',
+  status TEXT NOT NULL DEFAULT 'ready', created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL);`
+	if _, err := legacy.Exec(legacyDDL); err != nil {
+		t.Fatalf("legacy ddl: %v", err)
+	}
+	now := time.Now().Unix()
+	if _, err := legacy.Exec(
+		`INSERT INTO agents (id,org,name,model,instructions,description,tools,status,created_at,updated_at)
+		 VALUES ('old-id','acme','legacy','m','i','d','[]','ready',?,?)`, now, now); err != nil {
+		t.Fatalf("legacy insert: %v", err)
+	}
+	_ = legacy.Close()
+
+	// Open through the real store TWICE — the first migrates, the second proves
+	// idempotency (no error re-adding existing columns).
+	for i := 0; i < 2; i++ {
+		st, err := openStore(path)
+		if err != nil {
+			t.Fatalf("open #%d migrate failed: %v", i, err)
+		}
+		got, err := st.Get(context.Background(), "acme", "legacy")
+		if err != nil {
+			t.Fatalf("open #%d: legacy row lost: %v", i, err)
+		}
+		if got.ExecutionMode != ModeOneShot {
+			t.Fatalf("open #%d: migrated row default mode = %q, want %q", i, got.ExecutionMode, ModeOneShot)
+		}
+		if got.Schedule != "" || got.ComputeRef != "" || got.ServiceAccountID != "" {
+			t.Fatalf("open #%d: migrated defaults not empty: %+v", i, got)
+		}
+		_ = st.Close()
+	}
+}

clients/aihttp.go

@@ -0,0 +1,136 @@
+package clients
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	openai "github.com/sashabaranov/go-openai"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/clientcredentials"
+
+	"github.com/hanzoai/cloud/types"
+)
+
+// aiTracer emits the LLM/agent GenAI spans (OTel gen_ai.* semantic conventions)
+// shipped over the ZAP wire to o11y. One tracer for the whole clients package.
+var aiTracer = otel.Tracer("hanzo.ai/cloud")
+
+// httpAI is the real, in-process types.AIClient: it runs chat completions
+// against an OpenAI-compatible endpoint — the Hanzo LLM gateway
+// (https://api.hanzo.ai/v1). This is the ONE concrete inference client the
+// agents subsystem executes runs through; without it deps.AI is the fail-closed
+// stub and every POST /v1/agents/:name/run fail-closes rather than executing.
+//
+// Model routing is the gateway's job. The only cloud-side fallback is: an empty
+// request model → the operator-configured default. There is deliberately NO
+// in-code model aliasing (e.g. a "zen" → "zen3-nano" map) — that is config in
+// code, and the gateway already owns model resolution across its served set.
+type httpAI struct {
+	client       *openai.Client
+	defaultModel string
+}
+
+// aiHTTPTimeout bounds a single completion so a hung upstream cannot wedge an
+// agent run (or a scheduler tick) indefinitely. It is applied as a derived
+// deadline on the caller's context, so a caller carrying a tighter deadline
+// still wins — this is only a ceiling.
+const aiHTTPTimeout = 120 * time.Second
+
+// AIHTTPAt returns a types.AIClient that POSTs OpenAI-compatible chat
+// completions to baseURL, authenticated with apiKey. baseURL is the gateway
+// /v1 root (the go-openai client appends /chat/completions). defaultModel is
+// substituted when a ChatRequest carries no explicit model.
+//
+// apiKey is a KMS-injected secret and is NEVER logged: it lives only inside the
+// go-openai client's Authorization header. Callers log the base URL and default
+// model, never the key.
+func AIHTTPAt(baseURL, apiKey, defaultModel string) types.AIClient {
+	cfg := openai.DefaultConfig(apiKey)
+	cfg.BaseURL = strings.TrimRight(baseURL, "/")
+	return &httpAI{client: openai.NewClientWithConfig(cfg), defaultModel: defaultModel}
+}
+
+// AIHTTPM2M returns a types.AIClient that authenticates to the gateway with an
+// IAM client-credentials (M2M) token instead of a static key. This is the
+// durable Hanzo credential path: the cloud binary mints and auto-refreshes a
+// short-lived token from its OWN service identity (IAM_CLIENT_ID/SECRET), so
+// there is NO static key to rotate and no expiry cliff. On the Hanzo deployment
+// that identity resolves to admin/hanzo-cloud, which the gateway treats as
+// balance-exempt — so cloud's own per-org ResourceMeter stays the single
+// revenue debit (no double-bill).
+//
+// tokenURL is the IAM token endpoint ({issuer}/v1/iam/oauth/token). clientSecret
+// is a KMS-injected secret and is NEVER logged: it lives only inside the oauth2
+// token source. The token is fetched lazily on first use (boot never blocks on
+// IAM) and cached+refreshed automatically by the oauth2 client.
+//
+// go-openai sets its own Authorization header only when its authToken is
+// non-empty; here it is empty, so the sole auth header is the fresh Bearer the
+// oauth2 transport injects on every request.
+func AIHTTPM2M(baseURL, tokenURL, clientID, clientSecret, defaultModel string) types.AIClient {
+	cc := &clientcredentials.Config{
+		ClientID:     clientID,
+		ClientSecret: clientSecret,
+		TokenURL:     tokenURL,
+		// hanzo.id (Casdoor) expects the credentials in the form body, not Basic
+		// auth — matches the proven client_credentials call.
+		AuthStyle: oauth2.AuthStyleInParams,
+	}
+	cfg := openai.DefaultConfig("") // empty authToken → go-openai adds no header
+	cfg.BaseURL = strings.TrimRight(baseURL, "/")
+	cfg.HTTPClient = cc.Client(context.Background()) // caches + auto-refreshes
+	return &httpAI{client: openai.NewClientWithConfig(cfg), defaultModel: defaultModel}
+}
+
+// ChatCompletion maps a types.ChatRequest to a single user-message chat
+// completion and returns the assistant content. On a transport failure, a
+// non-2xx upstream status, or a response with no choices it returns an explicit
+// wrapped error — executeRun records that as an honest error-status run, never a
+// fabricated "ok". The error text names the model but never the key or prompt.
+func (a *httpAI) ChatCompletion(ctx context.Context, req *types.ChatRequest) (*types.ChatResponse, error) {
+	model := strings.TrimSpace(req.Model)
+	if model == "" {
+		model = a.defaultModel
+	}
+
+	// GenAI client span (OTel semantic conventions) — one span per LLM call,
+	// nested under any active agent-run span carried on ctx.
+	ctx, span := aiTracer.Start(ctx, "chat "+model, trace.WithSpanKind(trace.SpanKindClient))
+	defer span.End()
+	span.SetAttributes(
+		attribute.String("gen_ai.system", "hanzo"),
+		attribute.String("gen_ai.operation.name", "chat"),
+		attribute.String("gen_ai.request.model", model),
+	)
+
+	ctx, cancel := context.WithTimeout(ctx, aiHTTPTimeout)
+	defer cancel()
+
+	resp, err := a.client.CreateChatCompletion(ctx, openai.ChatCompletionRequest{
+		Model: model,
+		Messages: []openai.ChatCompletionMessage{
+			{Role: openai.ChatMessageRoleUser, Content: req.Prompt},
+		},
+	})
+	if err != nil {
+		span.RecordError(err)
+		span.SetStatus(codes.Error, "chat completion failed")
+		return nil, fmt.Errorf("cloud: chat completion (model %q): %w", model, err)
+	}
+	span.SetAttributes(
+		attribute.String("gen_ai.response.model", resp.Model),
+		attribute.Int("gen_ai.usage.input_tokens", resp.Usage.PromptTokens),
+		attribute.Int("gen_ai.usage.output_tokens", resp.Usage.CompletionTokens),
+	)
+	if len(resp.Choices) == 0 {
+		span.SetStatus(codes.Error, "no choices")
+		return nil, fmt.Errorf("cloud: chat completion (model %q): upstream returned no choices", model)
+	}
+	return &types.ChatResponse{Content: resp.Choices[0].Message.Content}, nil
+}

clients/aihttp_test.go

@@ -0,0 +1,193 @@
+package clients
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/hanzoai/cloud/types"
+)
+
+// TestAIHTTP_DefaultModelAndContent asserts the two happy-path contracts the
+// agents run path depends on: (1) an empty request model is replaced by the
+// configured default before the call leaves the process, an explicit model is
+// passed through verbatim, and (2) the assistant content is parsed out of
+// choices[0].message.content. It also asserts the key rides only in the
+// Authorization header (Bearer <key>) — the wiring the gateway authenticates.
+func TestAIHTTP_DefaultModelAndContent(t *testing.T) {
+	const defaultModel = "deepseek-v4-flash"
+	var gotModel, gotAuth, gotPath string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.Path
+		gotAuth = r.Header.Get("Authorization")
+		body, _ := io.ReadAll(r.Body)
+		var req struct {
+			Model    string `json:"model"`
+			Messages []struct {
+				Role    string `json:"role"`
+				Content string `json:"content"`
+			} `json:"messages"`
+		}
+		if err := json.Unmarshal(body, &req); err != nil {
+			t.Errorf("decode request: %v", err)
+		}
+		gotModel = req.Model
+		if len(req.Messages) != 1 || req.Messages[0].Role != "user" {
+			t.Errorf("want one user message, got %+v", req.Messages)
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"id": "chatcmpl-x", "object": "chat.completion", "created": 1, "model": req.Model,
+			"choices": []map[string]any{{
+				"index":         0,
+				"message":       map[string]string{"role": "assistant", "content": "hi there"},
+				"finish_reason": "stop",
+			}},
+		})
+	}))
+	defer srv.Close()
+
+	ai := AIHTTPAt(srv.URL, "sk-test", defaultModel)
+
+	// (1) empty model → default substituted; content parsed.
+	got, err := ai.ChatCompletion(context.Background(), &types.ChatRequest{Prompt: "say hi"})
+	if err != nil {
+		t.Fatalf("ChatCompletion: %v", err)
+	}
+	if gotModel != defaultModel {
+		t.Errorf("default model: got %q want %q", gotModel, defaultModel)
+	}
+	if got.Content != "hi there" {
+		t.Errorf("content: got %q want %q", got.Content, "hi there")
+	}
+	if gotAuth != "Bearer sk-test" {
+		t.Errorf("auth header: got %q want %q", gotAuth, "Bearer sk-test")
+	}
+	if gotPath != "/chat/completions" {
+		t.Errorf("path: got %q want /chat/completions", gotPath)
+	}
+
+	// (2) explicit model wins over the default.
+	if _, err := ai.ChatCompletion(context.Background(), &types.ChatRequest{Model: "zen3-nano", Prompt: "x"}); err != nil {
+		t.Fatalf("ChatCompletion explicit model: %v", err)
+	}
+	if gotModel != "zen3-nano" {
+		t.Errorf("explicit model: got %q want zen3-nano", gotModel)
+	}
+}
+
+// TestAIHTTP_UpstreamErrorMapped asserts a non-2xx upstream (429) becomes an
+// explicit wrapped error — executeRun renders it as an error-status run, never
+// a fabricated "ok".
+func TestAIHTTP_UpstreamErrorMapped(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusTooManyRequests)
+		_, _ = w.Write([]byte(`{"error":{"message":"rate limited","type":"rate_limit_error"}}`))
+	}))
+	defer srv.Close()
+	_, err := AIHTTPAt(srv.URL, "sk-test", "deepseek-v4-flash").
+		ChatCompletion(context.Background(), &types.ChatRequest{Prompt: "x"})
+	if err == nil {
+		t.Fatal("expected error on 429, got nil")
+	}
+	if !strings.Contains(err.Error(), "chat completion") {
+		t.Errorf("error not wrapped by client: %v", err)
+	}
+}
+
+// TestAIHTTP_ServerErrorMapped asserts a 5xx upstream also maps to an error.
+func TestAIHTTP_ServerErrorMapped(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusInternalServerError)
+		_, _ = w.Write([]byte(`{"error":{"message":"boom"}}`))
+	}))
+	defer srv.Close()
+	if _, err := AIHTTPAt(srv.URL, "sk-test", "deepseek-v4-flash").
+		ChatCompletion(context.Background(), &types.ChatRequest{Prompt: "x"}); err == nil {
+		t.Fatal("expected error on 500, got nil")
+	}
+}
+
+// TestAIHTTP_M2M asserts the M2M path mints a client-credentials token from the
+// IAM token endpoint and presents it as the completion's Bearer — the durable
+// no-static-key credential. One httptest server plays both roles: the token
+// endpoint (form-encoded client_credentials -> {access_token}) and the
+// completions endpoint (asserts Authorization == the minted token).
+func TestAIHTTP_M2M(t *testing.T) {
+	const minted = "iam-access-token-xyz"
+	var tokenHits int
+	var sawAuth string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/v1/iam/oauth/token":
+			tokenHits++
+			_ = r.ParseForm()
+			if r.PostFormValue("grant_type") != "client_credentials" {
+				t.Errorf("grant_type: got %q", r.PostFormValue("grant_type"))
+			}
+			if r.PostFormValue("client_id") != "hanzo-cloud" || r.PostFormValue("client_secret") != "s3cr3t" {
+				t.Errorf("creds not in form body: id=%q", r.PostFormValue("client_id"))
+			}
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = w.Write([]byte(`{"access_token":"` + minted + `","token_type":"Bearer","expires_in":3600}`))
+		case "/chat/completions":
+			sawAuth = r.Header.Get("Authorization")
+			w.Header().Set("Content-Type", "application/json")
+			_ = json.NewEncoder(w).Encode(map[string]any{
+				"id": "chatcmpl-m2m", "object": "chat.completion", "model": "deepseek-v4-flash",
+				"choices": []map[string]any{{"index": 0, "message": map[string]string{"role": "assistant", "content": "pong"}, "finish_reason": "stop"}},
+			})
+		default:
+			t.Errorf("unexpected path %q", r.URL.Path)
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer srv.Close()
+
+	ai := AIHTTPM2M(srv.URL /*baseURL*/, srv.URL+"/v1/iam/oauth/token" /*tokenURL*/, "hanzo-cloud", "s3cr3t", "deepseek-v4-flash")
+	got, err := ai.ChatCompletion(context.Background(), &types.ChatRequest{Prompt: "ping"})
+	if err != nil {
+		t.Fatalf("M2M ChatCompletion: %v", err)
+	}
+	if got.Content != "pong" {
+		t.Errorf("content: got %q want pong", got.Content)
+	}
+	if sawAuth != "Bearer "+minted {
+		t.Errorf("completion Authorization: got %q want %q", sawAuth, "Bearer "+minted)
+	}
+	if tokenHits == 0 {
+		t.Error("token endpoint was never called — M2M token was not minted")
+	}
+
+	// Second call reuses the cached token (no re-mint within its lifetime).
+	if _, err := ai.ChatCompletion(context.Background(), &types.ChatRequest{Prompt: "ping2"}); err != nil {
+		t.Fatalf("M2M second call: %v", err)
+	}
+	if tokenHits != 1 {
+		t.Errorf("expected token cached (1 mint), got %d mints", tokenHits)
+	}
+}
+
+// TestAIHTTP_EmptyChoices asserts a 200 with an empty choices array is a hard
+// error, not a silent empty completion.
+func TestAIHTTP_EmptyChoices(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"id":"x","object":"chat.completion","choices":[]}`))
+	}))
+	defer srv.Close()
+	_, err := AIHTTPAt(srv.URL, "sk-test", "deepseek-v4-flash").
+		ChatCompletion(context.Background(), &types.ChatRequest{Prompt: "x"})
+	if err == nil {
+		t.Fatal("expected error on empty choices, got nil")
+	}
+	if !strings.Contains(err.Error(), "no choices") {
+		t.Errorf("expected no-choices error, got: %v", err)
+	}
+}

clients/analytics/analytics.go

@@ -0,0 +1,418 @@
+// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package analytics mounts the Hanzo Cloud /v1/analytics/* surface: a native-Go,
+// per-org analytics read API over the `hanzo` ClickHouse warehouse (the
+// `datastore` cluster). It is the backend for the console Native Analytics module
+// (unified-analytics.md §5) — two read lenses over one warehouse:
+//
+//   - LLM lens (REAL today): hanzo.cloud_usage, the live per-org usage ledger the
+//     cloud o11y path already writes (requests, tokens, spend, models, errors).
+//   - Web/commerce lens (honest-empty until the collector emits): hanzo.events.
+//
+// ONE ClickHouse client. This package does NOT open a second connection: it rides
+// the SAME clickhouse-go/v2 client the ai subsystem's o11y ledger opens in the
+// shared Bootstrap (ai/object.InitDatastore → object.DatastoreQuery). DRY: one
+// transport, one pool, one set of KMS-injected DATASTORE_* creds — never
+// hard-coded, never a second design.
+//
+// TENANT ISOLATION is the security bar and is enforced SERVER-SIDE on every
+// request. The org is c.Org() — the value SanitizeIdentity minted from the
+// VALIDATED bearer owner claim (HIP-0026), never a client header — AND every
+// request must carry a validated principal (c.User() set, which SanitizeIdentity
+// sets ONLY for a verified bearer). This closes the Phase-1 "no-bearer + forged
+// X-Org-Id direct-to-pod" cross-tenant read exactly as clients/s3 does. Every
+// ClickHouse query binds the org POSITIONALLY (query.go llmWhere/eventsWhere), so
+// a maxpower token can NEVER read another org's analytics.
+//
+// Surface (all org-scoped; /v1 only; read-only):
+//
+//	GET /v1/analytics/overview     per-org KPIs (llm real; web/commerce honest-empty)
+//	GET /v1/analytics/timeseries   requests/tokens/spend over time (hour|day buckets)
+//	GET /v1/analytics/top          top models (real) + top products (honest-empty)
+//	GET /v1/analytics/health       subsystem health (datastore connectivity + lens tables)
+//
+// Registered as "analyticssvc" (NOT "analytics") + order 132: the name diverges
+// from the /v1/analytics route prefix so serve.go's generic GET /v1/<name>/health
+// liveness route parks at /v1/analyticssvc/health and our REAL /v1/analytics/health
+// (below) owns the probe — the same health-shadow-avoidance the kmssvc/s3svc
+// subsystems use. Order 132 binds /v1/analytics/* before the ai subsystem's /v1/*
+// catch-all (150).
+package analytics
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	aiobject "github.com/hanzoai/ai/object"
+	"github.com/hanzoai/cloud"
+	luxlog "github.com/luxfi/log"
+	"github.com/zap-proto/zip"
+)
+
+const (
+	// defaultTop / maxTop bound the /top result cardinality.
+	defaultTop = 10
+	maxTop     = 100
+	// probeTimeout bounds the health-endpoint table-existence probes so an
+	// unauthenticated liveness hit can never hang on a slow warehouse.
+	probeTimeout = 3 * time.Second
+)
+
+type svc struct {
+	log luxlog.Logger
+}
+
+// Mount wires the analytics surface onto app per HIP-0106.
+func Mount(app *zip.App, deps cloud.Deps) error {
+	if app == nil {
+		return fmt.Errorf("analytics.Mount: nil zip.App")
+	}
+	log := deps.Logger
+	if log == nil {
+		return fmt.Errorf("analytics.Mount: nil deps.Logger")
+	}
+	log = log.New("subsystem", "analytics")
+	s := &svc{log: log}
+
+	// Health owns /v1/analytics/health explicitly (not JWT-gated: liveness must be
+	// probe-able). The data endpoints are all org-gated in-handler.
+	app.Get("/v1/analytics/health", s.health)
+	app.Get("/v1/analytics/overview", s.overview)
+	app.Get("/v1/analytics/timeseries", s.timeseries)
+	app.Get("/v1/analytics/top", s.top)
+
+	log.Info("analytics mounted", "warehouse", "hanzo", "brand", deps.Brand)
+	return nil
+}
+
+func init() {
+	cloud.Register("analyticssvc", 132, func(app any, deps cloud.Deps) error {
+		a, ok := app.(*zip.App)
+		if !ok {
+			return fmt.Errorf("analytics.Mount: app is %T, want *zip.App", app)
+		}
+		return Mount(a, deps)
+	})
+}
+
+// ── shared helpers ──────────────────────────────────────────────────────────
+
+// tenant resolves the org — the tenant-isolation KEY — for a request, and refuses
+// the forgeable data path. It REQUIRES a validated principal: c.User() (X-User-Id)
+// is set by SanitizeIdentity ONLY when it verified a bearer/cookie; on the Phase-1
+// no-principal path it may RESTORE a client's raw X-Org-Id but leaves X-User-Id
+// empty. Gating on c.User() therefore refuses an in-cluster caller that forges
+// `X-Org-Id: victim` with NO bearer — the same defense clients/s3 uses — while
+// breaking no legitimate caller (all reach this via a user-bound bearer).
+//
+// The org is used EXACTLY as minted (no case-fold/normalize): the cloud_usage
+// ledger stored `organization` verbatim from the same owner claim, so an exact
+// match is required to see one's own rows (normalizing could collapse or miss).
+func tenant(c *zip.Ctx) (string, bool) {
+	if strings.TrimSpace(c.User()) == "" {
+		return "", false // no validated principal — refuse the forgeable data path
+	}
+	org := strings.TrimSpace(c.Org())
+	if org == "" || len(org) > 128 {
+		return "", false
+	}
+	return org, true
+}
+
+// window resolves the [start,end) window + bucket interval from ?range/?start/?end,
+// reusing ai/object.ResolveCloudUsageWindow so analytics and the console2 Overview
+// share ONE window grammar (24h|7d|30d|custom). A bad range is a 400.
+func window(c *zip.Ctx) (time.Time, time.Time, string, string, error) {
+	rangeLabel := strings.TrimSpace(c.Query("range"))
+	start, end, interval, err := aiobject.ResolveCloudUsageWindow(rangeLabel, c.Query("start"), c.Query("end"), time.Now())
+	if err != nil {
+		return time.Time{}, time.Time{}, "", "", zip.ErrBadRequest(err.Error())
+	}
+	if rangeLabel == "" {
+		rangeLabel = "24h"
+	}
+	return start, end, interval, rangeLabel, nil
+}
+
+// requireDatastore returns the honest 503 when the ClickHouse ledger is not
+// connected, rather than fabricating zeros. Mirrors ai/object's read gate.
+func requireDatastore() error {
+	if !aiobject.DatastoreEnabled() {
+		return zip.Errorf(http.StatusServiceUnavailable, "analytics warehouse unavailable: datastore (ClickHouse) not connected")
+	}
+	return nil
+}
+
+// warehouseErr maps a ClickHouse query failure to the HONEST HTTP status. A
+// connectivity failure — the warehouse became unreachable mid-request (dial /
+// i/o timeout / refused / reset / EOF) — is a transient 503 "unavailable", the
+// SAME contract requireDatastore() uses when the pool never connected. Only a
+// REACHABLE warehouse that rejected the query (bad SQL, protocol error) is a 502
+// bad-gateway. This is the fix for /v1/analytics/* surfacing a raw 502 on a
+// ClickHouse `:9000` i/o timeout — the caller now gets an honest 503 it can retry.
+func warehouseErr(kind string, err error) error {
+	if isWarehouseUnreachable(err) {
+		return zip.Errorf(http.StatusServiceUnavailable, "analytics warehouse unavailable: %s: %v", kind, err)
+	}
+	return zip.Errorf(http.StatusBadGateway, "analytics %s query: %v", kind, err)
+}
+
+// isWarehouseUnreachable reports whether err is a transport/connectivity failure
+// to ClickHouse (as opposed to a query the warehouse actively rejected). It checks
+// the typed context/net signals first, then the connectivity strings the
+// clickhouse-go driver surfaces without a typed net.Error wrapper.
+func isWarehouseUnreachable(err error) bool {
+	if err == nil {
+		return false
+	}
+	if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
+		return true
+	}
+	var ne net.Error
+	if errors.As(err, &ne) && ne.Timeout() {
+		return true
+	}
+	s := strings.ToLower(err.Error())
+	for _, sub := range []string{
+		"i/o timeout", "timeout", "connection refused", "connection reset",
+		"no route to host", "broken pipe", "eof", "network is unreachable",
+		"no such host", "dial ", "connect: ", "read: connection", "write: connection",
+	} {
+		if strings.Contains(s, sub) {
+			return true
+		}
+	}
+	return false
+}
+
+func topLimit(c *zip.Ctx) int {
+	n, err := strconv.Atoi(strings.TrimSpace(c.Query("limit")))
+	if err != nil || n <= 0 {
+		return defaultTop
+	}
+	if n > maxTop {
+		return maxTop
+	}
+	return n
+}
+
+// ── /v1/analytics/overview ──────────────────────────────────────────────────
+
+func (s *svc) overview(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("valid bearer required")
+	}
+	start, end, interval, rangeLabel, err := window(c)
+	if err != nil {
+		return err
+	}
+	if err := requireDatastore(); err != nil {
+		return err
+	}
+	ctx := c.Context()
+	// Ensure the ai-owned ledger table exists (idempotent, latched) so a fresh
+	// warehouse yields honest zeros, not an error. We NEVER create hanzo.events —
+	// that table is operator-owned (unified-analytics.md §3.1).
+	if err := aiobject.EnsureCloudUsageTable(ctx); err != nil {
+		return zip.Errorf(http.StatusServiceUnavailable, "analytics warehouse unavailable: %v", err)
+	}
+
+	// LLM lens — REAL per-org KPIs.
+	where, args := llmWhere(org, start, end)
+	llmSQL := "SELECT count() AS requests, sum(total_tokens) AS tokens, " +
+		"sum(prompt_tokens) AS prompt_tokens, sum(completion_tokens) AS completion_tokens, " +
+		"sum(cost_cents) AS cost_cents, uniqExact(model) AS models, uniqExact(provider) AS providers, " +
+		"countIf(status = 'error') AS errors FROM " + llmTable + " WHERE " + where
+	llmRows, err := aiobject.DatastoreQuery(ctx, llmSQL, args...)
+	if err != nil {
+		return warehouseErr("llm", err)
+	}
+	llm := buildLLMOverview(firstRow(llmRows))
+
+	// Web/commerce lens — one events query; degrades to honest-empty if the events
+	// table is absent (not yet provisioned) or errors.
+	ewhere, eargs := eventsWhere(org, start, end)
+	eventsSQL := "SELECT countIf(event = '$pageview') AS pageviews, uniqExact(distinct_id) AS visitors, " +
+		"uniqExact(session_id) AS sessions, countIf(event = 'order_completed') AS orders, " +
+		"toFloat64(sum(revenue)) AS revenue FROM " + eventsTable + " WHERE " + ewhere
+	eventsRows, eerr := aiobject.DatastoreQuery(ctx, eventsSQL, eargs...)
+	eventsOK := eerr == nil
+	if eerr != nil {
+		s.log.Debug("events lens unavailable (honest-empty)", "err", eerr)
+	}
+	erow := firstRow(eventsRows)
+
+	return c.JSON(http.StatusOK, Overview{
+		Range:    rangeLabel,
+		Start:    start.UTC().Format(time.RFC3339),
+		End:      end.UTC().Format(time.RFC3339),
+		Interval: interval,
+		Scope:    Scope{Org: org},
+		LLM:      llm,
+		Web:      buildWebOverview(erow, eventsOK),
+		Commerce: buildCommerceOverview(erow, eventsOK),
+	})
+}
+
+// ── /v1/analytics/timeseries ────────────────────────────────────────────────
+
+func (s *svc) timeseries(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("valid bearer required")
+	}
+	start, end, interval, rangeLabel, err := window(c)
+	if err != nil {
+		return err
+	}
+	if err := requireDatastore(); err != nil {
+		return err
+	}
+	ctx := c.Context()
+	if err := aiobject.EnsureCloudUsageTable(ctx); err != nil {
+		return zip.Errorf(http.StatusServiceUnavailable, "analytics warehouse unavailable: %v", err)
+	}
+
+	// bucketFn is a CLOSED server-chosen enum (never user input), so interpolating
+	// it is injection-safe; the org + time bounds stay bound parameters.
+	bucketFn := "Hour"
+	if interval == "day" {
+		bucketFn = "Day"
+	}
+	where, args := llmWhere(org, start, end)
+	seriesSQL := fmt.Sprintf("SELECT toStartOf%s(timestamp, 'UTC') AS bucket, count() AS requests, "+
+		"sum(total_tokens) AS tokens, sum(cost_cents) AS cost_cents FROM %s WHERE %s GROUP BY bucket ORDER BY bucket",
+		bucketFn, llmTable, where)
+	rows, err := aiobject.DatastoreQuery(ctx, seriesSQL, args...)
+	if err != nil {
+		return warehouseErr("timeseries", err)
+	}
+
+	return c.JSON(http.StatusOK, Timeseries{
+		Range:    rangeLabel,
+		Start:    start.UTC().Format(time.RFC3339),
+		End:      end.UTC().Format(time.RFC3339),
+		Interval: interval,
+		Scope:    Scope{Org: org},
+		Series:   buildSeries(start, end, interval, rows),
+		Source:   llmTable,
+	})
+}
+
+// ── /v1/analytics/top ───────────────────────────────────────────────────────
+
+func (s *svc) top(c *zip.Ctx) error {
+	org, ok := tenant(c)
+	if !ok {
+		return zip.ErrForbidden("valid bearer required")
+	}
+	start, end, _, rangeLabel, err := window(c)
+	if err != nil {
+		return err
+	}
+	if err := requireDatastore(); err != nil {
+		return err
+	}
+	ctx := c.Context()
+	if err := aiobject.EnsureCloudUsageTable(ctx); err != nil {
+		return zip.Errorf(http.StatusServiceUnavailable, "analytics warehouse unavailable: %v", err)
+	}
+	limit := topLimit(c)
+
+	// Top models — REAL. limit is a validated int (never user text) so %d is safe;
+	// org + time stay bound parameters.
+	where, args := llmWhere(org, start, end)
+	modelSQL := fmt.Sprintf("SELECT model, any(provider) AS provider, count() AS requests, "+
+		"sum(total_tokens) AS tokens, sum(cost_cents) AS cost_cents FROM %s WHERE %s "+
+		"GROUP BY model ORDER BY cost_cents DESC, requests DESC LIMIT %d", llmTable, where, limit)
+	modelRows, err := aiobject.DatastoreQuery(ctx, modelSQL, args...)
+	if err != nil {
+		return warehouseErr("top-models", err)
+	}
+
+	// Top products — honest-empty until commerce emits order events.
+	ewhere, eargs := eventsWhere(org, start, end)
+	prodSQL := fmt.Sprintf("SELECT product_id AS productId, countIf(event = 'order_completed') AS orders, "+
+		"toFloat64(sum(revenue)) AS revenue, sum(quantity) AS units FROM %s WHERE %s AND product_id != '' "+
+		"GROUP BY product_id ORDER BY revenue DESC LIMIT %d", eventsTable, ewhere, limit)
+	prodRows, perr := aiobject.DatastoreQuery(ctx, prodSQL, eargs...)
+
+	return c.JSON(http.StatusOK, Top{
+		Range:    rangeLabel,
+		Start:    start.UTC().Format(time.RFC3339),
+		End:      end.UTC().Format(time.RFC3339),
+		Scope:    Scope{Org: org},
+		Models:   buildTopModels(modelRows),
+		Products: buildTopProducts(prodRows, perr == nil),
+	})
+}
+
+// ── /v1/analytics/health ────────────────────────────────────────────────────
+
+// health is a REAL probe: it reports datastore connectivity (the load-bearing
+// signal) and, when connected, the availability of each lens table. Not
+// JWT-gated (liveness must be probe-able) and it NEVER reads tenant data — only
+// table existence. 503 when the warehouse is unreachable so a readiness probe
+// can gate; 200 otherwise even if the events lens is not yet provisioned (that is
+// honest-empty, not a failure).
+func (s *svc) health(c *zip.Ctx) error {
+	connected := aiobject.DatastoreEnabled()
+	res := map[string]any{
+		"service":   "analytics",
+		"status":    "ok",
+		"datastore": connected,
+		"warehouse": "hanzo",
+	}
+	if !connected {
+		res["status"] = "degraded"
+		res["reason"] = "datastore (ClickHouse) not connected"
+		return c.JSON(http.StatusServiceUnavailable, res)
+	}
+	ctx, cancel := context.WithTimeout(c.Context(), probeTimeout)
+	defer cancel()
+	res["lenses"] = map[string]any{
+		"llm":    map[string]any{"table": llmTable, "available": tableExists(ctx, llmTable)},
+		"events": map[string]any{"table": eventsTable, "available": tableExists(ctx, eventsTable)},
+	}
+	return c.JSON(http.StatusOK, res)
+}
+
+// tableExists probes ClickHouse for a table's presence. The name is a package
+// constant (never user input), so `EXISTS TABLE` is safe. Any error → false
+// (honest "not available") rather than surfacing.
+func tableExists(ctx context.Context, qualified string) bool {
+	rows, err := aiobject.DatastoreQuery(ctx, "EXISTS TABLE "+qualified)
+	if err != nil || len(rows) == 0 {
+		return false
+	}
+	for _, v := range rows[0] {
+		return aInt64(v) == 1
+	}
+	return false
+}
+
+func firstRow(rows []map[string]any) map[string]any {
+	if len(rows) == 0 {
+		return map[string]any{}
+	}
+	return rows[0]
+}

clients/analytics/analytics_test.go

@@ -0,0 +1,185 @@
+// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package analytics
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+// TestLLMWhereBindsOrgPositionally is THE tenant-isolation proof at the SQL
+// boundary: the org is ALWAYS the trailing bound parameter (never interpolated),
+// the predicate is "organization = ?", and the org value NEVER appears in the SQL
+// string. So a maxpower query and an acme query differ ONLY in a bound arg — one
+// tenant can never read another's rows, and a hostile org slug can't escape into
+// SQL.
+func TestLLMWhereBindsOrgPositionally(t *testing.T) {
+	start := time.Date(2026, 6, 24, 0, 0, 0, 0, time.UTC)
+	end := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
+
+	for _, org := range []string{"maxpower", "acme", "o'; DROP TABLE hanzo.cloud_usage; --"} {
+		sql, args := llmWhere(org, start, end)
+		if !strings.Contains(sql, "organization = ?") {
+			t.Fatalf("llmWhere sql must bind organization: %q", sql)
+		}
+		if strings.Contains(sql, org) {
+			t.Fatalf("org %q must NOT be interpolated into sql: %q", org, sql)
+		}
+		if len(args) != 3 {
+			t.Fatalf("want 3 bound args (start,end,org), got %d: %v", len(args), args)
+		}
+		if got, ok := args[2].(string); !ok || got != org {
+			t.Fatalf("org must be the trailing bound arg verbatim, want %q got %v", org, args[2])
+		}
+		// Time bounds are also bound (as CH DateTime literals), never interpolated.
+		if !strings.Contains(sql, "timestamp >= ? AND timestamp < ?") {
+			t.Fatalf("time bounds must be parameterized: %q", sql)
+		}
+	}
+}
+
+// TestEventsWhereBindsOrgPositionally: the events lens keys on tenant_id, same
+// bound-parameter discipline.
+func TestEventsWhereBindsOrgPositionally(t *testing.T) {
+	start := time.Date(2026, 6, 24, 0, 0, 0, 0, time.UTC)
+	end := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
+	sql, args := eventsWhere("maxpower", start, end)
+	if !strings.Contains(sql, "tenant_id = ?") {
+		t.Fatalf("eventsWhere must bind tenant_id: %q", sql)
+	}
+	if strings.Contains(sql, "maxpower") {
+		t.Fatalf("org must not be interpolated: %q", sql)
+	}
+	if got, ok := args[2].(string); !ok || got != "maxpower" {
+		t.Fatalf("org must be trailing bound arg, got %v", args[2])
+	}
+}
+
+// TestBuildLLMOverviewRealNumbers: the flagship assembler over a realistic row
+// (maxpower's live shape ≈ 21 req / 3.2K tokens / $1.20 / 3 models). Proves the
+// KPIs and the errorRate math are exact.
+func TestBuildLLMOverviewRealNumbers(t *testing.T) {
+	// Mimics the direct ClickHouse driver's native scan types (uint64 aggregates).
+	row := map[string]any{
+		"requests":          uint64(21),
+		"tokens":            uint64(3200),
+		"prompt_tokens":     uint64(2100),
+		"completion_tokens": uint64(1100),
+		"cost_cents":        uint64(120),
+		"models":            uint64(3),
+		"providers":         uint64(2),
+		"errors":            uint64(0),
+	}
+	o := buildLLMOverview(row)
+	if !o.Available {
+		t.Fatal("llm overview must be available when the datastore answered")
+	}
+	if o.Requests != 21 || o.Tokens != 3200 || o.SpendCents != 120 || o.Models != 3 || o.Providers != 2 {
+		t.Fatalf("KPI mismatch: %+v", o)
+	}
+	if o.PromptTokens != 2100 || o.CompletionTokens != 1100 {
+		t.Fatalf("token split mismatch: %+v", o)
+	}
+	if o.ErrorRate != 0 {
+		t.Fatalf("errorRate want 0, got %v", o.ErrorRate)
+	}
+	if o.Source != "hanzo.cloud_usage" {
+		t.Fatalf("source want hanzo.cloud_usage, got %q", o.Source)
+	}
+}
+
+// TestBuildLLMOverviewHonestEmpty: an empty aggregate (no usage in the window)
+// yields honest zeros — never fabricated, and NOT unavailable (the datastore did
+// answer; there is just nothing).
+func TestBuildLLMOverviewHonestEmpty(t *testing.T) {
+	o := buildLLMOverview(map[string]any{})
+	if !o.Available {
+		t.Fatal("empty window must still be Available (honest-zero, not unavailable)")
+	}
+	if o.Requests != 0 || o.Tokens != 0 || o.SpendCents != 0 || o.Models != 0 || o.ErrorRate != 0 {
+		t.Fatalf("empty overview must be all-zero, got %+v", o)
+	}
+}
+
+// TestErrorRate: errors/requests, rounded to 3 places.
+func TestErrorRate(t *testing.T) {
+	o := buildLLMOverview(map[string]any{"requests": uint64(10), "errors": uint64(2)})
+	if o.ErrorRate != 0.2 {
+		t.Fatalf("errorRate want 0.2, got %v", o.ErrorRate)
+	}
+}
+
+// TestOrgAOverviewDiffersFromOrgB: combined with the where-isolation proof, each
+// org's query returns only its own rows; distinct rows assemble to distinct
+// overviews. (org A's overview != org B's.)
+func TestOrgAOverviewDiffersFromOrgB(t *testing.T) {
+	a := buildLLMOverview(map[string]any{"requests": uint64(21), "tokens": uint64(3200), "cost_cents": uint64(120)})
+	b := buildLLMOverview(map[string]any{"requests": uint64(4), "tokens": uint64(500), "cost_cents": uint64(9)})
+	if a.Requests == b.Requests || a.Tokens == b.Tokens || a.SpendCents == b.SpendCents {
+		t.Fatalf("distinct orgs must assemble distinct overviews: a=%+v b=%+v", a, b)
+	}
+}
+
+// TestBuildSeriesGapFill: sparse ClickHouse buckets become an evenly-spaced,
+// gap-filled series across the window (zeros where no data, real where present).
+func TestBuildSeriesGapFill(t *testing.T) {
+	start := time.Date(2026, 6, 28, 0, 0, 0, 0, time.UTC)
+	end := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC) // 3 daily buckets: 28, 29, 30
+	rows := []map[string]any{
+		{"bucket": time.Date(2026, 6, 29, 0, 0, 0, 0, time.UTC), "requests": uint64(5), "tokens": uint64(100), "cost_cents": uint64(10)},
+	}
+	series := buildSeries(start, end, "day", rows)
+	if len(series) != 3 {
+		t.Fatalf("want 3 gap-filled daily points, got %d: %+v", len(series), series)
+	}
+	if series[0].Requests != 0 || series[0].T != "2026-06-28T00:00:00Z" {
+		t.Fatalf("first bucket must be honest-zero 06-28, got %+v", series[0])
+	}
+	if series[1].Requests != 5 || series[1].Tokens != 100 || series[1].SpendCents != 10 {
+		t.Fatalf("06-29 bucket must carry real data, got %+v", series[1])
+	}
+	if series[2].Requests != 0 {
+		t.Fatalf("06-30 bucket must be honest-zero, got %+v", series[2])
+	}
+}
+
+// TestBuildTopModelsSortAndPct: models sort by spend desc and each pct is its
+// share of total spend.
+func TestBuildTopModelsSortAndPct(t *testing.T) {
+	rows := []map[string]any{
+		{"model": "gpt-4o-mini", "provider": "do-ai", "requests": uint64(3), "tokens": uint64(200), "cost_cents": uint64(20)},
+		{"model": "claude-sonnet-4-5", "provider": "anthropic", "requests": uint64(9), "tokens": uint64(3000), "cost_cents": uint64(80)},
+	}
+	top := buildTopModels(rows)
+	if !top.Available || len(top.Items) != 2 {
+		t.Fatalf("want 2 models available, got %+v", top)
+	}
+	if top.Items[0].Model != "claude-sonnet-4-5" {
+		t.Fatalf("highest-spend model must sort first, got %q", top.Items[0].Model)
+	}
+	// 80 of 100 total = 80%, 20 of 100 = 20%.
+	if top.Items[0].Pct != 80 || top.Items[1].Pct != 20 {
+		t.Fatalf("pct shares wrong: %v / %v", top.Items[0].Pct, top.Items[1].Pct)
+	}
+}
+
+// TestBuildTopProductsHonestEmpty: when the events table is absent (ok=false) the
+// products lens is honestly reported unavailable with an empty (non-nil) list.
+func TestBuildTopProductsHonestEmpty(t *testing.T) {
+	tp := buildTopProducts(nil, false)
+	if tp.Available {
+		t.Fatal("products must be unavailable when events table is absent")
+	}
+	if tp.Items == nil || len(tp.Items) != 0 {
+		t.Fatalf("items must be an empty (non-nil) slice, got %#v", tp.Items)
+	}
+	if tp.Reason == "" || tp.Source != "hanzo.events" {
+		t.Fatalf("must carry honest reason + source, got %+v", tp)
+	}
+}

clients/analytics/http_test.go

@@ -0,0 +1,131 @@
+// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package analytics
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/hanzoai/cloud"
+	luxlog "github.com/luxfi/log"
+	"github.com/zap-proto/zip"
+)
+
+func mountApp(t *testing.T) *zip.App {
+	t.Helper()
+	app := zip.New(zip.Config{Logger: luxlog.New("test")})
+	if err := Mount(app, cloud.Deps{Logger: luxlog.New("test")}); err != nil {
+		t.Fatalf("Mount: %v", err)
+	}
+	return app
+}
+
+// do issues a request. user simulates the SanitizeIdentity-minted X-User-Id
+// (present ONLY for a validated bearer); org simulates the minted X-Org-Id. In the
+// harness there is no middleware, so c.User()/c.Org() read these headers directly
+// — exactly the values SanitizeIdentity would set downstream.
+func do(t *testing.T, app *zip.App, method, path, user, org string) (int, []byte) {
+	t.Helper()
+	req := httptest.NewRequest(method, path, nil)
+	if user != "" {
+		req.Header.Set("X-User-Id", user)
+	}
+	if org != "" {
+		req.Header.Set("X-Org-Id", org)
+	}
+	resp, err := app.Fiber().Test(req)
+	if err != nil {
+		t.Fatalf("Test %s %s: %v", method, path, err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	b, _ := io.ReadAll(resp.Body)
+	return resp.StatusCode, b
+}
+
+var dataEndpoints = []string{
+	"/v1/analytics/overview",
+	"/v1/analytics/timeseries",
+	"/v1/analytics/top",
+}
+
+// TestNoPrincipalForbidden: no validated principal (no X-User-Id) → 403 on every
+// data endpoint. This is the "no-Bearer → 403" contract.
+func TestNoPrincipalForbidden(t *testing.T) {
+	app := mountApp(t)
+	for _, p := range dataEndpoints {
+		if code, _ := do(t, app, http.MethodGet, p, "", ""); code != http.StatusForbidden {
+			t.Fatalf("no-principal GET %s want 403, got %d", p, code)
+		}
+	}
+}
+
+// TestForgedOrgWithoutBearerForbidden: THE cross-tenant-forge proof. A caller that
+// reaches the pod directly with a raw `X-Org-Id: maxpower` but NO validated
+// principal (no X-User-Id) is refused 403 — it can never read maxpower's analytics
+// off the Phase-1 header-passthrough path. (SanitizeIdentity leaves X-User-Id
+// empty on that path; our tenant() gate rejects it.)
+func TestForgedOrgWithoutBearerForbidden(t *testing.T) {
+	app := mountApp(t)
+	for _, p := range dataEndpoints {
+		if code, _ := do(t, app, http.MethodGet, p, "", "maxpower"); code != http.StatusForbidden {
+			t.Fatalf("forged-org-no-bearer GET %s want 403, got %d", p, code)
+		}
+	}
+}
+
+// TestDatastoreDisabledHonest503: a VALIDATED principal, but the ClickHouse ledger
+// is not connected (DatastoreEnabled()==false in this harness) → honest 503, never
+// a fake 200 with zeros. Proves the "no fabricated metrics" invariant.
+func TestDatastoreDisabledHonest503(t *testing.T) {
+	app := mountApp(t)
+	for _, p := range dataEndpoints {
+		code, body := do(t, app, http.MethodGet, p, "user-dave", "maxpower")
+		if code != http.StatusServiceUnavailable {
+			t.Fatalf("datastore-down GET %s want 503, got %d (%s)", p, code, body)
+		}
+	}
+}
+
+// TestBadRangeIs400: a validated principal with an unknown ?range → 400 (before
+// the datastore is even consulted).
+func TestBadRangeIs400(t *testing.T) {
+	app := mountApp(t)
+	code, _ := do(t, app, http.MethodGet, "/v1/analytics/overview?range=bogus", "user-dave", "maxpower")
+	if code != http.StatusBadRequest {
+		t.Fatalf("bad range want 400, got %d", code)
+	}
+}
+
+// TestHealthOwnedByAnalyticsHonest: /v1/analytics/health is the analytics
+// subsystem's REAL probe (service=analytics, datastore bool), NOT serve.go's
+// generic GET /v1/<name>/health fake-200 (which never mounts here because we
+// register as "analyticssvc"). With the datastore down it 503s honestly. Health
+// needs no principal — liveness must be probe-able.
+func TestHealthOwnedByAnalyticsHonest(t *testing.T) {
+	app := mountApp(t)
+	code, body := do(t, app, http.MethodGet, "/v1/analytics/health", "", "")
+	if code != http.StatusServiceUnavailable {
+		t.Fatalf("health (datastore down) want 503, got %d (%s)", code, body)
+	}
+	var h map[string]any
+	if err := json.Unmarshal(body, &h); err != nil {
+		t.Fatalf("health json: %v (%s)", err, body)
+	}
+	if h["service"] != "analytics" {
+		t.Fatalf("health service want analytics (not generic liveness), got %v", h["service"])
+	}
+	if h["datastore"] != false {
+		t.Fatalf("health datastore want false when disconnected, got %v", h["datastore"])
+	}
+	if h["status"] != "degraded" {
+		t.Fatalf("health status want degraded when datastore down, got %v", h["status"])
+	}
+}

clients/analytics/query.go

@@ -0,0 +1,431 @@
+// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Pure core of the analytics lens: SQL predicate builders, ClickHouse value
+// coercers, and the pure assemblers that turn raw ClickHouse rows into the
+// response structs. Everything here is I/O-free so the tests drive it with mock
+// rows — no ClickHouse needed — exactly as ai/object/cloud_usage.go proves out
+// its Overview assembler. The handlers (analytics.go) are the thin orchestration
+// that fetches the rows and calls these.
+//
+// THE ONE TENANCY INVARIANT lives here: llmWhere / eventsWhere ALWAYS emit
+// "… = ?" with the org bound POSITIONALLY (never interpolated), so no query this
+// package builds can read a tenant other than the caller's, and a hostile org
+// slug can never escape into SQL. The isolation test asserts this directly.
+package analytics
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Warehouse + tables (the ONE analytics warehouse per unified-analytics.md §1).
+const (
+	llmTable    = "hanzo.cloud_usage" // live LLM usage ledger (real data today)
+	eventsTable = "hanzo.events"      // web/commerce/UI wide event table (honest-empty until the collector emits)
+)
+
+// ── Tenancy predicates (the isolation boundary) ─────────────────────────────
+//
+// Both builders bind the org POSITIONALLY. The time bounds are bound too (as
+// ClickHouse DateTime string literals, the proven cloud_usage.go transport), so
+// NOTHING user-derived is ever interpolated. cloud_usage keys the tenant on
+// `organization`; hanzo.events keys it on `tenant_id` (== the IAM org slug).
+
+// llmWhere is the org-scoped time predicate for hanzo.cloud_usage. org is the
+// validated IAM owner slug, passed EXACTLY (the ledger stored it verbatim); it is
+// always the trailing bound parameter.
+func llmWhere(org string, start, end time.Time) (string, []any) {
+	return "timestamp >= ? AND timestamp < ? AND organization = ?",
+		[]any{tsLiteral(start), tsLiteral(end), org}
+}
+
+// eventsWhere is the org-scoped time predicate for hanzo.events. Same shape as
+// llmWhere but keyed on `tenant_id` (the events table's canonical org column).
+func eventsWhere(org string, start, end time.Time) (string, []any) {
+	return "timestamp >= ? AND timestamp < ? AND tenant_id = ?",
+		[]any{tsLiteral(start), tsLiteral(end), org}
+}
+
+// tsLiteral formats a time as a ClickHouse DateTime literal (UTC). Bound as a
+// string arg — identical to ai/object/cloud_usage.go's cloudUsageTS.
+func tsLiteral(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05") }
+
+// ── Response types ──────────────────────────────────────────────────────────
+
+type Scope struct {
+	Org string `json:"org"`
+}
+
+// LLMOverview is the flagship lens: real per-org KPIs from hanzo.cloud_usage.
+type LLMOverview struct {
+	Available        bool    `json:"available"`
+	Requests         int64   `json:"requests"`
+	Tokens           int64   `json:"tokens"`
+	PromptTokens     int64   `json:"promptTokens"`
+	CompletionTokens int64   `json:"completionTokens"`
+	SpendCents       int64   `json:"spendCents"`
+	Models           int64   `json:"models"`
+	Providers        int64   `json:"providers"`
+	Errors           int64   `json:"errors"`
+	ErrorRate        float64 `json:"errorRate"` // 0..1, errors/requests
+	Source           string  `json:"source"`
+}
+
+// WebOverview is the web lens over hanzo.events. Honest-empty (Available=false)
+// until the collector emits web events.
+type WebOverview struct {
+	Available bool   `json:"available"`
+	Reason    string `json:"reason,omitempty"`
+	Pageviews int64  `json:"pageviews"`
+	Visitors  int64  `json:"visitors"`
+	Sessions  int64  `json:"sessions"`
+	Source    string `json:"source"`
+}
+
+// CommerceOverview is the commerce lens over hanzo.events. Honest-empty until
+// commerce emits order events.
+type CommerceOverview struct {
+	Available bool    `json:"available"`
+	Reason    string  `json:"reason,omitempty"`
+	Orders    int64   `json:"orders"`
+	Revenue   float64 `json:"revenue"`
+	AOV       float64 `json:"aov"` // revenue/orders
+	Source    string  `json:"source"`
+}
+
+type Overview struct {
+	Range    string           `json:"range"`
+	Start    string           `json:"start"`
+	End      string           `json:"end"`
+	Interval string           `json:"interval"`
+	Scope    Scope            `json:"scope"`
+	LLM      LLMOverview      `json:"llm"`
+	Web      WebOverview      `json:"web"`
+	Commerce CommerceOverview `json:"commerce"`
+}
+
+type SeriesPoint struct {
+	T          string `json:"t"` // RFC3339 bucket start (UTC)
+	Requests   int64  `json:"requests"`
+	Tokens     int64  `json:"tokens"`
+	SpendCents int64  `json:"spendCents"`
+}
+
+type Timeseries struct {
+	Range    string        `json:"range"`
+	Start    string        `json:"start"`
+	End      string        `json:"end"`
+	Interval string        `json:"interval"`
+	Scope    Scope         `json:"scope"`
+	Series   []SeriesPoint `json:"series"`
+	Source   string        `json:"source"`
+}
+
+type ModelRow struct {
+	Model      string  `json:"model"`
+	Provider   string  `json:"provider"`
+	Requests   int64   `json:"requests"`
+	Tokens     int64   `json:"tokens"`
+	SpendCents int64   `json:"spendCents"`
+	Pct        float64 `json:"pct"` // share of total spend, 0..100
+}
+
+type TopModels struct {
+	Available bool       `json:"available"`
+	Items     []ModelRow `json:"items"`
+	Source    string     `json:"source"`
+}
+
+type ProductRow struct {
+	ProductID string  `json:"productId"`
+	Orders    int64   `json:"orders"`
+	Revenue   float64 `json:"revenue"`
+	Units     int64   `json:"units"`
+}
+
+type TopProducts struct {
+	Available bool         `json:"available"`
+	Reason    string       `json:"reason,omitempty"`
+	Items     []ProductRow `json:"items"`
+	Source    string       `json:"source"`
+}
+
+type Top struct {
+	Range    string      `json:"range"`
+	Start    string      `json:"start"`
+	End      string      `json:"end"`
+	Scope    Scope       `json:"scope"`
+	Models   TopModels   `json:"models"`
+	Products TopProducts `json:"products"`
+}
+
+// ── Pure assemblers ─────────────────────────────────────────────────────────
+
+// buildLLMOverview assembles the LLM KPI block from the single aggregate row.
+// A nil/empty row yields honest zeros (Available is still true — the datastore
+// answered; there is simply no usage in the window). Pure.
+func buildLLMOverview(row map[string]any) LLMOverview {
+	requests := aInt64(row["requests"])
+	errors := aInt64(row["errors"])
+	o := LLMOverview{
+		Available:        true,
+		Requests:         requests,
+		Tokens:           aInt64(row["tokens"]),
+		PromptTokens:     aInt64(row["prompt_tokens"]),
+		CompletionTokens: aInt64(row["completion_tokens"]),
+		SpendCents:       aInt64(row["cost_cents"]),
+		Models:           aInt64(row["models"]),
+		Providers:        aInt64(row["providers"]),
+		Errors:           errors,
+		Source:           llmTable,
+	}
+	if requests > 0 {
+		o.ErrorRate = round3(float64(errors) / float64(requests))
+	}
+	return o
+}
+
+// buildWebOverview / buildCommerceOverview assemble the events lenses. The
+// handler passes ok=false when the events query failed (table absent) so the
+// lens is honestly reported unavailable rather than as fabricated zeros.
+func buildWebOverview(row map[string]any, ok bool) WebOverview {
+	w := WebOverview{Available: ok, Source: eventsTable}
+	if !ok {
+		w.Reason = "no web analytics events yet"
+		return w
+	}
+	w.Pageviews = aInt64(row["pageviews"])
+	w.Visitors = aInt64(row["visitors"])
+	w.Sessions = aInt64(row["sessions"])
+	return w
+}
+
+func buildCommerceOverview(row map[string]any, ok bool) CommerceOverview {
+	c := CommerceOverview{Available: ok, Source: eventsTable}
+	if !ok {
+		c.Reason = "no commerce events yet"
+		return c
+	}
+	c.Orders = aInt64(row["orders"])
+	c.Revenue = aFloat64(row["revenue"])
+	if c.Orders > 0 {
+		c.AOV = round2(c.Revenue / float64(c.Orders))
+	}
+	return c
+}
+
+// buildSeries turns sparse ClickHouse buckets into an evenly-spaced, gap-filled
+// series so the client charts a continuous line. Bucket alignment matches
+// toStartOf{Hour,Day}(…, 'UTC'): Go's Truncate over the step lands on the same
+// UTC boundaries. Pure (mirrors ai/object buildCloudUsageSeries).
+func buildSeries(start, end time.Time, interval string, rows []map[string]any) []SeriesPoint {
+	step := stepOf(interval)
+
+	type agg struct{ requests, tokens, spend int64 }
+	idx := make(map[int64]agg, len(rows))
+	for _, r := range rows {
+		bt := aTime(r["bucket"]).Truncate(step)
+		idx[bt.Unix()] = agg{
+			requests: aInt64(r["requests"]),
+			tokens:   aInt64(r["tokens"]),
+			spend:    aInt64(r["cost_cents"]),
+		}
+	}
+
+	out := make([]SeriesPoint, 0, 64)
+	for t := start.UTC().Truncate(step); t.Before(end); t = t.Add(step) {
+		a := idx[t.Unix()]
+		out = append(out, SeriesPoint{
+			T:          t.UTC().Format(time.RFC3339),
+			Requests:   a.requests,
+			Tokens:     a.tokens,
+			SpendCents: a.spend,
+		})
+	}
+	return out
+}
+
+func stepOf(interval string) time.Duration {
+	if strings.EqualFold(interval, "day") {
+		return 24 * time.Hour
+	}
+	return time.Hour
+}
+
+// buildTopModels assembles the top-models table, computing each model's share of
+// total spend. Rows arrive already ordered by the query, but we sort defensively
+// so the pct/ordering is correct regardless of driver row order. Pure.
+func buildTopModels(rows []map[string]any) TopModels {
+	items := make([]ModelRow, 0, len(rows))
+	var totalCents int64
+	for _, r := range rows {
+		spend := aInt64(r["cost_cents"])
+		totalCents += spend
+		items = append(items, ModelRow{
+			Model:      aString(r["model"]),
+			Provider:   aString(r["provider"]),
+			Requests:   aInt64(r["requests"]),
+			Tokens:     aInt64(r["tokens"]),
+			SpendCents: spend,
+		})
+	}
+	sort.SliceStable(items, func(i, j int) bool {
+		if items[i].SpendCents != items[j].SpendCents {
+			return items[i].SpendCents > items[j].SpendCents
+		}
+		return items[i].Requests > items[j].Requests
+	})
+	for i := range items {
+		items[i].Pct = pctOf(items[i].SpendCents, totalCents)
+	}
+	return TopModels{Available: true, Items: items, Source: llmTable}
+}
+
+// buildTopProducts assembles the top-products table from hanzo.events. ok=false
+// (events table absent) → honest-empty. Pure.
+func buildTopProducts(rows []map[string]any, ok bool) TopProducts {
+	if !ok {
+		return TopProducts{Available: false, Reason: "no commerce events yet", Items: []ProductRow{}, Source: eventsTable}
+	}
+	items := make([]ProductRow, 0, len(rows))
+	for _, r := range rows {
+		items = append(items, ProductRow{
+			ProductID: aString(r["productId"]),
+			Orders:    aInt64(r["orders"]),
+			Revenue:   aFloat64(r["revenue"]),
+			Units:     aInt64(r["units"]),
+		})
+	}
+	return TopProducts{Available: true, Items: items, Source: eventsTable}
+}
+
+// ── Value coercion ──────────────────────────────────────────────────────────
+//
+// The direct ClickHouse driver decodes each column to its native Go scan type
+// (uint64 for count()/sum(UInt*), float64 for toFloat64, time.Time for DateTime,
+// string for String). These coercers accept those natives AND the JSON-transport
+// fallbacks (float64/json.Number/string) so a transport change can't crash a read.
+
+func aInt64(v any) int64 {
+	switch n := v.(type) {
+	case nil:
+		return 0
+	case int:
+		return int64(n)
+	case int64:
+		return n
+	case int32:
+		return int64(n)
+	case uint:
+		return int64(n)
+	case uint64:
+		return int64(n)
+	case uint32:
+		return int64(n)
+	case uint16:
+		return int64(n)
+	case uint8:
+		return int64(n)
+	case float64:
+		return int64(n)
+	case float32:
+		return int64(n)
+	case json.Number:
+		i, _ := n.Int64()
+		return i
+	case string:
+		if i, err := strconv.ParseInt(strings.TrimSpace(n), 10, 64); err == nil {
+			return i
+		}
+		if f, err := strconv.ParseFloat(strings.TrimSpace(n), 64); err == nil {
+			return int64(f)
+		}
+		return 0
+	default:
+		return 0
+	}
+}
+
+func aFloat64(v any) float64 {
+	switch n := v.(type) {
+	case nil:
+		return 0
+	case float64:
+		return n
+	case float32:
+		return float64(n)
+	case int:
+		return float64(n)
+	case int64:
+		return float64(n)
+	case uint64:
+		return float64(n)
+	case json.Number:
+		f, _ := n.Float64()
+		return f
+	case string:
+		f, _ := strconv.ParseFloat(strings.TrimSpace(n), 64)
+		return f
+	default:
+		return 0
+	}
+}
+
+func aString(v any) string {
+	switch s := v.(type) {
+	case nil:
+		return ""
+	case string:
+		return s
+	case fmt.Stringer:
+		return s.String()
+	default:
+		return fmt.Sprintf("%v", s)
+	}
+}
+
+func aTime(v any) time.Time {
+	if t, ok := v.(time.Time); ok {
+		return t.UTC()
+	}
+	s := strings.TrimSpace(aString(v))
+	if s != "" {
+		for _, layout := range []string{"2006-01-02 15:04:05", time.RFC3339, "2006-01-02"} {
+			if t, err := time.Parse(layout, s); err == nil {
+				return t.UTC()
+			}
+		}
+	}
+	if n := aInt64(v); n > 0 {
+		return time.Unix(n, 0).UTC()
+	}
+	return time.Time{}
+}
+
+func pctOf(part, total int64) float64 {
+	if total <= 0 {
+		return 0
+	}
+	return round1(float64(part) / float64(total) * 100)
+}
+
+func round1(f float64) float64 { return math.Round(f*10) / 10 }
+func round2(f float64) float64 { return math.Round(f*100) / 100 }
+func round3(f float64) float64 { return math.Round(f*1000) / 1000 }

clients/analytics/warehouse_err_test.go

@@ -0,0 +1,57 @@
+// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+
+package analytics
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"testing"
+
+	"github.com/zap-proto/zip"
+)
+
+// TestWarehouseErrStatus locks the honest-status contract: a ClickHouse
+// connectivity failure (the observed `:9000` i/o timeout) degrades to 503
+// "unavailable" — never a raw 502 — while a query the warehouse actively
+// rejected stays a 502 bad-gateway.
+func TestWarehouseErrStatus(t *testing.T) {
+	timeoutErr := &net.OpError{Op: "dial", Net: "tcp", Err: &timeoutError{}}
+	cases := []struct {
+		name string
+		err  error
+		want int
+	}{
+		{"clickhouse i/o timeout", fmt.Errorf("read tcp 10.0.0.1:9000: i/o timeout"), http.StatusServiceUnavailable},
+		{"connection refused", fmt.Errorf("dial tcp 10.0.0.1:9000: connect: connection refused"), http.StatusServiceUnavailable},
+		{"context deadline", context.DeadlineExceeded, http.StatusServiceUnavailable},
+		{"typed net timeout", timeoutErr, http.StatusServiceUnavailable},
+		{"connection reset", fmt.Errorf("read: connection reset by peer"), http.StatusServiceUnavailable},
+		{"bad SQL (reachable)", fmt.Errorf("code: 47, DB::Exception: Unknown identifier: foo"), http.StatusBadGateway},
+		{"unknown table (reachable)", fmt.Errorf("code: 60, DB::Exception: Table hanzo.x doesn't exist"), http.StatusBadGateway},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			he, ok := warehouseErr("llm", tc.err).(*zip.HTTPError)
+			if !ok {
+				t.Fatalf("warehouseErr did not return *zip.HTTPError")
+			}
+			if he.Status != tc.want {
+				t.Fatalf("warehouseErr(%q) status = %d, want %d", tc.err, he.Status, tc.want)
+			}
+		})
+	}
+}
+
+type timeoutError struct{}
+
+func (timeoutError) Error() string   { return "i/o timeout" }
+func (timeoutError) Timeout() bool   { return true }
+func (timeoutError) Temporary() bool { return true }

clients/auto/auto.go

@@ -0,0 +1,80 @@
+// Package auto mounts the workflow-automation surface at /v1/auto/* in the unified
+// cloud binary (HIP-0106). It is a per-org REVERSE PROXY to the standalone Hanzo Auto
+// engine (github.com/hanzoai/auto), which runs as its own in-cluster Deployment on
+// its own base/SQLite store + tasks worker. cloud does NOT re-embed that engine — one
+// engine, one store — it makes the engine reachable IN-PLATFORM, on api.hanzo.ai,
+// scoped to the caller's validated tenant. So a signed-in user sees their workflows,
+// the connector catalog, and can run flows through the SAME origin as the rest of the
+// platform, without a second login.
+//
+// THE TENANT BOUNDARY (why a bare proxy is NOT enough here).
+//
+// The auto engine trusts X-Org-Id UNCONDITIONALLY — its route layer scopes every
+// query by that header and does no JWT validation of its own (it was built to sit
+// behind a gateway that mints identity, HIP-0026). That makes THIS proxy the trust
+// boundary: whatever X-Org-Id we forward IS the tenant the engine serves.
+//
+// cloud's SanitizeIdentity middleware (middleware_identity.go) has already run before
+// this handler: it strips EVERY client-supplied authority header and re-sets X-User-Id
+// / X-Org-Id ONLY from a VALIDATED credential. But on the bearer-less "Phase-1 data"
+// path it RESTORES the client's raw X-Org-Id while leaving X-User-Id EMPTY — exactly
+// the anonymous-forge (an off-gateway caller sending `X-Org-Id: victim` with no
+// credential). If we forwarded that, an anonymous attacker would read/drive victim's
+// workflows. So this proxy GATES on a validated principal (X-User-Id present — the
+// same signal principal.Validated uses) and refuses the bearer-less-forge path with
+// 403. Past the gate, X-Org-Id is the SanitizeIdentity-pinned validated owner, so the
+// engine only ever serves the caller's own org. The gate + outbound identity
+// re-stamping live in the pure sub-package clients/auto/proxy (unit-tested in
+// isolation); this file only wires them into cloud.Registry.
+package auto
+
+import (
+	"os"
+	"strings"
+
+	"github.com/hanzoai/cloud"
+	"github.com/hanzoai/cloud/clients/auto/proxy"
+	"github.com/zap-proto/zip"
+)
+
+// defaultUpstream is the in-cluster address of the auto engine's Service
+// (auto.hanzo.svc, port 80 -> container 8080). Overridable via AUTO_UPSTREAM.
+const defaultUpstream = "http://auto.hanzo.svc.cluster.local:80"
+
+func upstream() string {
+	if v := strings.TrimSpace(os.Getenv("AUTO_UPSTREAM")); v != "" {
+		return v
+	}
+	return defaultUpstream
+}
+
+// Mount wires the /v1/auto/* reverse proxy onto app per HIP-0106.
+func Mount(app *zip.App, deps cloud.Deps) error {
+	h, err := proxy.NewHandler(upstream())
+	if err != nil {
+		return err
+	}
+	app.Mount("/v1/auto", proxy.Gate(h))
+	if deps.Logger != nil {
+		deps.Logger.New("subsystem", "auto").
+			Info("auto workflow surface mounted (reverse proxy)", "upstream", upstream())
+	}
+	return nil
+}
+
+func init() {
+	// Order 140: after the framework (129) and kb (130) so /v1/kb/* resolves at its
+	// own subsystem, and before the AI /v1/* catch-all (150) so /v1/auto/* reaches
+	// this proxy rather than the AI fallthrough.
+	cloud.Register("auto", 140, func(app any, deps cloud.Deps) error {
+		a, ok := app.(*zip.App)
+		if !ok {
+			return &mountError{"auto.Mount: app is not *zip.App"}
+		}
+		return Mount(a, deps)
+	})
+}
+
+type mountError struct{ msg string }
+
+func (e *mountError) Error() string { return e.msg }

clients/auto/proxy/proxy.go

@@ -0,0 +1,79 @@
+// Package proxy is the pure, dependency-free reverse-proxy mechanism behind the
+// /v1/auto subsystem. It is split from the cloud-registration wrapper (clients/auto)
+// so the tenant-boundary behavior — the validated-principal gate and the outbound
+// identity re-stamping the header-trusting auto engine depends on — is unit-testable
+// WITHOUT linking the cloud root package (which transitively pulls conflicting SQLite
+// drivers into a test binary). Separation of concerns: this file is the security
+// mechanism; clients/auto only wires it into cloud.Registry.
+package proxy
+
+import (
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"strings"
+)
+
+// EngineTrustHeaders are the identity headers the auto engine reads. The proxy
+// re-derives them from the gate-validated inbound request so the engine sees only
+// server-authoritative values.
+var EngineTrustHeaders = []string{"X-Org-Id", "X-User-Id", "X-User-Email"}
+
+// StrippedHeaders are identity aliases an attacker might smuggle that we delete on the
+// outbound request (a superset beyond what the engine needs) so nothing identity-ish
+// that cloud's SanitizeIdentity didn't set can reach the header-trusting engine.
+var StrippedHeaders = []string{
+	"X-Roles", "X-User-Permissions", "X-Phone-Number", "X-User-IsAdmin",
+	"X-User-Role", "X-User-Roles", "X-User-Name", "X-Tenant-Id", "X-Tenant-ID", "X-Org",
+}
+
+// NewHandler builds the reverse-proxy handler targeting the auto engine at rawURL.
+// Pure (URL in, handler out). The path is forwarded UNCHANGED: /v1/auto/* maps to
+// /v1/auto/* on the engine. The Director re-stamps the outbound identity headers from
+// the (gate-validated) inbound values, deleting every identity alias first, so the
+// engine — which trusts X-Org-Id absolutely — only ever receives the validated tenant.
+func NewHandler(rawURL string) (http.Handler, error) {
+	target, err := url.Parse(rawURL)
+	if err != nil {
+		return nil, err
+	}
+	rp := httputil.NewSingleHostReverseProxy(target)
+	base := rp.Director
+	rp.Director = func(r *http.Request) {
+		org := r.Header.Get("X-Org-Id")
+		user := r.Header.Get("X-User-Id")
+		email := r.Header.Get("X-User-Email")
+
+		base(r)
+		r.Host = target.Host
+
+		for _, h := range EngineTrustHeaders {
+			r.Header.Del(h)
+		}
+		for _, h := range StrippedHeaders {
+			r.Header.Del(h)
+		}
+		r.Header.Set("X-Org-Id", org)
+		r.Header.Set("X-User-Id", user)
+		if email != "" {
+			r.Header.Set("X-User-Email", email)
+		}
+	}
+	return rp, nil
+}
+
+// Gate refuses any request with no validated principal (empty X-User-Id, the signal
+// cloud's SanitizeIdentity sets only from a verified credential) before it reaches the
+// header-trusting engine. This closes the anonymous-forge path (a client-restored
+// X-Org-Id with no credential) that would otherwise drive a victim org's workflows.
+func Gate(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.TrimSpace(r.Header.Get("X-User-Id")) == "" {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusForbidden)
+			_, _ = w.Write([]byte(`{"status":"error","msg":"no validated principal"}`))
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}

clients/auto/proxy/proxy_test.go

@@ -0,0 +1,151 @@
+package proxy
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+// captureUpstream is a stub auto engine that records the headers it receives, so tests
+// can assert exactly what the proxy forwarded — the auto engine trusts X-Org-Id
+// absolutely, so what reaches here IS the tenant boundary.
+func captureUpstream(t *testing.T) (*httptest.Server, func() http.Header) {
+	t.Helper()
+	var last http.Header
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		last = r.Header.Clone()
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(`{"ok":true}`))
+	}))
+	t.Cleanup(srv.Close)
+	return srv, func() http.Header { return last }
+}
+
+// TestGate_RefusesAnonymousForge is THE core tenant test: a request with a restored
+// client X-Org-Id but NO validated principal (empty X-User-Id) — the exact
+// off-gateway forge — must be refused with 403 and NEVER reach the engine.
+func TestGate_RefusesAnonymousForge(t *testing.T) {
+	up, _ := captureUpstream(t)
+	h, err := NewHandler(up.URL)
+	if err != nil {
+		t.Fatalf("NewHandler: %v", err)
+	}
+	gated := Gate(h)
+
+	req := httptest.NewRequest(http.MethodGet, "/v1/auto/flows", nil)
+	req.Header.Set("X-Org-Id", "victim") // client-forged org, no credential
+	// X-User-Id deliberately absent — no validated principal.
+	rec := httptest.NewRecorder()
+	gated.ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusForbidden {
+		t.Fatalf("forged anon request: status = %d, want 403", rec.Code)
+	}
+	if strings.Contains(rec.Body.String(), "ok") {
+		t.Error("forged request reached the engine (body indicates upstream hit)")
+	}
+}
+
+// TestGate_AllowsValidatedPrincipal proves a request WITH a validated principal
+// (X-User-Id present, set only by SanitizeIdentity from a verified credential) passes
+// the gate and reaches the engine.
+func TestGate_AllowsValidatedPrincipal(t *testing.T) {
+	up, seen := captureUpstream(t)
+	h, err := NewHandler(up.URL)
+	if err != nil {
+		t.Fatalf("NewHandler: %v", err)
+	}
+	gated := Gate(h)
+
+	req := httptest.NewRequest(http.MethodGet, "/v1/auto/flows", nil)
+	req.Header.Set("X-User-Id", "user-123")
+	req.Header.Set("X-Org-Id", "acme")
+	rec := httptest.NewRecorder()
+	gated.ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("validated request: status = %d, want 200", rec.Code)
+	}
+	if got := seen().Get("X-Org-Id"); got != "acme" {
+		t.Errorf("engine saw X-Org-Id = %q, want acme", got)
+	}
+}
+
+// TestProxy_StripsSmuggledAuthorityHeaders proves the Director deletes every identity
+// alias an attacker might smuggle and forwards ONLY the validated trust headers — so
+// an engine that trusts headers absolutely can never be handed a forged X-User-IsAdmin
+// / X-Tenant-Id / X-Org and the like.
+func TestProxy_StripsSmuggledAuthorityHeaders(t *testing.T) {
+	up, seen := captureUpstream(t)
+	h, err := NewHandler(up.URL)
+	if err != nil {
+		t.Fatalf("NewHandler: %v", err)
+	}
+	gated := Gate(h)
+
+	req := httptest.NewRequest(http.MethodGet, "/v1/auto/connections", nil)
+	req.Header.Set("X-User-Id", "user-123")
+	req.Header.Set("X-Org-Id", "acme")
+	// Attacker-smuggled aliases that MUST NOT reach the engine.
+	req.Header.Set("X-User-IsAdmin", "true")
+	req.Header.Set("X-Tenant-Id", "victim")
+	req.Header.Set("X-Org", "victim")
+	req.Header.Set("X-Roles", "admin")
+	rec := httptest.NewRecorder()
+	gated.ServeHTTP(rec, req)
+
+	got := seen()
+	for _, h := range []string{"X-User-Isadmin", "X-Tenant-Id", "X-Org", "X-Roles"} {
+		if v := got.Get(h); v != "" {
+			t.Errorf("smuggled header %s reached engine with value %q", h, v)
+		}
+	}
+	// The validated org still reaches the engine.
+	if got.Get("X-Org-Id") != "acme" {
+		t.Errorf("validated X-Org-Id = %q, want acme", got.Get("X-Org-Id"))
+	}
+}
+
+// TestProxy_PerOrg_ForwardsValidatedOrgOnly proves two different validated tenants each
+// reach the engine tagged as THEMSELVES — the per-org boundary the engine relies on.
+func TestProxy_PerOrg_ForwardsValidatedOrgOnly(t *testing.T) {
+	up, seen := captureUpstream(t)
+	h, _ := NewHandler(up.URL)
+	gated := Gate(h)
+
+	for _, org := range []string{"acme", "globex"} {
+		req := httptest.NewRequest(http.MethodGet, "/v1/auto/flows", nil)
+		req.Header.Set("X-User-Id", "u")
+		req.Header.Set("X-Org-Id", org)
+		rec := httptest.NewRecorder()
+		gated.ServeHTTP(rec, req)
+		if got := seen().Get("X-Org-Id"); got != org {
+			t.Errorf("engine saw org %q, want %q", got, org)
+		}
+	}
+}
+
+// TestProxy_PreservesPath proves /v1/auto/* is forwarded UNCHANGED (the engine
+// registers its routes at that exact path).
+func TestProxy_PreservesPath(t *testing.T) {
+	var gotPath string
+	up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.Path
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer up.Close()
+	h, _ := NewHandler(up.URL)
+	gated := Gate(h)
+
+	req := httptest.NewRequest(http.MethodPost, "/v1/auto/pieces/notion/run", nil)
+	req.Header.Set("X-User-Id", "u")
+	req.Header.Set("X-Org-Id", "acme")
+	rec := httptest.NewRecorder()
+	gated.ServeHTTP(rec, req)
+	if gotPath != "/v1/auto/pieces/notion/run" {
+		t.Errorf("engine saw path %q, want /v1/auto/pieces/notion/run", gotPath)
+	}
+	_ = io.Discard
+}

clients/automations/automations.go

@@ -0,0 +1,810 @@
+// Package automations mounts the Hanzo Cloud /v1/automations/* surface: the
+// Connectors+Automations engine (HIP-0106, task #51). It composes THREE existing
+// seams rather than reinventing them:
+//
+//   - clients/integrations — per-org connector credentials (KMS-sealed). Connectors
+//     reach a token ONLY through integrations.TokenFor, never KMS directly.
+//   - cloud.EmbeddedTasks   — the ONE shared in-process durable engine. A flow runs
+//     as a durable workflow in the OWNER's namespace (engine.go).
+//   - clients/principal     — the ONE tenant gate. Every data handler resolves the
+//     org from principal.Tenant; a client-forged X-Org-Id with no bearer is refused.
+//
+// Surface (all under /v1/automations/*, all org-gated except the compose-root
+// generic GET /v1/automations/health):
+//
+//	GET    /v1/automations/pieces                     the piece catalogue (org-gated)
+//	GET    /v1/automations/flows                      list flows
+//	POST   /v1/automations/flows                      create a flow (+ initial draft version)
+//	GET    /v1/automations/flows/:id                  flow + latest version
+//	PATCH  /v1/automations/flows/:id                  update flow metadata
+//	DELETE /v1/automations/flows/:id                  delete a flow (+ versions + runs)
+//	GET    /v1/automations/flows/:id/versions         list versions
+//	POST   /v1/automations/flows/:id/versions         create a draft version
+//	POST   /v1/automations/flows/:id/operations       apply a FlowOperation
+//	POST   /v1/automations/flows/:id/run              start a durable run
+//	POST   /v1/automations/flows/:id/enable           enable (POLLING → CreateSchedule)
+//	POST   /v1/automations/flows/:id/disable          disable (POLLING → DeleteSchedule)
+//	GET    /v1/automations/runs                        list runs
+//	GET    /v1/automations/runs/:id                    run detail (refreshed from engine)
+//	POST   /v1/automations/runs/:id/resume             resume a paused run (SignalWorkflow)
+//	POST   /v1/automations/mcp                          MCP JSON-RPC tool surface
+package automations
+
+import (
+	"context"
+	_ "embed"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/hanzoai/cloud"
+	"github.com/hanzoai/cloud/audit"
+	"github.com/hanzoai/cloud/clients/principal"
+	luxlog "github.com/luxfi/log"
+	"github.com/zap-proto/zip"
+)
+
+const (
+	// defaultLimit / maxLimit bound list responses.
+	defaultLimit = 200
+	maxLimit     = 1000
+	// maxField caps a single text field so an unbounded body can't amplify the store.
+	maxField = 2048
+
+	// meterKind is the commerce meter key (product=automations). feeEnvPrefix lets
+	// ops price a flow-run / tool-call unit per deployment (0 ⇒ free). One unit is
+	// metered per flow-run start and per MCP tool call.
+	meterKind    = "automations.run"
+	feeEnvPrefix = "CLOUD_AUTOMATIONS_FEE_CENTS"
+
+	// Noisy-neighbor bounds (MED-3 / LOW-2 / LOW-4). A flow tree is capped in step
+	// count AND total serialized size at every write; the resume payload is bounded;
+	// and each org gets a front-door concurrency limit so one tenant cannot exhaust
+	// worker goroutines (notably via a burst of synchronous MCP core.delay calls).
+	maxSteps            = 256
+	maxTriggerBytes     = 512 * 1024
+	maxResumePayload    = 64 * 1024
+	maxConcurrentPerOrg = 32
+)
+
+// orgRunLimiter bounds concurrent in-flight run-starts + MCP tool executions PER
+// ORG — a front-door DoS/noisy-neighbor guard (LOW-2). Per-org so one tenant's burst
+// never starves another; independent of the durable engine's own worker concurrency.
+var orgRunLimiter = newConcurrencyLimiter(maxConcurrentPerOrg)
+
+// catalogJSON is the go:embed'd Tier-A piece catalogue served at
+// /v1/automations/pieces. A separate agent later OVERWRITES this file with the full
+// 701-piece set at the SAME schema+path; the Catalog unmarshal here is the contract.
+//
+//go:embed catalog/catalog.json
+var catalogJSON []byte
+
+type svc struct {
+	store   *Store
+	bill    *cloud.ResourceMeter
+	audit   *audit.Recorder
+	catalog Catalog
+	log     luxlog.Logger
+}
+
+// mounted is the active service so Shutdown can release the store.
+var mounted *svc
+
+// Mount wires /v1/automations/* onto app per HIP-0106.
+func Mount(app *zip.App, deps cloud.Deps) error {
+	if app == nil {
+		return fmt.Errorf("automations.Mount: nil zip.App")
+	}
+	if deps.Logger == nil {
+		return fmt.Errorf("automations.Mount: nil deps.Logger")
+	}
+	log := deps.Logger.New("subsystem", "automations")
+	if deps.DataDir == "" {
+		return fmt.Errorf("automations.Mount: empty DataDir")
+	}
+	if err := os.MkdirAll(deps.DataDir, 0o755); err != nil {
+		return fmt.Errorf("automations.Mount: data dir: %w", err)
+	}
+	store, err := openStore(filepath.Join(deps.DataDir, "automations.db"))
+	if err != nil {
+		return fmt.Errorf("automations.Mount: open store: %w", err)
+	}
+
+	// Parse the embedded catalogue at boot: a schema mismatch is a build-time fault,
+	// surfaced as a mount error rather than a runtime 500.
+	var catalog Catalog
+	if err := json.Unmarshal(catalogJSON, &catalog); err != nil {
+		_ = store.Close()
+		return fmt.Errorf("automations.Mount: catalog: %w", err)
+	}
+
+	s := &svc{
+		store:   store,
+		bill:    cloud.NewResourceMeter(deps, "automations"),
+		audit:   deps.Audit,
+		catalog: catalog,
+		log:     log,
+	}
+	mounted = s
+
+	app.Get("/v1/automations/pieces", s.pieces)
+
+	app.Get("/v1/automations/flows", s.listFlows)
+	app.Post("/v1/automations/flows", s.createFlow)
+	app.Get("/v1/automations/flows/:id", s.getFlow)
+	app.Patch("/v1/automations/flows/:id", s.updateFlow)
+	app.Delete("/v1/automations/flows/:id", s.deleteFlow)
+	app.Get("/v1/automations/flows/:id/versions", s.listVersions)
+	app.Post("/v1/automations/flows/:id/versions", s.createVersion)
+	app.Post("/v1/automations/flows/:id/operations", s.applyOperation)
+	app.Post("/v1/automations/flows/:id/run", s.runFlow)
+	app.Post("/v1/automations/flows/:id/enable", s.enableFlow)
+	app.Post("/v1/automations/flows/:id/disable", s.disableFlow)
+
+	app.Get("/v1/automations/runs", s.listRuns)
+	app.Get("/v1/automations/runs/:id", s.getRun)
+	app.Post("/v1/automations/runs/:id/resume", s.resumeRun)
+
+	app.Post("/v1/automations/mcp", s.mcp)
+
+	log.Info("automations mounted", "pieces", catalog.PieceCount, "connectors", len(registry), "brand", deps.Brand)
+	return nil
+}
+
+func init() {
+	// Order 148: after integrations (137), BEFORE ai (150) so /v1/automations/* wins
+	// over ai's /v1/* catch-all. RegisterWithShutdown so the store closes on stop.
+	cloud.RegisterWithShutdown("automations", 148, func(app any, deps cloud.Deps) error {
+		a, ok := app.(*zip.App)
+		if !ok {
+			return fmt.Errorf("automations.Mount: app is %T, want *zip.App", app)
+		}
+		return Mount(a, deps)
+	}, func(ctx context.Context) error {
+		return Shutdown(ctx)
+	})
+}
+
+// Shutdown closes the store. Idempotent — safe when nothing is mounted.
+func Shutdown(_ context.Context) error {
+	if mounted == nil {
+		return nil
+	}
+	var err error
+	if mounted.store != nil {
+		err = mounted.store.Close()
+	}
+	mounted = nil
+	return err
+}
+
+// ── pieces ────────────────────────────────────────────────────────────────────
+
+func (s *svc) pieces(c *zip.Ctx) error {
+	if _, ok := principal.Tenant(c); !ok {
+		return zip.ErrForbidden("a validated principal is required")
+	}
+	return c.JSON(http.StatusOK, s.catalog)
+}
+
+// ── flows ─────────────────────────────────────────────────────────────────────
+
+// populatedFlow is a flow plus its latest version — the shape the builder consumes.
+type populatedFlow struct {
+	Flow
+	Version *FlowVersion `json:"version,omitempty"`
+}
+
+type createFlowReq struct {
+	DisplayName string       `json:"displayName"`
+	ExternalID  string       `json:"externalId"`
+	FolderID    string       `json:"folderId"`
+	Trigger     *FlowTrigger `json:"trigger"`
+}
+
+func (s *svc) createFlow(c *zip.Ctx) error {
+	org, ok := s.tenant(c)
+	if !ok {
+		return zip.ErrForbidden("a validated principal is required")
+	}
+	var body createFlowReq
+	if err := c.Bind(&body); err != nil {
+		return err
+	}
+	if err := validateTrigger(body.Trigger); err != nil {
+		return zip.Errorf(http.StatusUnprocessableEntity, "%v", err)
+	}
+	now := time.Now().UnixMilli()
+	flowID, err := genID("flow")
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "rng: %v", err)
+	}
+	verID, err := genID("ver")
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "rng: %v", err)
+	}
+	f := Flow{
+		ID: flowID, Org: org, ExternalID: clip(body.ExternalID), FolderID: clip(body.FolderID),
+		Status: FlowDisabled, Created: now, Updated: now,
+	}
+	if _, err := s.store.CreateFlow(c.Context(), f); err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "create flow: %v", err)
+	}
+	v := FlowVersion{
+		ID: verID, Org: org, FlowID: flowID, DisplayName: clip(body.DisplayName),
+		Trigger: body.Trigger, Valid: body.Trigger != nil, State: VersionDraft,
+		SchemaVersion: LatestFlowSchemaVersion, Created: now, Updated: now,
+	}
+	saved, err := s.store.CreateVersion(c.Context(), v)
+	if err != nil {
+		return mapStoreErr(err, "flow not found")
+	}
+	return c.JSON(http.StatusCreated, populatedFlow{Flow: f, Version: &saved})
+}
+
+func (s *svc) listFlows(c *zip.Ctx) error {
+	org, ok := s.tenant(c)
+	if !ok {
+		return zip.ErrForbidden("a validated principal is required")
+	}
+	rows, err := s.store.ListFlows(c.Context(), org, limitOf(c))
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "list: %v", err)
+	}
+	return c.JSON(http.StatusOK, map[string]any{"data": rows})
+}
+
+func (s *svc) getFlow(c *zip.Ctx) error {
+	org, ok := s.tenant(c)
+	if !ok {
+		return zip.ErrForbidden("a validated principal is required")
+	}
+	f, err := s.store.GetFlow(c.Context(), org, idParam(c))
+	if err != nil {
+		return mapStoreErr(err, "flow not found")
+	}
+	out := populatedFlow{Flow: f}
+	if v, verr := s.store.LatestVersion(c.Context(), org, f.ID); verr == nil {
+		out.Version = &v
+	}
+	return c.JSON(http.StatusOK, out)
+}
+
+type patchFlowReq struct {
+	FolderID           *string         `json:"folderId"`
+	ExternalID         *string         `json:"externalId"`
+	PublishedVersionID *string         `json:"publishedVersionId"`
+	Metadata           json.RawMessage `json:"metadata"`
+}
+
+func (s *svc) updateFlow(c *zip.Ctx) error {
+	org, ok := s.tenant(c)
+	if !ok {
+		return zip.ErrForbidden("a validated principal is required")
+	}
+	f, err := s.store.GetFlow(c.Context(), org, idParam(c))
+	if err != nil {
+		return mapStoreErr(err, "flow not found")
+	}
+	var body patchFlowReq
+	if err := c.Bind(&body); err != nil {
+		return err
+	}
+	if body.FolderID != nil {
+		f.FolderID = clip(*body.FolderID)
+	}
+	if body.ExternalID != nil {
+		f.ExternalID = clip(*body.ExternalID)
+	}
+	if body.PublishedVersionID != nil {
+		pv := clip(*body.PublishedVersionID)
+		// LOW-3: a published version must be an EXISTING version OF THIS FLOW in THIS
+		// org — never an unvalidated (possibly cross-tenant / dangling) id. Empty clears it.
+		if pv != "" {
+			ver, verr := s.store.GetVersion(c.Context(), org, pv)
+			if verr != nil || ver.FlowID != f.ID {
+				return zip.Errorf(http.StatusUnprocessableEntity, "publishedVersionId must name a version of this flow")
+			}
+		}
+		f.PublishedVersionID = pv
+	}
+	if body.Metadata != nil {
+		f.Metadata = body.Metadata
+	}
+	f.Updated = time.Now().UnixMilli()
+	saved, err := s.store.UpdateFlow(c.Context(), f)
+	if err != nil {
+		return mapStoreErr(err, "flow not found")
+	}
+	return c.JSON(http.StatusOK, saved)
+}
+
+func (s *svc) deleteFlow(c *zip.Ctx) error {
+	org, ok := s.tenant(c)
+	if !ok {
+		return zip.ErrForbidden("a validated principal is required")
+	}
+	deleted, err := s.store.DeleteFlow(c.Context(), org, idParam(c))
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "delete: %v", err)
+	}
+	if !deleted {
+		return zip.ErrNotFound("flow not found")
+	}
+	return c.NoContent(http.StatusNoContent)
+}
+
+// ── versions ──────────────────────────────────────────────────────────────────
+
+func (s *svc) listVersions(c *zip.Ctx) error {
+	org, ok := s.tenant(c)
+	if !ok {
+		return zip.ErrForbidden("a validated principal is required")
+	}
+	rows, err := s.store.ListVersions(c.Context(), org, idParam(c), limitOf(c))
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "list versions: %v", err)
+	}
+	return c.JSON(http.StatusOK, map[string]any{"data": rows})
+}
+
+type createVersionReq struct {
+	DisplayName string       `json:"displayName"`
+	Trigger     *FlowTrigger `json:"trigger"`
+}
+
+func (s *svc) createVersion(c *zip.Ctx) error {
+	org, ok := s.tenant(c)
+	if !ok {
+		return zip.ErrForbidden("a validated principal is required")
+	}
+	flowID := idParam(c)
+	var body createVersionReq
+	if err := c.Bind(&body); err != nil {
+		return err
+	}
+	if err := validateTrigger(body.Trigger); err != nil {
+		return zip.Errorf(http.StatusUnprocessableEntity, "%v", err)
+	}
+	verID, err := genID("ver")
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "rng: %v", err)
+	}
+	now := time.Now().UnixMilli()
+	v := FlowVersion{
+		ID: verID, Org: org, FlowID: flowID, DisplayName: clip(body.DisplayName),
+		Trigger: body.Trigger, Valid: body.Trigger != nil, State: VersionDraft,
+		SchemaVersion: LatestFlowSchemaVersion, Created: now, Updated: now,
+	}
+	saved, err := s.store.CreateVersion(c.Context(), v)
+	if err != nil {
+		return mapStoreErr(err, "flow not found")
+	}
+	return c.JSON(http.StatusCreated, saved)
+}
+
+// applyOperation applies a FlowOperation. CHANGE_STATUS is flow-scoped (routes to
+// enable/disable); every other op mutates the flow's latest version's step tree.
+func (s *svc) applyOperation(c *zip.Ctx) error {
+	org, ok := s.tenant(c)
+	if !ok {
+		return zip.ErrForbidden("a validated principal is required")
+	}
+	flowID := idParam(c)
+	var op FlowOperation
+	if err := c.Bind(&op); err != nil {
+		return err
+	}
+
+	if op.Type == OpChangeStatus {
+		var r ChangeStatusRequest
+		if err := json.Unmarshal(op.Request, &r); err != nil {
+			return zip.ErrBadRequest("decode CHANGE_STATUS")
+		}
+		if r.Status == FlowEnabled {
+			return s.setEnabled(c, org, flowID, true)
+		}
+		return s.setEnabled(c, org, flowID, false)
+	}
+
+	v, err := s.store.LatestVersion(c.Context(), org, flowID)
+	if err != nil {
+		return mapStoreErr(err, "flow has no version")
+	}
+	updated, err := applyVersionOperation(&v, op)
+	if err != nil {
+		return mapOpErr(err)
+	}
+	// Re-bound the resulting tree so a sequence of ADD_ACTION ops can't grow a flow
+	// past the step/size caps one operation at a time (MED-3).
+	if err := validateTrigger(updated.Trigger); err != nil {
+		return zip.Errorf(http.StatusUnprocessableEntity, "%v", err)
+	}
+	updated.Updated = time.Now().UnixMilli()
+	saved, err := s.store.UpdateVersion(c.Context(), *updated)
+	if err != nil {
+		return mapStoreErr(err, "version not found")
+	}
+	return c.JSON(http.StatusOK, saved)
+}
+
+// ── runs ──────────────────────────────────────────────────────────────────────
+
+// runVersion resolves the version a run executes: the published version if set,
+// else the latest.
+func (s *svc) runVersion(ctx context.Context, org string, f Flow) (FlowVersion, error) {
+	if f.PublishedVersionID != "" {
+		if v, err := s.store.GetVersion(ctx, org, f.PublishedVersionID); err == nil {
+			return v, nil
+		}
+	}
+	return s.store.LatestVersion(ctx, org, f.ID)
+}
+
+func (s *svc) runFlow(c *zip.Ctx) error {
+	org, ok := s.tenant(c)
+	if !ok {
+		return zip.ErrForbidden("a validated principal is required")
+	}
+	f, err := s.store.GetFlow(c.Context(), org, idParam(c))
+	if err != nil {
+		return mapStoreErr(err, "flow not found")
+	}
+	v, err := s.runVersion(c.Context(), org, f)
+	if err != nil {
+		return mapStoreErr(err, "flow has no runnable version")
+	}
+	// Per-org front-door concurrency bound (LOW-2).
+	if !orgRunLimiter.acquire(org) {
+		return zip.Errorf(http.StatusTooManyRequests, "too many concurrent automation requests for this org")
+	}
+	defer orgRunLimiter.release(org)
+
+	runID, err := genID("run")
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "rng: %v", err)
+	}
+	in := FlowRunInput{
+		Owner:         org, // VALIDATED org — the cred scope; NEVER from the body
+		FlowID:        f.ID,
+		FlowVersionID: v.ID,
+		RunID:         runID,
+		Steps:         flattenSteps(&v),
+	}
+	if _, err := executeFlow(c.Context(), in); err != nil {
+		return engineErr(err)
+	}
+	now := time.Now().UnixMilli()
+	run := FlowRun{
+		ID: runID, Org: org, FlowID: f.ID, FlowVersionID: v.ID, WorkflowID: runID,
+		Status: RunRunning, StartTime: now, Created: now, Updated: now,
+	}
+	// Persist the row for IMMEDIATE visibility (getRun/listRuns), but do NOT meter or
+	// audit here: the durable run-start activity is the SINGLE owner of run
+	// bookkeeping and bills the run exactly once (MED-1), so the manual path never
+	// double-records. CreateRunIfAbsent leaves metered=0 for the activity to claim.
+	if _, err := s.store.CreateRunIfAbsent(c.Context(), run); err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "persist run: %v", err)
+	}
+	return c.JSON(http.StatusCreated, run)
+}
+
+func (s *svc) listRuns(c *zip.Ctx) error {
+	org, ok := s.tenant(c)
+	if !ok {
+		return zip.ErrForbidden("a validated principal is required")
+	}
+	rows, err := s.store.ListRuns(c.Context(), org, clip(c.Query("flowId")), limitOf(c))
+	if err != nil {
+		return zip.Errorf(http.StatusInternalServerError, "list runs: %v", err)
+	}
+	return c.JSON(http.StatusOK, map[string]any{"data": rows})
+}
+
+// getRun returns a run, refreshing a non-terminal status from the engine (scoped to
+// the org's namespace) so the caller sees live progress.
+func (s *svc) getRun(c *zip.Ctx) error {
+	org, ok := s.tenant(c)
+	if !ok {
+		return zip.ErrForbidden("a validated principal is required")
+	}
+	run, err := s.store.GetRun(c.Context(), org, idParam(c))
+	if err != nil {
+		return mapStoreErr(err, "run not found")
+	}
+	if !terminal(run.Status) {
+		if st, derr := describeRunStatus(c.Context(), org, run.WorkflowID); derr == nil && st != run.Status {
+			finish := run.FinishTime
+			if terminal(st) {
+				finish = time.Now().UnixMilli()
+			}
+			if uerr := s.store.UpdateRunStatus(c.Context(), org, run.ID, st, finish, time.Now().UnixMilli()); uerr == nil {
+				run.Status, run.FinishTime = st, finish
+			}
+		}
+	}
+	return c.JSON(http.StatusOK, run)
+}
+
+func (s *svc) resumeRun(c *zip.Ctx) error {
+	org, ok := s.tenant(c)
+	if !ok {
+		return zip.ErrForbidden("a validated principal is required")
+	}
+	run, err := s.store.GetRun(c.Context(), org, idParam(c))
+	if err != nil {
+		return mapStoreErr(err, "run not found")
+	}
+	// LOW-4: bound the resume payload — it is delivered verbatim into the workflow
+	// as the waitpoint's output, so an unbounded body must not amplify engine state.
+	if len(c.Body()) > maxResumePayload {
+		return zip.Errorf(http.StatusRequestEntityTooLarge, "resume payload exceeds the %d-byte limit", maxResumePayload)
+	}
+	var payload any
+	if len(c.Body()) > 0 {
+		if err := json.Unmarshal(c.Body(), &payload); err != nil {
+			return zip.ErrBadRequest("resume payload must be JSON")
+		}
+	}
+	if err := signalResume(c.Context(), org, run.WorkflowID, payload); err != nil {
+		return engineErr(err)
+	}
+	s.auditEvent(c, org, "automations.run.resume", run.ID, "ok", http.StatusOK)
+	return c.JSON(http.StatusOK, map[string]any{"resumed": true})
+}
+
+// ── enable / disable ──────────────────────────────────────────────────────────
+
+func (s *svc) enableFlow(c *zip.Ctx) error {
+	org, ok := s.tenant(c)
+	if !ok {
+		return zip.ErrForbidden("a validated principal is required")
+	}
+	return s.setEnabled(c, org, idParam(c), true)
+}
+
+func (s *svc) disableFlow(c *zip.Ctx) error {
+	org, ok := s.tenant(c)
+	if !ok {
+		return zip.ErrForbidden("a validated principal is required")
+	}
+	return s.setEnabled(c, org, idParam(c), false)
+}
+
+// setEnabled flips a flow's status and wires its POLLING schedule to the engine
+// (CreateSchedule on enable, DeleteSchedule on disable). Non-POLLING flows are a
+// pure status flip — no engine needed. A POLLING enable requires the engine (503 if
+// not ready). Shared by /enable, /disable, and the CHANGE_STATUS operation.
+func (s *svc) setEnabled(c *zip.Ctx, org, flowID string, enable bool) error {
+	f, err := s.store.GetFlow(c.Context(), org, flowID)
+	if err != nil {
+		return mapStoreErr(err, "flow not found")
+	}
+	v, verr := s.runVersion(c.Context(), org, f)
+	cron, polling := pollingCron(v, verr)
+
+	if enable {
+		f.Status = FlowEnabled
+	} else {
+		f.Status = FlowDisabled
+	}
+	f.Updated = time.Now().UnixMilli()
+	saved, err := s.store.UpdateFlow(c.Context(), f)
+	if err != nil {
+		return mapStoreErr(err, "flow not found")
+	}
+
+	scheduleID := "flow-" + flowID
+	if polling {
+		if enable {
+			in := FlowRunInput{Owner: org, FlowID: f.ID, FlowVersionID: v.ID, RunID: "sched-" + flowID, Steps: flattenSteps(&v)}
+			if serr := enableSchedule(c.Context(), org, scheduleID, cron, in); serr != nil {
+				return engineErr(serr)
+			}
+		} else {
+			// Best-effort: local status is authoritative; a schedule-delete failure is logged.
+			if serr := disableSchedule(c.Context(), org, scheduleID); serr != nil {
+				s.log.Warn("schedule delete failed (continuing)", "flow", flowID, "err", serr)
+			}
+		}
+	}
+	action := "automations.flow.disable"
+	if enable {
+		action = "automations.flow.enable"
+	}
+	s.auditEvent(c, org, action, f.ID, "ok", http.StatusOK)
+	return c.JSON(http.StatusOK, saved)
+}
+
+// pollingCron reports whether a flow's trigger is a POLLING schedule and returns its
+// cron expression. A version-load error or non-POLLING trigger yields (,"" false).
+func pollingCron(v FlowVersion, verr error) (string, bool) {
+	if verr != nil || v.Trigger == nil || v.Trigger.Strategy != StrategyPolling {
+		return "", false
+	}
+	cron := ""
+	if c, ok := v.Trigger.Settings.Input["cron"].(string); ok {
+		cron = c
+	}
+	if cron == "" {
+		return "", false
+	}
+	return cron, true
+}
+
+// ── run bookkeeping (MED-1: the SINGLE owner, exactly-once) ─────────────────────
+
+// recordRunStart is the exactly-once run bookkeeping the durable run-start activity
+// runs for EVERY entrypoint (manual, MCP, scheduled cron). It ensures the run row
+// exists (idempotent by run id) and meters+audits ONLY the caller that wins the
+// metered-flag claim — so a run is billed at most once no matter how many paths race
+// to record it. mounted-nil-safe via the activity wrapper.
+func (s *svc) recordRunStart(ctx context.Context, in RunStartInput) error {
+	now := time.Now().UnixMilli()
+	if _, err := s.store.CreateRunIfAbsent(ctx, FlowRun{
+		ID: in.RunID, Org: in.Owner, FlowID: in.FlowID, FlowVersionID: in.FlowVersionID,
+		WorkflowID: in.RunID, Status: RunRunning, StartTime: now, Created: now, Updated: now,
+	}); err != nil {
+		return err
+	}
+	won, err := s.store.ClaimMeter(ctx, in.Owner, in.RunID)
+	if err != nil {
+		return err
+	}
+	if !won {
+		return nil // another path already metered this run — exactly once
+	}
+	// meter + audit fire together, behind the SAME won-guard, so the audit trail's
+	// count of automations.flow.run records is an exact proxy for the meter count.
+	s.meterRun(in.Owner)
+	s.auditRun(ctx, in.Owner, in.FlowID, in.RunID)
+	return nil
+}
+
+// recordRunEnd records a run's terminal status so listRuns reflects it without a
+// getRun refresh. Best-effort.
+func (s *svc) recordRunEnd(ctx context.Context, in RunEndInput) error {
+	now := time.Now().UnixMilli()
+	return s.store.UpdateRunStatus(ctx, in.Owner, in.RunID, FlowRunStatus(in.Status), now, now)
+}
+
+// ── billing + audit ───────────────────────────────────────────────────────────
+
+// meterUnit records one metered unit for an HTTP caller's org. Nil/disabled meter → no-op.
+func (s *svc) meterUnit(org string, c *zip.Ctx) {
+	s.bill.Meter(org, meterKind, cloud.ResourceFeeCents(feeEnvPrefix, meterKind), c.RequestID(), cloud.ClientIP(c))
+}
+
+// meterRun records one metered unit for a flow run from the durable path (no HTTP
+// context). Nil/disabled meter → no-op.
+func (s *svc) meterRun(org string) {
+	s.bill.Meter(org, meterKind, cloud.ResourceFeeCents(feeEnvPrefix, meterKind), "", "")
+}
+
+// auditEvent appends one tamper-evident audit record for an HTTP action. result is
+// "ok"|"error"; status is the HTTP status. Nil recorder → no-op.
+func (s *svc) auditEvent(c *zip.Ctx, org, action, resourceID, result string, status int) {
+	if s.audit == nil {
+		return
+	}
+	rec := audit.Record{
+		Actor:     audit.Actor{Org: org, Sub: c.User(), Email: c.UserEmail()},
+		Action:    action,
+		Resource:  audit.Resource{Type: "automations", ID: resourceID},
+		Auth:      audit.AuthContext{Method: "gateway", IsAdmin: c.IsAdmin()},
+		Outcome:   audit.Outcome{Result: result, Status: status},
+		Method:    c.Method(),
+		Path:      c.Path(),
+		SourceIP:  cloud.ClientIP(c),
+		RequestID: c.RequestID(),
+	}
+	if _, err := s.audit.Append(c.Context(), rec); err != nil {
+		s.log.Warn("audit append failed", "err", err, "action", action)
+	}
+}
+
+// auditRun appends the flow-run audit record from the durable path (no HTTP context,
+// so no actor sub/email/ip). Nil recorder → no-op.
+func (s *svc) auditRun(ctx context.Context, org, flowID, runID string) {
+	if s.audit == nil {
+		return
+	}
+	rec := audit.Record{
+		Actor:    audit.Actor{Org: org},
+		Action:   "automations.flow.run",
+		Resource: audit.Resource{Type: "automations", ID: flowID},
+		Auth:     audit.AuthContext{Method: "durable"},
+		Outcome:  audit.Outcome{Result: "ok", Status: http.StatusCreated},
+	}
+	if _, err := s.audit.Append(ctx, rec); err != nil {
+		s.log.Warn("audit append failed", "err", err, "action", "automations.flow.run", "run", runID)
+	}
+}
+
+// ── helpers ───────────────────────────────────────────────────────────────────
+
+// tenant resolves the caller's org, additionally validOrg-checking it because the
+// org is folded into per-org engine namespaces + store keys.
+func (s *svc) tenant(c *zip.Ctx) (string, bool) {
+	org, ok := principal.Tenant(c)
+	if !ok || !validOrg(org) {
+		return "", false
+	}
+	return org, true
+}
+
+func idParam(c *zip.Ctx) string { return clip(c.Param("id")) }
+
+func limitOf(c *zip.Ctx) int {
+	n := 0
+	_, _ = fmt.Sscanf(c.Query("limit"), "%d", &n)
+	if n <= 0 {
+		return defaultLimit
+	}
+	if n > maxLimit {
+		return maxLimit
+	}
+	return n
+}
+
+// clip trims and bounds a text field.
+func clip(s string) string {
+	if len(s) > maxField {
+		s = s[:maxField]
+	}
+	for len(s) > 0 && (s[0] == ' ' || s[0] == '\t' || s[0] == '\n' || s[0] == '\r') {
+		s = s[1:]
+	}
+	for len(s) > 0 {
+		last := s[len(s)-1]
+		if last != ' ' && last != '\t' && last != '\n' && last != '\r' {
+			break
+		}
+		s = s[:len(s)-1]
+	}
+	return s
+}
+
+func terminal(s FlowRunStatus) bool {
+	switch s {
+	case RunSucceeded, RunFailed, RunCanceled, RunTimeout:
+		return true
+	default:
+		return false
+	}
+}
+
+// mapStoreErr maps a store sentinel to the right HTTP error.
+func mapStoreErr(err error, notFoundMsg string) error {
+	switch err {
+	case errNotFound:
+		return zip.ErrNotFound(notFoundMsg)
+	case errBadRef:
+		return zip.Errorf(http.StatusUnprocessableEntity, "referenced record not found in org")
+	default:
+		return zip.Errorf(http.StatusInternalServerError, "%v", err)
+	}
+}
+
+// mapOpErr maps an operation-apply error to HTTP.
+func mapOpErr(err error) error {
+	switch err {
+	case errNotFound:
+		return zip.ErrNotFound("step not found")
+	case errUnsupportedOp:
+		return zip.Errorf(http.StatusUnprocessableEntity, "%v", err)
+	default:
+		return zip.ErrBadRequest(err.Error())
+	}
+}
+
+// engineErr maps an engine dial/exec error to HTTP: not-ready → 503 (honest), else 500.
+func engineErr(err error) error {
+	if err == ErrEngineNotReady {
+		return zip.Errorf(http.StatusServiceUnavailable, "automation engine not ready")
+	}
+	return zip.Errorf(http.StatusInternalServerError, "engine: %v", err)
+}

clients/automations/connector.go

@@ -0,0 +1,137 @@
+package automations
+
+import (
+	"context"
+	"fmt"
+	"sort"
+)
+
+// Connector framework — Tier-A, first-class Go connectors. ONE registry, N
+// connectors; a connector self-registers from its file's init() (mirroring
+// clients/integrations' register()). Adding a connector is a new file, never a
+// change to the dispatch code.
+//
+// A Connector's Name equals the clients/integrations provider id wherever its
+// actions need custodied credentials ("slack","github",…) — so RunContext.Token
+// resolves against the SAME per-org KMS custody the OAuth plane sealed. Connectors
+// that need no external creds (core) name a scope that no provider owns; their
+// actions never call Token.
+
+// RunContext is what an action's Run receives. It is the connector's ENTIRE view
+// of the world: the org (for logging/attribution only — never a place to widen
+// scope), the resolved input, the prior steps' outputs (threaded), and Token —
+// the ONLY door to a credential. Token is bound at dispatch time to the VALIDATED
+// org (StepInput.Owner) and the connector's own provider id, so a connector can
+// reach no other tenant's and no other provider's secret.
+type RunContext struct {
+	Org         string
+	Input       map[string]any
+	PrevOutputs map[string]any
+	Token       func(secretName string) ([]byte, error)
+}
+
+// Action is one invocable capability of a connector. Props declares its inputs
+// (drives the catalogue + the MCP tool input schema). Run is the side-effecting
+// body; it MUST handle every error explicitly and MUST NOT leak another org's data.
+type Action struct {
+	Name        string
+	DisplayName string
+	Description string
+	Props       []PropSpec
+	Run         func(ctx context.Context, rc RunContext) (any, error)
+}
+
+// Trigger is one entry point of a connector. Strategy selects POLLING (cron) vs
+// WEBHOOK vs MANUAL. Phase-1 triggers are catalogue/metadata only (the entry wiring
+// lives in the flow's root trigger + enable/disable); a Trigger carries no Run.
+type Trigger struct {
+	Name        string
+	DisplayName string
+	Description string
+	Strategy    TriggerStrategy
+	Props       []PropSpec
+}
+
+// Connector is one connectable capability provider. Name == the integrations
+// provider id where credentials apply.
+type Connector struct {
+	Name        string
+	DisplayName string
+	AuthType    string // "none" | "bot_token" | "oauth2" — for the catalogue card
+	AuthReq     bool
+	Actions     map[string]*Action
+	Triggers    map[string]*Trigger
+}
+
+// registry is populated by each connector file's register() from its init(). Go
+// initializes the map before any init() runs, so every connector is present by the
+// time Mount snapshots it.
+var registry = map[string]*Connector{}
+
+// register adds a connector to the package registry. A nil/empty connector or a
+// duplicate name is a programming error and panics at init — two connectors cannot
+// own the same name (which would make credential + tool routing ambiguous).
+func register(c *Connector) {
+	if c == nil || c.Name == "" {
+		panic("automations: register nil/empty connector")
+	}
+	if _, dup := registry[c.Name]; dup {
+		panic("automations: duplicate connector " + c.Name)
+	}
+	if c.Actions == nil {
+		c.Actions = map[string]*Action{}
+	}
+	if c.Triggers == nil {
+		c.Triggers = map[string]*Trigger{}
+	}
+	// INF-1: MCP tool names are "<connector>_<action>", and both halves may contain
+	// underscores (e.g. "google_sheets"+"append_row"), so two DIFFERENT pairs could
+	// collide into ONE tool name and make resolveTool ambiguous. Refuse at init — a
+	// future connector can never silently shadow another's tool.
+	for an := range c.Actions {
+		tool := c.Name + "_" + an
+		for exName, ex := range registry {
+			for exAn := range ex.Actions {
+				if exName+"_"+exAn == tool {
+					panic("automations: tool-name collision " + tool + " (" + c.Name + "." + an + " vs " + exName + "." + exAn + ")")
+				}
+			}
+		}
+	}
+	registry[c.Name] = c
+}
+
+// lookupAction resolves (connector, action) from the registry, or an honest error
+// naming exactly what was unknown. Used by the durable activity AND the MCP surface
+// — ONE resolution path, so the two can never diverge on what a tool means.
+func lookupAction(connector, action string) (*Connector, *Action, error) {
+	c, ok := registry[connector]
+	if !ok {
+		return nil, nil, fmt.Errorf("unknown connector %q", connector)
+	}
+	a, ok := c.Actions[action]
+	if !ok {
+		return nil, nil, fmt.Errorf("unknown action %q on connector %q", action, connector)
+	}
+	return c, a, nil
+}
+
+// sortedConnectors returns the connectors by name for stable catalogue/tool output.
+func sortedConnectors() []*Connector {
+	out := make([]*Connector, 0, len(registry))
+	for _, c := range registry {
+		out = append(out, c)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
+	return out
+}
+
+// sortedActions returns a connector's actions by name for stable output.
+func sortedActions(c *Connector) []*Action {
+	out := make([]*Action, 0, len(c.Actions))
+	for _, a := range c.Actions {
+		out = append(out, a)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
+	return out
+}

clients/automations/connector_core.go

@@ -0,0 +1,324 @@
+package automations
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+	"syscall"
+	"time"
+)
+
+// corePiece is the built-in connector: control-flow + generic I/O steps that need
+// NO external credentials. It is the connector the durable-run green test exercises
+// end-to-end, so it must run without integrations mounted.
+const corePiece = "core"
+
+func init() {
+	register(&Connector{
+		Name:        corePiece,
+		DisplayName: "Core",
+		AuthType:    "none",
+		AuthReq:     false,
+		Actions: map[string]*Action{
+			"http_request": {
+				Name:        "http_request",
+				DisplayName: "HTTP Request",
+				Description: "Issue an HTTP GET/POST and return {status, headers, body}.",
+				Props: []PropSpec{
+					{Name: "url", Type: "string", Required: true, Description: "Absolute http(s) URL."},
+					{Name: "method", Type: "string", Description: "GET (default) or POST."},
+					{Name: "body", Type: "string", Description: "Request body for POST."},
+					{Name: "headers", Type: "object", Description: "Optional request headers."},
+				},
+				Run: runHTTPRequest,
+			},
+			"delay": {
+				Name:        "delay",
+				DisplayName: "Delay",
+				Description: "Pause the flow for a bounded number of seconds.",
+				Props: []PropSpec{
+					{Name: "seconds", Type: "number", Required: true, Description: "Seconds to wait (capped)."},
+				},
+				Run: runDelay,
+			},
+			"code": {
+				Name:        "code",
+				DisplayName: "Transform",
+				Description: "Return the (reference-resolved) input map — a Go data-mapper transform.",
+				Props: []PropSpec{
+					{Name: "input", Type: "object", Description: "Values, optionally {{step.path}} references to prior outputs."},
+				},
+				Run: runCode,
+			},
+			"wait_for_approval": {
+				Name:        "wait_for_approval",
+				DisplayName: "Wait for Approval",
+				Description: "A manual waitpoint. Inside a flow it blocks until a resume signal; invoked directly it approves.",
+				Props: []PropSpec{
+					{Name: "reason", Type: "string", Description: "Why approval is requested."},
+				},
+				Run: runWaitForApproval,
+			},
+		},
+		Triggers: map[string]*Trigger{
+			"schedule": {
+				Name:        "schedule",
+				DisplayName: "Schedule",
+				Description: "Start the flow on a cron schedule.",
+				Strategy:    StrategyPolling,
+				Props: []PropSpec{
+					{Name: "cron", Type: "string", Required: true, Description: "Cron expression, e.g. \"0 3 * * *\"."},
+				},
+			},
+			"manual": {
+				Name:        "manual",
+				DisplayName: "Manual",
+				Description: "Start the flow only on an explicit /run.",
+				Strategy:    StrategyManual,
+			},
+		},
+	})
+}
+
+// ── core.http_request ────────────────────────────────────────────────────────
+
+const (
+	// maxHTTPBody bounds a fetched response so a hostile/huge upstream can't exhaust
+	// memory. 5 MiB is generous for an automation step.
+	maxHTTPBody = 5 << 20
+	// httpAllowPrivateEnv, when "1", lifts the SSRF guard (private/loopback targets
+	// allowed). Off by default and intended ONLY for tests that hit a loopback stub;
+	// production leaves it unset, so a flow can never reach cluster-internal or
+	// cloud-metadata addresses.
+	httpAllowPrivateEnv = "AUTOMATIONS_HTTP_ALLOW_PRIVATE"
+)
+
+// ssrfDialer is a net.Dialer whose Control hook runs AFTER DNS resolution, on the
+// concrete IP:port about to be connected — so it closes the DNS-rebinding TOCTOU
+// window a resolve-then-check would leave open. A non-public target is refused
+// unless httpAllowPrivateEnv=="1".
+var ssrfDialer = &net.Dialer{
+	Timeout: 10 * time.Second,
+	Control: func(network, address string, _ syscall.RawConn) error {
+		if os.Getenv(httpAllowPrivateEnv) == "1" {
+			return nil
+		}
+		host, _, err := net.SplitHostPort(address)
+		if err != nil {
+			return fmt.Errorf("ssrf: bad address %q", address)
+		}
+		ip := net.ParseIP(host)
+		if ip == nil || !isPublicIP(ip) {
+			return fmt.Errorf("ssrf: refusing non-public address %s", host)
+		}
+		return nil
+	},
+}
+
+// httpClient is the ONE bounded client core.http_request uses: a real timeout
+// (fail-secure — never an infinite hang), the SSRF dialer, and a capped redirect
+// chain (each hop is re-checked by the dialer, so a redirect to a private host is
+// also refused).
+var httpClient = &http.Client{
+	Timeout: 15 * time.Second,
+	Transport: &http.Transport{
+		DialContext:           ssrfDialer.DialContext,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ResponseHeaderTimeout: 10 * time.Second,
+		DisableKeepAlives:     true,
+	},
+	CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		if len(via) >= 5 {
+			return fmt.Errorf("too many redirects")
+		}
+		return nil
+	},
+}
+
+// specialUseCIDRs are the IANA special-use ranges Go's net helpers do NOT cover.
+// Go's IsPrivate is only RFC1918 + ULA; IsLinkLocal* covers 169.254/16 + fe80::/10.
+// These add the ranges an SSRF attacker reaches for — notably CGNAT 100.64/10
+// (Alibaba's 100.100.100.200 metadata endpoint) and the reserved/benchmark/test
+// blocks — so the blocklist is honest about what it stops. Parsed once at init.
+var specialUseCIDRs = mustCIDRs(
+	"0.0.0.0/8",       // "this host on this network" (RFC 1122)
+	"100.64.0.0/10",   // CGNAT (RFC 6598) — Alibaba metadata 100.100.100.200
+	"192.0.0.0/24",    // IETF protocol assignments (RFC 6890)
+	"192.0.2.0/24",    // TEST-NET-1 (RFC 5737)
+	"192.88.99.0/24",  // 6to4 relay anycast (RFC 7526)
+	"198.18.0.0/15",   // benchmarking (RFC 2544)
+	"198.51.100.0/24", // TEST-NET-2 (RFC 5737)
+	"203.0.113.0/24",  // TEST-NET-3 (RFC 5737)
+	"240.0.0.0/4",     // reserved / former class E (RFC 1112)
+	"64:ff9b::/96",    // NAT64 well-known prefix (RFC 6052)
+)
+
+// isPublicIP reports whether ip is a routable PUBLIC address. It rejects loopback,
+// private (RFC1918/ULA), link-local, unspecified, multicast (Go's net helpers) PLUS
+// the IANA special-use ranges above. This is NOT a claim of a complete cloud-metadata
+// blocklist — 169.254.169.254 (AWS/GCP/Azure) is covered as link-local and
+// 100.100.100.200 (Alibaba) as CGNAT, but the guard is a general special-use/private
+// blocklist, not a per-cloud allowlist. Called by the dialer Control hook on the
+// concrete resolved IP, so it also closes the DNS-rebind TOCTOU window.
+func isPublicIP(ip net.IP) bool {
+	if ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() ||
+		ip.IsLinkLocalMulticast() || ip.IsMulticast() || ip.IsUnspecified() {
+		return false
+	}
+	// Normalize an IPv4-mapped IPv6 (::ffff:a.b.c.d) to its v4 form so a mapped
+	// private/special address can't slip past the v4 CIDR checks.
+	if v4 := ip.To4(); v4 != nil {
+		ip = v4
+	}
+	for _, n := range specialUseCIDRs {
+		if n.Contains(ip) {
+			return false
+		}
+	}
+	return true
+}
+
+// mustCIDRs parses CIDR literals at init; a bad literal is a programming error.
+func mustCIDRs(cidrs ...string) []*net.IPNet {
+	out := make([]*net.IPNet, 0, len(cidrs))
+	for _, c := range cidrs {
+		_, n, err := net.ParseCIDR(c)
+		if err != nil {
+			panic("automations: bad special-use CIDR " + c + ": " + err.Error())
+		}
+		out = append(out, n)
+	}
+	return out
+}
+
+func runHTTPRequest(ctx context.Context, rc RunContext) (any, error) {
+	rawURL := strInput(rc.Input, "url")
+	if rawURL == "" {
+		return nil, fmt.Errorf("http_request: url is required")
+	}
+	if !strings.HasPrefix(rawURL, "http://") && !strings.HasPrefix(rawURL, "https://") {
+		return nil, fmt.Errorf("http_request: url must be absolute http(s)")
+	}
+	method := strings.ToUpper(strInput(rc.Input, "method"))
+	if method == "" {
+		method = http.MethodGet
+	}
+	if method != http.MethodGet && method != http.MethodPost {
+		return nil, fmt.Errorf("http_request: method must be GET or POST")
+	}
+
+	var body io.Reader
+	if b := strInput(rc.Input, "body"); b != "" {
+		body = strings.NewReader(b)
+	}
+	req, err := http.NewRequestWithContext(ctx, method, rawURL, body)
+	if err != nil {
+		return nil, fmt.Errorf("http_request: %w", err)
+	}
+	if hdrs, ok := rc.Input["headers"].(map[string]any); ok {
+		for k, v := range hdrs {
+			if sv, ok := v.(string); ok {
+				req.Header.Set(k, sv)
+			}
+		}
+	}
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("http_request: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	raw, err := io.ReadAll(io.LimitReader(resp.Body, maxHTTPBody))
+	if err != nil {
+		return nil, fmt.Errorf("http_request: read: %w", err)
+	}
+	return map[string]any{
+		"status": resp.StatusCode,
+		"body":   string(raw),
+	}, nil
+}
+
+// ── core.delay ───────────────────────────────────────────────────────────────
+
+// maxDelay caps a single delay step so a flow can't pin a worker slot indefinitely.
+const maxDelay = 60 * time.Second
+
+func runDelay(ctx context.Context, rc RunContext) (any, error) {
+	secs := numInput(rc.Input, "seconds")
+	if secs < 0 {
+		secs = 0
+	}
+	d := time.Duration(secs * float64(time.Second))
+	if d > maxDelay {
+		d = maxDelay
+	}
+	select {
+	case <-time.After(d):
+		return map[string]any{"delayed": d.Seconds()}, nil
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
+
+// ── core.code ────────────────────────────────────────────────────────────────
+
+// runCode is the data-mapper transform: it returns its (already
+// reference-resolved, see resolveRefs) input map. No JS engine — a Go value map,
+// so later steps can read {{thisStep.field}}.
+func runCode(_ context.Context, rc RunContext) (any, error) {
+	if rc.Input == nil {
+		return map[string]any{}, nil
+	}
+	return rc.Input, nil
+}
+
+// ── core.wait_for_approval ───────────────────────────────────────────────────
+
+// runWaitForApproval is the DIRECT-invocation body (MCP tools/call). Inside a flow
+// the workflow special-cases this step onto a durable "resume" signal and never
+// calls Run — so this path represents an immediate, explicit approval.
+func runWaitForApproval(_ context.Context, rc RunContext) (any, error) {
+	return map[string]any{"approved": true, "reason": strInput(rc.Input, "reason")}, nil
+}
+
+// ── input helpers ────────────────────────────────────────────────────────────
+
+func strInput(in map[string]any, key string) string {
+	if in == nil {
+		return ""
+	}
+	if v, ok := in[key].(string); ok {
+		return strings.TrimSpace(v)
+	}
+	return ""
+}
+
+func numInput(in map[string]any, key string) float64 {
+	if in == nil {
+		return 0
+	}
+	switch v := in[key].(type) {
+	case float64:
+		return v
+	case int:
+		return float64(v)
+	case json.Number:
+		f, _ := v.Float64()
+		return f
+	}
+	return 0
+}
+
+// jsonBytes is a small helper for connectors that POST a JSON body.
+func jsonBytes(v any) (*bytes.Reader, error) {
+	b, err := json.Marshal(v)
+	if err != nil {
+		return nil, err
+	}
+	return bytes.NewReader(b), nil
+}

clients/automations/connector_github.go

@@ -0,0 +1,125 @@
+package automations
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+)
+
+// GitHub connector. Name == the clients/integrations provider id "github", which
+// is SCAFFOLDED there but custodies NO token yet. So RunContext.Token fails closed,
+// and every action here returns an honest "github not connected" — the code below
+// is complete and correct; it simply never reaches the API until integrations seals
+// a GitHub App installation token under githubTokenSecret.
+
+// githubTokenSecret is the KMS secret name a GitHub installation token WILL be
+// custodied under once integrations mints it. Named here so the connector is
+// complete the day that lands; today Token returns an error and this is unused
+// beyond the fail-closed check.
+const githubTokenSecret = "installation_token"
+
+var githubAPIBase = "https://api.github.com"
+
+var githubClient = &http.Client{Timeout: 15 * time.Second}
+
+func init() {
+	register(&Connector{
+		Name:        "github",
+		DisplayName: "GitHub",
+		AuthType:    "oauth2",
+		AuthReq:     true,
+		Actions: map[string]*Action{
+			"create_issue": {
+				Name:        "create_issue",
+				DisplayName: "Create Issue",
+				Description: "Open an issue on a repository.",
+				Props: []PropSpec{
+					{Name: "owner", Type: "string", Required: true, Description: "Repo owner/org."},
+					{Name: "repo", Type: "string", Required: true, Description: "Repository name."},
+					{Name: "title", Type: "string", Required: true, Description: "Issue title."},
+					{Name: "body", Type: "string", Description: "Issue body."},
+				},
+				Run: runGithubCreateIssue,
+			},
+			"list_repos": {
+				Name:        "list_repos",
+				DisplayName: "List Repositories",
+				Description: "List repositories the installation can access.",
+				Props:       []PropSpec{},
+				Run:         runGithubListRepos,
+			},
+		},
+	})
+}
+
+// githubToken fetches the org's GitHub installation token, failing closed with an
+// honest error when GitHub is not connected (the Phase-1 reality — no token is
+// custodied yet). ONE resolution point both actions share.
+func githubToken(rc RunContext) (string, error) {
+	tok, err := rc.Token(githubTokenSecret)
+	if err != nil {
+		return "", fmt.Errorf("github not connected: %w", err)
+	}
+	return string(tok), nil
+}
+
+func runGithubCreateIssue(ctx context.Context, rc RunContext) (any, error) {
+	tok, err := githubToken(rc)
+	if err != nil {
+		return nil, err
+	}
+	owner := strInput(rc.Input, "owner")
+	repo := strInput(rc.Input, "repo")
+	title := strInput(rc.Input, "title")
+	if owner == "" || repo == "" || title == "" {
+		return nil, fmt.Errorf("github create_issue: owner, repo, title are required")
+	}
+	payload, err := jsonBytes(map[string]any{"title": title, "body": strInput(rc.Input, "body")})
+	if err != nil {
+		return nil, fmt.Errorf("github create_issue: encode: %w", err)
+	}
+	url := fmt.Sprintf("%s/repos/%s/%s/issues", githubAPIBase, owner, repo)
+	return githubDo(ctx, http.MethodPost, url, tok, payload)
+}
+
+func runGithubListRepos(ctx context.Context, rc RunContext) (any, error) {
+	tok, err := githubToken(rc)
+	if err != nil {
+		return nil, err
+	}
+	return githubDo(ctx, http.MethodGet, githubAPIBase+"/installation/repositories", tok, nil)
+}
+
+// githubDo issues one GitHub REST call and returns the decoded JSON. Bounded body
+// read; explicit non-2xx handling. Unreachable in Phase 1 (no token), but complete.
+func githubDo(ctx context.Context, method, url, token string, body io.Reader) (any, error) {
+	req, err := http.NewRequestWithContext(ctx, method, url, body)
+	if err != nil {
+		return nil, fmt.Errorf("github: %w", err)
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	req.Header.Set("Accept", "application/vnd.github+json")
+	if body != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	resp, err := githubClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("github: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	raw, err := io.ReadAll(io.LimitReader(resp.Body, 5<<20))
+	if err != nil {
+		return nil, fmt.Errorf("github: read: %w", err)
+	}
+	if resp.StatusCode/100 != 2 {
+		return nil, fmt.Errorf("github http %d", resp.StatusCode)
+	}
+	var out any
+	if err := json.Unmarshal(raw, &out); err != nil {
+		return nil, fmt.Errorf("github: decode: %w", err)
+	}
+	return out, nil
+}

clients/automations/connector_google.go

@@ -0,0 +1,129 @@
+package automations
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+// Google connectors (Sheets + Drive). Google is NOT yet a provider in
+// clients/integrations, so RunContext.Token — bound to (in.Owner, "google_sheets"
+// / "google_drive") — resolves an UNKNOWN provider and fails closed. Every action
+// therefore returns an honest "google not connected" until a google provider lands
+// in integrations. The API code below is complete; it simply never runs in Phase 1.
+
+const googleTokenSecret = "access_token"
+
+var (
+	googleSheetsAPIBase = "https://sheets.googleapis.com/v4"
+	googleDriveAPIBase  = "https://www.googleapis.com/drive/v3"
+)
+
+var googleClient = &http.Client{Timeout: 15 * time.Second}
+
+func init() {
+	register(&Connector{
+		Name:        "google_sheets",
+		DisplayName: "Google Sheets",
+		AuthType:    "oauth2",
+		AuthReq:     true,
+		Actions: map[string]*Action{
+			"append_row": {
+				Name:        "append_row",
+				DisplayName: "Append Row",
+				Description: "Append a row of values to a spreadsheet range.",
+				Props: []PropSpec{
+					{Name: "spreadsheetId", Type: "string", Required: true, Description: "Target spreadsheet id."},
+					{Name: "range", Type: "string", Required: true, Description: "A1 range, e.g. Sheet1!A:C."},
+					{Name: "values", Type: "array", Required: true, Description: "Row cell values."},
+				},
+				Run: runSheetsAppendRow,
+			},
+		},
+	})
+	register(&Connector{
+		Name:        "google_drive",
+		DisplayName: "Google Drive",
+		AuthType:    "oauth2",
+		AuthReq:     true,
+		Actions: map[string]*Action{
+			"list_files": {
+				Name:        "list_files",
+				DisplayName: "List Files",
+				Description: "List files matching an optional query.",
+				Props: []PropSpec{
+					{Name: "query", Type: "string", Description: "Drive query, e.g. name contains 'report'."},
+				},
+				Run: runDriveListFiles,
+			},
+		},
+	})
+}
+
+func runSheetsAppendRow(ctx context.Context, rc RunContext) (any, error) {
+	tok, err := rc.Token(googleTokenSecret)
+	if err != nil {
+		return nil, fmt.Errorf("google not connected: %w", err)
+	}
+	sheet := strInput(rc.Input, "spreadsheetId")
+	rng := strInput(rc.Input, "range")
+	if sheet == "" || rng == "" {
+		return nil, fmt.Errorf("sheets append_row: spreadsheetId and range are required")
+	}
+	values, ok := rc.Input["values"].([]any)
+	if !ok {
+		return nil, fmt.Errorf("sheets append_row: values must be an array")
+	}
+	payload, err := jsonBytes(map[string]any{"values": [][]any{values}})
+	if err != nil {
+		return nil, fmt.Errorf("sheets append_row: encode: %w", err)
+	}
+	u := fmt.Sprintf("%s/spreadsheets/%s/values/%s:append?valueInputOption=USER_ENTERED",
+		googleSheetsAPIBase, url.PathEscape(sheet), url.PathEscape(rng))
+	return googleDo(ctx, http.MethodPost, u, string(tok), payload)
+}
+
+func runDriveListFiles(ctx context.Context, rc RunContext) (any, error) {
+	tok, err := rc.Token(googleTokenSecret)
+	if err != nil {
+		return nil, fmt.Errorf("google not connected: %w", err)
+	}
+	u := googleDriveAPIBase + "/files"
+	if q := strInput(rc.Input, "query"); q != "" {
+		u += "?q=" + url.QueryEscape(q)
+	}
+	return googleDo(ctx, http.MethodGet, u, string(tok), nil)
+}
+
+// googleDo issues one Google API call. Bounded read, explicit non-2xx handling.
+func googleDo(ctx context.Context, method, u, token string, body io.Reader) (any, error) {
+	req, err := http.NewRequestWithContext(ctx, method, u, body)
+	if err != nil {
+		return nil, fmt.Errorf("google: %w", err)
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	if body != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	resp, err := googleClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("google: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	raw, err := io.ReadAll(io.LimitReader(resp.Body, 5<<20))
+	if err != nil {
+		return nil, fmt.Errorf("google: read: %w", err)
+	}
+	if resp.StatusCode/100 != 2 {
+		return nil, fmt.Errorf("google http %d", resp.StatusCode)
+	}
+	var out any
+	if err := json.Unmarshal(raw, &out); err != nil {
+		return nil, fmt.Errorf("google: decode: %w", err)
+	}
+	return out, nil
+}

clients/automations/connector_slack.go

@@ -0,0 +1,104 @@
+package automations
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// Slack connector. Name == the clients/integrations provider id "slack", so
+// RunContext.Token resolves the SAME per-org bot token the OAuth plane sealed into
+// KMS. Fails closed if the org has not connected Slack (Token returns an error).
+
+// slackBotTokenSecret is the KMS secret name Slack's bot token is custodied under.
+// It MUST equal the unexported const of the same name in
+// clients/integrations/slack.go ("bot_token"); duplicated here because Go cannot
+// import an unexported identifier. The integrations OAuth exchange seals the
+// xoxb-… token under exactly this name.
+const slackBotTokenSecret = "bot_token"
+
+// slackAPIBase is Slack's Web API root. A var so a happy-path test can point it at
+// an httptest stub; production hits the fixed public endpoint.
+var slackAPIBase = "https://slack.com/api"
+
+// slackClient bounds every Slack call with a real timeout (fail-secure).
+var slackClient = &http.Client{Timeout: 15 * time.Second}
+
+func init() {
+	register(&Connector{
+		Name:        "slack",
+		DisplayName: "Slack",
+		AuthType:    "bot_token",
+		AuthReq:     true,
+		Actions: map[string]*Action{
+			"send_message": {
+				Name:        "send_message",
+				DisplayName: "Send Message",
+				Description: "Post a message to a Slack channel via chat.postMessage.",
+				Props: []PropSpec{
+					{Name: "channel", Type: "string", Required: true, Description: "Channel id or name."},
+					{Name: "text", Type: "string", Required: true, Description: "Message text."},
+				},
+				Run: runSlackSendMessage,
+			},
+		},
+	})
+}
+
+func runSlackSendMessage(ctx context.Context, rc RunContext) (any, error) {
+	// Credential first: fail closed if Slack is not connected for THIS org. Token is
+	// bound to (in.Owner, "slack"), so it can only ever return this tenant's token.
+	tok, err := rc.Token(slackBotTokenSecret)
+	if err != nil {
+		return nil, fmt.Errorf("slack not connected: %w", err)
+	}
+	channel := strInput(rc.Input, "channel")
+	text := strInput(rc.Input, "text")
+	if channel == "" || text == "" {
+		return nil, fmt.Errorf("slack send_message: channel and text are required")
+	}
+
+	payload, err := jsonBytes(map[string]any{"channel": channel, "text": text})
+	if err != nil {
+		return nil, fmt.Errorf("slack send_message: encode: %w", err)
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, slackAPIBase+"/chat.postMessage", payload)
+	if err != nil {
+		return nil, fmt.Errorf("slack send_message: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+	req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(string(tok)))
+	resp, err := slackClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("slack send_message: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	if err != nil {
+		return nil, fmt.Errorf("slack send_message: read: %w", err)
+	}
+	var r struct {
+		OK      bool   `json:"ok"`
+		Error   string `json:"error"`
+		Channel string `json:"channel"`
+		TS      string `json:"ts"`
+	}
+	if err := json.Unmarshal(body, &r); err != nil {
+		return nil, fmt.Errorf("slack send_message: decode: %w", err)
+	}
+	if !r.OK {
+		return nil, fmt.Errorf("slack send_message error: %s", nonEmpty(r.Error, "unknown_error"))
+	}
+	return map[string]any{"ok": true, "channel": r.Channel, "ts": r.TS}, nil
+}
+
+func nonEmpty(s, fallback string) string {
+	if strings.TrimSpace(s) == "" {
+		return fallback
+	}
+	return s
+}

clients/automations/engine.go

@@ -0,0 +1,389 @@
+package automations
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/hanzoai/cloud"
+	"github.com/hanzoai/cloud/clients/integrations"
+	tasksclient "github.com/hanzoai/tasks/pkg/sdk/client"
+	"github.com/hanzoai/tasks/pkg/sdk/temporal"
+	tasksworker "github.com/hanzoai/tasks/pkg/sdk/worker"
+	"github.com/hanzoai/tasks/pkg/sdk/workflow"
+)
+
+// Durable execution on the ONE shared in-process hanzoai/tasks engine
+// (cloud.EmbeddedTasks). A flow runs as a durable workflow in the OWNER's
+// namespace: crash-recovering, retried, event-sourced. This mirrors ai's
+// object/ingest_tasks.go per-org lazy-worker precedent exactly.
+
+const (
+	// automationsTaskQueue is this workflow family's own queue (<service>-<purpose>);
+	// nothing unrelated multiplexes onto it.
+	automationsTaskQueue = "automations-flow"
+	// resumeSignal is the signal a paused (wait_for_approval) flow blocks on; the
+	// /v1/automations/runs/:id/resume endpoint sends it.
+	resumeSignal = "resume"
+	// flowRunWorkflowType is the registered workflow name — MUST equal the reflected
+	// Go name of FlowRunWorkflow, so a cron schedule (which starts by name) resolves it.
+	flowRunWorkflowType = "FlowRunWorkflow"
+)
+
+// ErrEngineNotReady is returned when cloud.EmbeddedTasks() is still nil (the engine
+// is wired after MountAll). Handlers render it as 503 "automation engine not ready".
+var ErrEngineNotReady = errors.New("automations: engine not ready")
+
+// tokenSource is the ONE door to per-org credential custody. It defaults to
+// integrations.TokenFor (KMS-sealed, fail-closed). It is a package var ONLY so a
+// test can OBSERVE the (org,provider) a step tokenizes and prove the activity scopes
+// every credential fetch to in.Owner — never to a client-supplied field. Production
+// never reassigns it.
+var tokenSource = integrations.TokenFor
+
+// ── the durable workflow ─────────────────────────────────────────────────────
+
+// RunStartInput / RunEndInput are the durable run-bookkeeping activity payloads. The
+// durable path is the SINGLE owner of run bookkeeping (MED-1): whichever entrypoint
+// drives a workflow — manual /run, MCP, or a scheduled cron tick — records the run
+// exactly once here, so metering/audit never double-count and every execution lands
+// a FlowRun row visible to listRuns/getRun.
+type (
+	RunStartInput struct {
+		RunID         string `json:"runId"`
+		Owner         string `json:"owner"`
+		FlowID        string `json:"flowId"`
+		FlowVersionID string `json:"flowVersionId"`
+	}
+	RunEndInput struct {
+		RunID  string `json:"runId"`
+		Owner  string `json:"owner"`
+		Status string `json:"status"`
+	}
+)
+
+// FlowRunWorkflow walks the flow's flattened step chain. Each side-effecting step
+// runs as a retried activity (ExecuteStepActivity); a core.wait_for_approval step
+// is a durable PAUSE that blocks on the resume signal. Prior step outputs are
+// threaded so later steps can reference them ({{step.field}}). Deterministic: it
+// executes steps strictly in order, Get-ing each before dispatching the next.
+//
+// It also owns run bookkeeping (MED-1): the per-EXECUTION run id is the workflow id
+// (workflow.GetInfo) — manual runs set it to the run id; a scheduled cron mints a
+// fresh one per tick — so a run-start activity persists+meters+audits the run
+// EXACTLY once, and a run-end activity records the terminal status.
+func FlowRunWorkflow(ctx workflow.Context, in FlowRunInput) (FlowRunResult, error) {
+	// The workflow id keys exactly-once bookkeeping. NOT in.RunID: a schedule embeds
+	// one fixed FlowRunInput, so every tick shares in.RunID but gets a distinct
+	// workflow id — the id that makes each tick its own metered run.
+	runID := workflow.GetInfo(ctx).WorkflowID
+	if runID == "" {
+		runID = in.RunID
+	}
+
+	bkCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
+		StartToCloseTimeout: 30 * time.Second,
+		RetryPolicy: &temporal.RetryPolicy{
+			InitialInterval: time.Second, BackoffCoefficient: 2.0,
+			MaximumInterval: 10 * time.Second, MaximumAttempts: 5,
+		},
+	})
+	// Run-start bookkeeping is best-effort: a failure never blocks execution (the
+	// activity is idempotent + retried, so transient errors recover on their own).
+	_ = workflow.ExecuteActivity(bkCtx, RecordRunStartActivity, RunStartInput{
+		RunID: runID, Owner: in.Owner, FlowID: in.FlowID, FlowVersionID: in.FlowVersionID,
+	}).Get(bkCtx, nil)
+
+	actCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
+		// A step (HTTP call, provider API) can take a while; cap at 10m (never above).
+		StartToCloseTimeout: 10 * time.Minute,
+		RetryPolicy: &temporal.RetryPolicy{
+			InitialInterval:    2 * time.Second,
+			BackoffCoefficient: 2.0,
+			MaximumInterval:    2 * time.Minute,
+			MaximumAttempts:    3,
+		},
+	})
+
+	outputs := make(map[string]any, len(in.Steps))
+	for _, step := range in.Steps {
+		// wait_for_approval is the manual waitpoint: block on the durable resume
+		// signal rather than executing an activity. The engine persists the signal,
+		// so a resume survives a worker crash.
+		if step.PieceName == corePiece && step.ActionName == "wait_for_approval" {
+			var payload any
+			workflow.GetSignalChannel(ctx, resumeSignal).Receive(ctx, &payload)
+			outputs[step.Name] = payload
+			continue
+		}
+
+		var out StepOutput
+		si := StepInput{
+			Owner:       in.Owner, // the isolation boundary — copied verbatim, never widened
+			RunID:       runID,
+			Name:        step.Name,
+			PieceName:   step.PieceName,
+			ActionName:  step.ActionName,
+			Input:       step.Input,
+			PrevOutputs: outputs,
+		}
+		if err := workflow.ExecuteActivity(actCtx, ExecuteStepActivity, si).Get(actCtx, &out); err != nil {
+			_ = workflow.ExecuteActivity(bkCtx, RecordRunEndActivity, RunEndInput{RunID: runID, Owner: in.Owner, Status: string(RunFailed)}).Get(bkCtx, nil)
+			return FlowRunResult{RunID: runID, Status: RunFailed, Outputs: outputs, Steps: len(outputs)}, err
+		}
+		outputs[step.Name] = out.Output
+	}
+	_ = workflow.ExecuteActivity(bkCtx, RecordRunEndActivity, RunEndInput{RunID: runID, Owner: in.Owner, Status: string(RunSucceeded)}).Get(bkCtx, nil)
+	return FlowRunResult{RunID: runID, Status: RunSucceeded, Outputs: outputs, Steps: len(outputs)}, nil
+}
+
+// RecordRunStartActivity persists the run row + meters + audits EXACTLY once
+// (idempotent by run id via the store's metered-flag claim). The single owner of run
+// bookkeeping for every entrypoint. mounted==nil (an engine-only test without Mount)
+// is a no-op — nothing to record against.
+func RecordRunStartActivity(ctx context.Context, in RunStartInput) error {
+	if mounted == nil {
+		return nil
+	}
+	return mounted.recordRunStart(ctx, in)
+}
+
+// RecordRunEndActivity records a run's terminal status so listRuns reflects it
+// without a getRun refresh. Best-effort; mounted==nil is a no-op.
+func RecordRunEndActivity(ctx context.Context, in RunEndInput) error {
+	if mounted == nil {
+		return nil
+	}
+	return mounted.recordRunEnd(ctx, in)
+}
+
+// ExecuteStepActivity is the side-effecting body of ONE step. It is THE isolation
+// boundary: RunContext.Token is bound to in.Owner (the VALIDATED org set at
+// flow-start) and the step's own connector id, so a flow authored by org A can
+// never reach org B's connection nor another provider's secret. in.Owner is used
+// for token custody and NOTHING from in.Input can change it.
+func ExecuteStepActivity(ctx context.Context, in StepInput) (StepOutput, error) {
+	_, action, err := lookupAction(in.PieceName, in.ActionName)
+	if err != nil {
+		return StepOutput{}, err
+	}
+	// Resolve {{step.field}} references against the threaded prior outputs before the
+	// connector sees its input.
+	resolved, _ := resolveRefs(in.Input, in.PrevOutputs).(map[string]any)
+
+	owner := in.Owner // capture: the SOLE credential scope for this step
+	rc := RunContext{
+		Org:         owner,
+		Input:       resolved,
+		PrevOutputs: in.PrevOutputs,
+		Token: func(secretName string) ([]byte, error) {
+			return tokenSource(ctx, owner, in.PieceName, secretName)
+		},
+	}
+	out, err := action.Run(ctx, rc)
+	if err != nil {
+		return StepOutput{}, err
+	}
+	return StepOutput{Name: in.Name, Output: out}, nil
+}
+
+// ── per-org lazy worker (mirror ai/object/ingest_tasks.go) ───────────────────
+
+var (
+	provMu     sync.Mutex
+	orgWorkers sync.Map // org → tasksclient.Client (its automations worker already started)
+)
+
+// orgEngineClient returns the org's tasks client, lazily provisioning its per-org
+// automations worker on first use (CONTRACT §6: one client+worker per active org).
+// provMu serializes first-touch so an org never gets two workers; the fast path is a
+// lock-free sync.Map hit. Fails closed with ErrEngineNotReady until the shared engine
+// is wired.
+func orgEngineClient(org string) (tasksclient.Client, error) {
+	if c, ok := orgWorkers.Load(org); ok {
+		return c.(tasksclient.Client), nil
+	}
+	provMu.Lock()
+	defer provMu.Unlock()
+	if c, ok := orgWorkers.Load(org); ok {
+		return c.(tasksclient.Client), nil
+	}
+	eng := cloud.EmbeddedTasks()
+	if eng == nil {
+		return nil, ErrEngineNotReady
+	}
+	cli, err := tasksclient.Dial(tasksclient.Options{
+		HostPort:  fmt.Sprintf("127.0.0.1:%d", eng.ZAPPort()),
+		Namespace: org,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("automations dial engine: %w", err)
+	}
+	w := tasksworker.New(cli, automationsTaskQueue, tasksworker.Options{})
+	w.RegisterWorkflow(FlowRunWorkflow)
+	w.RegisterActivity(ExecuteStepActivity)
+	w.RegisterActivity(RecordRunStartActivity)
+	w.RegisterActivity(RecordRunEndActivity)
+	if err := w.Start(); err != nil {
+		cli.Close()
+		return nil, fmt.Errorf("automations worker start: %w", err)
+	}
+	orgWorkers.Store(org, cli)
+	return cli, nil
+}
+
+// executeFlow starts a FlowRunWorkflow in the owner's namespace and returns the
+// handle immediately (the caller never blocks on the run). in.Owner is the
+// per-namespace scope; in.RunID is the workflow id (resume/describe address it).
+func executeFlow(ctx context.Context, in FlowRunInput) (tasksclient.WorkflowRun, error) {
+	cli, err := orgEngineClient(in.Owner)
+	if err != nil {
+		return nil, err
+	}
+	return cli.ExecuteWorkflow(ctx, tasksclient.StartWorkflowOptions{
+		ID:        in.RunID,
+		TaskQueue: automationsTaskQueue,
+	}, FlowRunWorkflow, in)
+}
+
+// signalResume delivers the resume signal (with an optional approval payload) to a
+// paused run in the org's namespace.
+func signalResume(ctx context.Context, org, workflowID string, payload any) error {
+	cli, err := orgEngineClient(org)
+	if err != nil {
+		return err
+	}
+	return cli.SignalWorkflow(ctx, workflowID, "", resumeSignal, payload)
+}
+
+// describeRunStatus reads the engine's current status for a run and maps it onto a
+// FlowRunStatus. Scoped to the org's namespace, so it can only observe that org's runs.
+func describeRunStatus(ctx context.Context, org, workflowID string) (FlowRunStatus, error) {
+	cli, err := orgEngineClient(org)
+	if err != nil {
+		return "", err
+	}
+	info, err := cli.DescribeWorkflow(ctx, workflowID, "")
+	if err != nil {
+		return "", err
+	}
+	return mapWorkflowStatus(info.Status), nil
+}
+
+// enableSchedule registers a cron schedule that starts FlowRunWorkflow each tick in
+// the org's namespace. disableSchedule removes it. NO bespoke ticker — the engine
+// owns the cron.
+func enableSchedule(ctx context.Context, org, scheduleID, cron string, in FlowRunInput) error {
+	cli, err := orgEngineClient(org)
+	if err != nil {
+		return err
+	}
+	// WorkflowID is left unset so the engine mints a unique id per tick (a fixed id
+	// would make every tick collide on one execution). Input carries the flattened
+	// run so each tick starts an identical FlowRunWorkflow.
+	return cli.CreateSchedule(ctx, tasksclient.CreateScheduleOptions{
+		ID:              scheduleID,
+		CronExpressions: []string{cron},
+		WorkflowType:    flowRunWorkflowType,
+		TaskQueue:       automationsTaskQueue,
+		Input:           []any{in},
+	})
+}
+
+func disableSchedule(ctx context.Context, org, scheduleID string) error {
+	cli, err := orgEngineClient(org)
+	if err != nil {
+		return err
+	}
+	return cli.DeleteSchedule(ctx, scheduleID)
+}
+
+// mapWorkflowStatus maps the engine's workflow status onto the flow-run status.
+func mapWorkflowStatus(s tasksclient.WorkflowStatus) FlowRunStatus {
+	switch s {
+	case tasksclient.WorkflowStatusCompleted:
+		return RunSucceeded
+	case tasksclient.WorkflowStatusFailed:
+		return RunFailed
+	case tasksclient.WorkflowStatusCanceled, tasksclient.WorkflowStatusTerminated:
+		return RunCanceled
+	case tasksclient.WorkflowStatusTimedOut:
+		return RunTimeout
+	default:
+		return RunRunning
+	}
+}
+
+// ── reference threading (data-mapper) ────────────────────────────────────────
+
+// refRE matches a {{ step.path }} reference. The path is a dot-walk into the prior
+// step outputs (e.g. "http1.status", "code.user.name").
+var refRE = regexp.MustCompile(`\{\{\s*([^}]+?)\s*\}\}`)
+
+// resolveRefs deep-resolves {{step.path}} references in a value against the threaded
+// prior outputs. A string that is EXACTLY one reference is replaced by the referenced
+// VALUE (type preserved); a string with embedded references gets string substitution.
+// Maps and slices are resolved recursively. Non-string scalars pass through.
+func resolveRefs(v any, outputs map[string]any) any {
+	switch x := v.(type) {
+	case string:
+		return resolveStringRefs(x, outputs)
+	case map[string]any:
+		m := make(map[string]any, len(x))
+		for k, val := range x {
+			m[k] = resolveRefs(val, outputs)
+		}
+		return m
+	case []any:
+		s := make([]any, len(x))
+		for i, val := range x {
+			s[i] = resolveRefs(val, outputs)
+		}
+		return s
+	default:
+		return v
+	}
+}
+
+func resolveStringRefs(s string, outputs map[string]any) any {
+	m := refRE.FindStringSubmatch(s)
+	// Whole-string single reference → return the resolved value with its type.
+	if m != nil && strings.TrimSpace(s) == m[0] {
+		if val, ok := lookupPath(outputs, m[1]); ok {
+			return val
+		}
+		return s
+	}
+	// Embedded references → string substitution.
+	if m == nil {
+		return s
+	}
+	return refRE.ReplaceAllStringFunc(s, func(match string) string {
+		sub := refRE.FindStringSubmatch(match)
+		if val, ok := lookupPath(outputs, sub[1]); ok {
+			return fmt.Sprintf("%v", val)
+		}
+		return match
+	})
+}
+
+// lookupPath walks a dot path ("step.field.sub") into a nested output map.
+func lookupPath(outputs map[string]any, path string) (any, bool) {
+	parts := strings.Split(strings.TrimSpace(path), ".")
+	var cur any = outputs
+	for _, p := range parts {
+		m, ok := cur.(map[string]any)
+		if !ok {
+			return nil, false
+		}
+		cur, ok = m[p]
+		if !ok {
+			return nil, false
+		}
+	}
+	return cur, true
+}

clients/automations/engine_test.go

@@ -0,0 +1,231 @@
+package automations
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	tasksclient "github.com/hanzoai/tasks/pkg/sdk/client"
+	tasksworker "github.com/hanzoai/tasks/pkg/sdk/worker"
+	tasksengine "github.com/hanzoai/tasks/pkg/tasks"
+)
+
+// probe is a TEST-ONLY connector (registered only in _test builds) whose record
+// action captures the resolved input it received — so a durable flow can PROVE, end
+// to end, that a later step saw an earlier step's threaded output.
+var (
+	probeMu   sync.Mutex
+	probeSeen []string
+)
+
+func init() {
+	register(&Connector{
+		Name:        "probe",
+		DisplayName: "Probe",
+		AuthType:    "none",
+		Actions: map[string]*Action{
+			"record": {
+				Name: "record", DisplayName: "Record", Description: "test probe: record resolved input",
+				Run: func(_ context.Context, rc RunContext) (any, error) {
+					probeMu.Lock()
+					probeSeen = append(probeSeen, strInput(rc.Input, "seen"))
+					probeMu.Unlock()
+					return map[string]any{"recorded": strInput(rc.Input, "seen")}, nil
+				},
+			},
+		},
+	})
+}
+
+func probeReset() { probeMu.Lock(); probeSeen = nil; probeMu.Unlock() }
+func probeSnapshot() []string {
+	probeMu.Lock()
+	defer probeMu.Unlock()
+	return append([]string(nil), probeSeen...)
+}
+
+// TestDurableFlowRunSucceeds proves a flow runs DURABLY on hanzoai/tasks: it embeds a
+// real tasks engine (like examples/embed.go), registers FlowRunWorkflow +
+// ExecuteStepActivity on a worker, runs a flow of core steps (code + delay + a test
+// probe), and asserts the run reaches SUCCEEDED with the step outputs THREADED (the
+// probe saw the code step's output resolved through {{step1.greeting}}).
+func TestDurableFlowRunSucceeds(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	port := l.Addr().(*net.TCPAddr).Port
+	_ = l.Close()
+
+	srv, err := tasksengine.Embed(ctx, tasksengine.EmbedConfig{ZAPPort: port, Namespace: "acme", DataDir: t.TempDir()})
+	if err != nil {
+		t.Fatalf("embed: %v", err)
+	}
+	defer func() { _ = srv.Stop(context.Background()) }()
+
+	cli, err := tasksclient.Dial(tasksclient.Options{
+		HostPort: fmt.Sprintf("127.0.0.1:%d", port), Namespace: "acme",
+		DialTimeout: 5 * time.Second, CallTimeout: 5 * time.Second,
+	})
+	if err != nil {
+		t.Fatalf("dial: %v", err)
+	}
+	defer cli.Close()
+
+	w := tasksworker.New(cli, automationsTaskQueue, tasksworker.Options{})
+	w.RegisterWorkflow(FlowRunWorkflow)
+	w.RegisterActivity(ExecuteStepActivity)
+	w.RegisterActivity(RecordRunStartActivity)
+	w.RegisterActivity(RecordRunEndActivity)
+	if err := w.Start(); err != nil {
+		t.Fatalf("worker start: %v", err)
+	}
+	defer w.Stop()
+
+	probeReset()
+	in := FlowRunInput{
+		Owner: "acme", FlowID: "f1", FlowVersionID: "v1", RunID: "run_durable_1",
+		Steps: []FlowStep{
+			{Name: "step1", PieceName: corePiece, ActionName: "code", Input: map[string]any{"greeting": "hi"}},
+			{Name: "delay1", PieceName: corePiece, ActionName: "delay", Input: map[string]any{"seconds": 0.01}},
+			{Name: "probe1", PieceName: "probe", ActionName: "record", Input: map[string]any{"seen": "{{step1.greeting}} there"}},
+		},
+	}
+	run, err := cli.ExecuteWorkflow(ctx, tasksclient.StartWorkflowOptions{ID: in.RunID, TaskQueue: automationsTaskQueue}, FlowRunWorkflow, in)
+	if err != nil {
+		t.Fatalf("execute workflow: %v", err)
+	}
+	// Get(nil) blocks until a terminal state; a non-Completed terminal returns an error.
+	if err := run.Get(ctx, nil); err != nil {
+		t.Fatalf("flow did not reach SUCCEEDED: %v", err)
+	}
+
+	info, err := cli.DescribeWorkflow(ctx, in.RunID, "")
+	if err != nil {
+		t.Fatalf("describe: %v", err)
+	}
+	if got := mapWorkflowStatus(info.Status); got != RunSucceeded {
+		t.Fatalf("run status want SUCCEEDED, got %q (engine status %d)", got, info.Status)
+	}
+
+	// Threading proven end-to-end: the probe step saw step1's output.
+	seen := probeSnapshot()
+	if len(seen) != 1 || seen[0] != "hi there" {
+		t.Fatalf("step outputs not threaded through the durable run: probe saw %v", seen)
+	}
+}
+
+// TestTokenIsolation is the cross-tenant credential-leak test RED must scrutinize: an
+// activity's ONLY credential scope is in.Owner (the VALIDATED org). A FORGED
+// in.Input.owner cannot widen it. The step's connector id is the ONLY provider.
+func TestTokenIsolation(t *testing.T) {
+	type call struct{ org, provider string }
+	var (
+		mu    sync.Mutex
+		calls []call
+	)
+	orig := tokenSource
+	tokenSource = func(_ context.Context, org, provider, _ string) ([]byte, error) {
+		mu.Lock()
+		calls = append(calls, call{org, provider})
+		mu.Unlock()
+		return nil, fmt.Errorf("not connected")
+	}
+	t.Cleanup(func() { tokenSource = orig })
+
+	// Org A runs a slack step; a hostile Input carries owner=globex to try to smuggle
+	// another tenant's scope. The activity must tokenize ONLY as (acme, slack).
+	in := StepInput{
+		Owner: "acme", PieceName: "slack", ActionName: "send_message",
+		Input: map[string]any{"owner": "globex", "org": "globex", "channel": "C1", "text": "hi"},
+	}
+	if _, err := ExecuteStepActivity(context.Background(), in); err == nil {
+		t.Fatal("slack step must fail closed when not connected")
+	}
+
+	mu.Lock()
+	defer mu.Unlock()
+	if len(calls) != 1 {
+		t.Fatalf("expected exactly one credential fetch, got %d: %+v", len(calls), calls)
+	}
+	if calls[0].org != "acme" {
+		t.Fatalf("credential scope MUST be in.Owner=acme; forged input widened it to %q", calls[0].org)
+	}
+	if calls[0].provider != "slack" {
+		t.Fatalf("credential provider MUST be the step's connector=slack, got %q", calls[0].provider)
+	}
+}
+
+// TestStepOutputsThreaded proves the reference-resolution mechanism directly: an
+// activity resolves {{step.path}} against the threaded prior outputs, preserving type
+// for a whole-string reference.
+func TestStepOutputsThreaded(t *testing.T) {
+	// String-embedded reference → string substitution.
+	out, err := ExecuteStepActivity(context.Background(), StepInput{
+		Owner: "acme", PieceName: corePiece, ActionName: "code", Name: "step2",
+		Input:       map[string]any{"echo": "{{step1.greeting}} world"},
+		PrevOutputs: map[string]any{"step1": map[string]any{"greeting": "hello"}},
+	})
+	if err != nil {
+		t.Fatalf("code activity: %v", err)
+	}
+	m, ok := out.Output.(map[string]any)
+	if !ok || m["echo"] != "hello world" {
+		t.Fatalf("embedded reference not threaded, got %+v", out.Output)
+	}
+
+	// Whole-string reference → value with type preserved (number stays number).
+	out2, err := ExecuteStepActivity(context.Background(), StepInput{
+		Owner: "acme", PieceName: corePiece, ActionName: "code", Name: "step3",
+		Input:       map[string]any{"n": "{{step1.count}}"},
+		PrevOutputs: map[string]any{"step1": map[string]any{"count": float64(5)}},
+	})
+	if err != nil {
+		t.Fatalf("code activity 2: %v", err)
+	}
+	m2 := out2.Output.(map[string]any)
+	if m2["n"] != float64(5) {
+		t.Fatalf("whole-string reference must preserve type, got %T %v", m2["n"], m2["n"])
+	}
+}
+
+// TestHTTPRequestSSRFBlocked proves the http_request SSRF guard refuses a non-public
+// target by default (the DNS-rebind TOCTOU window is closed by the dialer Control hook).
+func TestHTTPRequestSSRFBlocked(t *testing.T) {
+	_, err := runHTTPRequest(context.Background(), RunContext{Input: map[string]any{"url": "http://127.0.0.1:9/x"}})
+	if err == nil || !strings.Contains(err.Error(), "ssrf") {
+		t.Fatalf("loopback target must be SSRF-refused, got err=%v", err)
+	}
+}
+
+// TestHTTPRequestHappyPath proves the success path (with the private-allow override a
+// test uses to hit a loopback stub): status + body are returned.
+func TestHTTPRequestHappyPath(t *testing.T) {
+	t.Setenv(httpAllowPrivateEnv, "1")
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(`{"ok":true}`))
+	}))
+	defer ts.Close()
+
+	out, err := runHTTPRequest(context.Background(), RunContext{Input: map[string]any{"url": ts.URL}})
+	if err != nil {
+		t.Fatalf("http_request: %v", err)
+	}
+	m := out.(map[string]any)
+	if m["status"] != http.StatusOK {
+		t.Fatalf("status want 200, got %v", m["status"])
+	}
+	if !strings.Contains(m["body"].(string), "ok") {
+		t.Fatalf("body want ok, got %v", m["body"])
+	}
+}

clients/automations/fixes_test.go

@@ -0,0 +1,315 @@
+package automations
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/hanzoai/cloud/audit"
+	tasksclient "github.com/hanzoai/tasks/pkg/sdk/client"
+	tasksworker "github.com/hanzoai/tasks/pkg/sdk/worker"
+	tasksengine "github.com/hanzoai/tasks/pkg/tasks"
+	"github.com/zap-proto/zip"
+)
+
+// ── MED-1: exactly-once run bookkeeping across every entrypoint ──────────────
+
+// TestScheduledRunMeteredExactlyOnce proves the durable path is the single owner of
+// run bookkeeping: a scheduled tick (a FlowRunWorkflow started by workflow id, as the
+// cron scheduler does) meters+audits the run EXACTLY once and the run shows up in
+// listRuns — keyed on the per-tick workflow id, so distinct ticks are distinct runs
+// even though a schedule embeds one fixed FlowRunInput.RunID.
+func TestScheduledRunMeteredExactlyOnce(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	app, rec := newAppWithAudit(t) // Mount sets the package `mounted` used by the bookkeeping activity
+
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	port := l.Addr().(*net.TCPAddr).Port
+	_ = l.Close()
+
+	srv, err := tasksengine.Embed(ctx, tasksengine.EmbedConfig{ZAPPort: port, Namespace: "acme", DataDir: t.TempDir()})
+	if err != nil {
+		t.Fatalf("embed: %v", err)
+	}
+	defer func() { _ = srv.Stop(context.Background()) }()
+
+	cli, err := tasksclient.Dial(tasksclient.Options{HostPort: fmt.Sprintf("127.0.0.1:%d", port), Namespace: "acme", DialTimeout: 5 * time.Second, CallTimeout: 5 * time.Second})
+	if err != nil {
+		t.Fatalf("dial: %v", err)
+	}
+	defer cli.Close()
+
+	w := tasksworker.New(cli, automationsTaskQueue, tasksworker.Options{})
+	w.RegisterWorkflow(FlowRunWorkflow)
+	w.RegisterActivity(ExecuteStepActivity)
+	w.RegisterActivity(RecordRunStartActivity)
+	w.RegisterActivity(RecordRunEndActivity)
+	if err := w.Start(); err != nil {
+		t.Fatalf("worker start: %v", err)
+	}
+	defer w.Stop()
+
+	// A tick starts FlowRunWorkflow with a per-tick workflow id; in.RunID is the
+	// schedule's FIXED id (same for every tick) — proving the run key is the workflow id.
+	tick := func(wfID string) {
+		in := FlowRunInput{Owner: "acme", FlowID: "f1", FlowVersionID: "v1", RunID: "sched-f1",
+			Steps: []FlowStep{{Name: "s1", PieceName: corePiece, ActionName: "code", Input: map[string]any{"k": "v"}}}}
+		run, err := cli.ExecuteWorkflow(ctx, tasksclient.StartWorkflowOptions{ID: wfID, TaskQueue: automationsTaskQueue}, FlowRunWorkflow, in)
+		if err != nil {
+			t.Fatalf("tick %s: %v", wfID, err)
+		}
+		if err := run.Get(ctx, nil); err != nil {
+			t.Fatalf("tick %s get: %v", wfID, err)
+		}
+	}
+
+	tick("tick-1")
+	// Exactly one flow.run audit record ⇒ exactly one meter (same won-guard).
+	if n := auditCount(t, rec, "acme", "automations.flow.run"); n != 1 {
+		t.Fatalf("first tick metered %d times, want 1", n)
+	}
+	runs := listRunsHTTP(t, app, "acme")
+	if len(runs) != 1 || runs[0].ID != "tick-1" {
+		t.Fatalf("listRuns want [tick-1], got %+v", runs)
+	}
+
+	tick("tick-2")
+	if n := auditCount(t, rec, "acme", "automations.flow.run"); n != 2 {
+		t.Fatalf("after two ticks metered %d total, want 2", n)
+	}
+	runs = listRunsHTTP(t, app, "acme")
+	ids := map[string]bool{}
+	for _, r := range runs {
+		ids[r.ID] = true
+	}
+	if len(runs) != 2 || !ids["tick-1"] || !ids["tick-2"] {
+		t.Fatalf("two ticks must be two distinct runs, got %+v", runs)
+	}
+}
+
+// TestRunStartBookkeepingIdempotent proves recordRunStart is idempotent by run id:
+// calling it twice for the same run (the manual handler's row-create + the durable
+// activity, or an activity retry) meters + audits ONCE and lands ONE run row.
+func TestRunStartBookkeepingIdempotent(t *testing.T) {
+	_, rec := newAppWithAudit(t)
+	ctx := context.Background()
+	in := RunStartInput{RunID: "r1", Owner: "acme", FlowID: "f1", FlowVersionID: "v1"}
+
+	if err := mounted.recordRunStart(ctx, in); err != nil {
+		t.Fatalf("recordRunStart 1: %v", err)
+	}
+	if err := mounted.recordRunStart(ctx, in); err != nil {
+		t.Fatalf("recordRunStart 2 (retry): %v", err)
+	}
+	if n := auditCount(t, rec, "acme", "automations.flow.run"); n != 1 {
+		t.Fatalf("recordRunStart double-billed: %d flow.run records, want 1", n)
+	}
+	runs, _ := mounted.store.ListRuns(ctx, "acme", "", 10)
+	if len(runs) != 1 || runs[0].ID != "r1" {
+		t.Fatalf("want exactly one run row r1, got %+v", runs)
+	}
+}
+
+// listRunsHTTP exercises the real GET /v1/automations/runs endpoint (org-gated) and
+// returns the rows — proving a run "shows up in listRuns", not just in the store.
+func listRunsHTTP(t *testing.T, app *zip.App, org string) []FlowRun {
+	t.Helper()
+	r := req(t, app, http.MethodGet, "/v1/automations/runs", org, nil)
+	if r.Code != http.StatusOK {
+		t.Fatalf("listRuns want 200, got %d (%s)", r.Code, r.Body)
+	}
+	var out struct {
+		Data []FlowRun `json:"data"`
+	}
+	if err := json.Unmarshal(r.Body, &out); err != nil {
+		t.Fatalf("listRuns body: %v (%s)", err, r.Body)
+	}
+	return out.Data
+}
+
+// ── MED-2: SSRF blocklist covers IANA special-use ranges ────────────────────
+
+func TestIsPublicIP(t *testing.T) {
+	blocked := []string{
+		"127.0.0.1", "10.0.0.1", "172.16.0.1", "192.168.1.1", // loopback + RFC1918
+		"169.254.169.254",    // AWS/GCP/Azure metadata (link-local)
+		"100.100.100.200",    // Alibaba metadata (CGNAT)
+		"100.64.0.1",         // CGNAT
+		"192.0.0.1",          // IETF protocol
+		"192.0.2.5",          // TEST-NET-1
+		"192.88.99.1",        // 6to4 relay anycast
+		"198.18.0.1",         // benchmarking
+		"198.51.100.7",       // TEST-NET-2
+		"203.0.113.9",        // TEST-NET-3
+		"240.0.0.1",          // reserved / class E
+		"0.0.0.0", "0.1.2.3", // this-network
+		"::1", "fe80::1", "fc00::1", // v6 loopback/link-local/ULA
+		"64:ff9b::808:808",       // NAT64 of 8.8.8.8
+		"::ffff:10.0.0.1",        // v4-mapped private
+		"::ffff:169.254.169.254", // v4-mapped metadata
+	}
+	for _, s := range blocked {
+		ip := net.ParseIP(s)
+		if ip == nil {
+			t.Fatalf("bad test IP %q", s)
+		}
+		if isPublicIP(ip) {
+			t.Fatalf("%s must be blocked (non-public)", s)
+		}
+	}
+	allowed := []string{"8.8.8.8", "1.1.1.1", "93.184.216.34", "2606:4700:4700::1111"}
+	for _, s := range allowed {
+		if !isPublicIP(net.ParseIP(s)) {
+			t.Fatalf("%s must be allowed (public)", s)
+		}
+	}
+}
+
+// ── MED-3 + LOW-4: noisy-neighbor caps ──────────────────────────────────────
+
+func TestFlowStepCapRejected(t *testing.T) {
+	// A chain past the step cap is rejected at validation.
+	var head *FlowAction
+	for i := 0; i <= maxSteps; i++ {
+		head = &FlowAction{Name: fmt.Sprintf("a%d", i), Type: ActionTypeCode, NextAction: head}
+	}
+	over := &FlowTrigger{Name: "t", Type: TriggerTypePiece, NextAction: head}
+	if err := validateTrigger(over); err == nil {
+		t.Fatalf("a flow of %d steps must be rejected", maxSteps+1)
+	}
+	// And over HTTP the create is an honest 422.
+	app := newApp(t)
+	r := req(t, app, http.MethodPost, "/v1/automations/flows", "acme", map[string]any{"displayName": "big", "trigger": over})
+	if r.Code != http.StatusUnprocessableEntity {
+		t.Fatalf("oversized flow create want 422, got %d (%s)", r.Code, r.Body)
+	}
+
+	// A within-cap flow passes.
+	small := &FlowTrigger{Name: "t", Type: TriggerTypePiece, NextAction: &FlowAction{Name: "a", Type: ActionTypeCode}}
+	if err := validateTrigger(small); err != nil {
+		t.Fatalf("a small flow must pass: %v", err)
+	}
+
+	// A giant serialized tree (huge input blob) is rejected on size.
+	big := &FlowTrigger{Name: "t", Type: TriggerTypePiece, NextAction: &FlowAction{
+		Name: "a", Type: ActionTypeCode, Settings: StepSettings{Input: map[string]any{"blob": strings.Repeat("Z", maxTriggerBytes+1)}},
+	}}
+	if err := validateTrigger(big); err == nil {
+		t.Fatal("an oversized flow tree must be rejected on size")
+	}
+}
+
+func TestResumePayloadBounded(t *testing.T) {
+	app := newApp(t)
+	ctx := context.Background()
+	if _, err := mounted.store.CreateRun(ctx, FlowRun{ID: "r1", Org: "acme", FlowID: "f1", WorkflowID: "r1", Status: RunPaused, Created: 1, Updated: 1}); err != nil {
+		t.Fatalf("seed run: %v", err)
+	}
+	body := `{"note":"` + strings.Repeat("x", maxResumePayload+1) + `"}`
+	r := reqRaw(t, app, "/v1/automations/runs/r1/resume", "acme", body)
+	if r.Code != http.StatusRequestEntityTooLarge {
+		t.Fatalf("oversized resume want 413, got %d (%s)", r.Code, r.Body)
+	}
+}
+
+// ── LOW-2: per-org concurrency cap ──────────────────────────────────────────
+
+func TestConcurrencyLimiter(t *testing.T) {
+	l := newConcurrencyLimiter(2)
+	if !l.acquire("a") || !l.acquire("a") {
+		t.Fatal("first two acquisitions must succeed")
+	}
+	if l.acquire("a") {
+		t.Fatal("third acquisition must be refused at the cap")
+	}
+	if !l.acquire("b") {
+		t.Fatal("a different org must be independent of a's cap")
+	}
+	l.release("a")
+	if !l.acquire("a") {
+		t.Fatal("after a release a slot must free")
+	}
+}
+
+// ── LOW-1: MCP outcome is derived from the real result, after Run ───────────
+
+func TestMCPAuditOutcome(t *testing.T) {
+	app, rec := newAppWithAudit(t)
+	// Success: core_code runs and returns.
+	reqRaw(t, app, "/v1/automations/mcp", "acme",
+		`{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"core_code","arguments":{"k":"v"}}}`)
+	// Failure: slack is not connected → Run errors.
+	reqRaw(t, app, "/v1/automations/mcp", "acme",
+		`{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"slack_send_message","arguments":{"channel":"C","text":"hi"}}}`)
+
+	rows, _, err := rec.Query(context.Background(), audit.Filter{Org: "acme", Action: "automations.mcp.call", Limit: 100})
+	if err != nil {
+		t.Fatalf("audit query: %v", err)
+	}
+	var ok, bad int
+	for _, r := range rows {
+		switch r.Outcome.Result {
+		case "ok":
+			ok++
+		case "error":
+			bad++
+		}
+	}
+	if ok != 1 || bad != 1 {
+		t.Fatalf("want 1 ok + 1 error mcp.call audit (outcome from real result), got ok=%d error=%d (total %d)", ok, bad, len(rows))
+	}
+}
+
+// ── LOW-3: publishedVersionId must name an in-org version of the flow ───────
+
+func TestUpdateFlowPublishedVersionValidated(t *testing.T) {
+	app := newApp(t)
+	mk := func() populatedFlow {
+		r := req(t, app, http.MethodPost, "/v1/automations/flows", "acme", map[string]any{
+			"displayName": "F", "trigger": map[string]any{"name": "trigger", "type": TriggerTypePiece, "strategy": string(StrategyManual)},
+		})
+		var pf populatedFlow
+		_ = json.Unmarshal(r.Body, &pf)
+		return pf
+	}
+	a := mk()
+	b := mk()
+
+	// Valid: a's own version → 200.
+	if r := req(t, app, http.MethodPatch, "/v1/automations/flows/"+a.ID, "acme", map[string]any{"publishedVersionId": a.Version.ID}); r.Code != http.StatusOK {
+		t.Fatalf("valid publishedVersionId want 200, got %d (%s)", r.Code, r.Body)
+	}
+	// Bogus id → 422.
+	if r := req(t, app, http.MethodPatch, "/v1/automations/flows/"+a.ID, "acme", map[string]any{"publishedVersionId": "ver_bogus"}); r.Code != http.StatusUnprocessableEntity {
+		t.Fatalf("bogus publishedVersionId want 422, got %d", r.Code)
+	}
+	// A version that belongs to ANOTHER flow (same org) → 422.
+	if r := req(t, app, http.MethodPatch, "/v1/automations/flows/"+a.ID, "acme", map[string]any{"publishedVersionId": b.Version.ID}); r.Code != http.StatusUnprocessableEntity {
+		t.Fatalf("cross-flow publishedVersionId want 422, got %d", r.Code)
+	}
+}
+
+// ── INF-1: tool-name collision refused at registration ──────────────────────
+
+func TestToolNameCollisionPanics(t *testing.T) {
+	defer func() {
+		if recover() == nil {
+			t.Fatal("a connector whose tool name collides with an existing one must panic at register")
+		}
+	}()
+	// "core" already owns action "http_request" ⇒ tool "core_http_request". A
+	// connector "core_http" with action "request" would collide into the SAME tool.
+	register(&Connector{Name: "core_http", Actions: map[string]*Action{
+		"request": {Name: "request", Run: func(context.Context, RunContext) (any, error) { return nil, nil }},
+	}})
+}

clients/automations/harness_test.go

@@ -0,0 +1,120 @@
+package automations
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	fiber "github.com/gofiber/fiber/v3"
+	"github.com/hanzoai/cloud"
+	"github.com/hanzoai/cloud/audit"
+	luxlog "github.com/luxfi/log"
+	"github.com/zap-proto/zip"
+)
+
+// newApp mounts the automations subsystem on a fresh zip.App with a temp DataDir,
+// mirroring clients/integrations' test harness. No KMS/engine is wired: the HTTP
+// tests exercise the org gate, the store, the catalogue, and the core-connector MCP
+// dispatch — none of which need custody or the durable engine.
+func newApp(t *testing.T) *zip.App {
+	t.Helper()
+	app := zip.New(zip.Config{Logger: luxlog.New("test")})
+	deps := cloud.Deps{Logger: luxlog.New("test"), DataDir: t.TempDir()}
+	if err := Mount(app, deps); err != nil {
+		t.Fatalf("Mount: %v", err)
+	}
+	t.Cleanup(func() { _ = Shutdown(context.Background()) })
+	return app
+}
+
+// newAppWithAudit mounts the subsystem with a REAL in-memory audit recorder so a
+// test can read the tamper-evident trail back and assert outcomes (LOW-1) and
+// exactly-once run bookkeeping (MED-1). Returns the recorder for querying.
+func newAppWithAudit(t *testing.T) (*zip.App, *audit.Recorder) {
+	t.Helper()
+	rec, err := audit.Open(":memory:", nil)
+	if err != nil {
+		t.Fatalf("audit.Open: %v", err)
+	}
+	app := zip.New(zip.Config{Logger: luxlog.New("test")})
+	deps := cloud.Deps{Logger: luxlog.New("test"), DataDir: t.TempDir(), Audit: rec}
+	if err := Mount(app, deps); err != nil {
+		t.Fatalf("Mount: %v", err)
+	}
+	t.Cleanup(func() { _ = Shutdown(context.Background()) })
+	return app, rec
+}
+
+// auditCount returns how many records match (org, action) in the recorder.
+func auditCount(t *testing.T, rec *audit.Recorder, org, action string) int {
+	t.Helper()
+	rows, _, err := rec.Query(context.Background(), audit.Filter{Org: org, Action: action, Limit: 1000})
+	if err != nil {
+		t.Fatalf("audit query: %v", err)
+	}
+	return len(rows)
+}
+
+type httpResult struct {
+	Code int
+	Body []byte
+}
+
+// req issues one request. org != "" sets the gateway identity headers (X-Org-Id +
+// a validated X-User-Id) exactly as SanitizeIdentity would; org == "" sends NO
+// identity — the anonymous-forge path the 403 tests need.
+func req(t *testing.T, app *zip.App, method, path, org string, body any) httpResult {
+	t.Helper()
+	var r io.Reader
+	if body != nil {
+		b, _ := json.Marshal(body)
+		r = bytes.NewReader(b)
+	}
+	rq := httptest.NewRequest(method, path, r)
+	if body != nil {
+		rq.Header.Set("Content-Type", "application/json")
+	}
+	if org != "" {
+		rq.Header.Set("X-Org-Id", org)
+		rq.Header.Set("X-User-Id", "u-"+org)
+	}
+	resp, err := app.Fiber().Test(rq, fiber.TestConfig{Timeout: 0})
+	if err != nil {
+		t.Fatalf("Test %s %s: %v", method, path, err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	b, _ := io.ReadAll(resp.Body)
+	return httpResult{Code: resp.StatusCode, Body: b}
+}
+
+// reqRaw is req for a raw (already-encoded) JSON body — used by the MCP JSON-RPC tests.
+func reqRaw(t *testing.T, app *zip.App, path, org string, raw string) httpResult {
+	t.Helper()
+	rq := httptest.NewRequest(http.MethodPost, path, bytes.NewReader([]byte(raw)))
+	rq.Header.Set("Content-Type", "application/json")
+	if org != "" {
+		rq.Header.Set("X-Org-Id", org)
+		rq.Header.Set("X-User-Id", "u-"+org)
+	}
+	resp, err := app.Fiber().Test(rq, fiber.TestConfig{Timeout: 0})
+	if err != nil {
+		t.Fatalf("Test POST %s: %v", path, err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	b, _ := io.ReadAll(resp.Body)
+	return httpResult{Code: resp.StatusCode, Body: b}
+}
+
+func testStore(t *testing.T) *Store {
+	t.Helper()
+	s, err := openStore(t.TempDir() + "/automations.db")
+	if err != nil {
+		t.Fatalf("openStore: %v", err)
+	}
+	t.Cleanup(func() { _ = s.Close() })
+	return s
+}

clients/automations/http_test.go

@@ -0,0 +1,181 @@
+package automations
+
+import (
+	"encoding/json"
+	"net/http"
+	"testing"
+)
+
+// TestOrgGating403 proves every data endpoint refuses a request with NO validated
+// principal (the anonymous-forge path). A data plane never serves an unauthenticated
+// principal.
+func TestOrgGating403(t *testing.T) {
+	app := newApp(t)
+	gated := []struct {
+		method, path string
+	}{
+		{http.MethodGet, "/v1/automations/pieces"},
+		{http.MethodGet, "/v1/automations/flows"},
+		{http.MethodPost, "/v1/automations/flows"},
+		{http.MethodGet, "/v1/automations/flows/x"},
+		{http.MethodGet, "/v1/automations/runs"},
+		{http.MethodPost, "/v1/automations/flows/x/run"},
+		{http.MethodPost, "/v1/automations/mcp"},
+	}
+	for _, g := range gated {
+		if r := req(t, app, g.method, g.path, "", nil); r.Code != http.StatusForbidden {
+			t.Fatalf("%s %s without principal want 403, got %d (%s)", g.method, g.path, r.Code, r.Body)
+		}
+	}
+}
+
+// TestPiecesCatalog: the /pieces endpoint returns the full piece catalogue and,
+// within it, the Tier-A executable connectors (core/slack/github/google_*). The
+// count is not pinned to the seed — the full 701-piece catalogue is embedded — so
+// the invariant is PieceCount==len(Pieces) and every Tier-A connector is present.
+func TestPiecesCatalog(t *testing.T) {
+	app := newApp(t)
+	r := req(t, app, http.MethodGet, "/v1/automations/pieces", "acme", nil)
+	if r.Code != http.StatusOK {
+		t.Fatalf("pieces want 200, got %d (%s)", r.Code, r.Body)
+	}
+	var cat Catalog
+	if err := json.Unmarshal(r.Body, &cat); err != nil {
+		t.Fatalf("pieces body: %v (%s)", err, r.Body)
+	}
+	if cat.PieceCount != len(cat.Pieces) || len(cat.Pieces) < 5 {
+		t.Fatalf("catalogue inconsistent/too small: count=%d len=%d", cat.PieceCount, len(cat.Pieces))
+	}
+	byName := map[string]PieceMetadata{}
+	for _, p := range cat.Pieces {
+		byName[p.Name] = p
+	}
+	for _, want := range []string{"core", "slack", "github", "google_sheets", "google_drive"} {
+		if _, ok := byName[want]; !ok {
+			t.Fatalf("catalogue missing %q: %+v", want, cat.Pieces)
+		}
+	}
+	if byName["slack"].Auth.Type != "bot_token" || !byName["slack"].Auth.Required {
+		t.Fatalf("slack auth descriptor wrong: %+v", byName["slack"].Auth)
+	}
+}
+
+// TestFlowCRUDHTTP exercises the flow lifecycle over HTTP with a validated principal,
+// and proves cross-tenant invisibility at the HTTP boundary.
+func TestFlowCRUDHTTP(t *testing.T) {
+	app := newApp(t)
+
+	// Create a flow with an initial draft version.
+	create := req(t, app, http.MethodPost, "/v1/automations/flows", "acme", map[string]any{
+		"displayName": "Nightly Sync",
+		"trigger": map[string]any{
+			"name": "trigger", "type": TriggerTypePiece, "displayName": "Start",
+			"strategy": string(StrategyManual),
+			"settings": map[string]any{"pieceName": "core", "triggerName": "manual"},
+		},
+	})
+	if create.Code != http.StatusCreated {
+		t.Fatalf("create flow want 201, got %d (%s)", create.Code, create.Body)
+	}
+	var pf populatedFlow
+	if err := json.Unmarshal(create.Body, &pf); err != nil {
+		t.Fatalf("create body: %v (%s)", err, create.Body)
+	}
+	if pf.Org != "acme" {
+		t.Fatalf("flow projectId must be the org, got %q", pf.Org)
+	}
+	if pf.Version == nil || pf.Version.DisplayName != "Nightly Sync" {
+		t.Fatalf("create must return the initial version: %+v", pf.Version)
+	}
+	flowID := pf.ID
+
+	// GET returns the flow + latest version.
+	get := req(t, app, http.MethodGet, "/v1/automations/flows/"+flowID, "acme", nil)
+	if get.Code != http.StatusOK {
+		t.Fatalf("get flow want 200, got %d (%s)", get.Code, get.Body)
+	}
+
+	// A different org cannot see it.
+	if r := req(t, app, http.MethodGet, "/v1/automations/flows/"+flowID, "globex", nil); r.Code != http.StatusNotFound {
+		t.Fatalf("globex GET acme flow want 404, got %d", r.Code)
+	}
+	// A different org's list is empty.
+	rl := req(t, app, http.MethodGet, "/v1/automations/flows", "globex", nil)
+	var listOut struct {
+		Data []Flow `json:"data"`
+	}
+	_ = json.Unmarshal(rl.Body, &listOut)
+	if len(listOut.Data) != 0 {
+		t.Fatalf("globex must see zero flows, got %d", len(listOut.Data))
+	}
+
+	// DELETE.
+	if r := req(t, app, http.MethodDelete, "/v1/automations/flows/"+flowID, "acme", nil); r.Code != http.StatusNoContent {
+		t.Fatalf("delete flow want 204, got %d (%s)", r.Code, r.Body)
+	}
+	if r := req(t, app, http.MethodGet, "/v1/automations/flows/"+flowID, "acme", nil); r.Code != http.StatusNotFound {
+		t.Fatalf("deleted flow GET want 404, got %d", r.Code)
+	}
+}
+
+// TestOperationsApply proves the FlowOperation apply mutates the flow's version tree
+// (ADD_ACTION into the linear chain) and honestly rejects an unsupported tree op.
+func TestOperationsApply(t *testing.T) {
+	app := newApp(t)
+
+	create := req(t, app, http.MethodPost, "/v1/automations/flows", "acme", map[string]any{
+		"displayName": "Builder Flow",
+		"trigger": map[string]any{
+			"name": "trigger", "type": TriggerTypePiece, "displayName": "Start",
+			"strategy": string(StrategyManual),
+			"settings": map[string]any{"pieceName": "core", "triggerName": "manual"},
+		},
+	})
+	var pf populatedFlow
+	_ = json.Unmarshal(create.Body, &pf)
+	flowID := pf.ID
+
+	// ADD_ACTION: append a core.http_request step after the trigger.
+	addReq := map[string]any{
+		"type": string(OpAddAction),
+		"request": map[string]any{
+			"parentStep": "",
+			"action": map[string]any{
+				"name": "http1", "type": ActionTypePiece, "displayName": "Fetch",
+				"settings": map[string]any{"pieceName": "core", "actionName": "http_request", "input": map[string]any{"url": "https://example.com"}},
+			},
+		},
+	}
+	r := req(t, app, http.MethodPost, "/v1/automations/flows/"+flowID+"/operations", "acme", addReq)
+	if r.Code != http.StatusOK {
+		t.Fatalf("ADD_ACTION want 200, got %d (%s)", r.Code, r.Body)
+	}
+	var v FlowVersion
+	if err := json.Unmarshal(r.Body, &v); err != nil {
+		t.Fatalf("operation body: %v (%s)", err, r.Body)
+	}
+	if v.Trigger == nil || v.Trigger.NextAction == nil || v.Trigger.NextAction.Name != "http1" {
+		t.Fatalf("ADD_ACTION must insert http1 into the chain: %+v", v.Trigger)
+	}
+
+	// CHANGE_NAME.
+	nameReq := map[string]any{"type": string(OpChangeName), "request": map[string]any{"displayName": "Renamed"}}
+	rn := req(t, app, http.MethodPost, "/v1/automations/flows/"+flowID+"/operations", "acme", nameReq)
+	var v2 FlowVersion
+	_ = json.Unmarshal(rn.Body, &v2)
+	if v2.DisplayName != "Renamed" {
+		t.Fatalf("CHANGE_NAME must rename, got %q", v2.DisplayName)
+	}
+
+	// An ADD_ACTION planting a ROUTER (tree op) is honestly rejected 422.
+	badReq := map[string]any{
+		"type": string(OpAddAction),
+		"request": map[string]any{
+			"parentStep": "http1",
+			"action":     map[string]any{"name": "router1", "type": ActionTypeRouter, "displayName": "Route"},
+		},
+	}
+	if rb := req(t, app, http.MethodPost, "/v1/automations/flows/"+flowID+"/operations", "acme", badReq); rb.Code != http.StatusUnprocessableEntity {
+		t.Fatalf("ROUTER add want 422 (unsupported), got %d (%s)", rb.Code, rb.Body)
+	}
+}

clients/automations/mcp.go

@@ -0,0 +1,173 @@
+package automations
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/hanzoai/cloud/clients/principal"
+	"github.com/zap-proto/zip"
+)
+
+// MCP — the HIP-0300 JSON-RPC 2.0 tool surface at POST /v1/automations/mcp. Every
+// connector action is exposed as a tool named "<connector>_<action>", so /v1/agents
+// can invoke a connector as an agent tool. GATED: a request without a validated
+// principal is refused 403 (never the unscoped tool plane); tools/call dispatches to
+// the action's Run with RunContext bound to the caller's VALIDATED org — the same
+// isolation boundary the durable activity uses.
+
+// mcpRequest is the JSON-RPC 2.0 request envelope (mirrors tasks/pkg/tasks/mcp.go).
+type mcpRequest struct {
+	JSONRPC string `json:"jsonrpc"`
+	ID      any    `json:"id,omitempty"`
+	Method  string `json:"method"`
+	Params  any    `json:"params,omitempty"`
+}
+
+// mcp is the single JSON-RPC endpoint. Org-gated at the top: no validated principal
+// → 403, so a client-forged X-Org-Id with no bearer can never reach a tool.
+func (s *svc) mcp(c *zip.Ctx) error {
+	org, ok := principal.Tenant(c)
+	if !ok {
+		return zip.ErrForbidden("a validated principal is required")
+	}
+	if !validOrg(org) {
+		return zip.ErrBadRequest("org must be a DNS-1123 label")
+	}
+	var req mcpRequest
+	if err := json.Unmarshal(c.Body(), &req); err != nil {
+		return c.JSON(http.StatusOK, mcpErrorObj(nil, -32700, "parse error: "+err.Error()))
+	}
+	switch req.Method {
+	case "initialize":
+		return c.JSON(http.StatusOK, mcpResultObj(req.ID, map[string]any{
+			"protocolVersion": "2025-06-18",
+			"serverInfo":      map[string]any{"name": "hanzo-automations", "version": "1.0.0"},
+			"capabilities":    map[string]any{"tools": map[string]any{}},
+		}))
+	case "ping":
+		return c.JSON(http.StatusOK, mcpResultObj(req.ID, map[string]any{}))
+	case "tools/list":
+		return c.JSON(http.StatusOK, mcpResultObj(req.ID, map[string]any{"tools": mcpTools()}))
+	case "tools/call":
+		return s.mcpToolCall(c, org, req)
+	default:
+		return c.JSON(http.StatusOK, mcpErrorObj(req.ID, -32601, "method not found: "+req.Method))
+	}
+}
+
+// mcpToolCall dispatches a tool to its connector action's Run. RunContext.Org and
+// the Token closure are pinned to the VALIDATED org — a caller can never invoke a
+// tool against another tenant's credentials. One metered unit + one audit record per
+// call.
+func (s *svc) mcpToolCall(c *zip.Ctx, org string, req mcpRequest) error {
+	var p struct {
+		Name      string         `json:"name"`
+		Arguments map[string]any `json:"arguments"`
+	}
+	body, _ := json.Marshal(req.Params)
+	_ = json.Unmarshal(body, &p)
+
+	connector, action, ok := resolveTool(p.Name)
+	if !ok {
+		return c.JSON(http.StatusOK, mcpErrorObj(req.ID, -32601, "unknown tool: "+p.Name))
+	}
+	_, act, err := lookupAction(connector, action)
+	if err != nil {
+		return c.JSON(http.StatusOK, mcpErrorObj(req.ID, -32601, err.Error()))
+	}
+
+	// Per-org concurrency bound (LOW-2): the tool executes SYNCHRONOUSLY here, so a
+	// burst of core.delay calls would otherwise pin a goroutine each for up to the
+	// delay cap. Held across Run, released after.
+	if !orgRunLimiter.acquire(org) {
+		return c.JSON(http.StatusOK, mcpErrorObj(req.ID, -32005, "too many concurrent tool calls for this org"))
+	}
+	defer orgRunLimiter.release(org)
+
+	ctx := c.Context()
+	rc := RunContext{
+		Org:   org,
+		Input: p.Arguments,
+		Token: func(secretName string) ([]byte, error) {
+			return tokenSource(ctx, org, connector, secretName)
+		},
+	}
+
+	// Meter + audit AFTER Run, deriving the outcome from the real result (LOW-1): a
+	// failed / SSRF-blocked / not-connected call is audited as an error and is NOT
+	// billed as a successful unit.
+	out, err := act.Run(ctx, rc)
+	if err != nil {
+		s.auditEvent(c, org, "automations.mcp.call", p.Name, "error", http.StatusFailedDependency)
+		return c.JSON(http.StatusOK, mcpErrorObj(req.ID, -32000, err.Error()))
+	}
+	s.meterUnit(org, c)
+	s.auditEvent(c, org, "automations.mcp.call", p.Name, "ok", http.StatusOK)
+
+	text, _ := json.Marshal(out)
+	return c.JSON(http.StatusOK, mcpResultObj(req.ID, map[string]any{
+		"content": []map[string]any{{"type": "text", "text": string(text)}},
+	}))
+}
+
+// mcpTools is the tool catalogue: every connector action, name "<connector>_<action>",
+// with an input schema derived from its Props. Stable order (sorted).
+func mcpTools() []map[string]any {
+	tools := make([]map[string]any, 0, 16)
+	for _, c := range sortedConnectors() {
+		for _, a := range sortedActions(c) {
+			tools = append(tools, map[string]any{
+				"name":        c.Name + "_" + a.Name,
+				"description": a.Description,
+				"inputSchema": propsToSchema(a.Props),
+			})
+		}
+	}
+	return tools
+}
+
+// propsToSchema derives a JSON-Schema object from an action's Props — the ONE
+// mapping from PropSpec to the wire schema, shared by every tool.
+func propsToSchema(props []PropSpec) map[string]any {
+	properties := make(map[string]any, len(props))
+	required := make([]string, 0, len(props))
+	for _, p := range props {
+		properties[p.Name] = map[string]any{"type": jsonType(p.Type), "description": p.Description}
+		if p.Required {
+			required = append(required, p.Name)
+		}
+	}
+	return map[string]any{"type": "object", "properties": properties, "required": required}
+}
+
+// jsonType normalizes a PropSpec type to a JSON-Schema type (default "string").
+func jsonType(t string) string {
+	switch t {
+	case "number", "boolean", "object", "array", "string":
+		return t
+	default:
+		return "string"
+	}
+}
+
+// resolveTool maps a "<connector>_<action>" tool name back to its (connector,action)
+// pair. Connector and action names both contain underscores, so an unambiguous
+// resolution walks the registry rather than splitting the string.
+func resolveTool(name string) (connector, action string, ok bool) {
+	for cn, c := range registry {
+		for an := range c.Actions {
+			if cn+"_"+an == name {
+				return cn, an, true
+			}
+		}
+	}
+	return "", "", false
+}
+
+func mcpResultObj(id any, result any) map[string]any {
+	return map[string]any{"jsonrpc": "2.0", "id": id, "result": result}
+}
+
+func mcpErrorObj(id any, code int, msg string) map[string]any {
+	return map[string]any{"jsonrpc": "2.0", "id": id, "error": map[string]any{"code": code, "message": msg}}
+}

clients/automations/mcp_test.go

@@ -0,0 +1,100 @@
+package automations
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+)
+
+// TestMCPToolsList: tools/list (with a validated principal) exposes every connector
+// action as a "<connector>_<action>" tool.
+func TestMCPToolsList(t *testing.T) {
+	app := newApp(t)
+	r := reqRaw(t, app, "/v1/automations/mcp", "acme", `{"jsonrpc":"2.0","id":1,"method":"tools/list"}`)
+	if r.Code != 200 {
+		t.Fatalf("tools/list want 200, got %d (%s)", r.Code, r.Body)
+	}
+	var out struct {
+		Result struct {
+			Tools []struct {
+				Name        string         `json:"name"`
+				InputSchema map[string]any `json:"inputSchema"`
+			} `json:"tools"`
+		} `json:"result"`
+	}
+	if err := json.Unmarshal(r.Body, &out); err != nil {
+		t.Fatalf("tools/list body: %v (%s)", err, r.Body)
+	}
+	names := map[string]bool{}
+	for _, tl := range out.Result.Tools {
+		names[tl.Name] = true
+	}
+	for _, want := range []string{"slack_send_message", "core_http_request", "core_code", "github_create_issue", "google_sheets_append_row"} {
+		if !names[want] {
+			t.Fatalf("tools/list missing %q; got %v", want, names)
+		}
+	}
+}
+
+// TestMCPToolCallGated: tools/call with NO validated principal is refused 403 — the
+// tool plane never serves an unauthenticated caller.
+func TestMCPToolCallGated(t *testing.T) {
+	app := newApp(t)
+	r := reqRaw(t, app, "/v1/automations/mcp", "",
+		`{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"core_code","arguments":{"x":1}}}`)
+	if r.Code != 403 {
+		t.Fatalf("tools/call without principal want 403, got %d (%s)", r.Code, r.Body)
+	}
+}
+
+// TestMCPToolCallDispatches: tools/call with a principal dispatches to the action's
+// Run end-to-end (core_code echoes its resolved input).
+func TestMCPToolCallDispatches(t *testing.T) {
+	app := newApp(t)
+	r := reqRaw(t, app, "/v1/automations/mcp", "acme",
+		`{"jsonrpc":"2.0","id":7,"method":"tools/call","params":{"name":"core_code","arguments":{"greeting":"hello"}}}`)
+	if r.Code != 200 {
+		t.Fatalf("tools/call want 200, got %d (%s)", r.Code, r.Body)
+	}
+	var out struct {
+		Result struct {
+			Content []struct {
+				Type string `json:"type"`
+				Text string `json:"text"`
+			} `json:"content"`
+		} `json:"result"`
+		Error *struct {
+			Message string `json:"message"`
+		} `json:"error"`
+	}
+	if err := json.Unmarshal(r.Body, &out); err != nil {
+		t.Fatalf("tools/call body: %v (%s)", err, r.Body)
+	}
+	if out.Error != nil {
+		t.Fatalf("tools/call unexpected error: %s", out.Error.Message)
+	}
+	if len(out.Result.Content) == 0 || !strings.Contains(out.Result.Content[0].Text, `"greeting":"hello"`) {
+		t.Fatalf("core_code must echo its input, got %+v", out.Result.Content)
+	}
+}
+
+// TestMCPToolCallSlackFailsClosed: invoking slack_send_message with no connection
+// (integrations not mounted) fails closed with an honest error — never a fake success
+// and never another tenant's token.
+func TestMCPToolCallSlackFailsClosed(t *testing.T) {
+	app := newApp(t)
+	r := reqRaw(t, app, "/v1/automations/mcp", "acme",
+		`{"jsonrpc":"2.0","id":9,"method":"tools/call","params":{"name":"slack_send_message","arguments":{"channel":"C1","text":"hi"}}}`)
+	if r.Code != 200 {
+		t.Fatalf("tools/call want 200 (JSON-RPC error in body), got %d (%s)", r.Code, r.Body)
+	}
+	var out struct {
+		Error *struct {
+			Message string `json:"message"`
+		} `json:"error"`
+	}
+	_ = json.Unmarshal(r.Body, &out)
+	if out.Error == nil || !strings.Contains(out.Error.Message, "slack not connected") {
+		t.Fatalf("slack must fail closed with 'slack not connected', got %s", r.Body)
+	}
+}

clients/automations/operations.go

@@ -0,0 +1,180 @@
+package automations
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// FlowOperation apply over the LINEAR step chain (trigger → action → action …).
+// This is the exact model the durable engine executes in Phase 1, so the operations
+// implemented here are complete and orthogonal to it. A tree-restructuring op that
+// the linear model cannot represent (branch/loop/router) is honestly REJECTED
+// (errUnsupportedOp) — never silently faked — so the builder gets a truthful answer.
+
+// errUnsupportedOp is returned for a well-formed operation whose semantics require
+// the branch/loop/router tree the Phase-1 linear engine does not execute. Handlers
+// render it 422.
+var errUnsupportedOp = fmt.Errorf("automations: operation not supported by the phase-1 linear flow model")
+
+// applyVersionOperation mutates a version's step tree per op and returns it. It
+// covers the version-scoped ops (UPDATE_TRIGGER, ADD_ACTION, UPDATE_ACTION,
+// DELETE_ACTION, MOVE_ACTION, CHANGE_NAME). CHANGE_STATUS is flow-scoped and handled
+// by the operations HTTP handler, not here.
+func applyVersionOperation(v *FlowVersion, op FlowOperation) (*FlowVersion, error) {
+	switch op.Type {
+	case OpChangeName:
+		var r ChangeNameRequest
+		if err := json.Unmarshal(op.Request, &r); err != nil {
+			return nil, fmt.Errorf("decode CHANGE_NAME: %w", err)
+		}
+		v.DisplayName = r.DisplayName
+		return v, nil
+
+	case OpUpdateTrigger:
+		var r UpdateTriggerRequest
+		if err := json.Unmarshal(op.Request, &r); err != nil {
+			return nil, fmt.Errorf("decode UPDATE_TRIGGER: %w", err)
+		}
+		trig := FlowTrigger(r)
+		// Preserve the existing action chain unless the request supplies a new one.
+		if trig.NextAction == nil && v.Trigger != nil {
+			trig.NextAction = v.Trigger.NextAction
+		}
+		v.Trigger = &trig
+		return v, nil
+
+	case OpAddAction:
+		var r AddActionRequest
+		if err := json.Unmarshal(op.Request, &r); err != nil {
+			return nil, fmt.Errorf("decode ADD_ACTION: %w", err)
+		}
+		if r.Action == nil {
+			return nil, fmt.Errorf("ADD_ACTION: action is required")
+		}
+		if err := requireLinear(r.Action.Type); err != nil {
+			return nil, err
+		}
+		if err := insertAfter(v, r.ParentStep, r.Action); err != nil {
+			return nil, err
+		}
+		return v, nil
+
+	case OpUpdateAction:
+		var r UpdateActionRequest
+		if err := json.Unmarshal(op.Request, &r); err != nil {
+			return nil, fmt.Errorf("decode UPDATE_ACTION: %w", err)
+		}
+		a := findAction(v, r.Name)
+		if a == nil {
+			return nil, errNotFound
+		}
+		if r.Type != "" {
+			if err := requireLinear(r.Type); err != nil {
+				return nil, err
+			}
+			a.Type = r.Type
+		}
+		a.Settings = r.Settings
+		return v, nil
+
+	case OpDeleteAction:
+		var r DeleteActionRequest
+		if err := json.Unmarshal(op.Request, &r); err != nil {
+			return nil, fmt.Errorf("decode DELETE_ACTION: %w", err)
+		}
+		for _, name := range r.Names {
+			if _, ok := detachAction(v, name); !ok {
+				return nil, errNotFound
+			}
+		}
+		return v, nil
+
+	case OpMoveAction:
+		var r MoveActionRequest
+		if err := json.Unmarshal(op.Request, &r); err != nil {
+			return nil, fmt.Errorf("decode MOVE_ACTION: %w", err)
+		}
+		node, ok := detachAction(v, r.Name)
+		if !ok {
+			return nil, errNotFound
+		}
+		if err := insertAfter(v, r.NewParentStep, node); err != nil {
+			return nil, err
+		}
+		return v, nil
+
+	default:
+		// A recognized builder op the linear model does not execute.
+		return nil, errUnsupportedOp
+	}
+}
+
+// requireLinear rejects the tree action types the Phase-1 linear engine cannot
+// execute, so ADD/UPDATE never plants a step that would silently never run.
+func requireLinear(actionType string) error {
+	switch actionType {
+	case ActionTypePiece, ActionTypeCode, "":
+		return nil
+	default: // ROUTER / LOOP_ON_ITEMS / BRANCH
+		return errUnsupportedOp
+	}
+}
+
+// ── linear-chain surgery ──────────────────────────────────────────────────────
+
+// findAction returns the action named name in the chain, or nil.
+func findAction(v *FlowVersion, name string) *FlowAction {
+	if v == nil || v.Trigger == nil {
+		return nil
+	}
+	for a := v.Trigger.NextAction; a != nil; a = a.NextAction {
+		if a.Name == name {
+			return a
+		}
+	}
+	return nil
+}
+
+// insertAfter links node into the chain immediately after the step named parent.
+// parent=="" (or the trigger's name) inserts node as the trigger's first action,
+// pushing the existing chain down. Returns errNotFound if parent is named but absent.
+func insertAfter(v *FlowVersion, parent string, node *FlowAction) error {
+	if v == nil || v.Trigger == nil {
+		return errNotFound
+	}
+	if parent == "" || parent == v.Trigger.Name {
+		node.NextAction = v.Trigger.NextAction
+		v.Trigger.NextAction = node
+		return nil
+	}
+	p := findAction(v, parent)
+	if p == nil {
+		return errNotFound
+	}
+	node.NextAction = p.NextAction
+	p.NextAction = node
+	return nil
+}
+
+// detachAction removes the step named name from the chain, relinking around it, and
+// returns the detached node (its NextAction cleared) plus whether it was found.
+func detachAction(v *FlowVersion, name string) (*FlowAction, bool) {
+	if v == nil || v.Trigger == nil {
+		return nil, false
+	}
+	// Head of the chain (trigger's direct child).
+	if head := v.Trigger.NextAction; head != nil && head.Name == name {
+		v.Trigger.NextAction = head.NextAction
+		head.NextAction = nil
+		return head, true
+	}
+	for p := v.Trigger.NextAction; p != nil && p.NextAction != nil; p = p.NextAction {
+		if p.NextAction.Name == name {
+			node := p.NextAction
+			p.NextAction = node.NextAction
+			node.NextAction = nil
+			return node, true
+		}
+	}
+	return nil, false
+}

clients/automations/store.go

@@ -0,0 +1,461 @@
+package automations
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	// github.com/hanzoai/sqlite is the ONE Hanzo SQLite driver: it registers the
+	// "sqlite" database/sql name under both build tags (cgo → mattn+SQLCipher,
+	// encrypted at rest; !cgo → pure-Go modernc). Blank import registers it. This
+	// mirrors clients/crm exactly — the ONE storage pattern.
+	_ "github.com/hanzoai/sqlite"
+)
+
+// Sentinel errors mapped to HTTP status by the handlers: errNotFound → 404,
+// errBadRef → 422.
+var (
+	errNotFound = errors.New("automations: not found")
+	errBadRef   = errors.New("automations: referenced record not found in org")
+)
+
+// Store is the automations database. ONE SQLite file ({DataDir}/automations.db)
+// holds every org's flows, versions, and runs; tenant isolation is the `org`
+// column, physical on EVERY uniqueness + lookup index (each leads with org).
+// MaxOpenConns(1) serializes writes against the single-writer WAL file.
+type Store struct {
+	db *sql.DB
+}
+
+func openStore(path string) (*Store, error) {
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		return nil, fmt.Errorf("open sqlite %q: %w", path, err)
+	}
+	db.SetMaxOpenConns(1)
+	for _, pragma := range []string{
+		"PRAGMA busy_timeout=5000",
+		"PRAGMA journal_mode=WAL",
+		"PRAGMA foreign_keys=ON",
+	} {
+		if _, err := db.Exec(pragma); err != nil {
+			_ = db.Close()
+			return nil, fmt.Errorf("pragma %q: %w", pragma, err)
+		}
+	}
+	s := &Store{db: db}
+	if err := s.migrate(); err != nil {
+		_ = db.Close()
+		return nil, err
+	}
+	return s, nil
+}
+
+// migrate creates the three tables. Idempotent (IF NOT EXISTS). Every uniqueness
+// + lookup index leads with `org`, so tenant isolation is a physical property of
+// the index, not just a WHERE clause.
+func (s *Store) migrate() error {
+	const ddl = `
+CREATE TABLE IF NOT EXISTS automations_flows (
+  id                   TEXT PRIMARY KEY,
+  org                  TEXT NOT NULL,
+  external_id          TEXT NOT NULL DEFAULT '',
+  folder_id            TEXT NOT NULL DEFAULT '',
+  status               TEXT NOT NULL DEFAULT 'DISABLED',
+  published_version_id TEXT NOT NULL DEFAULT '',
+  metadata             TEXT NOT NULL DEFAULT '',
+  created              INTEGER NOT NULL,
+  updated              INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS ix_auto_flows_org_updated  ON automations_flows(org, updated);
+CREATE INDEX IF NOT EXISTS ix_auto_flows_org_status   ON automations_flows(org, status);
+
+CREATE TABLE IF NOT EXISTS automations_flow_versions (
+  id             TEXT PRIMARY KEY,
+  org            TEXT NOT NULL,
+  flow_id        TEXT NOT NULL,
+  display_name   TEXT NOT NULL DEFAULT '',
+  trigger_json   TEXT NOT NULL DEFAULT '',
+  valid          INTEGER NOT NULL DEFAULT 0,
+  state          TEXT NOT NULL DEFAULT 'DRAFT',
+  schema_version TEXT NOT NULL DEFAULT '',
+  created        INTEGER NOT NULL,
+  updated        INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS ix_auto_versions_org_flow    ON automations_flow_versions(org, flow_id, created);
+
+CREATE TABLE IF NOT EXISTS automations_runs (
+  id              TEXT PRIMARY KEY,
+  org             TEXT NOT NULL,
+  flow_id         TEXT NOT NULL,
+  flow_version_id TEXT NOT NULL,
+  workflow_id     TEXT NOT NULL,
+  status          TEXT NOT NULL DEFAULT 'RUNNING',
+  start_time      INTEGER NOT NULL DEFAULT 0,
+  finish_time     INTEGER NOT NULL DEFAULT 0,
+  -- metered is the EXACTLY-ONCE billing/audit idempotency flag: a run is metered
+  -- + audited by whichever caller flips it 0→1 (ClaimMeter), so the manual, MCP,
+  -- and scheduled-cron entrypoints together bill a run at most once (MED-1).
+  metered         INTEGER NOT NULL DEFAULT 0,
+  created         INTEGER NOT NULL,
+  updated         INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS ix_auto_runs_org_created ON automations_runs(org, created);
+CREATE INDEX IF NOT EXISTS ix_auto_runs_org_flow    ON automations_runs(org, flow_id, created);
+`
+	if _, err := s.db.Exec(ddl); err != nil {
+		return fmt.Errorf("automations migrate: %w", err)
+	}
+	return nil
+}
+
+// Close closes the underlying database. Idempotent-safe via sql.DB.
+func (s *Store) Close() error { return s.db.Close() }
+
+// ── Flow ────────────────────────────────────────────────────────────────────
+
+const flowCols = `id,org,external_id,folder_id,status,published_version_id,metadata,created,updated`
+
+func scanFlow(sc interface{ Scan(...any) error }) (Flow, error) {
+	var f Flow
+	var meta string
+	err := sc.Scan(&f.ID, &f.Org, &f.ExternalID, &f.FolderID, &f.Status,
+		&f.PublishedVersionID, &meta, &f.Created, &f.Updated)
+	if meta != "" {
+		f.Metadata = json.RawMessage(meta)
+	}
+	return f, err
+}
+
+func (s *Store) CreateFlow(ctx context.Context, f Flow) (Flow, error) {
+	if _, err := s.db.ExecContext(ctx,
+		`INSERT INTO automations_flows (`+flowCols+`) VALUES (?,?,?,?,?,?,?,?,?)`,
+		f.ID, f.Org, f.ExternalID, f.FolderID, f.Status, f.PublishedVersionID,
+		string(f.Metadata), f.Created, f.Updated); err != nil {
+		return Flow{}, fmt.Errorf("insert flow: %w", err)
+	}
+	return f, nil
+}
+
+func (s *Store) GetFlow(ctx context.Context, org, id string) (Flow, error) {
+	row := s.db.QueryRowContext(ctx, `SELECT `+flowCols+` FROM automations_flows WHERE org=? AND id=?`, org, id)
+	f, err := scanFlow(row)
+	if errors.Is(err, sql.ErrNoRows) {
+		return Flow{}, errNotFound
+	}
+	if err != nil {
+		return Flow{}, fmt.Errorf("get flow: %w", err)
+	}
+	return f, nil
+}
+
+func (s *Store) ListFlows(ctx context.Context, org string, limit int) ([]Flow, error) {
+	rows, err := s.db.QueryContext(ctx,
+		`SELECT `+flowCols+` FROM automations_flows WHERE org=? ORDER BY updated DESC LIMIT ?`, org, limit)
+	if err != nil {
+		return nil, fmt.Errorf("list flows: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+	out := make([]Flow, 0, 16)
+	for rows.Next() {
+		f, err := scanFlow(rows)
+		if err != nil {
+			return nil, fmt.Errorf("scan flow: %w", err)
+		}
+		out = append(out, f)
+	}
+	return out, rows.Err()
+}
+
+// UpdateFlow persists the mutable flow fields (status, folder, published version,
+// metadata) for (org,id). RowsAffected==0 ⇒ errNotFound (cross-tenant or missing).
+func (s *Store) UpdateFlow(ctx context.Context, f Flow) (Flow, error) {
+	res, err := s.db.ExecContext(ctx,
+		`UPDATE automations_flows SET external_id=?,folder_id=?,status=?,published_version_id=?,metadata=?,updated=? WHERE org=? AND id=?`,
+		f.ExternalID, f.FolderID, f.Status, f.PublishedVersionID, string(f.Metadata), f.Updated, f.Org, f.ID)
+	if err != nil {
+		return Flow{}, fmt.Errorf("update flow: %w", err)
+	}
+	if n, _ := res.RowsAffected(); n == 0 {
+		return Flow{}, errNotFound
+	}
+	return s.GetFlow(ctx, f.Org, f.ID)
+}
+
+// DeleteFlow removes a flow and all its versions + runs within the org. One
+// transaction so a partial delete never strands a version/run.
+func (s *Store) DeleteFlow(ctx context.Context, org, id string) (bool, error) {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return false, fmt.Errorf("begin: %w", err)
+	}
+	defer func() { _ = tx.Rollback() }()
+	res, err := tx.ExecContext(ctx, `DELETE FROM automations_flows WHERE org=? AND id=?`, org, id)
+	if err != nil {
+		return false, fmt.Errorf("delete flow: %w", err)
+	}
+	if _, err := tx.ExecContext(ctx, `DELETE FROM automations_flow_versions WHERE org=? AND flow_id=?`, org, id); err != nil {
+		return false, fmt.Errorf("delete versions: %w", err)
+	}
+	if _, err := tx.ExecContext(ctx, `DELETE FROM automations_runs WHERE org=? AND flow_id=?`, org, id); err != nil {
+		return false, fmt.Errorf("delete runs: %w", err)
+	}
+	n, _ := res.RowsAffected()
+	if err := tx.Commit(); err != nil {
+		return false, fmt.Errorf("commit: %w", err)
+	}
+	return n > 0, nil
+}
+
+// ── FlowVersion ─────────────────────────────────────────────────────────────
+
+const versionCols = `id,org,flow_id,display_name,trigger_json,valid,state,schema_version,created,updated`
+
+func scanVersion(sc interface{ Scan(...any) error }) (FlowVersion, error) {
+	var v FlowVersion
+	var trig string
+	var valid int
+	err := sc.Scan(&v.ID, &v.Org, &v.FlowID, &v.DisplayName, &trig, &valid,
+		&v.State, &v.SchemaVersion, &v.Created, &v.Updated)
+	if err != nil {
+		return FlowVersion{}, err
+	}
+	v.Valid = valid != 0
+	if trig != "" {
+		var t FlowTrigger
+		if uerr := json.Unmarshal([]byte(trig), &t); uerr != nil {
+			return FlowVersion{}, fmt.Errorf("decode trigger: %w", uerr)
+		}
+		v.Trigger = &t
+	}
+	return v, nil
+}
+
+func b2i(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+// CreateVersion inserts a version. The flow must exist in the SAME org (validated
+// here, not by a SQL FK, so a cross-tenant flow_id can never anchor a version).
+func (s *Store) CreateVersion(ctx context.Context, v FlowVersion) (FlowVersion, error) {
+	ok, err := s.flowExists(ctx, v.Org, v.FlowID)
+	if err != nil {
+		return FlowVersion{}, err
+	}
+	if !ok {
+		return FlowVersion{}, errBadRef
+	}
+	trig, err := marshalTrigger(v.Trigger)
+	if err != nil {
+		return FlowVersion{}, err
+	}
+	if _, err := s.db.ExecContext(ctx,
+		`INSERT INTO automations_flow_versions (`+versionCols+`) VALUES (?,?,?,?,?,?,?,?,?,?)`,
+		v.ID, v.Org, v.FlowID, v.DisplayName, trig, b2i(v.Valid), v.State, v.SchemaVersion, v.Created, v.Updated); err != nil {
+		return FlowVersion{}, fmt.Errorf("insert version: %w", err)
+	}
+	return v, nil
+}
+
+func (s *Store) GetVersion(ctx context.Context, org, id string) (FlowVersion, error) {
+	row := s.db.QueryRowContext(ctx, `SELECT `+versionCols+` FROM automations_flow_versions WHERE org=? AND id=?`, org, id)
+	v, err := scanVersion(row)
+	if errors.Is(err, sql.ErrNoRows) {
+		return FlowVersion{}, errNotFound
+	}
+	if err != nil {
+		return FlowVersion{}, fmt.Errorf("get version: %w", err)
+	}
+	return v, nil
+}
+
+func (s *Store) ListVersions(ctx context.Context, org, flowID string, limit int) ([]FlowVersion, error) {
+	rows, err := s.db.QueryContext(ctx,
+		`SELECT `+versionCols+` FROM automations_flow_versions WHERE org=? AND flow_id=? ORDER BY created DESC LIMIT ?`, org, flowID, limit)
+	if err != nil {
+		return nil, fmt.Errorf("list versions: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+	out := make([]FlowVersion, 0, 8)
+	for rows.Next() {
+		v, err := scanVersion(rows)
+		if err != nil {
+			return nil, fmt.Errorf("scan version: %w", err)
+		}
+		out = append(out, v)
+	}
+	return out, rows.Err()
+}
+
+// LatestVersion returns the most-recently-created version for a flow, or
+// errNotFound if the flow has none.
+func (s *Store) LatestVersion(ctx context.Context, org, flowID string) (FlowVersion, error) {
+	row := s.db.QueryRowContext(ctx,
+		`SELECT `+versionCols+` FROM automations_flow_versions WHERE org=? AND flow_id=? ORDER BY created DESC LIMIT 1`, org, flowID)
+	v, err := scanVersion(row)
+	if errors.Is(err, sql.ErrNoRows) {
+		return FlowVersion{}, errNotFound
+	}
+	if err != nil {
+		return FlowVersion{}, fmt.Errorf("latest version: %w", err)
+	}
+	return v, nil
+}
+
+// UpdateVersion replaces a version's editable content (display name, trigger tree,
+// valid, state). RowsAffected==0 ⇒ errNotFound.
+func (s *Store) UpdateVersion(ctx context.Context, v FlowVersion) (FlowVersion, error) {
+	trig, err := marshalTrigger(v.Trigger)
+	if err != nil {
+		return FlowVersion{}, err
+	}
+	res, err := s.db.ExecContext(ctx,
+		`UPDATE automations_flow_versions SET display_name=?,trigger_json=?,valid=?,state=?,schema_version=?,updated=? WHERE org=? AND id=?`,
+		v.DisplayName, trig, b2i(v.Valid), v.State, v.SchemaVersion, v.Updated, v.Org, v.ID)
+	if err != nil {
+		return FlowVersion{}, fmt.Errorf("update version: %w", err)
+	}
+	if n, _ := res.RowsAffected(); n == 0 {
+		return FlowVersion{}, errNotFound
+	}
+	return s.GetVersion(ctx, v.Org, v.ID)
+}
+
+// ── FlowRun ─────────────────────────────────────────────────────────────────
+
+const runCols = `id,org,flow_id,flow_version_id,workflow_id,status,start_time,finish_time,created,updated`
+
+func scanRun(sc interface{ Scan(...any) error }) (FlowRun, error) {
+	var r FlowRun
+	err := sc.Scan(&r.ID, &r.Org, &r.FlowID, &r.FlowVersionID, &r.WorkflowID,
+		&r.Status, &r.StartTime, &r.FinishTime, &r.Created, &r.Updated)
+	return r, err
+}
+
+func (s *Store) CreateRun(ctx context.Context, r FlowRun) (FlowRun, error) {
+	if _, err := s.db.ExecContext(ctx,
+		`INSERT INTO automations_runs (`+runCols+`) VALUES (?,?,?,?,?,?,?,?,?,?)`,
+		r.ID, r.Org, r.FlowID, r.FlowVersionID, r.WorkflowID, r.Status,
+		r.StartTime, r.FinishTime, r.Created, r.Updated); err != nil {
+		return FlowRun{}, fmt.Errorf("insert run: %w", err)
+	}
+	return r, nil
+}
+
+// CreateRunIfAbsent inserts a run row keyed on its id, doing NOTHING if it already
+// exists. It reports whether THIS call created the row. Idempotent by run id (=the
+// workflow execution id), so a retried run-start bookkeeping step, or a manual
+// handler + the durable path racing to record the same run, converge on one row.
+func (s *Store) CreateRunIfAbsent(ctx context.Context, r FlowRun) (bool, error) {
+	res, err := s.db.ExecContext(ctx,
+		`INSERT INTO automations_runs (`+runCols+`) VALUES (?,?,?,?,?,?,?,?,?,?) ON CONFLICT(id) DO NOTHING`,
+		r.ID, r.Org, r.FlowID, r.FlowVersionID, r.WorkflowID, r.Status,
+		r.StartTime, r.FinishTime, r.Created, r.Updated)
+	if err != nil {
+		return false, fmt.Errorf("insert run if absent: %w", err)
+	}
+	n, _ := res.RowsAffected()
+	return n > 0, nil
+}
+
+// ClaimMeter atomically flips the run's metered flag 0→1 for (org,id) and reports
+// whether THIS call won the flip. It is the exactly-once billing gate: only the
+// winner meters + audits the run, so no entrypoint double-bills a run (MED-1). The
+// (org,id) predicate keeps the claim tenant-scoped.
+func (s *Store) ClaimMeter(ctx context.Context, org, id string) (bool, error) {
+	res, err := s.db.ExecContext(ctx,
+		`UPDATE automations_runs SET metered=1 WHERE org=? AND id=? AND metered=0`, org, id)
+	if err != nil {
+		return false, fmt.Errorf("claim meter: %w", err)
+	}
+	n, _ := res.RowsAffected()
+	return n == 1, nil
+}
+
+func (s *Store) GetRun(ctx context.Context, org, id string) (FlowRun, error) {
+	row := s.db.QueryRowContext(ctx, `SELECT `+runCols+` FROM automations_runs WHERE org=? AND id=?`, org, id)
+	r, err := scanRun(row)
+	if errors.Is(err, sql.ErrNoRows) {
+		return FlowRun{}, errNotFound
+	}
+	if err != nil {
+		return FlowRun{}, fmt.Errorf("get run: %w", err)
+	}
+	return r, nil
+}
+
+func (s *Store) ListRuns(ctx context.Context, org, flowID string, limit int) ([]FlowRun, error) {
+	var (
+		rows *sql.Rows
+		err  error
+	)
+	if flowID == "" {
+		rows, err = s.db.QueryContext(ctx,
+			`SELECT `+runCols+` FROM automations_runs WHERE org=? ORDER BY created DESC LIMIT ?`, org, limit)
+	} else {
+		rows, err = s.db.QueryContext(ctx,
+			`SELECT `+runCols+` FROM automations_runs WHERE org=? AND flow_id=? ORDER BY created DESC LIMIT ?`, org, flowID, limit)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("list runs: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+	out := make([]FlowRun, 0, 16)
+	for rows.Next() {
+		r, err := scanRun(rows)
+		if err != nil {
+			return nil, fmt.Errorf("scan run: %w", err)
+		}
+		out = append(out, r)
+	}
+	return out, rows.Err()
+}
+
+// UpdateRunStatus persists a terminal/observed status transition for (org,id).
+func (s *Store) UpdateRunStatus(ctx context.Context, org, id string, status FlowRunStatus, finish, updated int64) error {
+	_, err := s.db.ExecContext(ctx,
+		`UPDATE automations_runs SET status=?,finish_time=?,updated=? WHERE org=? AND id=?`,
+		status, finish, updated, org, id)
+	if err != nil {
+		return fmt.Errorf("update run status: %w", err)
+	}
+	return nil
+}
+
+// ── helpers ─────────────────────────────────────────────────────────────────
+
+// flowExists reports whether (org,flowID) names a flow — used to keep a version's
+// flow_id inside the tenant before a write (a cross-tenant ref would be errBadRef).
+func (s *Store) flowExists(ctx context.Context, org, flowID string) (bool, error) {
+	if flowID == "" {
+		return false, nil
+	}
+	var one int
+	err := s.db.QueryRowContext(ctx,
+		`SELECT 1 FROM automations_flows WHERE org=? AND id=?`, org, flowID).Scan(&one)
+	if errors.Is(err, sql.ErrNoRows) {
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("flow exists: %w", err)
+	}
+	return true, nil
+}
+
+// marshalTrigger encodes a version's trigger tree for storage. A nil trigger
+// stores an empty string (an EMPTY/unconfigured flow), never the literal "null".
+func marshalTrigger(t *FlowTrigger) (string, error) {
+	if t == nil {
+		return "", nil
+	}
+	b, err := json.Marshal(t)
+	if err != nil {
+		return "", fmt.Errorf("encode trigger: %w", err)
+	}
+	return string(b), nil
+}

clients/automations/store_test.go

@@ -0,0 +1,148 @@
+package automations
+
+import (
+	"context"
+	"errors"
+	"testing"
+)
+
+// TestStoreOrgIsolation is the load-bearing tenant-isolation test: two orgs write
+// flows/versions/runs; neither can list, get, or mutate the other's rows. The org is
+// folded into every key, so a cross-tenant read is a physical miss, not a filtered one.
+func TestStoreOrgIsolation(t *testing.T) {
+	ctx := context.Background()
+	s := testStore(t)
+
+	fa, err := s.CreateFlow(ctx, Flow{ID: "flow_a", Org: "acme", Status: FlowDisabled, Created: 1, Updated: 1})
+	if err != nil {
+		t.Fatalf("create acme flow: %v", err)
+	}
+	if _, err := s.CreateFlow(ctx, Flow{ID: "flow_b", Org: "globex", Status: FlowDisabled, Created: 1, Updated: 1}); err != nil {
+		t.Fatalf("create globex flow: %v", err)
+	}
+
+	// acme lists exactly its own.
+	list, err := s.ListFlows(ctx, "acme", 100)
+	if err != nil {
+		t.Fatalf("list acme: %v", err)
+	}
+	if len(list) != 1 || list[0].ID != "flow_a" {
+		t.Fatalf("acme must see [flow_a], got %+v", list)
+	}
+
+	// globex cannot GET acme's flow (cross-tenant read → not found).
+	if _, err := s.GetFlow(ctx, "globex", fa.ID); !errors.Is(err, errNotFound) {
+		t.Fatalf("globex GET acme flow want errNotFound, got %v", err)
+	}
+	// globex cannot UPDATE acme's flow (cross-tenant write → not found, no mutation).
+	if _, err := s.UpdateFlow(ctx, Flow{ID: fa.ID, Org: "globex", Status: FlowEnabled, Updated: 2}); !errors.Is(err, errNotFound) {
+		t.Fatalf("globex UPDATE acme flow want errNotFound, got %v", err)
+	}
+	got, _ := s.GetFlow(ctx, "acme", fa.ID)
+	if got.Status != FlowDisabled {
+		t.Fatalf("acme flow must be unchanged, got status %q", got.Status)
+	}
+	// globex cannot DELETE acme's flow.
+	deleted, err := s.DeleteFlow(ctx, "globex", fa.ID)
+	if err != nil {
+		t.Fatalf("globex delete: %v", err)
+	}
+	if deleted {
+		t.Fatal("globex must not delete acme's flow")
+	}
+	if _, err := s.GetFlow(ctx, "acme", fa.ID); err != nil {
+		t.Fatalf("acme flow must survive globex delete: %v", err)
+	}
+
+	// Versions + runs isolate identically.
+	if _, err := s.CreateVersion(ctx, FlowVersion{ID: "ver_a", Org: "acme", FlowID: fa.ID, State: VersionDraft, Created: 1, Updated: 1}); err != nil {
+		t.Fatalf("create acme version: %v", err)
+	}
+	if _, err := s.GetVersion(ctx, "globex", "ver_a"); !errors.Is(err, errNotFound) {
+		t.Fatalf("globex GET acme version want errNotFound, got %v", err)
+	}
+	if _, err := s.CreateRun(ctx, FlowRun{ID: "run_a", Org: "acme", FlowID: fa.ID, WorkflowID: "run_a", Status: RunRunning, Created: 1, Updated: 1}); err != nil {
+		t.Fatalf("create acme run: %v", err)
+	}
+	if _, err := s.GetRun(ctx, "globex", "run_a"); !errors.Is(err, errNotFound) {
+		t.Fatalf("globex GET acme run want errNotFound, got %v", err)
+	}
+	if runs, _ := s.ListRuns(ctx, "globex", "", 100); len(runs) != 0 {
+		t.Fatalf("globex must see zero runs, got %d", len(runs))
+	}
+}
+
+// TestVersionCrossTenantRefRejected: a version's flow_id must resolve INSIDE the
+// org; anchoring a version to another tenant's flow is errBadRef, never a link.
+func TestVersionCrossTenantRefRejected(t *testing.T) {
+	ctx := context.Background()
+	s := testStore(t)
+
+	if _, err := s.CreateFlow(ctx, Flow{ID: "flow_acme", Org: "acme", Created: 1, Updated: 1}); err != nil {
+		t.Fatalf("seed acme flow: %v", err)
+	}
+	// globex tries to anchor a version to acme's flow → errBadRef.
+	if _, err := s.CreateVersion(ctx, FlowVersion{ID: "ver_x", Org: "globex", FlowID: "flow_acme", State: VersionDraft, Created: 1, Updated: 1}); !errors.Is(err, errBadRef) {
+		t.Fatalf("cross-tenant version ref want errBadRef, got %v", err)
+	}
+	// A version anchored to a missing flow in-org is also errBadRef.
+	if _, err := s.CreateVersion(ctx, FlowVersion{ID: "ver_y", Org: "acme", FlowID: "flow_ghost", State: VersionDraft, Created: 1, Updated: 1}); !errors.Is(err, errBadRef) {
+		t.Fatalf("missing-flow version ref want errBadRef, got %v", err)
+	}
+}
+
+// TestFlowLifecycleRoundTrip exercises the full flow+version+run lifecycle, including
+// the trigger-tree JSON round-trip through SQLite.
+func TestFlowLifecycleRoundTrip(t *testing.T) {
+	ctx := context.Background()
+	s := testStore(t)
+
+	if _, err := s.CreateFlow(ctx, Flow{ID: "f1", Org: "o", Status: FlowDisabled, Created: 10, Updated: 10}); err != nil {
+		t.Fatalf("create flow: %v", err)
+	}
+	trig := &FlowTrigger{
+		Name: "trigger", Type: TriggerTypePiece, DisplayName: "Start", Strategy: StrategyManual,
+		Settings: StepSettings{PieceName: corePiece, TriggerName: "manual"},
+		NextAction: &FlowAction{
+			Name: "step1", Type: ActionTypeCode, DisplayName: "Transform",
+			Settings: StepSettings{Input: map[string]any{"x": float64(1)}},
+		},
+	}
+	v, err := s.CreateVersion(ctx, FlowVersion{ID: "v1", Org: "o", FlowID: "f1", DisplayName: "My Flow", Trigger: trig, Valid: true, State: VersionDraft, Created: 10, Updated: 10})
+	if err != nil {
+		t.Fatalf("create version: %v", err)
+	}
+	got, err := s.GetVersion(ctx, "o", v.ID)
+	if err != nil {
+		t.Fatalf("get version: %v", err)
+	}
+	if got.Trigger == nil || got.Trigger.NextAction == nil || got.Trigger.NextAction.Name != "step1" {
+		t.Fatalf("trigger tree lost in round-trip: %+v", got.Trigger)
+	}
+	if got.Trigger.NextAction.Settings.Input["x"] != float64(1) {
+		t.Fatalf("action input lost in round-trip: %+v", got.Trigger.NextAction.Settings)
+	}
+
+	// Run status transition.
+	if _, err := s.CreateRun(ctx, FlowRun{ID: "r1", Org: "o", FlowID: "f1", FlowVersionID: "v1", WorkflowID: "r1", Status: RunRunning, StartTime: 11, Created: 11, Updated: 11}); err != nil {
+		t.Fatalf("create run: %v", err)
+	}
+	if err := s.UpdateRunStatus(ctx, "o", "r1", RunSucceeded, 12, 12); err != nil {
+		t.Fatalf("update run status: %v", err)
+	}
+	r, _ := s.GetRun(ctx, "o", "r1")
+	if r.Status != RunSucceeded || r.FinishTime != 12 {
+		t.Fatalf("run status transition lost: %+v", r)
+	}
+
+	// Deleting the flow cascades its versions + runs (within the org).
+	if _, err := s.DeleteFlow(ctx, "o", "f1"); err != nil {
+		t.Fatalf("delete flow: %v", err)
+	}
+	if _, err := s.GetVersion(ctx, "o", "v1"); !errors.Is(err, errNotFound) {
+		t.Fatalf("version must be cascaded, got %v", err)
+	}
+	if _, err := s.GetRun(ctx, "o", "r1"); !errors.Is(err, errNotFound) {
+		t.Fatalf("run must be cascaded, got %v", err)
+	}
+}

clients/automations/types.go

@@ -0,0 +1,382 @@
+// Package automations mounts the Hanzo Cloud /v1/automations/* surface: a
+// native-Go Connectors+Automations engine (HIP-0106, task #51) that runs an
+// org's flows durably on the ONE shared in-process hanzoai/tasks engine and
+// invokes third-party connectors whose credentials are custodied by
+// clients/integrations (KMS-sealed, per-org).
+//
+// This file ports the ActivePieces shared contract
+// (auto/packages/shared/src/lib/automation/) to plain Go structs + string-const
+// enums. The JSON tags match the TypeScript field names verbatim because the
+// reused web/ flow builder is the contract consumer — it authors the same
+// trigger→action tree this engine walks.
+//
+// TENANT ISOLATION is a physical property, not a policy: every stored row leads
+// its indexes with `org`, and the durable engine's ONLY credential scope is
+// FlowRunInput.Owner — the VALIDATED org resolved from principal.Tenant at
+// flow-start, never a client-supplied field. See engine.go (ExecuteStepActivity).
+package automations
+
+import "encoding/json"
+
+// ── Flow (flows/flow.ts) ────────────────────────────────────────────────────
+
+// FlowStatus mirrors flows/flow.ts FlowStatus.
+type FlowStatus string
+
+const (
+	FlowEnabled  FlowStatus = "ENABLED"
+	FlowDisabled FlowStatus = "DISABLED"
+)
+
+// Flow is an org-scoped automation. projectId IS the org (the TS `projectId`
+// field), always server-derived from principal.Tenant and NEVER trusted from a
+// request body — a caller can never author a flow into another tenant.
+type Flow struct {
+	ID                 string          `json:"id"`
+	Org                string          `json:"projectId"` // projectId == org (server-derived)
+	ExternalID         string          `json:"externalId"`
+	FolderID           string          `json:"folderId"`
+	Status             FlowStatus      `json:"status"`
+	PublishedVersionID string          `json:"publishedVersionId"`
+	Metadata           json.RawMessage `json:"metadata,omitempty"`
+	Created            int64           `json:"created"`
+	Updated            int64           `json:"updated"`
+}
+
+// ── FlowVersion (flows/flow-version.ts) ─────────────────────────────────────
+
+// FlowVersionState mirrors flows/flow-version.ts FlowVersionState.
+type FlowVersionState string
+
+const (
+	VersionDraft  FlowVersionState = "DRAFT"
+	VersionLocked FlowVersionState = "LOCKED"
+)
+
+// LatestFlowSchemaVersion is the shared schema stamp new versions carry, matching
+// flow-version.ts LATEST_FLOW_SCHEMA_VERSION.
+const LatestFlowSchemaVersion = "21"
+
+// FlowVersion is one editable revision of a flow: a display name plus the root
+// trigger of the step tree. Org is the isolation key (never serialized).
+type FlowVersion struct {
+	ID            string           `json:"id"`
+	Org           string           `json:"-"`
+	FlowID        string           `json:"flowId"`
+	DisplayName   string           `json:"displayName"`
+	Trigger       *FlowTrigger     `json:"trigger"`
+	Valid         bool             `json:"valid"`
+	State         FlowVersionState `json:"state"`
+	SchemaVersion string           `json:"schemaVersion"`
+	Created       int64            `json:"created"`
+	Updated       int64            `json:"updated"`
+}
+
+// ── Step tree (triggers/trigger.ts + actions/action.ts) ─────────────────────
+
+// TriggerStrategy mirrors trigger/index.ts TriggerStrategy. It selects HOW a
+// flow starts: POLLING → a cron schedule on the tasks engine; WEBHOOK/APP_WEBHOOK
+// → an inbound HTTP event; MANUAL → an explicit /run.
+type TriggerStrategy string
+
+const (
+	StrategyPolling    TriggerStrategy = "POLLING"
+	StrategyWebhook    TriggerStrategy = "WEBHOOK"
+	StrategyAppWebhook TriggerStrategy = "APP_WEBHOOK"
+	StrategyManual     TriggerStrategy = "MANUAL"
+)
+
+// FlowTriggerType / FlowActionType mirror the discriminants of the step nodes.
+const (
+	TriggerTypePiece = "PIECE_TRIGGER"
+	TriggerTypeEmpty = "EMPTY"
+
+	ActionTypePiece  = "PIECE"
+	ActionTypeCode   = "CODE"
+	ActionTypeBranch = "BRANCH"
+	ActionTypeLoop   = "LOOP_ON_ITEMS"
+	ActionTypeRouter = "ROUTER"
+)
+
+// StepSettings is the flattened union of PieceTriggerSettings / PieceActionSettings
+// / CodeActionSettings — the fields the engine actually reads to dispatch a step:
+// which piece, which action/trigger, and the input map. (The builder-only settings
+// — propertySettings, sampleData, errorHandlingOptions — round-trip opaquely via
+// the raw version JSON in the store; they are not modeled here.)
+type StepSettings struct {
+	PieceName    string         `json:"pieceName,omitempty"`
+	PieceVersion string         `json:"pieceVersion,omitempty"`
+	ActionName   string         `json:"actionName,omitempty"`
+	TriggerName  string         `json:"triggerName,omitempty"`
+	Input        map[string]any `json:"input,omitempty"`
+}
+
+// FlowTrigger is the root of the step tree (triggers/trigger.ts). Strategy drives
+// enable/disable (POLLING → CreateSchedule); NextAction chains into the action list.
+type FlowTrigger struct {
+	Name        string          `json:"name"`
+	Type        string          `json:"type"` // PIECE_TRIGGER | EMPTY
+	DisplayName string          `json:"displayName"`
+	Valid       bool            `json:"valid"`
+	Strategy    TriggerStrategy `json:"strategy,omitempty"`
+	Settings    StepSettings    `json:"settings"`
+	NextAction  *FlowAction     `json:"nextAction,omitempty"`
+}
+
+// FlowAction is an action node (actions/action.ts). The engine walks the linear
+// NextAction chain; the tree discriminants (ROUTER/LOOP) are modeled for contract
+// fidelity but branch/loop execution is out of Phase-1 scope (see operations.go).
+type FlowAction struct {
+	Name        string       `json:"name"`
+	Type        string       `json:"type"` // PIECE | CODE | ROUTER | LOOP_ON_ITEMS
+	DisplayName string       `json:"displayName"`
+	Valid       bool         `json:"valid"`
+	Skip        bool         `json:"skip,omitempty"`
+	Settings    StepSettings `json:"settings"`
+	NextAction  *FlowAction  `json:"nextAction,omitempty"`
+}
+
+// ── FlowRun (flow-run/flow-run.ts + execution/flow-execution.ts) ────────────
+
+// FlowRunStatus mirrors execution/flow-execution.ts FlowRunStatus (the subset the
+// engine emits; the full enum has memory/log/quota terminal states we map onto FAILED).
+type FlowRunStatus string
+
+const (
+	RunRunning   FlowRunStatus = "RUNNING"
+	RunSucceeded FlowRunStatus = "SUCCEEDED"
+	RunFailed    FlowRunStatus = "FAILED"
+	RunPaused    FlowRunStatus = "PAUSED"
+	RunQueued    FlowRunStatus = "QUEUED"
+	RunCanceled  FlowRunStatus = "CANCELED"
+	RunTimeout   FlowRunStatus = "TIMEOUT"
+)
+
+// FlowRun is an execution record (flow-run/flow-run.ts). WorkflowID (the tasks
+// engine handle) equals ID and is internal — resume/describe address the engine
+// through it, scoped to the org's namespace.
+type FlowRun struct {
+	ID            string        `json:"id"`
+	Org           string        `json:"-"`
+	FlowID        string        `json:"flowId"`
+	FlowVersionID string        `json:"flowVersionId"`
+	WorkflowID    string        `json:"-"`
+	Status        FlowRunStatus `json:"status"`
+	StartTime     int64         `json:"startTime"`
+	FinishTime    int64         `json:"finishTime"`
+	Created       int64         `json:"created"`
+	Updated       int64         `json:"updated"`
+}
+
+// ── Piece catalog (pieces/framework piece-metadata.ts, catalog wire schema) ──
+
+// Catalog is the browse catalogue served at GET /v1/automations/pieces. It is the
+// go:embed'd catalog/catalog.json, seeded here with the Tier-A pieces and later
+// overwritten by a separate agent with the full 701-piece set at this EXACT schema.
+type Catalog struct {
+	PieceCount int             `json:"pieceCount"`
+	Pieces     []PieceMetadata `json:"pieces"`
+}
+
+// PieceMetadata is one catalogue entry. NOTE: the framework's PieceMetadata models
+// actions/triggers as maps; the catalog WIRE schema (the one this must unmarshal,
+// including the 701-piece overwrite) uses arrays — so this Go shape uses arrays to
+// match the wire exactly, as the contract requires.
+type PieceMetadata struct {
+	Name        string         `json:"name"`
+	DisplayName string         `json:"displayName"`
+	Description string         `json:"description"`
+	LogoURL     string         `json:"logoUrl"`
+	Version     string         `json:"version"`
+	Categories  []string       `json:"categories"`
+	Auth        PieceAuth      `json:"auth"`
+	Actions     []PieceAction  `json:"actions"`
+	Triggers    []PieceTrigger `json:"triggers"`
+}
+
+// PieceAuth is the catalogue's auth descriptor: which credential a piece needs
+// ("none" for core, "oauth2"/"bot_token" for a connected provider) and whether it
+// is required.
+type PieceAuth struct {
+	Type     string `json:"type"`
+	Required bool   `json:"required"`
+}
+
+// PieceAction / PieceTrigger are the catalogue's action/trigger descriptors.
+type PieceAction struct {
+	Name        string     `json:"name"`
+	DisplayName string     `json:"displayName"`
+	Description string     `json:"description"`
+	Props       []PropSpec `json:"props"`
+}
+
+type PieceTrigger struct {
+	Name        string     `json:"name"`
+	DisplayName string     `json:"displayName"`
+	Description string     `json:"description"`
+	Strategy    string     `json:"strategy"`
+	Props       []PropSpec `json:"props"`
+}
+
+// PropSpec describes one input property of an action/trigger. It is the ONE prop
+// shape shared by the connector framework (connector.go), the catalogue, and the
+// MCP tool input-schema derivation (mcp.go) — one definition, three consumers.
+type PropSpec struct {
+	Name        string `json:"name"`
+	DisplayName string `json:"displayName,omitempty"`
+	Type        string `json:"type"` // string|number|boolean|object|array
+	Required    bool   `json:"required,omitempty"`
+	Description string `json:"description,omitempty"`
+}
+
+// ── FlowOperation envelope (flows/operations/index.ts) ──────────────────────
+
+// FlowOperationType mirrors operations/index.ts FlowOperationType — the builder's
+// edit-op discriminants. The engine applies the linear-chain subset (see
+// operations.go); tree-restructuring ops are honestly rejected, never faked.
+type FlowOperationType string
+
+const (
+	OpAddAction     FlowOperationType = "ADD_ACTION"
+	OpUpdateAction  FlowOperationType = "UPDATE_ACTION"
+	OpDeleteAction  FlowOperationType = "DELETE_ACTION"
+	OpMoveAction    FlowOperationType = "MOVE_ACTION"
+	OpUpdateTrigger FlowOperationType = "UPDATE_TRIGGER"
+	OpChangeName    FlowOperationType = "CHANGE_NAME"
+	OpChangeStatus  FlowOperationType = "CHANGE_STATUS"
+)
+
+// FlowOperation is the discriminated-union envelope the builder POSTs to
+// /v1/automations/flows/:id/operations: a type + an opaque request the apply
+// switch decodes per type.
+type FlowOperation struct {
+	Type    FlowOperationType `json:"type"`
+	Request json.RawMessage   `json:"request"`
+}
+
+// Operation request shapes (operations/index.ts). Only the linear-chain ops are
+// modeled; each is decoded from FlowOperation.Request by the apply switch.
+type (
+	// AddActionRequest inserts Action after the step named ParentStep (empty ⇒ append
+	// to the end of the chain, or set as the trigger's first action).
+	AddActionRequest struct {
+		ParentStep string      `json:"parentStep"`
+		Action     *FlowAction `json:"action"`
+	}
+	// UpdateActionRequest replaces the settings of the step named Name.
+	UpdateActionRequest struct {
+		Name     string       `json:"name"`
+		Type     string       `json:"type"`
+		Settings StepSettings `json:"settings"`
+	}
+	// DeleteActionRequest removes the named steps and relinks the chain.
+	DeleteActionRequest struct {
+		Names []string `json:"names"`
+	}
+	// MoveActionRequest moves Name to sit immediately after NewParentStep.
+	MoveActionRequest struct {
+		Name          string `json:"name"`
+		NewParentStep string `json:"newParentStep"`
+	}
+	// UpdateTriggerRequest replaces the root trigger node.
+	UpdateTriggerRequest FlowTrigger
+	// ChangeNameRequest renames the version.
+	ChangeNameRequest struct {
+		DisplayName string `json:"displayName"`
+	}
+	// ChangeStatusRequest flips ENABLED/DISABLED.
+	ChangeStatusRequest struct {
+		Status FlowStatus `json:"status"`
+	}
+)
+
+// ── Durable-execution I/O (engine.go) ───────────────────────────────────────
+// All JSON-serializable (no funcs/channels): they cross the tasks wire.
+
+// FlowRunInput is the durable workflow's typed input. Owner is the VALIDATED org
+// set at flow-start from principal.Tenant — the SOLE credential scope and the
+// cross-tenant isolation boundary; it is NEVER read from a request body. Steps is
+// the flattened trigger→action chain (side-effecting steps in order).
+type FlowRunInput struct {
+	Owner         string     `json:"owner"`
+	FlowID        string     `json:"flowId"`
+	FlowVersionID string     `json:"flowVersionId"`
+	RunID         string     `json:"runId"`
+	Steps         []FlowStep `json:"steps"`
+}
+
+// FlowStep is one resolved, executable step in the flattened chain.
+type FlowStep struct {
+	Name       string         `json:"name"`
+	PieceName  string         `json:"pieceName"`
+	ActionName string         `json:"actionName"`
+	Input      map[string]any `json:"input"`
+}
+
+// StepInput is the activity's per-step input. Owner is copied verbatim from
+// FlowRunInput.Owner by the workflow — an activity can never widen its own scope.
+type StepInput struct {
+	Owner       string         `json:"owner"`
+	RunID       string         `json:"runId"`
+	Name        string         `json:"name"`
+	PieceName   string         `json:"pieceName"`
+	ActionName  string         `json:"actionName"`
+	Input       map[string]any `json:"input"`
+	PrevOutputs map[string]any `json:"prevOutputs"`
+}
+
+// StepOutput is the activity's result for one step.
+type StepOutput struct {
+	Name   string `json:"name"`
+	Output any    `json:"output"`
+}
+
+// FlowRunResult is the workflow's terminal result: the final status plus every
+// step's output, keyed by step name (the threaded outputs).
+type FlowRunResult struct {
+	RunID   string         `json:"runId"`
+	Status  FlowRunStatus  `json:"status"`
+	Steps   int            `json:"steps"`
+	Outputs map[string]any `json:"outputs"`
+}
+
+// flattenSteps walks a version's trigger→nextAction chain into the ordered,
+// executable []FlowStep the workflow runs. The trigger node itself is an entry
+// point, not a side-effecting step, so it is not emitted; only PIECE/CODE actions
+// (the linear chain) are. Skipped actions are dropped. A nil trigger yields none.
+func flattenSteps(v *FlowVersion) []FlowStep {
+	if v == nil || v.Trigger == nil {
+		return nil
+	}
+	out := make([]FlowStep, 0, 8)
+	for a := v.Trigger.NextAction; a != nil; a = a.NextAction {
+		if a.Skip {
+			continue
+		}
+		piece, action := stepPieceAction(a)
+		if piece == "" {
+			continue
+		}
+		out = append(out, FlowStep{
+			Name:       a.Name,
+			PieceName:  piece,
+			ActionName: action,
+			Input:      a.Settings.Input,
+		})
+	}
+	return out
+}
+
+// stepPieceAction resolves the (piece, action) an action node dispatches to. A
+// PIECE node names them explicitly; a CODE node is the built-in core.code transform.
+func stepPieceAction(a *FlowAction) (piece, action string) {
+	switch a.Type {
+	case ActionTypeCode:
+		return corePiece, "code"
+	case ActionTypePiece:
+		return a.Settings.PieceName, a.Settings.ActionName
+	default:
+		// ROUTER/LOOP are not linear-executable in Phase 1 — skip (piece=="").
+		return "", ""
+	}
+}