Compare commits
204
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8c3ccb55ac | ||
|
|
6422afdcf7 | ||
|
|
e92cf60378 | ||
|
|
8316db1f9b | ||
|
|
f5aad4dd36 | ||
|
|
2c24ea104a | ||
|
|
99a00f69f0 | ||
|
|
1f62bf4aa8 | ||
|
|
ff86446070 | ||
|
|
82e48a3b30 | ||
|
|
75788dd07b | ||
|
|
57c2ee9e92 | ||
|
|
65d0dbcbeb | ||
|
|
5f518ae493 | ||
|
|
2dfce35465 | ||
|
|
be8651ae31 | ||
|
|
6058aa3532 | ||
|
|
8eff71c937 | ||
|
|
e6a1af1a53 | ||
|
|
9bafa850cd | ||
|
|
c1a36548e1 | ||
|
|
25c1a1d3f3 | ||
|
|
84a7f7b9a1 | ||
|
|
6f6e03ff81 | ||
|
|
716217670f | ||
|
|
514c4f8e2f | ||
|
|
5116fbf6fc | ||
|
|
1558023ec6 | ||
|
|
efaeedb82c | ||
|
|
8913ece0a6 | ||
|
|
94130dca61 | ||
|
|
d9a99e6a54 | ||
|
|
4ea8b06f79 | ||
|
|
829cdf72cc | ||
|
|
802bb6ad29 | ||
|
|
8a38817b5e | ||
|
|
c4047ee492 | ||
|
|
e72addafca | ||
|
|
08f2554969 | ||
|
|
5876cecb7a | ||
|
|
6e82c09897 | ||
|
|
3baf677016 | ||
|
|
bb50b2aeef | ||
|
|
517b8a0650 | ||
|
|
526172f200 | ||
|
|
71bead9cea | ||
|
|
70aae48c86 | ||
|
|
2c887dba3b | ||
|
|
704011d389 | ||
|
|
21cd20589e | ||
|
|
6c720908f4 | ||
|
|
2b3f5f6f58 | ||
|
|
26d5e0f433 | ||
|
|
06e399491b | ||
|
|
f0ab9e7fb9 | ||
|
|
dd20b05f41 | ||
|
|
98e05b6aa4 | ||
|
|
a1bf850c96 | ||
|
|
f0931cab5f | ||
|
|
fbe41ec92b | ||
|
|
f50f29a8bb | ||
|
|
cd5cbd9345 | ||
|
|
796ba9df68 | ||
|
|
0b46a4ff8d | ||
|
|
b5192af1e1 | ||
|
|
e12a781599 | ||
|
|
b6042582b8 | ||
|
|
249efbee86 | ||
|
|
69c5fca283 | ||
|
|
0c805b9367 | ||
|
|
6158f6b111 | ||
|
|
f0a1d510bf | ||
|
|
a4e4d551af | ||
|
|
e6d7c23237 | ||
|
|
de5782e48b | ||
|
|
c0b285183a | ||
|
|
06029d645f | ||
|
|
75e5615b0b | ||
|
|
ffaa319c6e | ||
|
|
d3a640bae8 | ||
|
|
4ffb8852b9 | ||
|
|
4efffd4944 | ||
|
|
a664c138b5 | ||
|
|
891f0c6dfa | ||
|
|
ed1e789634 | ||
|
|
a1d4235fc8 | ||
|
|
f074b4ad29 | ||
|
|
6a0d254028 | ||
|
|
e4f663aff8 | ||
|
|
fad100a90d | ||
|
|
1e344c0bf4 | ||
|
|
64911105ff | ||
|
|
cd78c5b36a | ||
|
|
743fc40b5d | ||
|
|
dfaa359511 | ||
|
|
1ed8f3b37d | ||
|
|
9fa690ad18 | ||
|
|
5309dd09d3 | ||
|
|
1286c41f16 | ||
|
|
bc04b675fa | ||
|
|
4ef2510254 | ||
|
|
6865c4c5a6 | ||
|
|
ea5ae54b46 | ||
|
|
a877eda205 | ||
|
|
01ee372a67 | ||
|
|
b8658dc543 | ||
|
|
bc3e9e52de | ||
|
|
93f20d40d9 | ||
|
|
8b961f3273 | ||
|
|
96833fde9a | ||
|
|
cfd1c5bf00 | ||
|
|
4aa2ea79cf | ||
|
|
2d77642de2 | ||
|
|
0fd3d62f71 | ||
|
|
62c58040a6 | ||
|
|
84ecd7b6f2 | ||
|
|
2a1d76ae4d | ||
|
|
734c52b93f | ||
|
|
1e8ec3e11a | ||
|
|
7f9ba10b16 | ||
|
|
5824068aa1 | ||
|
|
f429d7228d | ||
|
|
c549074bb8 | ||
|
|
3ecbcfe1c8 | ||
|
|
803267b226 | ||
|
|
031388bd1a | ||
|
|
d7f7077484 | ||
|
|
c5b69e2139 | ||
|
|
a0dbfc75a0 | ||
|
|
dba4666173 | ||
|
|
df1de096ff | ||
|
|
84281e3ddc | ||
|
|
e2cf2e978b | ||
|
|
d5c1a5b676 | ||
|
|
eed81e6c8e | ||
|
|
51eb4f281c | ||
|
|
945b2d5aba | ||
|
|
5b5c95fea9 | ||
|
|
41c2b3edb2 | ||
|
|
8938a22cde | ||
|
|
1bd58f0fc0 | ||
|
|
7140168680 | ||
|
|
dc48ee02c5 | ||
|
|
5ee3853c2d | ||
|
|
fe63e9bb4e | ||
|
|
5770ae8720 | ||
|
|
b5f1e2dcad | ||
|
|
874c328ca5 | ||
|
|
a832e1f3ba | ||
|
|
44d237a66a | ||
|
|
0c9cc4d46b | ||
|
|
df447d3470 | ||
|
|
046beb5ae8 | ||
|
|
a8b52baf9f | ||
|
|
8f0d633d24 | ||
|
|
41c73dd177 | ||
|
|
c3d28bd278 | ||
|
|
76a7b184b3 | ||
|
|
411865bcea | ||
|
|
f87bb69f57 | ||
|
|
7b2da28c5b | ||
|
|
3a8391235e | ||
|
|
87aa341e20 | ||
|
|
c18e20d625 | ||
|
|
43871c446a | ||
|
|
93dc54977c | ||
|
|
3010dccef2 | ||
|
|
e7ab2ee968 | ||
|
|
a80f92cca5 | ||
|
|
1198254ffd | ||
|
|
ea65ba9e48 | ||
|
|
73d51910a8 | ||
|
|
bcc84015a8 | ||
|
|
8baac66b43 | ||
|
|
4164cd33b0 | ||
|
|
2f482b1ce4 | ||
|
|
c7376993b9 | ||
|
|
52decbbc04 | ||
|
|
bb147ac06d | ||
|
|
d965a68885 | ||
|
|
aca6db042a | ||
|
|
9c56954ce6 | ||
|
|
27adcc80ea | ||
|
|
787664527c | ||
|
|
fe04a60187 | ||
|
|
61019879f4 | ||
|
|
d9f2735cf8 | ||
|
|
68b0e1a0dc | ||
|
|
aef466ee2a | ||
|
|
b93e7320b1 | ||
|
|
299dde6848 | ||
|
|
ae7330a539 | ||
|
|
7ad8940061 | ||
|
|
aa119d6108 | ||
|
|
e1158dc6eb | ||
|
|
30f4f4cd40 | ||
|
|
68d0399fbb | ||
|
|
910e18609c | ||
|
|
9d71c41059 | ||
|
|
a6dbbdf9f3 | ||
|
|
b9a9f856a4 | ||
|
|
5c8338c0be | ||
|
|
2b61d83ea8 | ||
|
|
d6ca4ba692 |
@@ -1,12 +0,0 @@
|
||||
# ~7-line canonical caller — all real config lives in /hanzo.yml.
|
||||
# Test gate on OUR arc pool; release.yml owns the image + v* tags.
|
||||
name: CI/CD
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
jobs:
|
||||
cicd:
|
||||
uses: hanzoai/ci/.github/workflows/build.yml@v1
|
||||
secrets: inherit
|
||||
@@ -1,684 +0,0 @@
|
||||
name: release
|
||||
|
||||
# Cuts a release of ghcr.io/hanzoai/cloud. The invariant this workflow exists to
|
||||
# enforce:
|
||||
#
|
||||
# a git tag v<X.Y.Z> exists ⇔ an image ghcr.io/hanzoai/cloud:v<X.Y.Z>
|
||||
# was pushed AND booted to "listening" in the smoke test.
|
||||
#
|
||||
# The tag is a RECEIPT for a proven image, minted only AFTER a successful push —
|
||||
# never a trigger for a build that might fail. The prior design triggered builds
|
||||
# FROM pushed tags, so a tag could exist with no image behind it (a failed or
|
||||
# never-run build) — universe would then try to roll that tag and the pods went
|
||||
# ImagePullBackOff (phantom v1.786.42/43). Here the order is inverted:
|
||||
#
|
||||
# main push → compute next version → build → SMOKE → push image → tag → notify
|
||||
#
|
||||
# so a push/smoke/build failure fails the run BEFORE the tag step and leaves no
|
||||
# tag; universe is only ever notified of a version whose image is proven present.
|
||||
#
|
||||
# DO NOT push v* tags by hand anymore. This workflow OWNS them. A hand-cut tag has
|
||||
# no image behind it (exactly the phantom this prevents) and won't build (there is
|
||||
# no `tags:` trigger). Every merge to main IS the release; skip one with the usual
|
||||
# `[skip ci]` in the commit/merge message (a docs-only change need not ship).
|
||||
#
|
||||
# concurrency: a single serialized lane (cancel-in-progress:false — a
|
||||
# mid-flight push must finish, never be killed between "image pushed" and "tag
|
||||
# created"). Two main pushes can therefore never compute the same next number:
|
||||
# the queued run starts only after the running one tags, re-reads the tags, and
|
||||
# lands on the next patch. Monotonic by construction.
|
||||
#
|
||||
# The next version is max(highest git tag, highest pushed container tag) + 1 (patch
|
||||
# bump only — never a major/minor jump). Folding in the container tags means we
|
||||
# never reuse a number that already has a pushed image, even if some earlier run
|
||||
# pushed an image but died before tagging.
|
||||
#
|
||||
# ── Infra notes (unchanged, still true) ─────────────────────────────────────────
|
||||
# Self-hosted arcd amd64 scale set — NEVER GitHub-hosted runners (this org's
|
||||
# GitHub-hosted Actions are billing-frozen). GHCR login uses GH_PAT, not the repo
|
||||
# GITHUB_TOKEN: the ghcr.io/hanzoai/cloud package is linked to a DIFFERENT repo
|
||||
# (hanzoai/ai, from the cloud→ai module rename), so this repo's GITHUB_TOKEN is
|
||||
# denied write to it (permission_denied: write_package). GH_PAT (admin:org +
|
||||
# write:packages) writes any hanzoai package regardless of package-repo linkage,
|
||||
# and is the BuildKit gh_token the Dockerfile uses to fetch private cross-org Go
|
||||
# modules. amd64-only: the cluster is amd64; one platform completes on the live
|
||||
# scale set without waiting on the arm64 pool.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
# Cut tags on the cloud repo (git tag push) → contents: write. packages: write
|
||||
# to push the image; id-token for provenance.
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
id-token: write
|
||||
|
||||
# One serialized release lane. Never cancel in-flight: a run killed between
|
||||
# "image pushed" and "tag created" is exactly the drift we are preventing.
|
||||
concurrency:
|
||||
group: release-cloud
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build-amd64:
|
||||
# ARC ephemeral runners match jobs targeting the scale-set NAME as a label.
|
||||
runs-on: [hanzo-build-linux-amd64]
|
||||
outputs:
|
||||
# The FINAL assigned version comes from the Tag step (atomic free-version
|
||||
# assignment), NOT the compute step — under concurrency the compute value may
|
||||
# have been superseded. notify-universe must roll the version that was actually
|
||||
# tagged + whose image was actually retagged.
|
||||
version: ${{ steps.tag.outputs.version }}
|
||||
version_v: ${{ steps.tag.outputs.version_v }}
|
||||
steps:
|
||||
- name: Checkout (full history + all tags — the version floor is read from tags)
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
- name: Compute next version (monotonic patch bump over git + container tags)
|
||||
id: ver
|
||||
env:
|
||||
GH_PAT: ${{ secrets.GH_PAT }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git fetch --tags --force --quiet
|
||||
|
||||
# Highest semver git tag (vX.Y.Z), normalised without the leading v.
|
||||
git_max="$(git tag -l 'v[0-9]*.[0-9]*.[0-9]*' \
|
||||
| sed 's/^v//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1 || true)"
|
||||
|
||||
# Best-effort: highest ALREADY-PUSHED container tag, so a number that has
|
||||
# an image (even from a run that died before tagging) is never reused.
|
||||
cont_max=""
|
||||
if command -v gh >/dev/null 2>&1; then
|
||||
cont_max="$(GH_TOKEN="$GH_PAT" gh api \
|
||||
'/orgs/hanzoai/packages/container/cloud/versions?per_page=100' \
|
||||
--jq '.[].metadata.container.tags[]?' 2>/dev/null \
|
||||
| sed 's/^v//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1 || true)"
|
||||
fi
|
||||
|
||||
# Floor = highest of the two; fall back to 1.786.0 only if the repo has
|
||||
# no tags at all (first release ever).
|
||||
max="$(printf '%s\n%s\n%s\n' "1.786.0" "$git_max" "$cont_max" \
|
||||
| grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1)"
|
||||
|
||||
major="${max%%.*}"; rest="${max#*.}"; minor="${rest%%.*}"; patch="${rest##*.}"
|
||||
version="${major}.${minor}.$((patch + 1))"
|
||||
|
||||
# This version is a STARTING HINT only. It labels the smoke-built image and
|
||||
# seeds the OCI metadata; the FINAL version is assigned atomically in the Tag
|
||||
# step (which recomputes + retries on collision), so a taken number here is
|
||||
# NOT fatal — the Tag step finds the next free one. Just note it and proceed.
|
||||
if git rev-parse -q --verify "refs/tags/v${version}" >/dev/null; then
|
||||
echo "note: hint v${version} already tagged — the Tag step will assign the next free version"
|
||||
fi
|
||||
|
||||
echo "version=${version}" >> "$GITHUB_OUTPUT"
|
||||
echo "version_v=v${version}" >> "$GITHUB_OUTPUT"
|
||||
echo "major_minor=${major}.${minor}" >> "$GITHUB_OUTPUT"
|
||||
echo "sha_short=$(git rev-parse --short "$GITHUB_SHA")" >> "$GITHUB_OUTPUT"
|
||||
echo "Next release: v${version} (git_max='${git_max:-none}' container_max='${cont_max:-none}')"
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
driver: docker-container
|
||||
driver-opts: network=host
|
||||
|
||||
- name: Mirror credential (registry.hanzo.ai)
|
||||
# Dual-host: pull the KMS deploy kubeconfig (same Universal Auth flow
|
||||
# the hanzoai/ci reusable uses), read the cluster-synced
|
||||
# registry-credentials dockerconfig, and log in. Best-effort: absent
|
||||
# creds → GHCR-only release, never a blocked tag.
|
||||
env:
|
||||
KMS_CLIENT_ID: ${{ secrets.KMS_CLIENT_ID }}
|
||||
KMS_CLIENT_SECRET: ${{ secrets.KMS_CLIENT_SECRET }}
|
||||
KMS_ENDPOINT: ${{ vars.KMS_ENDPOINT || 'https://kms.hanzo.ai' }}
|
||||
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
|
||||
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
# Direct credential first (repo/org secret — works on private repos,
|
||||
# where the Free plan hides org KMS secrets); KMS kubeconfig fallback.
|
||||
if [ -n "${REGISTRY_USER:-}" ] && [ -n "${REGISTRY_PASSWORD:-}" ]; then
|
||||
if echo "$REGISTRY_PASSWORD" | docker login registry.hanzo.ai -u "$REGISTRY_USER" --password-stdin; then
|
||||
echo "MIRROR_OK=1" >> "$GITHUB_ENV"
|
||||
else
|
||||
echo "mirror login failed (registry.hanzo.ai unreachable) — mirror skipped, release continues"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
[ -z "${KMS_CLIENT_ID:-}" ] && { echo "no KMS creds — mirror skipped"; exit 0; }
|
||||
TOKEN=$(curl -sf "$KMS_ENDPOINT/v1/kms/auth/login" -H 'Content-Type: application/json' -d "{\"clientId\":\"$KMS_CLIENT_ID\",\"clientSecret\":\"$KMS_CLIENT_SECRET\"}" | jq -r '.accessToken // empty')
|
||||
[ -z "$TOKEN" ] && { echo "KMS login failed — mirror skipped"; exit 0; }
|
||||
KC=$(curl -sf "$KMS_ENDPOINT/v1/kms/orgs/hanzo/secrets/deploy/KUBECONFIG?env=prod" -H "Authorization: Bearer $TOKEN" | jq -r '.secret.value // empty')
|
||||
[ -z "$KC" ] && { echo "no KUBECONFIG in KMS — mirror skipped"; exit 0; }
|
||||
echo "$KC" | base64 -d > "$RUNNER_TEMP/kubeconfig"
|
||||
command -v kubectl >/dev/null 2>&1 || {
|
||||
KVER=$(curl -fsSL https://dl.k8s.io/release/stable.txt)
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
curl -fsSL "https://dl.k8s.io/release/${KVER}/bin/linux/amd64/kubectl" -o "$HOME/.local/bin/kubectl" && chmod +x "$HOME/.local/bin/kubectl"
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
}
|
||||
CFG=$(KUBECONFIG="$RUNNER_TEMP/kubeconfig" kubectl -n hanzo get secret registry-credentials -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null | base64 -d || true)
|
||||
[ -z "$CFG" ] && { echo "registry-credentials unreadable — mirror skipped"; exit 0; }
|
||||
UP=$(echo "$CFG" | jq -r '.auths["registry.hanzo.ai"].auth // empty' | base64 -d)
|
||||
[ -z "$UP" ] && { echo "no registry auth — mirror skipped"; exit 0; }
|
||||
echo "::add-mask::${UP#*:}"
|
||||
if echo "${UP#*:}" | docker login registry.hanzo.ai -u "${UP%%:*}" --password-stdin; then
|
||||
echo "MIRROR_OK=1" >> "$GITHUB_ENV"
|
||||
else
|
||||
echo "mirror login failed (registry.hanzo.ai unreachable) — mirror skipped, release continues"
|
||||
fi
|
||||
|
||||
- name: Log in to ghcr.io (GH_PAT — writes the cloud package despite its ai-repo linkage)
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: hanzo-dev
|
||||
password: ${{ secrets.GH_PAT }}
|
||||
|
||||
- name: Resolve decomplection artifact digests (the Go-only build's prebuilt inputs)
|
||||
id: artifacts
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# cloud compiles ONLY Go; it pulls three prebuilt artifacts (console SPA,
|
||||
# agent-skills catalog, native flags staticlib). Resolve each published
|
||||
# :latest to an IMMUTABLE digest so THIS release is reproducible (pinned,
|
||||
# not floating :latest) AND a console/skills/flags change is picked up —
|
||||
# its CI republished :latest, so this resolves to the NEW digest. A MISSING
|
||||
# artifact FAILS the release HERE, before build/smoke/push/tag: the receipt
|
||||
# invariant means we never tag an image that couldn't embed the real console.
|
||||
command -v crane >/dev/null 2>&1 || {
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
curl -fsSL "https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz" \
|
||||
| tar -xz -C "$HOME/.local/bin" crane
|
||||
}
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
resolve() {
|
||||
local repo="$1" d
|
||||
d="$(crane digest "ghcr.io/hanzoai/${repo}:latest" 2>/dev/null || true)"
|
||||
[ -n "$d" ] || { echo "::error::decomplection artifact ghcr.io/hanzoai/${repo}:latest is not published — refusing to cut a release that would embed a stale/placeholder ${repo}"; return 1; }
|
||||
printf 'ghcr.io/hanzoai/%s@%s' "$repo" "$d"
|
||||
}
|
||||
CONSOLE_IMAGE="$(resolve console-embed)" || exit 1
|
||||
SKILLS_IMAGE="$(resolve agent-skills)" || exit 1
|
||||
FLAGS_IMAGE="$(resolve cloud-flags)" || exit 1
|
||||
{
|
||||
echo "console_image=${CONSOLE_IMAGE}"
|
||||
echo "skills_image=${SKILLS_IMAGE}"
|
||||
echo "flags_image=${FLAGS_IMAGE}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
echo "resolved: console=${CONSOLE_IMAGE} skills=${SKILLS_IMAGE} flags=${FLAGS_IMAGE}"
|
||||
|
||||
- name: OCI labels
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ghcr.io/hanzoai/cloud
|
||||
tags: type=raw,value=${{ steps.ver.outputs.version_v }}
|
||||
|
||||
# ── Build → SMOKE → push → tag ───────────────────────────────────────────
|
||||
# 1. Build once to a LOCAL tag (load into the daemon, do NOT push). Warms
|
||||
# the BuildKit cache — the expensive console/npm + Go layers land here.
|
||||
# 2. Boot that exact image and assert it reaches "listening" with no
|
||||
# startup-crash signature (the gate).
|
||||
# 3. Re-run build with push:true and the real tags: identical context /
|
||||
# platform / secrets, so every layer is a cache hit from step 1 and the
|
||||
# step only exports + pushes the already-tested image. Nothing that failed
|
||||
# the smoke test can reach the registry.
|
||||
# 4. Only after the push succeeds, mint + push the git tag (the receipt).
|
||||
- name: Build (load locally for the smoke test)
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64
|
||||
push: false
|
||||
load: true
|
||||
tags: cloud:smoke
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
# cloud compiles ONLY Go: pull the three prebuilt artifacts pinned to the
|
||||
# digests resolved above (reproducible, and fresh — a console/skills/flags
|
||||
# change is a new digest). No node/python/rust toolchain in this build.
|
||||
build-args: |
|
||||
CONSOLE_IMAGE=${{ steps.artifacts.outputs.console_image }}
|
||||
SKILLS_IMAGE=${{ steps.artifacts.outputs.skills_image }}
|
||||
FLAGS_IMAGE=${{ steps.artifacts.outputs.flags_image }}
|
||||
# GIT_AUTH_TOKEN: BuildKit secret the Dockerfile consumes to fetch private
|
||||
# cross-org Go modules (hanzoai/*, luxfi/*) over authenticated git.
|
||||
secrets: |
|
||||
GIT_AUTH_TOKEN=${{ secrets.GH_PAT }}
|
||||
|
||||
- name: Smoke test — the binary MUST boot to "listening" with no crash signature
|
||||
run: |
|
||||
set -euo pipefail
|
||||
IMAGE=cloud:smoke
|
||||
CID=""
|
||||
cleanup() { [ -n "$CID" ] && docker rm -f "$CID" >/dev/null 2>&1 || true; }
|
||||
trap cleanup EXIT
|
||||
|
||||
# Minimal, production-representative boot env:
|
||||
# • a writable ephemeral /data root — the audit store, the embedded
|
||||
# KMS secrets plane and every per-tenant SQLite open files under
|
||||
# CLOUD_DATA_DIR; an unwritable dir would fail EVERY image before
|
||||
# MountAll and the gate would stop discriminating good from bad; and
|
||||
# • a throwaway 32-byte KMS master key so the KMS plane mounts on its
|
||||
# normal ready path exactly as prod does (no real secret is used).
|
||||
# The subsystem that crashed the incident (metrics, mount order 40)
|
||||
# mounts AFTER kms (order 10), so the boot must get past kms for the
|
||||
# gate to observe the panic — this env does exactly that.
|
||||
KEY="$(head -c 32 /dev/urandom | base64 | tr -d '\n')"
|
||||
CID="$(docker run -d \
|
||||
--tmpfs /data:rw,size=64m \
|
||||
-e CLOUD_DATA_DIR=/data \
|
||||
-e CLOUD_ENV=smoke \
|
||||
-e CLOUD_KMS_MASTER_KEY_REF="$KEY" \
|
||||
"$IMAGE")"
|
||||
|
||||
# Poll up to 60s for boot to either finish ("listening" is logged once
|
||||
# every subsystem has mounted and both transports are about to bind) or
|
||||
# die (a Mount panic exits the process). A healthy boot is ~1-2s; the
|
||||
# ceiling only guards a cold daemon.
|
||||
listening=0
|
||||
for _ in $(seq 1 60); do
|
||||
logs="$(docker logs "$CID" 2>&1 || true)"
|
||||
if printf '%s' "$logs" | grep -q '"message":"listening"'; then listening=1; break; fi
|
||||
if [ "$(docker inspect -f '{{.State.Running}}' "$CID" 2>/dev/null || echo false)" != "true" ]; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
logs="$(docker logs "$CID" 2>&1 || true)"
|
||||
echo "::group::cloud:smoke boot logs"
|
||||
printf '%s\n' "$logs"
|
||||
echo "::endgroup::"
|
||||
|
||||
# (1) No startup-crash signature. Catches the incident's Mount
|
||||
# type-assert panic AND any generic Go panic — case-insensitive so a
|
||||
# re-worded variant can't slip through — BEFORE a byte is pushed.
|
||||
if printf '%s' "$logs" | grep -Eiq 'metrics\.Mount|mount metrics|panic|want \*zip\.App'; then
|
||||
echo "SMOKE FAIL: startup-crash signature in boot logs (see above)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# (2) Reached "listening" — proof that MountAll returned for every
|
||||
# enabled subsystem (a failed Mount returns before this line).
|
||||
if [ "$listening" -ne 1 ]; then
|
||||
echo "SMOKE FAIL: binary never reached \"listening\" (a subsystem did not mount)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# (3) Still alive — a server that logged "listening" then exited (e.g. a
|
||||
# listener bind failure) is not a healthy image.
|
||||
if [ "$(docker inspect -f '{{.State.Running}}' "$CID" 2>/dev/null || echo false)" != "true" ]; then
|
||||
echo "SMOKE FAIL: process exited after \"listening\""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "SMOKE PASS: cloud:smoke booted to \"listening\" with no crash signature"
|
||||
|
||||
# ── Functional smoke — authenticated per-subsystem probe (the REAL gate) ─────
|
||||
# The boot check above proves the process REACHES "listening"; this proves the
|
||||
# mounted HTTP surface actually WORKS. /smoke (cmd/smoke, baked into the image)
|
||||
# hits ONE side-effect-free read per core subsystem and FAILS the release on any
|
||||
# broken code — above all a 402 on a READ (the balance-gate-over-blocks-reads
|
||||
# regression) or a 5xx (a crash, e.g. the /v1/billing/usage self-dispatch 500).
|
||||
# So a release can never ship with chat/billing/projects/kms/... down.
|
||||
- name: Functional smoke — per-subsystem probe (fails the release if a core endpoint is broken)
|
||||
env:
|
||||
KMS_CLIENT_ID: ${{ secrets.KMS_CLIENT_ID }}
|
||||
KMS_CLIENT_SECRET: ${{ secrets.KMS_CLIENT_SECRET }}
|
||||
KMS_ENDPOINT: ${{ vars.KMS_ENDPOINT || 'https://kms.hanzo.ai' }}
|
||||
# A KMS-provisioned short-lived smoke bearer, injected as a secret (NEVER
|
||||
# hardcoded). Absent → the anonymous matrix still gates public/authed and
|
||||
# catches every 402-on-read / 5xx.
|
||||
SMOKE_TOKEN: ${{ secrets.SMOKE_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
IMAGE=cloud:smoke
|
||||
CID=""
|
||||
cleanup() { [ -n "$CID" ] && docker rm -f "$CID" >/dev/null 2>&1 || true; }
|
||||
trap cleanup EXIT
|
||||
|
||||
KEY="$(head -c 32 /dev/urandom | base64 | tr -d '\n')"
|
||||
CID="$(docker run -d \
|
||||
--tmpfs /data:rw,size=64m \
|
||||
-e CLOUD_DATA_DIR=/data -e CLOUD_ENV=smoke -e CLOUD_KMS_MASTER_KEY_REF="$KEY" \
|
||||
"$IMAGE")"
|
||||
|
||||
# Wait for the HTTP listener to bind (or the process to die).
|
||||
up=0
|
||||
for _ in $(seq 1 60); do
|
||||
lg="$(docker logs "$CID" 2>&1 || true)"
|
||||
printf '%s' "$lg" | grep -q '"message":"listening"' && { up=1; break; }
|
||||
[ "$(docker inspect -f '{{.State.Running}}' "$CID" 2>/dev/null || echo false)" != "true" ] && break
|
||||
sleep 1
|
||||
done
|
||||
if [ "$up" != 1 ]; then
|
||||
echo "::group::boot logs"; docker logs "$CID" 2>&1 || true; echo "::endgroup::"
|
||||
echo "FUNCTIONAL SMOKE INFRA FAIL: image never reached \"listening\""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Token: prefer the injected secret; else mint from KMS (a provisioned smoke
|
||||
# identity); else run the anonymous matrix. Never hardcoded.
|
||||
if [ -z "${SMOKE_TOKEN:-}" ] && [ -n "${KMS_CLIENT_ID:-}" ]; then
|
||||
KT=$(curl -sf "$KMS_ENDPOINT/v1/kms/auth/login" -H 'Content-Type: application/json' \
|
||||
-d "{\"clientId\":\"$KMS_CLIENT_ID\",\"clientSecret\":\"$KMS_CLIENT_SECRET\"}" | jq -r '.accessToken // empty' || true)
|
||||
[ -n "$KT" ] && SMOKE_TOKEN=$(curl -sf "$KMS_ENDPOINT/v1/kms/orgs/hanzo/secrets/smoke/TOKEN?env=prod" \
|
||||
-H "Authorization: Bearer $KT" | jq -r '.secret.value // empty' || true)
|
||||
fi
|
||||
if [ -n "${SMOKE_TOKEN:-}" ]; then echo "::add-mask::$SMOKE_TOKEN"; echo "smoke: AUTHENTICATED matrix"; else echo "smoke: ANONYMOUS matrix (no SMOKE_TOKEN wired)"; fi
|
||||
|
||||
# /smoke is baked into the image (Dockerfile) — exec it INSIDE the container,
|
||||
# so it probes the real mounted surface at localhost:8080 with no port/network
|
||||
# plumbing. A non-zero exit here fails the release BEFORE any image is pushed.
|
||||
docker exec \
|
||||
-e SMOKE_BASE_URL=http://127.0.0.1:8080 \
|
||||
-e SMOKE_TOKEN="${SMOKE_TOKEN:-}" \
|
||||
"$CID" /smoke
|
||||
|
||||
# ── Migration smoke — the gate the v1.800.1 crashloop would have tripped ─────
|
||||
# The plain smoke above boots on a FRESH /data, so every subsystem's migrate()
|
||||
# takes its CREATE-TABLE path and no forward-migration is exercised — which is
|
||||
# exactly why a DDL valid on a fresh store but broken on a pre-existing one (an
|
||||
# index over a not-yet-ADDed column: affiliates referrer_org in v1.800.1, wallets
|
||||
# project/agent before it) sailed through CI and took api.hanzo.ai down. This
|
||||
# step reproduces the REAL prod upgrade path: boot the PRIOR released image to lay
|
||||
# its on-disk (cek-encrypted) SQLite schema into a persistent volume, then boot
|
||||
# the candidate over that SAME volume and require it to still reach "listening".
|
||||
# A migrate() that assumes a fresh store dies here, before any image is pushed.
|
||||
- name: Migration smoke — candidate MUST boot over the PRIOR release's on-disk schema
|
||||
env:
|
||||
# The image whose on-disk schema a prod upgrade migrates FROM — the tag the
|
||||
# fleet runs today. Bump to the last-DEPLOYED tag as releases roll (override
|
||||
# without a code change via the SMOKE_MIGRATION_BASELINE repo/org variable).
|
||||
BASELINE: ${{ vars.SMOKE_MIGRATION_BASELINE }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
BASELINE="${BASELINE:-ghcr.io/hanzoai/cloud:v1.799.19}"
|
||||
CANDIDATE=cloud:smoke
|
||||
VOL="cloudmig-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
B1=""; B2=""
|
||||
cleanup() { docker rm -f "$B1" "$B2" >/dev/null 2>&1 || true; docker volume rm "$VOL" >/dev/null 2>&1 || true; }
|
||||
trap cleanup EXIT
|
||||
docker volume create "$VOL" >/dev/null
|
||||
|
||||
# ONE throwaway 32-byte master key for BOTH boots: cek seals each per-db DEK
|
||||
# under it on the baseline boot and unwraps it on the candidate boot. A
|
||||
# mismatched key fails closed (never opens), so sharing it is what puts the
|
||||
# MIGRATE path — not a decrypt error — under test.
|
||||
KEY="$(head -c 32 /dev/urandom | base64 | tr -d '\n')"
|
||||
|
||||
boot() { # $1=image $2=name -> prints container id
|
||||
docker run -d --name "$2" \
|
||||
-v "$VOL":/data \
|
||||
-e CLOUD_DATA_DIR=/data \
|
||||
-e CLOUD_ENV=smoke \
|
||||
-e CLOUD_KMS_MASTER_KEY_REF="$KEY" \
|
||||
"$1"
|
||||
}
|
||||
wait_listen() { # $1=container -> 0 if "listening", 1 if it died / timed out
|
||||
for _ in $(seq 1 90); do
|
||||
lg="$(docker logs "$1" 2>&1 || true)"
|
||||
printf '%s' "$lg" | grep -q '"message":"listening"' && return 0
|
||||
[ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || echo false)" != "true" ] && return 1
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
# A fresh docker named volume is root:root 0755, but the cloud image runs
|
||||
# NON-ROOT, so it cannot create cek's <db>.cek.lock under /data (the single-boot
|
||||
# smoke only worked because --tmpfs is world-writable). Make the shared volume
|
||||
# writable for BOTH boots, re-opening it between them so the candidate can read
|
||||
# the baseline's files even if their runtime UIDs differ.
|
||||
chmod_vol() { docker run --rm --user 0 -v "$VOL":/data --entrypoint sh "$CANDIDATE" -c 'chmod -R 0777 /data'; }
|
||||
|
||||
# Boot 1 — the prior release writes its real schema into the volume. It must
|
||||
# reach "listening" (proof every subsystem migrated + its DB is on disk); if
|
||||
# the pinned baseline can't boot in this env the gate is blind, so fail loud.
|
||||
echo "migration baseline: $BASELINE"
|
||||
pulled=0; for _ in 1 2 3; do if docker pull "$BASELINE"; then pulled=1; break; fi; sleep 5; done
|
||||
[ "$pulled" = 1 ] || { echo "MIGRATION SMOKE INFRA FAIL: cannot pull baseline $BASELINE"; exit 1; }
|
||||
chmod_vol
|
||||
B1="$(boot "$BASELINE" cloudmig_base)"
|
||||
if ! wait_listen "$B1"; then
|
||||
echo "::group::baseline boot logs"; docker logs "$B1" 2>&1 || true; echo "::endgroup::"
|
||||
echo "MIGRATION SMOKE INFRA FAIL: baseline $BASELINE did not reach \"listening\" — cannot stage the prior schema (inspect/bump SMOKE_MIGRATION_BASELINE)"
|
||||
exit 1
|
||||
fi
|
||||
docker stop "$B1" >/dev/null
|
||||
chmod_vol
|
||||
|
||||
# Boot 2 — the candidate migrates that on-disk schema IN PLACE. This is the gate.
|
||||
B2="$(boot "$CANDIDATE" cloudmig_cand)"
|
||||
listening=0; wait_listen "$B2" && listening=1
|
||||
logs="$(docker logs "$B2" 2>&1 || true)"
|
||||
echo "::group::candidate migration boot logs"; printf '%s\n' "$logs"; echo "::endgroup::"
|
||||
|
||||
# 'no such column'/'no such table' is the exact index-before-ADD-COLUMN crash;
|
||||
# 'panic' catches any generic Mount failure. The load-bearing check is the
|
||||
# "listening" assertion below — a migrate() crash exits before it.
|
||||
if printf '%s' "$logs" | grep -Eiq 'panic|no such column|no such table'; then
|
||||
echo "MIGRATION SMOKE FAIL: candidate logged a DDL/migration error over the prior schema (the v1.800.1-class regression)"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$listening" -ne 1 ]; then
|
||||
echo "MIGRATION SMOKE FAIL: candidate did NOT reach \"listening\" over the $BASELINE schema — a subsystem's migrate() crashes on a pre-existing store"
|
||||
exit 1
|
||||
fi
|
||||
echo "MIGRATION SMOKE PASS: candidate booted to \"listening\" over the $BASELINE on-disk schema"
|
||||
|
||||
- name: Push (cache hit from the smoke build — publishes the tested image)
|
||||
id: push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
# Push ONLY the immutable, per-commit sha- tag (always unique — never races)
|
||||
# and the floating latest. The v<X.Y.Z> version is NOT pushed here: the
|
||||
# compute-step version may be claimed by a concurrent release between compute
|
||||
# and now, and pushing it would clobber that release's :vX image (mutable tag
|
||||
# corruption). The version is assigned + the proven sha-image retagged to it
|
||||
# ATOMICALLY in the Tag step below, so :vX exists iff its git tag exists.
|
||||
tags: |
|
||||
ghcr.io/hanzoai/cloud:sha-${{ steps.ver.outputs.sha_short }}
|
||||
ghcr.io/hanzoai/cloud:latest
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
# SAME artifact digests as the smoke build → every layer is a cache hit from
|
||||
# step 1 and the pushed image is byte-identical to the one smoke proved.
|
||||
build-args: |
|
||||
CONSOLE_IMAGE=${{ steps.artifacts.outputs.console_image }}
|
||||
SKILLS_IMAGE=${{ steps.artifacts.outputs.skills_image }}
|
||||
FLAGS_IMAGE=${{ steps.artifacts.outputs.flags_image }}
|
||||
secrets: |
|
||||
GIT_AUTH_TOKEN=${{ secrets.GH_PAT }}
|
||||
|
||||
# THE RECEIPT + ATOMIC VERSION ASSIGNMENT (race-safe). Reached only because
|
||||
# build + smoke + push all succeeded, so a proven image exists under the unique
|
||||
# sha- tag. Here we assign the next FREE v<X.Y.Z> and retag that proven image to
|
||||
# it — atomically, with the git-tag push as the serialization point:
|
||||
# * Recompute the next version FRESH (compute-step's value may have been claimed
|
||||
# by a concurrent release in the build window).
|
||||
# * If that version's git tag already exists, bump and retry.
|
||||
# * Retag the proven sha-image → :vX (+ :X.Y.Z + :X.Y) via imagetools (metadata
|
||||
# only, NO rebuild — byte-identical to the smoke-passed image).
|
||||
# * Push the git tag; the FIRST pusher of vX wins, a loser deletes its local tag
|
||||
# and recomputes. So concurrent releases each grab a distinct free number and
|
||||
# the invariant "git tag vX ⇔ image :vX pushed+smoke-passed" holds under race.
|
||||
- name: Tag the proven image (atomic free-version assignment — race-safe)
|
||||
id: tag
|
||||
env:
|
||||
GH_PAT: ${{ secrets.GH_PAT }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git config user.name "hanzo-dev"
|
||||
git config user.email "dev@hanzo.ai"
|
||||
SHA_IMG="ghcr.io/hanzoai/cloud:sha-${{ steps.ver.outputs.sha_short }}"
|
||||
PUSH_URL="https://x-access-token:${GH_PAT}@github.com/${GITHUB_REPOSITORY}.git"
|
||||
for attempt in $(seq 1 8); do
|
||||
git fetch --tags --force --quiet
|
||||
git_max="$(git tag -l 'v[0-9]*.[0-9]*.[0-9]*' \
|
||||
| sed 's/^v//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1 || true)"
|
||||
# Newest page only — NOT --paginate. Container versions are created
|
||||
# newest-first and version tags are monotonic, so the highest version
|
||||
# is always among the most-recent versions; paginating the WHOLE
|
||||
# registry history is what livelocked this step as tags accumulated.
|
||||
# Fail-CLOSED. An ORPHANED container tag — image pushed by a run that
|
||||
# died or was cancelled after imagetools-create but before its git tag —
|
||||
# MUST raise the floor, or a later run reassigns that same number to a
|
||||
# different image (an ambiguous mutable prod tag; the v1.801.50 flip). A
|
||||
# git-only floor can't see the orphan, so if the container-tag lookup
|
||||
# ERRORS (vs legitimately returning no tags) we retry the whole attempt
|
||||
# rather than silently proceeding — a version with a pushed image is never
|
||||
# reused. (Reordering git-tag before imagetools-create is the WRONG fix: it
|
||||
# reintroduces the phantom "tag ⇔ no image" this workflow exists to prevent.)
|
||||
cont_max=""
|
||||
if command -v gh >/dev/null 2>&1; then
|
||||
if cont_raw="$(GH_TOKEN="$GH_PAT" gh api \
|
||||
'/orgs/hanzoai/packages/container/cloud/versions?per_page=100' \
|
||||
--jq '.[].metadata.container.tags[]?' 2>/dev/null)"; then
|
||||
cont_max="$(printf '%s\n' "$cont_raw" \
|
||||
| sed 's/^v//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1 || true)"
|
||||
else
|
||||
echo " container-tag lookup failed — retry so an orphaned tag can't be reused (attempt $attempt)"; sleep 3; continue
|
||||
fi
|
||||
fi
|
||||
max="$(printf '%s\n%s\n%s\n' "1.786.0" "$git_max" "$cont_max" \
|
||||
| grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1)"
|
||||
major="${max%%.*}"; rest="${max#*.}"; minor="${rest%%.*}"; patch="${rest##*.}"
|
||||
VER="${major}.${minor}.$((patch + 1))"; V="v${VER}"
|
||||
if git rev-parse -q --verify "refs/tags/$V" >/dev/null; then
|
||||
echo " $V already tagged — recomputing (attempt $attempt)"; sleep 3; continue
|
||||
fi
|
||||
docker buildx imagetools create \
|
||||
-t "ghcr.io/hanzoai/cloud:${V}" \
|
||||
-t "ghcr.io/hanzoai/cloud:${VER}" \
|
||||
-t "ghcr.io/hanzoai/cloud:${major}.${minor}" \
|
||||
"$SHA_IMG"
|
||||
# Dual-host: mirror the release tags to OUR fleet registry (server-
|
||||
# side copy) so the cluster never depends on GHCR to deploy. crane,
|
||||
# not buildx imagetools: the IAM token realm doesn't answer buildx's
|
||||
# multi-scope token request (spec gap, tracked), crane's single-scope
|
||||
# flow works. Best-effort — a mirror hiccup never blocks the receipt.
|
||||
if [ "${MIRROR_OK:-}" = "1" ]; then
|
||||
command -v crane >/dev/null 2>&1 || {
|
||||
curl -fsSL https://github.com/google/go-containerregistry/releases/download/v0.20.2/go-containerregistry_Linux_x86_64.tar.gz \
|
||||
| tar -xz -C "$HOME/.local/bin" crane 2>/dev/null || {
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
curl -fsSL https://github.com/google/go-containerregistry/releases/download/v0.20.2/go-containerregistry_Linux_x86_64.tar.gz \
|
||||
| tar -xz -C "$HOME/.local/bin" crane
|
||||
}
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
}
|
||||
for MT in "${V}" "${VER}" "${major}.${minor}"; do
|
||||
# Bounded: registry.hanzo.ai can *hang* (not just fail), and this
|
||||
# is best-effort — an unbounded crane copy once livelocked the whole
|
||||
# tag step and held the serialized release lane. timeout makes the
|
||||
# mirror truly best-effort so the git-tag receipt below always runs.
|
||||
timeout 120 crane copy "$SHA_IMG" "registry.hanzo.ai/hanzoai/cloud:${MT}" \
|
||||
|| echo "::warning::mirror registry.hanzo.ai/hanzoai/cloud:${MT} failed or timed out"
|
||||
done
|
||||
fi
|
||||
git tag -a "$V" -m "release $V — image ghcr.io/hanzoai/cloud:$V (retagged from sha-${{ steps.ver.outputs.sha_short }}, smoke-passed ${GITHUB_SHA})"
|
||||
if git push "$PUSH_URL" "$V" 2>/dev/null; then
|
||||
echo "Tagged $V → ghcr.io/hanzoai/cloud:$V"
|
||||
echo "version=${VER}" >> "$GITHUB_OUTPUT"
|
||||
echo "version_v=${V}" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo " push of $V lost the race — recomputing (attempt $attempt)"
|
||||
git tag -d "$V" >/dev/null 2>&1 || true
|
||||
sleep 3
|
||||
done
|
||||
echo "::error::could not acquire a free version tag after 8 attempts"
|
||||
exit 1
|
||||
|
||||
# ── Promote: the declared-tag bump that makes the release DEPLOY ─────────────
|
||||
# The tag minted above is the receipt for a pushed, smoke-passed image; THIS job
|
||||
# records it as the desired state Hanzo CD reconciles. The universe-crs ArgoCD
|
||||
# Application (ns hanzo-cd, `automated` sync + selfHeal) syncs
|
||||
# infra/k8s/operator/crs/*.yaml → cluster and the operator rolls the Deployment,
|
||||
# so a tag bump committed here reaches api.hanzo.ai with NO hand-dispatch and NO
|
||||
# hand-edit of the CR.
|
||||
#
|
||||
# This is the SAME yq-bump → `deploy(<svc>): <tag>` universe commit the hanzoai/ci
|
||||
# reusable (build.yml deploy step) does for every other service. cloud owns it
|
||||
# HERE because its image is built by this workflow, not the ci reusable — its
|
||||
# hanzo.yml carries no main `images:` entry and `# NO deploy`, so the shared
|
||||
# deploy step never bumps cloud's CR. A direct in-cluster CR patch is NOT enough:
|
||||
# ArgoCD selfHeal reverts any live edit not also recorded in git within ~45s.
|
||||
# The retired notify-universe repository_dispatch had no receiver after the
|
||||
# image-update.yml deploy hub was deleted in the Hanzo CD cutover; the git commit
|
||||
# IS the sanctioned path now.
|
||||
promote:
|
||||
needs: build-amd64
|
||||
# Only a real release promotes: build+smoke+push+tag all succeeded, so a
|
||||
# proven v* image exists. A failure earlier leaves version_v empty → skipped.
|
||||
if: ${{ needs.build-amd64.outputs.version_v != '' }}
|
||||
runs-on: [hanzo-build-linux-amd64]
|
||||
steps:
|
||||
- name: Record the proven tag in universe crs/cloud.yaml (Hanzo CD rolls it)
|
||||
env:
|
||||
# GH_PAT already pushes this repo's git tags above (contents:write on the
|
||||
# hanzoai org), so it writes hanzoai/universe too — the SAME token the ci
|
||||
# reusable falls back to for the universe deploy commit.
|
||||
GH_PAT: ${{ secrets.GH_PAT }}
|
||||
VERSION_V: ${{ needs.build-amd64.outputs.version_v }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[ -n "${GH_PAT:-}" ] || { echo "::error::no GH_PAT — cannot record the declared-tag bump in universe"; exit 1; }
|
||||
|
||||
# Bare arc runners ship no yq — provision the static binary (sudo-free,
|
||||
# same pattern the ci reusable and this workflow's kubectl/crane fetches use).
|
||||
if ! command -v yq >/dev/null 2>&1; then
|
||||
mkdir -p "$HOME/.local/bin"; export PATH="$HOME/.local/bin:$PATH"
|
||||
curl -fsSL https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 \
|
||||
-o "$HOME/.local/bin/yq" && chmod +x "$HOME/.local/bin/yq"
|
||||
fi
|
||||
|
||||
git clone -q --depth 1 \
|
||||
"https://x-access-token:${GH_PAT}@github.com/hanzoai/universe.git" \
|
||||
"$RUNNER_TEMP/universe"
|
||||
CR="$RUNNER_TEMP/universe/infra/k8s/operator/crs/cloud.yaml"
|
||||
[ -f "$CR" ] || { echo "::error::crs/cloud.yaml not found in universe"; exit 1; }
|
||||
|
||||
CUR="$(yq -r '.spec.image.tag // ""' "$CR")"
|
||||
echo "cloud CR: ${CUR:-<empty>} → ${VERSION_V}"
|
||||
|
||||
# Monotonic guard: never roll the CR BACKWARD. Release runs finish under a
|
||||
# serialized lane but a slow older run must never overwrite a newer promote.
|
||||
# Skip iff the CR already holds a semver >= the version we just cut.
|
||||
CURN="${CUR#v}"; NEWN="${VERSION_V#v}"
|
||||
if printf '%s' "$CURN" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then
|
||||
top="$(printf '%s\n%s\n' "$CURN" "$NEWN" | sort -V | tail -1)"
|
||||
if [ "$top" = "$CURN" ] && [ "$CURN" != "$NEWN" ]; then
|
||||
echo "::notice::cloud CR already at v${CURN} (≥ ${VERSION_V}) — not rolling back"; exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
yq -i ".spec.image.tag = \"${VERSION_V}\"" "$CR"
|
||||
if git -C "$RUNNER_TEMP/universe" diff --quiet; then
|
||||
echo "::notice::crs/cloud.yaml already at ${VERSION_V} — nothing to record"; exit 0
|
||||
fi
|
||||
git -C "$RUNNER_TEMP/universe" -c user.name=hanzo-ci -c user.email=dev@hanzo.ai \
|
||||
commit -qam "deploy(cloud): ${VERSION_V} (${GITHUB_REPOSITORY}@$(echo "${GITHUB_SHA}" | cut -c1-7))"
|
||||
|
||||
# Rebase-safe push: universe main advances on every service's deploy, so a
|
||||
# concurrent commit must not make cloud's promote lose the whole roll. Retry
|
||||
# a few times, rebasing between attempts.
|
||||
for attempt in $(seq 1 5); do
|
||||
if git -C "$RUNNER_TEMP/universe" push -q origin HEAD:main; then
|
||||
echo "recorded deploy(cloud): ${VERSION_V} — Hanzo CD (universe-crs) will roll it to api.hanzo.ai"
|
||||
exit 0
|
||||
fi
|
||||
echo " universe push lost the race — rebasing (attempt ${attempt})"
|
||||
git -C "$RUNNER_TEMP/universe" pull -q --rebase origin main || true
|
||||
sleep 3
|
||||
done
|
||||
echo "::error::could not record the cloud tag bump in universe after 5 attempts"; exit 1
|
||||
@@ -13,6 +13,14 @@
|
||||
coverage.txt
|
||||
coverage.html
|
||||
|
||||
# cek encryption sidecars are key material — never commit one from a source tree.
|
||||
# Tests seal their stores under t.TempDir(); a sidecar in a package dir (e.g. from a
|
||||
# test that opened ":memory:") is a mistake. The cek testdata fixtures are the one
|
||||
# intentional exception.
|
||||
*.dek
|
||||
*.cek.lock
|
||||
!cek/testdata/**
|
||||
|
||||
# Environment files
|
||||
.env
|
||||
.env.*
|
||||
|
||||
@@ -1,52 +1,84 @@
|
||||
name: containment
|
||||
|
||||
# Guards the Stage-1 byzantine ceremony containment (clients/controlplane,
|
||||
# build tag `controlplane`). Its increment-1 crypto is stub/forgeable BY
|
||||
# DESIGN (SHA256-of-public-inputs commitments, symmetric-HMAC
|
||||
# proof-of-possession, seed-derived threshold shares — see
|
||||
# clients/controlplane/doc.go) and MUST NEVER reach a release/serve binary.
|
||||
# Three independent checks; any one failing blocks the PR:
|
||||
name: CI/CD
|
||||
# The ONE pipeline for cloud, on our own runners against git.hanzo.ai.
|
||||
#
|
||||
# 1. grep (tag) — no build/release invocation anywhere in the repo
|
||||
# (Dockerfile, Makefile, shell scripts, any workflow) may pass a `-tags`
|
||||
# value containing `controlplane` to go build/vet/run/install. The
|
||||
# package's own `//go:build controlplane` tag declarations
|
||||
# (clients/controlplane/*) are the thing being guarded, not a violation,
|
||||
# and are excluded by path.
|
||||
# 2. grep (spoof) — no build/release invocation may pass
|
||||
# `-X testing.testBinary=1` (or any -ldflags containing it) to a REAL
|
||||
# `go build`. That linker flag is what `go test` itself uses to make
|
||||
# testing.Testing() report true (cmd/go/internal/load/test.go) — the
|
||||
# runtime guard in containment.go trusts that signal, so this is the one
|
||||
# concrete way to spoof it in a non-test binary. This grep is what turns
|
||||
# "someone could type this" into "CI fails the PR that types it".
|
||||
# 3. build — `go build ./...` (no tag — exactly what the Dockerfile
|
||||
# and Makefile run) must not link clients/controlplane into any cmd/
|
||||
# main, and `go build ./clients/controlplane/...` with no tag must match
|
||||
# zero buildable packages (proves the tag still gates every file in it).
|
||||
# THE LAW: `.github/workflows` holds exactly one file — a sync nudge that runs
|
||||
# zero CI. Everything that gates, builds or deploys lives here.
|
||||
#
|
||||
# Runtime belt-and-suspenders (defense in depth, not a substitute for the
|
||||
# above): clients/controlplane/containment.go fail-closed panics if its stub
|
||||
# crypto is ever constructed outside a go-test binary (testing.Testing()==
|
||||
# false) — see TestContainment_NonHarnessProcessRefuses in
|
||||
# containment_test.go. KNOWN RESIDUAL: testing.Testing() is a linker-set
|
||||
# string var (testing.testBinary), not cryptographically bound to "actually
|
||||
# is a test" — `-ldflags="-X testing.testBinary=1"` spoofs it in a real
|
||||
# binary. Check #2 above is the mitigation: it fails the PR that would ship
|
||||
# that flag. Closing the residual for real needs a signal `go build` cannot
|
||||
# produce at all (increment-2, tracked in doc.go) rather than one merely
|
||||
# absent by convention.
|
||||
|
||||
# It absorbs the three pipelines that used to run beside each other:
|
||||
#
|
||||
# .github/workflows/cicd.yml → `gate` (unchanged: the ~7-line caller; every
|
||||
# real detail still lives in the repo-root hanzo.yml, which
|
||||
# platform.hanzo.ai reads too, so no build logic moved)
|
||||
# .github/workflows/containment.yml → `containment` (verbatim; only its own
|
||||
# self-exclusion path follows the file)
|
||||
# .hanzo/workflows/deploy.yml → deleted, not moved (below)
|
||||
#
|
||||
# CI gates. It does not deploy, and deliberately builds no cloud image: that
|
||||
# image and its v* tags have ONE owner, clients/platform/release.go (POST
|
||||
# /v1/runner {release:true}) — a second builder on the same commit is the
|
||||
# double-build hanzo.yml's images: block already refuses for this reason.
|
||||
# deploy.yml claimed to do both and could do neither: it shelled
|
||||
# `buildctl-daemonless.sh`, absent from the image this fleet actually serves for
|
||||
# `hanzo-build-linux-amd64` (catthehacker/ubuntu:act-24.04 —
|
||||
# universe:infra/k8s/git-runner/statefulset.yaml), read a GIT_CLONE_TOKEN secret
|
||||
# that exists on neither the repo nor the org, and ended in a `kubectl patch app`
|
||||
# that cd.hanzo.ai's selfHeal undoes on its next poll. Rollout is, and stays, a
|
||||
# reviewed tag pin in hanzoai/universe.
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
pull_request:
|
||||
# The sync (sync-from-github.yml) dispatches this workflow by name after a
|
||||
# fast-forward: a push made with the workflow token does not trigger workflows,
|
||||
# so without this trigger every synced commit would gate nothing. deploy.yml
|
||||
# declared no dispatch trigger, which is why that curl could only 404.
|
||||
workflow_dispatch:
|
||||
concurrency:
|
||||
group: cicd-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
jobs:
|
||||
controlplane-containment:
|
||||
# Test gate + the decoupled native flags staticlib image, driven by hanzo.yml.
|
||||
gate:
|
||||
uses: hanzoai/ci/.hanzo/workflows/build.yml@v2
|
||||
secrets: inherit
|
||||
|
||||
# Guards the Stage-1 byzantine ceremony containment (clients/controlplane,
|
||||
# build tag `controlplane`). Its increment-1 crypto is stub/forgeable BY
|
||||
# DESIGN (SHA256-of-public-inputs commitments, symmetric-HMAC
|
||||
# proof-of-possession, seed-derived threshold shares — see
|
||||
# clients/controlplane/doc.go) and MUST NEVER reach a release/serve binary.
|
||||
# Three independent checks; any one failing blocks the merge:
|
||||
#
|
||||
# 1. grep (tag) — no build/release invocation anywhere in the repo
|
||||
# (Dockerfile, Makefile, shell scripts, any workflow) may pass a `-tags`
|
||||
# value containing `controlplane` to go build/vet/run/install. The
|
||||
# package's own `//go:build controlplane` tag declarations
|
||||
# (clients/controlplane/*) are the thing being guarded, not a violation,
|
||||
# and are excluded by path.
|
||||
# 2. grep (spoof) — no build/release invocation may pass
|
||||
# `-X testing.testBinary=1` (or any -ldflags containing it) to a REAL
|
||||
# `go build`. That linker flag is what `go test` itself uses to make
|
||||
# testing.Testing() report true (cmd/go/internal/load/test.go) — the
|
||||
# runtime guard in containment.go trusts that signal, so this is the one
|
||||
# concrete way to spoof it in a non-test binary. This grep is what turns
|
||||
# "someone could type this" into "CI fails the PR that types it".
|
||||
# 3. graph — no cmd/ main may reach clients/controlplane through its
|
||||
# untagged import graph (`go list -deps`), and
|
||||
# `go build ./clients/controlplane/...` with no tag must match zero
|
||||
# buildable packages (proves the tag still gates every file in it).
|
||||
#
|
||||
# Runtime belt-and-suspenders (defense in depth, not a substitute for the
|
||||
# above): clients/controlplane/containment.go fail-closed panics if its stub
|
||||
# crypto is ever constructed outside a go-test binary (testing.Testing()==
|
||||
# false) — see TestContainment_NonHarnessProcessRefuses in
|
||||
# containment_test.go. KNOWN RESIDUAL: testing.Testing() is a linker-set
|
||||
# string var (testing.testBinary), not cryptographically bound to "actually
|
||||
# is a test" — `-ldflags="-X testing.testBinary=1"` spoofs it in a real
|
||||
# binary. Check #2 above is the mitigation: it fails the PR that would ship
|
||||
# that flag. Closing the residual for real needs a signal `go build` cannot
|
||||
# produce at all (increment-2, tracked in doc.go) rather than one merely
|
||||
# absent by convention.
|
||||
containment:
|
||||
runs-on: [hanzo-build-linux-amd64]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -69,7 +101,7 @@ jobs:
|
||||
. 2>/dev/null \
|
||||
| grep -v '\.git/' \
|
||||
| grep -v 'clients/controlplane/' \
|
||||
| grep -v '.github/workflows/containment.yml:'; then
|
||||
| grep -v '.hanzo/workflows/cicd.yml:'; then
|
||||
echo "::error::found a build/release invocation passing -tags controlplane — clients/controlplane's stub crypto must never enter a release/serve binary (see clients/controlplane/doc.go)"
|
||||
hits=1
|
||||
fi
|
||||
@@ -77,7 +109,7 @@ jobs:
|
||||
if grep -RnE -- 'testing\.testBinary' \
|
||||
--exclude-dir=.git --exclude-dir=node_modules --exclude-dir=.claude --exclude-dir=vendor \
|
||||
. 2>/dev/null \
|
||||
| grep -v '.github/workflows/containment.yml:'; then
|
||||
| grep -v '.hanzo/workflows/cicd.yml:'; then
|
||||
echo "::error::found a reference to testing.testBinary outside the Go toolchain itself — this is the linker var that spoofs testing.Testing() in a real (non go-test) binary; the containment.go runtime guard trusts that signal, so setting it anywhere in a real build path defeats it (see doc.go)"
|
||||
hits=1
|
||||
fi
|
||||
@@ -137,8 +169,19 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
go build ./...
|
||||
|
||||
# Containment is an IMPORT-GRAPH property, so prove it with the import
|
||||
# graph. This step used to lead with `go build ./...`, which compiles
|
||||
# AND LINKS every cmd/ main — and linking needs
|
||||
# native/flags/target/release/libhanzo_flags.a, the Rust staticlib
|
||||
# clients/flags pulls in under cgo (clients/flags/engine.go). Nothing
|
||||
# in THIS job builds it: `make native` runs in the gate job, in that
|
||||
# job's own workspace. It passed on GitHub only because the arc
|
||||
# runners reuse a workspace and target/ is gitignored, so an earlier
|
||||
# gate job's leftover .a was still sitting there; on a container
|
||||
# runner with a fresh volume every cmd/ main died at
|
||||
# `ld: cannot find .../libhanzo_flags.a`. The compile was never the
|
||||
# proof anyway — `go list -deps` reads the same untagged file set
|
||||
# without linking, and the gate job is the ONE place that builds.
|
||||
for m in $(go list ./cmd/...); do
|
||||
if go list -deps "$m" | grep -qx 'github.com/hanzoai/cloud/clients/controlplane'; then
|
||||
echo "::error::$m links clients/controlplane into a real binary — containment breach"
|
||||
@@ -153,3 +196,4 @@ jobs:
|
||||
fi
|
||||
|
||||
echo "OK: containment holds — clients/controlplane has zero buildable files by default and is linked into no cmd/ binary"
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
name: Sync from GitHub
|
||||
# git.hanzo.ai is canonical and builds (cicd.yml); development also lands on
|
||||
# github.com/hanzoai/cloud. ONE deterministic direction: an in-cluster PULL. The
|
||||
# runner reaches both ends (GitHub outbound, this Gitea via the instance URL
|
||||
# actions/checkout already uses), so the sync has no ingress dependency.
|
||||
#
|
||||
# Fast-forward ONLY: a divergence fails loudly here instead of force-pushing
|
||||
# either side.
|
||||
#
|
||||
# Inert until hanzoai/cloud stops being a Gitea pull mirror — while it is one,
|
||||
# the forge overwrites main from GitHub on its own timer and rejects the push
|
||||
# below. That conversion is also what turns Actions on here (measured today:
|
||||
# mirror: true, has_actions: false ⇒ zero native runs).
|
||||
on:
|
||||
schedule:
|
||||
- cron: '*/10 * * * *'
|
||||
workflow_dispatch: {}
|
||||
concurrency:
|
||||
group: sync-from-github
|
||||
cancel-in-progress: false
|
||||
jobs:
|
||||
ff-main:
|
||||
runs-on: [hanzo-build-linux-amd64]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: true
|
||||
- name: Fast-forward main from github.com/hanzoai/cloud
|
||||
env:
|
||||
GH_PAT: ${{ secrets.GH_PAT }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git fetch --quiet "https://x-access-token:${GH_PAT}@github.com/hanzoai/cloud.git" main
|
||||
LOCAL="$(git rev-parse HEAD)"; REMOTE="$(git rev-parse FETCH_HEAD)"
|
||||
if [ "$LOCAL" = "$REMOTE" ]; then echo "in sync at $LOCAL"; exit 0; fi
|
||||
if git merge-base --is-ancestor "$LOCAL" "$REMOTE"; then
|
||||
echo "fast-forwarding $LOCAL -> $REMOTE"
|
||||
git push origin "$REMOTE:refs/heads/main"
|
||||
# A push made with the workflow token does not trigger workflows, so
|
||||
# synced commits would never build. Dispatch the pipeline explicitly.
|
||||
curl -fsS --max-time 20 -X POST \
|
||||
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${{ github.server_url }}/v1/repos/${{ github.repository }}/actions/workflows/cicd.yml/dispatches" \
|
||||
-d '{"ref":"main"}' \
|
||||
&& echo "CI/CD dispatched" || echo "CI/CD dispatch failed (non-fatal — next direct push builds)"
|
||||
elif git merge-base --is-ancestor "$REMOTE" "$LOCAL"; then
|
||||
echo "canonical is AHEAD of GitHub — nothing to pull (never force from here)."
|
||||
else
|
||||
echo "::error::main DIVERGED between GitHub ($REMOTE) and canonical ($LOCAL) — refusing to force. Reconcile manually."
|
||||
exit 1
|
||||
fi
|
||||
+37
-8
@@ -19,11 +19,26 @@
|
||||
# Pinned to ghcr.io so BOTH buildx lanes (release.yml + platform arcbuild) pull
|
||||
# it directly; the SAME tags are mirrored to registry.hanzo.ai (S3-backed) for
|
||||
# GET-flow consumers (docker/kaniko/crane). Override any pin with
|
||||
# --build-arg <NAME>_IMAGE=… — release.yml resolves CONSOLE_IMAGE to a fresh
|
||||
# console-embed digest, exactly as CONSOLE_CACHEBUST re-fetched console before.
|
||||
ARG CONSOLE_IMAGE=ghcr.io/hanzoai/console-embed:latest
|
||||
ARG SKILLS_IMAGE=ghcr.io/hanzoai/agent-skills:latest
|
||||
ARG FLAGS_IMAGE=ghcr.io/hanzoai/cloud-flags:latest
|
||||
# --build-arg <NAME>_IMAGE=… .
|
||||
#
|
||||
# IMMUTABLE per-commit tags, never `:latest`. These defaults are LOAD-BEARING:
|
||||
# the builder that actually runs our releases is the native one (POST /v1/runner
|
||||
# → launchDirectBuild → BuildKit), and it passes no --build-arg, so whatever is
|
||||
# written here is what gets baked. `release.yml`, which the previous comment said
|
||||
# would resolve a fresh digest, is a stub and resolves nothing.
|
||||
#
|
||||
# With `:latest` the embedded console was therefore decided by WHEN the build ran,
|
||||
# not by what we shipped — and it bit: cloud v1.801.215 was built ~12 minutes
|
||||
# before console CI finished publishing the console-embed carrying v8.5.26, so a
|
||||
# release whose whole purpose was that console change silently baked the previous
|
||||
# one and shipped green. Same image, two contents, no diff to show for it.
|
||||
#
|
||||
# BUMP: when a console/skills/flags change must reach production, move its pin
|
||||
# here in the same commit that claims it. That is what makes a cloud release
|
||||
# reproducible and makes "what console is in v1.801.N" answerable from git.
|
||||
ARG CONSOLE_IMAGE=ghcr.io/hanzoai/console-embed:sha-9f7042c-amd64
|
||||
ARG SKILLS_IMAGE=ghcr.io/hanzoai/agent-skills:sha-b931a11-amd64
|
||||
ARG FLAGS_IMAGE=ghcr.io/hanzoai/cloud-flags:sha-e1ca02a-amd64
|
||||
|
||||
# ── toolchain base images: the golang + alpine FROMs below pull from our own
|
||||
# GHCR mirror (ghcr.io/hanzoai/mirror/*), pinned by digest. WHY: public.ecr.aws
|
||||
@@ -163,10 +178,21 @@ LABEL org.opencontainers.image.revision="${REVISION}" \
|
||||
# libgcc: the hanzo-flags Rust staticlib (clients/featureflags FFI) references the
|
||||
# _Unwind_* unwinder symbols; musl needs libgcc_s at load time or the binary fails
|
||||
# relocation ("Error relocating /cloud: _Unwind_GetIP: symbol not found").
|
||||
RUN apk add --no-cache ca-certificates tzdata sqlcipher-libs git libgcc \
|
||||
# tini: /cloud runs as PID 1, and PID 1 inherits every orphaned descendant in the
|
||||
# container. git is not a single process — fetch/clone fan out to git-upload-pack,
|
||||
# git-index-pack, git-rev-list and git-pack-objects. When cloud Kill()s a wedged
|
||||
# direct child (gitPackStream.Close does exactly that, correctly), those
|
||||
# grandchildren orphan and reparent to PID 1. A Go program never reaps adopted
|
||||
# orphans, so each one becomes a permanent zombie holding a PID slot.
|
||||
# Measured 2026-07-26 on worker-xl-37bw71: 18,553 zombie `git` out of 18,741
|
||||
# processes, all parented to /cloud, which drove the node to PID pressure and
|
||||
# got cloud ITSELF evicted. A zombie costs no CPU and no memory, so nothing but
|
||||
# an eviction ever surfaces it. tini reaps them.
|
||||
RUN apk add --no-cache ca-certificates tzdata sqlcipher-libs git libgcc tini \
|
||||
&& SC="$(find /usr/lib /lib -name 'libsqlcipher.so*' 2>/dev/null | sort | head -1)" \
|
||||
&& test -n "$SC" \
|
||||
&& ln -sf "$SC" /usr/lib/libsqlite3.so.0
|
||||
&& ln -sf "$SC" /usr/lib/libsqlite3.so.0 \
|
||||
&& test -x /sbin/tini
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
COPY --from=build /usr/share/zoneinfo /usr/share/zoneinfo
|
||||
COPY --from=build /etc/passwd /etc/passwd
|
||||
@@ -175,4 +201,7 @@ COPY --from=build /cloud /cloud
|
||||
COPY --from=build /smoke /smoke
|
||||
EXPOSE 8080 9090 9653
|
||||
USER 65532:65532
|
||||
ENTRYPOINT ["/cloud"]
|
||||
# tini as PID 1 forwards signals to /cloud unchanged (so SIGTERM still drains
|
||||
# normally) and reaps the orphans described above. `--` keeps cloud's own args
|
||||
# untouched; the CR passes none today, but that stays true if it ever does.
|
||||
ENTRYPOINT ["/sbin/tini", "--", "/cloud"]
|
||||
|
||||
+192
@@ -0,0 +1,192 @@
|
||||
# IAM cutover — Casdoor pod → embedded IAM in cloud (supervised)
|
||||
|
||||
Flip `hanzo.id`'s identity plane from the standalone Casdoor pod to the clean-room
|
||||
IAM rewrite embedded in this binary (`clients/iam`, `github.com/hanzoai/iam`
|
||||
**v1.33.6**). This is the last step of HIP-0106 (one binary embeds IAM + KMS + o11y).
|
||||
|
||||
**This is a single supervised session. Do not run it piecemeal or in the
|
||||
background.** Every step below is: **action → verify → rollback**. Money/auth is on
|
||||
the line — a bad flip 401s every login and every metered request.
|
||||
|
||||
The blocking code work is **done and shipped**:
|
||||
- IAM v1.33.6 serves the legacy path aliases the deployed fleet hard-codes
|
||||
(`/v1/iam/oauth/access_token`, `/v1/iam/oauth/refresh_token`, `/v1/iam/userinfo`)
|
||||
next to the canonical paths — so no hard-coded caller 404s at the flip.
|
||||
- `cloud` pins `github.com/hanzoai/iam v1.33.8` (go.mod) and compiles it in.
|
||||
|
||||
> **`iam` is NOT staged.** `stagedSubsystems` in `config.go` holds only `ingress`, so
|
||||
> the empty-`CLOUD_ENABLE` "mount everything" default — **the live posture on
|
||||
> `universe/infra/k8s/operator/crs/cloud.yaml`** — mounts the embedded IAM. Step 1 is
|
||||
> therefore a **precondition of the next deploy**, not of a later flip: a cloud that
|
||||
> boots before the store is migrated opens an EMPTY `iam2.db` and seeds only from
|
||||
> `init_data.json`. `iam_edge.go` forwards `/v1/iam/*` to the standalone pod ONLY
|
||||
> under a non-empty `--enable` that omits `iam`, which is the one way to hold cloud
|
||||
> on the old plane while Step 1 runs.
|
||||
|
||||
## Preconditions (verify before touching anything)
|
||||
|
||||
1. Live image is at or above the tag that carries embedded IAM v1.33.6
|
||||
(`universe/infra/k8s/operator/crs/cloud.yaml` `spec.image.tag`). The subsystem is
|
||||
compiled in but inert until enabled — safe to deploy ahead of the flip.
|
||||
2. `spec.replicas: 1` and `strategy: Recreate` (already required — the embedded
|
||||
store is single-writer/single-open). **`config.go` refuses to boot iam-enabled
|
||||
above 1 replica.** Never scale up with iam on.
|
||||
3. `CLOUD_DATA_DIR=/var/lib/cloud` on the RWO `cloud-api-data` PVC. The embedded IAM
|
||||
store is **`/var/lib/cloud/iam/iam2.db`** (`clients/iam/iam.go` `paths()`) — the v2
|
||||
store. `iam.db` is a different database; opening it serves the wrong identities
|
||||
without failing.
|
||||
4. You have the live Casdoor store to migrate FROM and its KMS master key:
|
||||
- encrypted sharded root: `<dir>/iam.db` + `<dir>/orgs/*/iam.db` (+ `.dek` sidecars)
|
||||
- `IAM_KMS_MASTER_KEY` = the 64-hex master key (from KMS — never an arg, never logged)
|
||||
5. `migrate-v1` is built from the **iam** repo (`github.com/hanzoai/iam`,
|
||||
`cmd/migrate-v1`, same v1.33.6 tag) with a C `sqlcipher` binary on PATH (for
|
||||
`--wal-inclusive`).
|
||||
|
||||
## Step 1 — Migrate the live store into the embedded datadir (BEFORE any seed)
|
||||
|
||||
The embedded IAM seeds new-only from `init_data.json`. Migrating real rows must
|
||||
happen **before** the subsystem ever seeds, or the seed masks/collides with them.
|
||||
The source is opened **read-only** — the live Casdoor pod is untouched.
|
||||
|
||||
**1a. Dry-run = the drift/parity gate.** `--dry-run` runs the full extraction and
|
||||
prints the per-entity report WITHOUT writing. Require every entity's count to match
|
||||
the live source and **drift = 0** before proceeding.
|
||||
|
||||
```
|
||||
migrate-v1 \
|
||||
--src-datadir /path/to/live/casdoor/store \
|
||||
--src-master-key-env IAM_KMS_MASTER_KEY \
|
||||
--wal-inclusive \
|
||||
--dest /var/lib/cloud/iam/iam2.db \
|
||||
--dry-run
|
||||
```
|
||||
|
||||
- `--wal-inclusive` checkpoints each shard's uncheckpointed `-wal` via the C
|
||||
sqlcipher binary → COMPLETE extraction. Without it, uncheckpointed WAL rows are a
|
||||
hard error (or, with `--ignore-wal`, silently dropped — do NOT use for a real cutover).
|
||||
- `--dest` accepts either form and both land on the same file here (`storePath` in
|
||||
`iam/cmd/migrate-v1/main.go`): a `.db` path is taken verbatim, anything else is
|
||||
treated as a data-dir and gets `/iam2.db` appended. So `…/iam/iam2.db` and `…/iam`
|
||||
are equivalent. What matters is that the written file is exactly
|
||||
`/var/lib/cloud/iam/iam2.db` — the path `clients/iam` opens.
|
||||
|
||||
**Verify:** dry-run report shows expected counts for users, orgs, applications,
|
||||
providers, certs; zero drift; zero errors.
|
||||
**Rollback:** none needed — nothing written.
|
||||
|
||||
**1b. Real migration.** Same command **without** `--dry-run`, writing into the
|
||||
(empty) embedded datadir on the cloud PVC. Do this while iam is still staged OFF.
|
||||
|
||||
**Verify:** re-run `--dry-run` against `--src /var/lib/cloud/iam/iam2.db` (or open it
|
||||
read-only) and confirm counts equal the source.
|
||||
**Rollback:** `rm -f /var/lib/cloud/iam/iam2.db*` (only the freshly-written store) and
|
||||
re-run. Nothing else consumes it until cloud next boots iam-enabled.
|
||||
|
||||
## Step 2 — Boot cloud with the embedded IAM subsystem
|
||||
|
||||
`iam` is **not** staged, so the live CR's empty `CLOUD_ENABLE` already enables it —
|
||||
there is no env var to add. Deploying the image IS this step. Nothing here is
|
||||
additive or reversible by a flag: plan Step 1 to complete before the next roll.
|
||||
|
||||
Apply the CR; the operator rolls the Recreate Deployment (single pod, brief blip —
|
||||
expected). On boot, `clients/iam` opens `/var/lib/cloud/iam/iam2.db` (the migrated
|
||||
store), seeds new-only from `init_data.json` (idempotent — real rows already present,
|
||||
so seed only adds anything genuinely missing), and mounts the full `/v1/iam/*` surface
|
||||
IN-PROCESS. `iam_edge.go` stops mounting (`serve.go`: `if !cfg.Enabled("iam") …`), so
|
||||
there is **no double-mount** and no forward to the Casdoor pod.
|
||||
|
||||
**Verify (still on the internal Service, before repointing the edge):**
|
||||
```
|
||||
kubectl -n hanzo exec deploy/cloud -- \
|
||||
curl -s localhost:8000/v1/iam/.well-known/openid-configuration | jq .issuer
|
||||
# → "https://hanzo.id"
|
||||
kubectl -n hanzo logs deploy/cloud | grep 'iam embedded in-process'
|
||||
```
|
||||
A boot failure serves fail-closed 503 on `/v1/iam/*` (cloud + every other subsystem
|
||||
stay up) — it does NOT crash the binary.
|
||||
**Rollback:** set `CLOUD_ENABLE` to an explicit list that OMITS `iam` and re-apply the
|
||||
CR. Deleting an env var does not roll this back — an empty `CLOUD_ENABLE` is
|
||||
iam-ENABLED. An explicit list is an allowlist, so it must name every other subsystem
|
||||
this deployment serves; take it from the CR's own subsystem set, not from memory.
|
||||
With `iam` out of that list the edge re-mounts and forwards to Casdoor again.
|
||||
hanzo.id is unaffected (still pointed at Casdoor until Step 3).
|
||||
|
||||
## Step 3 — Repoint the hanzo.id identity backend at the edge
|
||||
|
||||
`universe/infra/k8s/ingress/routes.yaml`: router `hanzo-id-iam-api` (priority 100)
|
||||
matches `Host(hanzo.id) && (PathPrefix(/v1/iam) || PathPrefix(/oauth) ||
|
||||
PathPrefix(/.well-known))` → `service: iam-hanzo-ai`. Repoint that **service** from
|
||||
the Casdoor pod to embedded cloud:
|
||||
|
||||
```yaml
|
||||
iam-hanzo-ai:
|
||||
loadBalancer:
|
||||
passHostHeader: true
|
||||
servers:
|
||||
- url: http://cloud.hanzo.svc.cluster.local:8000 # was: http://iam.hanzo.svc.cluster.local:80
|
||||
```
|
||||
|
||||
Leave the `hanzo-id` service (`id.hanzo.svc:80`, the @hanzo/id login SPA) as-is — only
|
||||
the API backend moves. The ingress file-provider applies the ConfigMap edit **HOT**
|
||||
(fsnotify) — **do NOT `rollout restart deploy/ingress`** (that triggers the per-node
|
||||
ACME storm / TLS outage documented in `universe/CLAUDE.md`).
|
||||
|
||||
**Verify:** run the Step-4 parity checks against the public host `https://hanzo.id`.
|
||||
**Rollback:** revert the one `url:` back to `http://iam.hanzo.svc.cluster.local:80`;
|
||||
hot-reapplies in seconds. Instant, complete rollback to Casdoor.
|
||||
|
||||
## Step 4 — Playwright / curl parity (drive it, don't just curl a status)
|
||||
|
||||
Against `https://hanzo.id` (through the repointed edge). Use Playwright for the browser
|
||||
login (real interaction, not an HTTP status peek):
|
||||
|
||||
1. **Discovery + JWKS**: `/.well-known/openid-configuration`,
|
||||
`/v1/iam/.well-known/openid-configuration`, `/v1/iam/.well-known/jwks` — issuer
|
||||
`https://hanzo.id`, keys present.
|
||||
2. **Browser login → code → token** (Playwright): `/login/oauth/authorize` → sign in
|
||||
→ callback with `code` → token exchange → a verifiable JWT; the app authenticates.
|
||||
3. **client_credentials** (KMS bridge / gateway guards shape) at BOTH
|
||||
`/v1/iam/oauth/token` and the alias `/v1/iam/oauth/access_token` → 200 + token.
|
||||
4. **refresh** at the alias `/v1/iam/oauth/refresh_token` (the `hanzo` CLI shape) → 200
|
||||
+ rotated token.
|
||||
5. **userinfo** at BOTH `/v1/iam/oauth/userinfo` and the alias `/v1/iam/userinfo`
|
||||
(commerce shape) → same principal.
|
||||
6. **Real callers**: force one live KMS-bridge / gateway-guard token fetch and one
|
||||
`hanzo login` + `hanzo` CLI refresh against hanzo.id → all 200.
|
||||
|
||||
**Accepted delta (not a regression):** BARE `/oauth/token|access_token|userinfo`
|
||||
(no `/v1/iam` prefix) on hanzo.id 404 post-cutover — the rewrite serves the
|
||||
`/v1/iam/…`-prefixed canonical + alias paths only, discovery advertises those, and no
|
||||
live caller uses the bare form (grep-verified: every fleet caller uses `/v1/iam/oauth/*`
|
||||
or an app-local `/oauth/*` proxy that rewrites to `/v1/iam/*`). Bare `/oauth/authorize`
|
||||
still 302s to the login SPA via the priority-150 router (unchanged). Optionally tighten
|
||||
the `hanzo-id-iam-api` rule to drop the bare `/oauth` prefix in the same edit.
|
||||
|
||||
**If any parity check fails: roll back Step 3 immediately** (one `url:` revert) and
|
||||
diagnose with iam still embedded-but-unrouted.
|
||||
|
||||
## Step 5 — Retire the standalone Casdoor `iam` (final, only after parity holds)
|
||||
|
||||
With hanzo.id served by embedded IAM and parity green, remove the standalone Casdoor
|
||||
workload. It is operator-managed via its App/CR
|
||||
(`universe/infra/k8s/operator/crs/iam.yaml`, legacy
|
||||
`hanzo-operator/crs/iam.yaml` + `iam-v1.yaml`). **First scale to 0** (reversible),
|
||||
soak, then delete the CR + drop its basename from the Hanzo CD
|
||||
`universe-crs` `include` glob (so ArgoCD stops governing it).
|
||||
|
||||
**Verify:** hanzo.id fully green with the Casdoor pod at 0 replicas for a full soak
|
||||
(logins, token refresh, metered `/v1/*` traffic). Then delete.
|
||||
**Rollback (pre-delete):** scale the Casdoor Deployment back to 1 and revert Step 3's
|
||||
edge `url:` — hanzo.id is back on Casdoor in seconds. **After** the CR is deleted this
|
||||
is no longer a one-step rollback (re-apply the CR from git), so hold the scale-to-0
|
||||
soak until you are certain.
|
||||
|
||||
## Guardrails (do NOT, until this supervised session)
|
||||
|
||||
- Do not roll a cloud image onto the live CR before Step 1 completes — the empty
|
||||
`CLOUD_ENABLE` mounts embedded IAM on boot, so the deploy itself performs Step 2.
|
||||
- Do not repoint `iam-hanzo-ai` before Step-2 in-process verification passes.
|
||||
- Do not delete or scale down the Casdoor `iam` workload before Step-4 parity holds.
|
||||
- Do not run `migrate-v1` against prod without the read-only source + a green `--dry-run`.
|
||||
- Do not `rollout restart deploy/ingress` (ACME storm). The routes edit is hot-reloaded.
|
||||
- Keep `replicas: 1` / `strategy: Recreate` — embedded IAM is single-writer.
|
||||
@@ -1,10 +1,40 @@
|
||||
# LLM.md — hanzoai/cloud
|
||||
|
||||
Guidance for AI agents working in this repo. `hanzoai/cloud` (HIP-0106) is ONE Go
|
||||
binary + CLI that mounts every Hanzo subsystem into a single process; the same
|
||||
artifact serves `api.hanzo.ai`, `api.lux.cloud`, `api.zoo.cloud`, `api.osage.cloud`
|
||||
and every white-label reseller. Brand, enabled subsystems, and org scope are
|
||||
deployment configuration.
|
||||
**Canonical repo.** `hanzoai/cloud` (HIP-0106) is the Open AI Cloud as ONE Go
|
||||
binary + `hanzo` CLI: every Hanzo subsystem (iam, base, kms, ai, gateway,
|
||||
commerce, o11y, tasks, …) mounted into a single multi-org process. The same
|
||||
artifact serves `api.hanzo.ai`, `api.lux.cloud`, `api.zoo.cloud`,
|
||||
`api.osage.cloud`, and every white-label reseller — brand, enabled subsystems,
|
||||
and org scope are deployment configuration. This is the impl home for the cloud
|
||||
control plane; the OpenAPI it emits at `GET /v1/openapi.json` is the single
|
||||
source for the generated per-language SDKs.
|
||||
|
||||
## Role in the SDK model
|
||||
- Full Cloud SDK is GENERATED from THIS binary's OpenAPI; SDK impl lives in
|
||||
`hanzo-<lang>/sdk`, docs/wrappers in `hanzoai/<lang>-sdk`, meta in `hanzoai/sdk`.
|
||||
- AI/agents flagship lib is separate: Python `hanzo` (`hanzoai/python-sdk`),
|
||||
Node `@hanzo/ai` (`hanzo-js/ai`). Completeness: Python > Rust > C++ > Go.
|
||||
- DRY: one impl, one place; discovery repos link OUT, never duplicate impl.
|
||||
- Full spec: `~/work/hanzo/SDK-ARCHITECTURE.md`.
|
||||
|
||||
## Brand rules (hard)
|
||||
- Hanzo is a full AI cloud, NOT a proxy — never "LLM gateway", never position vs
|
||||
LiteLLM. Zen models are our OWN family; never name upstream models.
|
||||
- `/v1/` only, never `/api/`. Voice: "Hanzo — the Open AI Cloud."
|
||||
|
||||
## Install / run
|
||||
- `docker run -p 8080:8080 ghcr.io/hanzoai/cloud:vX.Y.Z` (pin a released tag) ·
|
||||
`go install github.com/hanzoai/cloud/cmd/hanzo@latest` · `brew install hanzoai/tap/hanzo`
|
||||
- Build in MODULE mode only: `make build` / `GOWORK=off go build ./...` — never
|
||||
workspace mode (see "Build & module graph" below).
|
||||
|
||||
## Key entry points
|
||||
- `cmd/cloud` — server binary · `cmd/hanzo` (`cli/`) — control CLI · `webui.go` — embedded console
|
||||
- `apps/apps.go:Wire()` — composition root (the one ordered subsystem slice)
|
||||
- `deps.go` / `cloud.Deps` — process-wide handles · `clients/<name>/` — every subsystem
|
||||
- `openapi/` — live spec derived from the router (no checked-in spec file)
|
||||
|
||||
---
|
||||
|
||||
## Open Cloud planes
|
||||
|
||||
@@ -22,6 +52,8 @@ reverse).
|
||||
| `/v1/compute/bots` | Hosting: `@hanzo/bot` Node containers | `clients/bots` | Shipped |
|
||||
| `/v1/tasks` | Durable engine | `clients/tasks` | Shipped |
|
||||
| `/v1/gpus` + fleet | BYO GPU presence | `clients/fleet` + `clients/visor` | Shipped |
|
||||
| `/v1/cloud` | Cloud accounts: link DO/AWS/GCP/Azure, discover native k8s clusters, fold into the fleet | `clients/venue` (new) | In flight (branch `feat/cloud-account-connectors`; blue-held for red) |
|
||||
| `/v1/blueprint` | Cost: OSS-template SBOM (compose→images) + compute-cost estimate | `clients/blueprint` (new) | Shipped |
|
||||
| IAM | Identity: users, orgs, roles | IAM | Shipped |
|
||||
| KMS | Secret custody: sealed secrets | `clients/kms` | Shipped |
|
||||
|
||||
@@ -86,6 +118,16 @@ Dockerfile's dedicated `-tags libsqlite3` CGO stage, and fail under `make test`
|
||||
by design (clients/git, kms, flags, x402, cmd/kmsreseal, finance). Bundle-embed
|
||||
tests (clients/tasks/ui) need `make deploy-ui` first (real bundle is gitignored).
|
||||
|
||||
Store-heavy subsystem tests are fsync-bound, not CPU-bound. A mount opens its own
|
||||
SQLite stores, so a test that mounts several subsystems commits many times, and
|
||||
`t.TempDir()` under `/tmp` puts every commit behind the ext4 journal — on a box with
|
||||
a concurrent build the same mount that costs milliseconds idle costs ~90s, at ~0%
|
||||
CPU, blocked in `jbd2_log_wait_commit`. Point `TMPDIR` at tmpfs to measure the real
|
||||
cost: `TMPDIR=/dev/shm/t GOWORK=off go test -p 1 ./clients/guide` runs the eight-seam
|
||||
cross-subsystem harness (`clients/guide/drivehome_e2e_test.go`) in under a second.
|
||||
Prefer one package per `go test` invocation regardless: `./...` links every main
|
||||
package at once (`cmd/cloud` alone links >6GB).
|
||||
|
||||
## Framework doctrine
|
||||
|
||||
One way to do everything. Composable, orthogonal, DRY. A new subsystem is a
|
||||
@@ -226,6 +268,39 @@ package under `clients/<name>` that obeys these seams — nothing more.
|
||||
curriculum is a machine-readable contract (embedded `default.yaml`; org-custom via
|
||||
PUT replaces it) so `hanzoai/marketing` can author the full `checklist.yaml`
|
||||
against the same `Step`/`Curriculum` shape.
|
||||
- **The EXPERIMENT is a composition, not a fourth engine (`clients/experiments`,
|
||||
`/v1/experiments`).** A/B testing is ONE value whatever the variant KIND (feature
|
||||
flag, ad creative, email subject, model id): the primitive owns only the
|
||||
experiment registry (definition + decision); it COMPOSES three planes it never
|
||||
duplicates. ASSIGNMENT = `flags.Assign(org,project,key,subject,props)` —
|
||||
subject→variant is a deterministic `engineEvaluate` (sha1 rollout hash), no second
|
||||
bucketing, no assignment store; create writes a multivariate flag def
|
||||
(`flags.PutDef`) and decide rewrites its weights to 100% for the winner
|
||||
(`flags.GetDef`+`PutDef`). MEASUREMENT = `analytics.Outcomes(...)` — one
|
||||
org-scoped `hanzo.events` query (the `eventsWhere` isolation invariant), no second
|
||||
event store; the analyze fold joins each subject's analytics outcome to its flags
|
||||
variant by `distinct_id`. EVIDENCE = `research.Record`/`research.List` — per-variant
|
||||
samples land as immutable `kind:"ab"` rows; significance (two-proportion z-test,
|
||||
`math.Erfc`, no dep) is a PURE function over them. `clients/campaign` runs a
|
||||
creative A/B by composing `experiments.Assign`/`experiments.Analyze` — it never
|
||||
reinvents assignment or evidence. Add a new variant KIND by putting a payload on
|
||||
the variant; the primitive does not care what it is.
|
||||
- **The OSS-template compute cost is DERIVED from the compose, not a fourth ledger
|
||||
(`clients/blueprint`, `/v1/blueprint`).** A blueprint's `docker-compose.yml` is
|
||||
parsed to its SBOM (the bill of container images) and its services' CPU/memory
|
||||
footprint priced through ONE documented rate card (microdollars per vCPU-/GB-hour,
|
||||
DigitalOcean-droplet-derived + platform margin; tunable via
|
||||
`CLOUD_BLUEPRINT_UCPU_HR`/`_UGB_HR`). Sizing is the declared
|
||||
`deploy.resources.reservations`/`limits` (or legacy `cpus`/`mem_*`) else a default
|
||||
footprint per inferred class (db/cache/web/worker/other). `blueprint.EstimateTemplate(id)`
|
||||
returns `{sbom, vcpuHr, gbHr, microUsdPerHour, estCentsPerMonth}`: `estCentsPerMonth`
|
||||
is the "~$X/mo to run" the console shows; `microUsdPerHour` is the exact rate the
|
||||
deploy path meters the deploying org on via the SAME commerce spine `resource_billing`
|
||||
uses. The author royalty (`clients/authors`, `defaultShareBps=2000`) already accrues
|
||||
20% of a deploying org's metered spend — this plane only DEFINES the compute component
|
||||
of that spend from a real rate card; it never touches the ledger or the accrual sweep.
|
||||
Distinct from `clients/sbom` (CycloneDX packages INSIDE one image, keyed by digest);
|
||||
this is the bill of IMAGES a stack runs, keyed by template.
|
||||
|
||||
## Identity vocabulary is IAM-native
|
||||
|
||||
@@ -300,3 +375,42 @@ app; `?org=` cannot widen it). The rolling restart needs `patch` on `apps/deploy
|
||||
here (TS-Dokploy contract, 404), and `/v1/platform/*` needs a co-resident IAM store this
|
||||
deployment does not fold in (IAM runs as a separate svc) so it 500s; the live apps backend
|
||||
is `/v1/paas`, whose board reads k8s directly with no IAM-store dependency.
|
||||
|
||||
## GTM: `/v1/campaign` orchestration → channels → connectors → analytics
|
||||
|
||||
The go-to-market stack decomplects a campaign from its execution. A **Campaign is a
|
||||
VALUE** (`clients/campaign`: `{name, audience, content[], schedule, budget, channels[],
|
||||
status}`) that SPANS channels; a **Channel is an EXECUTOR** (`channel.go`, the
|
||||
`Channel` interface) it fans out to. The three channels are orthogonal and each
|
||||
CONSUMES the connector plane via `integrations.TokenFor` — the campaign object never
|
||||
touches a credential:
|
||||
|
||||
- **paid → `/v1/ads`** — `ads.LaunchPaid/PaidSpend/PausePaid` (`clients/ads/provider.go`)
|
||||
resolve the org's ad token (`meta_ads`/`google_ads`/… via `TokenFor(org, <id>,
|
||||
"access_token")`) and run the campaign on the provider. Meta is executed for real;
|
||||
fail-closed when the org has not connected (424). This is the ONLY place `/v1/ads`
|
||||
touches the connector plane.
|
||||
- **organic → `/v1/publish`** (rename of `clients/social`) and **email → `/v1/marketing`**
|
||||
are DESIGNED follow-ons: register their executors the same way in `apps/wire_seams.go`
|
||||
(`campaign.RegisterChannel(campaign.NewChannel(kind, launch, spend, pause))`). Until
|
||||
wired, a fan-out records that channel "unavailable" (honest), never fabricated.
|
||||
|
||||
Channels are injected at the composition root (`apps/wire_seams.go`), the SAME
|
||||
injected-function decoupling the coding dispatcher uses — `campaign` never imports
|
||||
`ads`, `ads` never imports `campaign`. Fan-out (`launch.go` `fanOut`) is best-effort
|
||||
per channel; the org (the ONLY tenant key) is passed verbatim to every executor, so a
|
||||
campaign can only ever resolve its OWN org's token.
|
||||
|
||||
**Metrics = the ONE analytics plane, not a second store.** `GET /v1/campaign/:id/metrics`
|
||||
reads the funnel from `analytics.CampaignMetrics(org, campaignID, variant, start, end)`
|
||||
(`clients/analytics/campaign.go`) — an org+`utm_campaign`(+`utm_content`)-scoped query
|
||||
over `hanzo.events`, org and campaign bound POSITIONALLY (same tenancy invariant as
|
||||
every analytics query) — joined with each channel connector's reported spend
|
||||
(`Channel.Spend`). Derived KPIs: CTR/CVR/CAC/ROAS. Honest-empty when the warehouse is
|
||||
absent.
|
||||
|
||||
**Creative A/B composes the experiment primitive** (`experiment.go`), it does not
|
||||
reinvent it: a creative A/B is an experiment whose variant = a creative (tagged
|
||||
`utm_content`) and whose metric = the analytics read. The `AssignFunc`/`EvidenceFunc`
|
||||
seams are wired at the root to the flags-assignment + evidence primitive; nil-safe
|
||||
until it lands (single-creative honest default).
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
Hanzo Cloud
|
||||
Copyright (c) 2026 Hanzo Industries, Inc.
|
||||
|
||||
This product includes software from the following upstream projects. Their names
|
||||
appear here and NOWHERE ELSE in this codebase's user-visible surfaces — Hanzo
|
||||
products are named for Hanzo (Hanzo Analytics, Hanzo Insights, Hanzo IAM, …).
|
||||
|
||||
Umami (https://github.com/umami-software/umami), licensed under MIT:
|
||||
|
||||
Copyright (c) 2022 Umami Software, Inc. <hello@umami.is>
|
||||
|
||||
Hanzo Analytics (analytics.hanzo.ai) is a fork. The cloud destination adapter
|
||||
speaks its public collect contract.
|
||||
|
||||
PostHog (https://github.com/PostHog/posthog), licensed under MIT:
|
||||
|
||||
Copyright (c) 2020-present PostHog Inc.
|
||||
|
||||
Hanzo Insights (insights.hanzo.ai) is a fork. The cloud destination adapter
|
||||
speaks its capture contract.
|
||||
|
||||
Casibase (https://github.com/casibase/casibase), licensed under Apache-2.0:
|
||||
|
||||
Copyright (c) The Casibase Authors
|
||||
|
||||
The Hanzo AI module (the /v1 AI, RAG, and search surfaces) derives from it.
|
||||
|
||||
Casdoor (https://github.com/casdoor/casdoor), licensed under Apache-2.0:
|
||||
|
||||
Copyright (c) The Casdoor Authors
|
||||
|
||||
Hanzo IAM (hanzo.id) derives from it.
|
||||
@@ -1,21 +1,30 @@
|
||||
<p align="center"><img src=".github/hero.svg" alt="cloud" width="880"></p>
|
||||
<p align="center"><img src=".github/hero.svg" alt="Hanzo Cloud" width="880"></p>
|
||||
|
||||
# cloud
|
||||
# Hanzo Cloud
|
||||
|
||||
Unified Go control plane and binary for the Hanzo platform (HIP-0106).
|
||||
**The Open AI Cloud as one Go binary.** Identity, secrets, data, AI, gateway, observability, and the console — every Hanzo-native subsystem mounted into a single multi-org process. Per [HIP-0106](https://github.com/hanzoai/HIPs/blob/main/HIPs/hip-0106-unified-hanzo-cloud-binary.md).
|
||||
|
||||
[]()
|
||||
[]()
|
||||
|
||||
The same artifact serves `api.hanzo.ai`, `api.lux.cloud`, `api.zoo.cloud`, `api.osage.cloud`, and every white-label reseller. Brand, enabled subsystems, and org scope are deployment configuration — one binary, one origin, no sidecars.
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
docker run -p 8080:8080 ghcr.io/hanzoai/cloud:latest
|
||||
# Run the unified binary (pin a released version)
|
||||
docker run -p 8080:8080 ghcr.io/hanzoai/cloud:v1.801.206
|
||||
|
||||
# Or install the CLI + server
|
||||
go install github.com/hanzoai/cloud/cmd/hanzo@latest
|
||||
brew install hanzoai/tap/hanzo
|
||||
```
|
||||
|
||||
Open <http://localhost:8080> for the embedded console; the API is served under `/v1` on the same origin.
|
||||
|
||||
## What this is
|
||||
|
||||
`hanzoai/cloud` is one Go binary that mounts every Hanzo subsystem (iam, kms, base, gateway, ai, commerce, vfs, mq, dns, amqp, mcp, o11y, ...) into a single multi-org process. Same artifact serves `api.hanzo.ai`, `api.osage.cloud`, `api.lux.cloud`, `api.zoo.cloud`, and every white-label reseller. Brand, enabled subsystems, and org scope are deployment configuration.
|
||||
`hanzoai/cloud` is one Go binary that mounts every Hanzo subsystem (iam, kms, base, gateway, ai, commerce, vfs, mq, dns, amqp, mcp, o11y, tasks, …) into a single multi-org process. The same artifact serves `api.hanzo.ai`, `api.osage.cloud`, `api.lux.cloud`, `api.zoo.cloud`, and every white-label reseller. Brand, enabled subsystems, and org scope are deployment configuration.
|
||||
|
||||
## `hanzo` — cloud control CLI
|
||||
|
||||
@@ -47,16 +56,31 @@ authed (it cannot validate user tokens), so `apps`/`deploy`/`clusters` use
|
||||
|
||||
Install: `go install github.com/hanzoai/cloud/cmd/hanzo@latest`, or `brew install hanzoai/tap/hanzo`.
|
||||
|
||||
## Specs
|
||||
## Subsystems mounted
|
||||
|
||||
Implements:
|
||||
- HIP-0014 Application Deployment
|
||||
- HIP-0026 IAM
|
||||
- HIP-0027 KMS
|
||||
- HIP-0037 AI Cloud Platform
|
||||
- HIP-0105 In-Process Extension Runtime
|
||||
- HIP-0106 Unified Cloud Binary
|
||||
- HIP-0302 Encrypted SQLite + ZapDB Durability
|
||||
Each subsystem exposes `func Mount(app *zip.App, deps cloud.Deps) error` and wires its own `/v1/<name>/*` routes onto the shared app.
|
||||
|
||||
- `iam` — identity & access (users, orgs, roles, OIDC/JWKS per HIP-0026)
|
||||
- `base` — per-org SQLite + in-process extension runtimes (HIP-0105)
|
||||
- `kms` — secret custody (sealed secrets, HIP-0027)
|
||||
- `commerce` — checkout, billing, pricing, invoicing (light router; NOT in PCI-DSS scope)
|
||||
- `ai` — AI control plane: inference, RAG, model hub, agents, MCP management
|
||||
- `gateway` — HTTP routing + policy
|
||||
- `o11y` — metrics / traces / logs
|
||||
- `vfs` — virtual filesystem / object-store abstraction
|
||||
- `mq` — message queue
|
||||
- `dns`, `amqp`, `mcp`, `auto`, `tasks`, … — full list per HIP-0106
|
||||
|
||||
## Deployment modes
|
||||
|
||||
Same binary; different startup configuration:
|
||||
|
||||
```bash
|
||||
cloud --enable=iam,base,kms,commerce,ai,gateway,o11y --brand=hanzo --domain=hanzo.ai
|
||||
cloud --enable=iam,base,kms,commerce,ai,gateway,o11y --brand=osage --domain=osage.cloud
|
||||
cloud --enable=iam,base,kms,commerce,ai,gateway,o11y --brand=lux --domain=lux.cloud
|
||||
cloud --enable=iam,base,kms,commerce,ai,gateway,o11y --brand=zoo --domain=zoo.cloud
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
@@ -70,49 +94,15 @@ Implements:
|
||||
| Mount() | Mount() | Mount() | Mount() | Mount() |
|
||||
+----------+----------+----------+----------+----------+
|
||||
per-org SQLite (HIP-0302) | Hanzo IAM JWKS (HIP-0026)
|
||||
replicate -> S3 (HIP-0107) | ZAP inter-subsystem RPC
|
||||
replicate -> S3 (HIP-0107) | ZAP inter-subsystem RPC
|
||||
```
|
||||
|
||||
Every subsystem exposes `func Mount(app *zip.App, deps cloud.Deps) error`. White-label fork pattern: customers fork this repo to launch their own ecosystem.
|
||||
|
||||
|
||||
---
|
||||
|
||||
# Hanzo Cloud
|
||||
|
||||
The unified Go binary that imports every Hanzo-native subsystem and dispatches
|
||||
requests per deployment configuration. One artifact, many subsystems.
|
||||
|
||||
Per [HIP-0106](https://github.com/hanzoai/HIPs/blob/main/HIPs/hip-0106-unified-hanzo-cloud-binary.md).
|
||||
|
||||
## Subsystems mounted
|
||||
|
||||
- `iam` — identity & access
|
||||
- `base` — per-org SQLite + extension runtimes (per HIP-0105)
|
||||
- `kms` — secrets
|
||||
- `commerce` — checkout, billing, pricing, invoicing (light router; NOT in PCI-DSS scope)
|
||||
- `ai` — LLM control plane / RAG / model hub / MCP management (was hanzoai/cloud pre-rename)
|
||||
- `gateway` — HTTP routing + policy
|
||||
- `o11y` — metrics / traces / logs
|
||||
- `vfs` — virtual filesystem / object-store abstraction
|
||||
- `mq` — message queue
|
||||
- `dns`, `amqp`, `mcp`, `auto`, `tasks`, ... (full list per HIP-0106)
|
||||
|
||||
## Deployment modes
|
||||
|
||||
Same binary; different startup configuration:
|
||||
|
||||
```bash
|
||||
cloud --enable=iam,base,kms,commerce,ai,gateway,o11y --brand=hanzo --domain=hanzo.ai
|
||||
cloud --enable=iam,base,kms,commerce,ai,gateway,o11y --brand=osage --domain=osage.cloud
|
||||
cloud --enable=iam,base,kms,commerce,ai,gateway,o11y --brand=lux --domain=lux.cloud
|
||||
cloud --enable=iam,base,kms,commerce,ai,gateway,o11y --brand=zoo --domain=zoo.cloud
|
||||
```
|
||||
Every subsystem mounts through the same `Mount(app, deps)` seam. Cross-subsystem calls ride a narrow in-process interface; no subsystem reaches into another's store.
|
||||
|
||||
## White-label fork pattern
|
||||
|
||||
Customers fork `hanzoai/cloud` to launch their own ecosystem in one binary. Brand
|
||||
detection, enabled subsystems, ZAP endpoints (payments / vault backends) are all
|
||||
detection, enabled subsystems, and ZAP endpoints (payments / vault backends) are all
|
||||
deployment configuration.
|
||||
|
||||
## Web framework
|
||||
@@ -153,18 +143,39 @@ even without the Node toolchain. The image build overwrites `webui/dist` with th
|
||||
real console bundle. See `webui_test.go` for the boot-and-assert tests
|
||||
(`/` → shell, deep link → shell 200, `/v1/*` → API, unmatched `/v1` → 404).
|
||||
|
||||
Current state: `hanzoai/console` exposes `build:embed` (`scripts/build-embed.mjs`),
|
||||
which stashes its Next server route handlers (BFF proxies that collapse to the
|
||||
cloud `/v1/*` the SPA calls same-origin), wraps the client catch-all pages for
|
||||
`output: 'export'`, neutralizes the root layout's request-time `headers()` read,
|
||||
and emits a real static export at `out/` (a ~360 KB `index.html` + `_next/`
|
||||
chunks). The image build (and `make webui`) run it and overlay `webui/dist`, so
|
||||
`//go:embed` bakes the FULL `@hanzo/gui` console into the ONE binary. The
|
||||
Dockerfile console stage FAILS HARD if that bundle is missing or degenerate —
|
||||
the placeholder shell can never silently ship to prod (escape hatch:
|
||||
`--build-arg ALLOW_PLACEHOLDER=1` for a pure-Go dev image).
|
||||
The `hanzoai/console` `build:embed` script (`scripts/build-embed.mjs`) stashes its
|
||||
Next server route handlers (BFF proxies that collapse to the cloud `/v1/*` the SPA
|
||||
calls same-origin), wraps the client catch-all pages for `output: 'export'`,
|
||||
neutralizes the root layout's request-time `headers()` read, and emits a real
|
||||
static export at `out/` (a ~360 KB `index.html` + `_next/` chunks). The image
|
||||
build (and `make webui`) run it and overlay `webui/dist`, so `//go:embed` bakes
|
||||
the FULL `@hanzo/gui` console into the ONE binary. The Dockerfile console stage
|
||||
FAILS HARD if that bundle is missing or degenerate — the placeholder shell can
|
||||
never silently ship to prod (escape hatch: `--build-arg ALLOW_PLACEHOLDER=1` for a
|
||||
pure-Go dev image).
|
||||
|
||||
## Specs
|
||||
|
||||
Implements:
|
||||
- HIP-0014 Application Deployment
|
||||
- HIP-0026 IAM
|
||||
- HIP-0027 KMS
|
||||
- HIP-0037 AI Cloud Platform
|
||||
- HIP-0105 In-Process Extension Runtime
|
||||
- HIP-0106 Unified Cloud Binary
|
||||
- HIP-0129 Open Cloud Planes
|
||||
- HIP-0302 Encrypted SQLite + ZapDB Durability
|
||||
|
||||
## Status
|
||||
|
||||
Scaffold. The Mount(app, deps) integration for each subsystem lands per
|
||||
HIP-0106's migration phases.
|
||||
In production. The unified binary serves `api.hanzo.ai` and the white-label cloud
|
||||
surfaces today, with per-org SQLite (HIP-0302) and the embedded console. Subsystems
|
||||
continue to land per HIP-0106's migration phases; `apps/apps.go:Wire()` is the one
|
||||
ordered list of everything mounted. For repo-level engineering doctrine (module
|
||||
graph, route-table projections, cross-subsystem seams), see [`LLM.md`](./LLM.md).
|
||||
|
||||
## Hanzo — the Open AI Cloud
|
||||
|
||||
Open source · every language · on-chain settlement. [hanzo.ai](https://hanzo.ai) · [docs.hanzo.ai](https://docs.hanzo.ai)
|
||||
|
||||
**SDKs in every language** — [Python](https://github.com/hanzoai/python-sdk) (flagship) · [TypeScript](https://github.com/hanzo-js/sdk) · [Go](https://github.com/hanzo-go/sdk) · [Rust](https://github.com/hanzo-rs/sdk) · [C++](https://github.com/hanzo-cpp/sdk) · [Swift](https://github.com/hanzo-swift/sdk) · [Kotlin](https://github.com/hanzo-kt/sdk) · [umbrella](https://github.com/hanzoai/sdk)
|
||||
|
||||
+13
-5
@@ -15,8 +15,8 @@ import (
|
||||
//
|
||||
// THE COMPLECTION IT REMOVES. The operator SPA authenticates via /v1/signin (a cloud
|
||||
// PKCE session → the X-User-* principal the middleware mints) but read its IDENTITY
|
||||
// from /v1/get-account, which the embedded IAM (casibase) answers from ITS OWN session
|
||||
// cookie. A PKCE session is not a casibase session, so get-account returned
|
||||
// from the account read, which the embedded IAM (casibase) answers from ITS OWN session
|
||||
// cookie. A PKCE session is not a casibase session, so that read returned
|
||||
// owner:"hanzo" (anonymous) or "Unauthorized operation" — and the SPA's SuperAdmin
|
||||
// gate (owner == "admin" && isAdmin), reading that, bounced the operator UI to login
|
||||
// even though the SAME session got 200 from every /v1/admin/* route. Two session
|
||||
@@ -24,16 +24,24 @@ import (
|
||||
//
|
||||
// THE DECOMPLECTION. Identity is now the principal: when a VALIDATED principal is
|
||||
// present (X-User-Id is set ONLY by IdentityMiddleware from a real credential, never a
|
||||
// raw client header — see middleware_identity.go), /v1/get-account reflects it. owner
|
||||
// raw client header — see middleware_identity.go), /v1/ai/account reflects it. owner
|
||||
// is the HOME org (principal.Owner) so a SuperAdmin org-switched into a tenant stays a
|
||||
// SuperAdmin; isAdmin is the validated bit. With NO principal it falls through
|
||||
// (c.Next()) to the casibase account surface unchanged — the anonymous sign-in page
|
||||
// and any legacy casibase-session caller are untouched. One truth, additive, fail-open
|
||||
// to the old path. MUST be registered AFTER IdentityMiddleware (needs the minted
|
||||
// headers) and BEFORE MountAll (so it precedes the casibase /v1/get-account handler).
|
||||
// headers) and BEFORE MountAll (so it precedes the casibase account handler).
|
||||
// accountPath is the account read this middleware fronts. It is a named constant
|
||||
// because the interception is a PATH MATCH: when the /v1 surface was namespaced
|
||||
// (/v1/get-account → /v1/ai/account) a literal left un-updated here would not
|
||||
// error — the middleware would simply stop firing, fall through to the casibase
|
||||
// account surface, and hand the SPA the anonymous owner again. That is precisely
|
||||
// the bug this file exists to fix, silently restored.
|
||||
const accountPath = "/v1/ai/account"
|
||||
|
||||
func AccountFromPrincipal() zip.Handler {
|
||||
return func(c *zip.Ctx) error {
|
||||
if c.Method() != http.MethodGet || c.Path() != "/v1/get-account" {
|
||||
if c.Method() != http.MethodGet || c.Path() != accountPath {
|
||||
return c.Next()
|
||||
}
|
||||
user := c.User() // X-User-Id — minted only from a validated credential
|
||||
|
||||
+109
-14
@@ -57,29 +57,36 @@ import (
|
||||
"github.com/hanzoai/cloud/clients/agents"
|
||||
"github.com/hanzoai/cloud/clients/agentskills"
|
||||
"github.com/hanzoai/cloud/clients/analytics"
|
||||
"github.com/hanzoai/cloud/clients/ask"
|
||||
"github.com/hanzoai/cloud/clients/auditlog"
|
||||
"github.com/hanzoai/cloud/clients/authors"
|
||||
"github.com/hanzoai/cloud/clients/automations"
|
||||
"github.com/hanzoai/cloud/clients/base"
|
||||
"github.com/hanzoai/cloud/clients/benchmark"
|
||||
"github.com/hanzoai/cloud/clients/billing"
|
||||
"github.com/hanzoai/cloud/clients/blueprint"
|
||||
"github.com/hanzoai/cloud/clients/books"
|
||||
"github.com/hanzoai/cloud/clients/bots"
|
||||
"github.com/hanzoai/cloud/clients/campaign"
|
||||
"github.com/hanzoai/cloud/clients/captable"
|
||||
"github.com/hanzoai/cloud/clients/catalogsync"
|
||||
"github.com/hanzoai/cloud/clients/channels"
|
||||
"github.com/hanzoai/cloud/clients/cloudflare"
|
||||
"github.com/hanzoai/cloud/clients/code"
|
||||
"github.com/hanzoai/cloud/clients/company"
|
||||
"github.com/hanzoai/cloud/clients/compliance"
|
||||
"github.com/hanzoai/cloud/clients/content"
|
||||
"github.com/hanzoai/cloud/clients/crm"
|
||||
"github.com/hanzoai/cloud/clients/dataroom"
|
||||
"github.com/hanzoai/cloud/clients/deploy"
|
||||
"github.com/hanzoai/cloud/clients/destinations"
|
||||
"github.com/hanzoai/cloud/clients/dns"
|
||||
"github.com/hanzoai/cloud/clients/do"
|
||||
"github.com/hanzoai/cloud/clients/domain"
|
||||
"github.com/hanzoai/cloud/clients/entitlements"
|
||||
"github.com/hanzoai/cloud/clients/benchmark"
|
||||
"github.com/hanzoai/cloud/clients/eval"
|
||||
"github.com/hanzoai/cloud/clients/exec"
|
||||
"github.com/hanzoai/cloud/clients/experiments"
|
||||
"github.com/hanzoai/cloud/clients/flags"
|
||||
"github.com/hanzoai/cloud/clients/framework"
|
||||
"github.com/hanzoai/cloud/clients/functions"
|
||||
@@ -87,6 +94,7 @@ import (
|
||||
"github.com/hanzoai/cloud/clients/git"
|
||||
"github.com/hanzoai/cloud/clients/graph"
|
||||
"github.com/hanzoai/cloud/clients/guide"
|
||||
"github.com/hanzoai/cloud/clients/help"
|
||||
"github.com/hanzoai/cloud/clients/iam"
|
||||
"github.com/hanzoai/cloud/clients/ingress"
|
||||
"github.com/hanzoai/cloud/clients/integrations"
|
||||
@@ -94,6 +102,7 @@ import (
|
||||
"github.com/hanzoai/cloud/clients/kms"
|
||||
"github.com/hanzoai/cloud/clients/knowledge"
|
||||
"github.com/hanzoai/cloud/clients/leaderboard"
|
||||
"github.com/hanzoai/cloud/clients/legal"
|
||||
"github.com/hanzoai/cloud/clients/link"
|
||||
"github.com/hanzoai/cloud/clients/marketing"
|
||||
"github.com/hanzoai/cloud/clients/marketplace"
|
||||
@@ -104,6 +113,7 @@ import (
|
||||
"github.com/hanzoai/cloud/clients/plan"
|
||||
"github.com/hanzoai/cloud/clients/platform"
|
||||
"github.com/hanzoai/cloud/clients/plugin"
|
||||
"github.com/hanzoai/cloud/clients/prefs"
|
||||
"github.com/hanzoai/cloud/clients/pricing"
|
||||
"github.com/hanzoai/cloud/clients/product"
|
||||
"github.com/hanzoai/cloud/clients/projects"
|
||||
@@ -111,11 +121,13 @@ import (
|
||||
"github.com/hanzoai/cloud/clients/provisioning"
|
||||
"github.com/hanzoai/cloud/clients/pubsub"
|
||||
"github.com/hanzoai/cloud/clients/referrals"
|
||||
"github.com/hanzoai/cloud/clients/research"
|
||||
"github.com/hanzoai/cloud/clients/rollingcap"
|
||||
"github.com/hanzoai/cloud/clients/runtime"
|
||||
"github.com/hanzoai/cloud/clients/sbom"
|
||||
"github.com/hanzoai/cloud/clients/security"
|
||||
"github.com/hanzoai/cloud/clients/settings"
|
||||
"github.com/hanzoai/cloud/clients/share"
|
||||
"github.com/hanzoai/cloud/clients/sign"
|
||||
"github.com/hanzoai/cloud/clients/social"
|
||||
"github.com/hanzoai/cloud/clients/storage"
|
||||
@@ -128,26 +140,29 @@ import (
|
||||
"github.com/hanzoai/cloud/clients/treasury"
|
||||
"github.com/hanzoai/cloud/clients/usage"
|
||||
"github.com/hanzoai/cloud/clients/validators"
|
||||
"github.com/hanzoai/cloud/clients/venue"
|
||||
"github.com/hanzoai/cloud/clients/visor"
|
||||
"github.com/hanzoai/cloud/clients/wallets"
|
||||
"github.com/hanzoai/cloud/clients/webhooks"
|
||||
"github.com/hanzoai/cloud/clients/websearch"
|
||||
"github.com/hanzoai/cloud/clients/world"
|
||||
"github.com/hanzoai/cloud/clients/x402"
|
||||
"github.com/hanzoai/cloud/clients/zt"
|
||||
|
||||
// Framework CONTENT modules — NOT mount subsystems (they carry no HTTP surface
|
||||
// and are absent from Wire()). Each registers its DocType fixtures and, for erp,
|
||||
// its ledger-posting lifecycle hooks into the clients/framework DocType engine
|
||||
// from a package init() (framework.RegisterModule) — the idiomatic
|
||||
// register-into-a-registry pattern (cf. database/sql drivers). The framework
|
||||
// engine is mounted (always-on, /v1/framework/*) but its module registry is
|
||||
// populated ONLY by these blank imports. Dropping one silently strips that
|
||||
// lane's DocTypes and hooks — for erp, the immutable ledger postings — from the
|
||||
// binary with NO mount change and NO failing mount test. #248 dropped them;
|
||||
// TestFrameworkContentModulesLinked now guards against a recurrence. Keep.
|
||||
// Framework CONTENT modules — pure fixture lanes that carry no HTTP surface and
|
||||
// are absent from Wire(). Each registers its DocType fixtures and, for erp, its
|
||||
// ledger-posting lifecycle hooks into the clients/framework DocType engine from a
|
||||
// package init() (framework.RegisterModule) — the idiomatic register-into-a-
|
||||
// registry pattern (cf. database/sql drivers). The framework engine is mounted
|
||||
// (always-on, /v1/framework/*) but its module registry is populated ONLY by these
|
||||
// blank imports. Dropping one silently strips that lane's DocTypes and hooks — for
|
||||
// erp, the immutable ledger postings — from the binary with NO mount change and NO
|
||||
// failing mount test. #248 dropped them; TestFrameworkContentModulesLinked now
|
||||
// guards against a recurrence. Keep. (help is a framework lane too but ALSO mounts
|
||||
// a thin /v1/help public plane, so it is a real import + a Wire() spec below,
|
||||
// alongside knowledge — the other lane with a companion subsystem.)
|
||||
_ "github.com/hanzoai/cloud/clients/cms"
|
||||
_ "github.com/hanzoai/cloud/clients/erp"
|
||||
_ "github.com/hanzoai/cloud/clients/help"
|
||||
)
|
||||
|
||||
// init wires the cross-subsystem func seams — the composition root is the one place
|
||||
@@ -242,8 +257,8 @@ func Wire() []cloud.MountSpec {
|
||||
// (121) + the commerce embed (100). Same clients/account package as "account" (48).
|
||||
{Name: "account-bridge", Mount: account.MountBridge},
|
||||
{Name: "do", Mount: do.Mount},
|
||||
{Name: "platform", Mount: platform.Mount, OwnsHealth: true},
|
||||
{Name: "projects", Mount: projects.Mount},
|
||||
{Name: "platform", Mount: platform.Mount, Shutdown: ctxShutdown(platform.Shutdown), OwnsHealth: true},
|
||||
{Name: "projects", Mount: projects.Mount, Shutdown: ctxShutdown(projects.Shutdown)},
|
||||
// The /v1/dns forward head: relays the console DNS dashboard to the DNS
|
||||
// control plane under the caller's own validated bearer (clients/dns).
|
||||
{Name: "dns", Mount: dns.Mount},
|
||||
@@ -268,8 +283,24 @@ func Wire() []cloud.MountSpec {
|
||||
{Name: "functions", Mount: functions.Mount},
|
||||
{Name: "tracker", Mount: tracker.Mount},
|
||||
{Name: "templates", Mount: templates.Mount},
|
||||
// OSS-template compute-cost basis /v1/blueprint/* — parses a blueprint's
|
||||
// docker-compose into its SBOM (bill of container images) and prices the
|
||||
// stack's CPU/memory footprint through a documented rate card. The per-hour
|
||||
// rate the deploy path meters an org on, and the "~$X/mo to run" the console
|
||||
// shows; it is the compute cost the 20% author royalty (clients/authors) is
|
||||
// taken from. OwnsHealth: serves its own /v1/blueprint/health. Reference
|
||||
// content (embedded blueprints), no store → no Shutdown. After templates, its
|
||||
// sibling catalog concern; before the AI /v1/* catch-all.
|
||||
{Name: "blueprint", Mount: blueprint.Mount, OwnsHealth: true},
|
||||
{Name: "framework", Mount: framework.Mount, Shutdown: ctxShutdown(framework.Shutdown)},
|
||||
{Name: "knowledge", Mount: knowledge.Mount},
|
||||
// Hanzo Support PUBLIC plane /v1/help/* (help center KB read + customer ticket
|
||||
// intake) — the anonymous face the secure-by-default framework surface can't
|
||||
// serve. A framework lane like knowledge: its DocType fixtures register via
|
||||
// init() (help.Module), and this mounts the thin public subsystem. Owns no store
|
||||
// (delegates to the framework in-process API), so no Shutdown. After framework
|
||||
// (whose in-process API it calls at request time); before the AI /v1/* catch-all.
|
||||
{Name: "help", Mount: help.Mount},
|
||||
// Marketing content loop /v1/content/* (generate → CMS → transition → publish).
|
||||
// After framework (its DocType store the ops read/write) + knowledge (the sibling
|
||||
// framework lane); before the AI /v1/* catch-all so /v1/content/* resolves here.
|
||||
@@ -281,6 +312,13 @@ func Wire() []cloud.MountSpec {
|
||||
// After content (whose EnsureCatalogAsset it drives). Inert until CLOUD_COMMERCE_NATS_URL
|
||||
// names the NATS carrying commerce catalog events — the reverse of the forward edge.
|
||||
{Name: "catalogsync", Mount: catalogsync.Mount, Shutdown: catalogsync.Shutdown},
|
||||
// The platform-global webhook layer: /v1/webhooks registry (per-org SQLite) +
|
||||
// ONE durable JetStream consumer that delivers ANY bus event to org-registered
|
||||
// HTTP subscribers. A bus consumer like catalogsync — placed adjacent to it and
|
||||
// after the commerce embed whose COMMERCE stream it reads. Fail-soft: the
|
||||
// registry serves even with the bus down; the consumer retries in the background.
|
||||
// Owns per-org store handles + a worker pool → Shutdown drains both.
|
||||
{Name: "webhooks", Mount: webhooks.Mount, Shutdown: webhooks.Shutdown},
|
||||
{Name: "ml", Mount: ml.Mount, OwnsHealth: true},
|
||||
{Name: "usage", Mount: usage.Mount},
|
||||
// Gamified usage analytics: /v1/usage/leaderboard + /v1/usage/activity + the
|
||||
@@ -297,6 +335,13 @@ func Wire() []cloud.MountSpec {
|
||||
// twin of crm/marketing. Owns a DB handle, so its Shutdown closes it cleanly
|
||||
// on SIGTERM (ctxShutdown adapts func() error).
|
||||
{Name: "ads", Mount: ads.Mount, Shutdown: ctxShutdown(ads.Shutdown)},
|
||||
// Top-level GTM orchestration /v1/campaign/* — the capability layer that fans a
|
||||
// campaign VALUE out to its channels (paid→ads, organic→publish, email→marketing),
|
||||
// each CONSUMING the connector plane via integrations.TokenFor. Metrics read from
|
||||
// the ONE analytics plane (never a second store); creative A/B composes the
|
||||
// experiment seam. The paid channel executor is wired in wire_seams.go. Owns a DB
|
||||
// handle, so its Shutdown closes it cleanly on SIGTERM.
|
||||
{Name: "campaign", Mount: campaign.Mount, Shutdown: ctxShutdown(campaign.Shutdown)},
|
||||
// GDA/SDM validator onboarding /v1/validators/* — wallet-sig + ETH-mainnet
|
||||
// GenesisNFT ownerOf → seal luxd staking identity into KMS → write a NEW-node
|
||||
// LuxNetwork CR (node.lux.cloud, never the live luxd) → enqueue an owner-gated
|
||||
@@ -314,15 +359,32 @@ func Wire() []cloud.MountSpec {
|
||||
// DB handles, so its Shutdown closes them on SIGTERM.
|
||||
{Name: "sync", Mount: sync.Mount, Shutdown: ctxShutdown(sync.Shutdown)},
|
||||
{Name: "visor", Mount: visor.Mount},
|
||||
// Connect-a-cloud-account plane /v1/cloud/*: an org links its DigitalOcean /
|
||||
// AWS / GCP / Azure accounts (labeled, KMS-sealed, keyless where possible),
|
||||
// Hanzo DISCOVERS each account's native Kubernetes clusters and FOLDS them into
|
||||
// the ONE fleet (clients/fleet.Register) — so they surface in visor's
|
||||
// /v1/clusters and run work like any BYO/managed cluster. No second registry.
|
||||
{Name: "venue", Mount: venue.Mount},
|
||||
// Cap table on Base via goja. STAGED behind CLOUD_ENABLE.
|
||||
{Name: "captable", Mount: captable.Mount, Shutdown: captable.Shutdown},
|
||||
{Name: "code", Mount: code.Mount, Shutdown: code.Shutdown},
|
||||
{Name: "zero-trust", Mount: zt.Mount},
|
||||
// ngrok-native public sharing: /v1/share/* provisions a per-org zrok
|
||||
// account so `hanzo share <port>` publishes a local service to a public
|
||||
// https://<token>.share.hanzo.ai URL. Fail-closed until ZROK_ADMIN_TOKEN.
|
||||
{Name: "share", Mount: share.Mount},
|
||||
// Data rooms via goja + per-tenant Base. STAGED behind CLOUD_ENABLE. OwnsHealth.
|
||||
{Name: "dataroom", Mount: dataroom.Mount, Shutdown: dataroom.Shutdown, OwnsHealth: true},
|
||||
{Name: "graph", Mount: graph.Mount},
|
||||
{Name: "security", Mount: security.Mount, Shutdown: ctxShutdown(security.Shutdown), OwnsHealth: true},
|
||||
{Name: "integrations", Mount: integrations.Mount, Shutdown: integrations.Shutdown},
|
||||
// Marketing destinations /v1/destinations/* — the native fan-out that
|
||||
// TRANSLATES the canonical /v1/event stream to each connected ad/analytics
|
||||
// platform (GA4 Measurement Protocol, Meta Conversions API, X/LinkedIn/TikTok/
|
||||
// Reddit) and forwards it server-side. Mounts AFTER analytics (whose fan-out
|
||||
// sink it installs) and integrations (a destination may reuse an OAuth
|
||||
// connection's token via integrations.TokenFor). Owns a DB handle → Shutdown.
|
||||
{Name: "destinations", Mount: destinations.Mount, Shutdown: ctxShutdown(destinations.Shutdown)},
|
||||
// First-class per-org Cloudflare asset plane /v1/cloudflare/{zones,pages,workers,
|
||||
// ai,r2,kv,d1}/* (sibling of /v1/dns, /v1/domain). Mounts AFTER integrations
|
||||
// because it reads the org's Cloudflare token through the integrations custody
|
||||
@@ -332,6 +394,7 @@ func Wire() []cloud.MountSpec {
|
||||
{Name: "sbom", Mount: sbom.Mount, OwnsHealth: true},
|
||||
{Name: "team", Mount: team.Mount, Shutdown: ctxShutdown(team.Shutdown)},
|
||||
{Name: "settings", Mount: settings.Mount, Shutdown: settings.Shutdown},
|
||||
{Name: "prefs", Mount: prefs.Mount, Shutdown: prefs.Shutdown},
|
||||
{Name: "notify", Mount: notify.Mount, OwnsHealth: true},
|
||||
{Name: "channels", Mount: channels.Mount, Shutdown: channels.Shutdown},
|
||||
{Name: "gateway", Mount: gateway.Mount},
|
||||
@@ -351,6 +414,20 @@ func Wire() []cloud.MountSpec {
|
||||
{Name: "product", Mount: product.Mount},
|
||||
{Name: "evals", Mount: eval.Mount},
|
||||
{Name: "benchmark", Mount: benchmark.Mount},
|
||||
// The R&D EVIDENCE plane (HIP-0512) + its R&D Ops Board UI at /research —
|
||||
// the arena's sibling: benchmark measures, research is the versioned diary
|
||||
// every product's runs accrue into. Non-staged: mounts under the default.
|
||||
{Name: "research", Mount: research.Mount, Shutdown: ctxShutdown(research.Shutdown)},
|
||||
// The unified EXPERIMENT primitive (/v1/experiments): A/B testing as ONE value
|
||||
// whatever the variant kind (feature | ad creative | email | model). It is a
|
||||
// COMPOSITION — assignment via flags, measurement via analytics, evidence via
|
||||
// research — so it mounts AFTER all three. It owns only the experiment registry
|
||||
// store → Shutdown. clients/campaign composes it (experiments.Assign/Analyze).
|
||||
{Name: "experiments", Mount: experiments.Mount, Shutdown: ctxShutdown(experiments.Shutdown)},
|
||||
// The revenue BOOKS spine (/v1/books): a native double-entry general ledger that
|
||||
// reads commerce transactions (the sole posting source) and books the accounting twin —
|
||||
// plus bank import (PDF/OFX/CSV/Plaid/Teller), reconciliation, and the AI Ask brain.
|
||||
{Name: "books", Mount: books.Mount, Shutdown: ctxShutdown(books.Shutdown)},
|
||||
{Name: "treasury", Mount: treasury.Mount, Shutdown: ctxShutdown(treasury.Shutdown)},
|
||||
{Name: "admin", Mount: admin.Mount},
|
||||
// Launch-control gate (per-service waitlist): the COMPLETE feature — host→service
|
||||
@@ -391,6 +468,14 @@ func Wire() []cloud.MountSpec {
|
||||
// google token custody; captable/dataroom facades) and before the /v1/* AI
|
||||
// catch-all so its routes resolve here.
|
||||
{Name: "company", Mount: company.Mount, Shutdown: company.Shutdown},
|
||||
// The corporate back-office surfaces — orthogonal to company/captable/billing:
|
||||
// Hanzo Compliance (/v1/compliance) orchestrates KYC/KYB verification providers
|
||||
// + tracks accreditation state + surfaces the SOC 2 audit posture; Hanzo Legal
|
||||
// (/v1/legal) is the versioned template + generation engine + e-sign/filing seams.
|
||||
// Both are TOOLING with providers/professionals in the loop, never advice or
|
||||
// certification. They mount before the /v1/* AI catch-all so their routes resolve.
|
||||
{Name: "compliance", Mount: compliance.Mount, Shutdown: ctxShutdown(compliance.Shutdown), OwnsHealth: true},
|
||||
{Name: "legal", Mount: legal.Mount, Shutdown: ctxShutdown(legal.Shutdown), OwnsHealth: true},
|
||||
// Chat orchestrator — POST /v1/chat: ONE LLM tool-calling round over the tool
|
||||
// plane. It COMPOSES the ai completion path (in-process, so per-org billing
|
||||
// runs) + the unified tool registry, and splits the model's tool calls into
|
||||
@@ -399,6 +484,16 @@ func Wire() []cloud.MountSpec {
|
||||
// beego /v1/chat alias behind its /v1/* glob is thereby shadowed, while ai
|
||||
// keeps /v1/chat/completions + /v1/completions.
|
||||
{Name: "agent", Mount: agent.Mount},
|
||||
// The UNIFIED GROUNDED ADVISOR — POST /v1/ask. DISTINCT from /v1/chat/completions
|
||||
// (ai's RAW model) and /v1/agent (tool-calling): it routes a plain-language question
|
||||
// to the domain(s) that can GROUND it, reads the REAL figures from each domain's own
|
||||
// endpoint IN-PROCESS under the caller's own creds (agent.go's replay pattern, so
|
||||
// per-tenant isolation is inherited), hands the model the EXACT figures, and returns
|
||||
// the grounded answer + figures + the domain reads that backed them. The model NEVER
|
||||
// invents a figure. Contributors plug in via a registry (books today; o11y/billing
|
||||
// next) WITHOUT a router edit. Mounts BEFORE the ai /v1/* catch-all so /v1/ask wins
|
||||
// Fiber's first-match; after books/agent so the domains it composes are wired.
|
||||
{Name: "ask", Mount: ask.Mount},
|
||||
// The bare /v1/* AI catch-all — the LAST route position. Every owning subsystem above
|
||||
// wins its own namespace (Fiber first-match); AI is the fallback for the rest of /v1/*.
|
||||
// zen mounts as a /v1-scoped Claim middleware BEFORE ai: it routes zen* models
|
||||
|
||||
@@ -314,6 +314,99 @@ func mountCommerce(app *zip.App, deps cloud.Deps) error {
|
||||
commercebilling.DeleteSpendAlert,
|
||||
)
|
||||
|
||||
// POST /v1/billing/topup/token — the INLINE Square card top-up (the console's
|
||||
// "Billing → Credits → add credits": the Square Web Payments SDK tokenizes the card
|
||||
// IN THE BROWSER → a single-use nonce → this endpoint charges it and credits the
|
||||
// caller's balance). commerce's api.Route() billing bundle is NOT compiled into the
|
||||
// co-resident embed, so — exactly like plans/invoices/spend-alerts above — without
|
||||
// this registration the POST fell through to the account bridge's /v1/billing/*
|
||||
// wildcard (order 122). That wildcard is service-token-forwardable for topup/token
|
||||
// (billing.go billingForwardable), so billingData re-forwarded it to COMMERCE_URL
|
||||
// (default the public api.hanzo.ai edge = THIS binary) over commerceinproc's
|
||||
// self-routing transport, re-entering the SAME wildcard until the depth-8 guard
|
||||
// refused → the "commerceinproc: in-process dispatch depth 8 exceeded" 502 that broke
|
||||
// top-up outright. Registering commerce's real TopupWithToken co-resident here
|
||||
// (order 100 < 122) shadows the wildcard and serves the charge in-process at depth 1
|
||||
// — no HTTP hop, no self-dispatch. topup/token STAYS in billingForwardable as the
|
||||
// split-deploy fallback (a standalone commerce still serves it); co-residence just
|
||||
// wins first.
|
||||
//
|
||||
// Chain — the browser money-WRITE posture, byte-for-byte what the bridge applied:
|
||||
// RequireCSRF — the ambient-cookie anti-CSRF gate the bridge's requireCSRF
|
||||
// wrapped POST /v1/billing/* with (a Bearer/gateway caller is
|
||||
// not CSRF-able; an ambient-cookie write needs the token).
|
||||
// RequestContext — the gated request context commerce's handler + ledger read.
|
||||
// IAMTokenRequired — resolves the org from the gateway-validated X-Org-Id into
|
||||
// Locals("organization"), which TopupWithToken.GetOrganization
|
||||
// + topupDestination read as the org billing key.
|
||||
// PinBillingSubject — pins ?user= to the caller's OWN account.Payer subject (the
|
||||
// SAME rule the ai spend-gate debits and billingData pins), so
|
||||
// the credit lands on the caller's subject (person=org/name) and
|
||||
// can never be widened; fail-closed for an unvalidated caller —
|
||||
// the IDOR boundary stays exactly where billingData put it.
|
||||
// The card PAN never touches this binary: TopupWithToken charges the Square nonce only,
|
||||
// and the settled charge itself is the mint authority (mintauth.WithAuthorized).
|
||||
app.Post("/v1/billing/topup/token",
|
||||
accountclient.RequireCSRF(),
|
||||
commercemid.RequestContext(),
|
||||
iammiddleware.IAMTokenRequired(),
|
||||
accountclient.PinBillingSubject(),
|
||||
commercebilling.TopupWithToken,
|
||||
)
|
||||
|
||||
// The remaining console billing WRITES that share topup/token's self-dispatch loop
|
||||
// class — each is a POST the console makes (billingForwardable in billing.go), each had
|
||||
// NO co-resident handler, so each fell through to the account bridge's /v1/billing/*
|
||||
// wildcard (order 122) and re-entered it over commerceinproc until the depth-8 guard
|
||||
// refused (the same "in-process dispatch depth 8 exceeded" 502 that broke top-up). Each
|
||||
// commerce handler exists in the vendored module (v1.49.13); registering them co-resident
|
||||
// (order 100 < 122) shadows the wildcard and serves the write in-process at depth 1. They
|
||||
// STAY in billingForwardable as the split-deploy fallback (same precedent as topup/token
|
||||
// + spend-alerts). Chain matches the bridge's write posture byte-for-byte:
|
||||
//
|
||||
// - RequireCSRF — the ambient-cookie anti-CSRF gate the bridge wrapped POST
|
||||
// /v1/billing/* with (Bearer/gateway callers are not CSRF-able).
|
||||
// - RequestContext — the gated request context commerce's handlers read.
|
||||
// - IAMTokenRequired — resolves the org from the gateway-validated X-Org-Id into
|
||||
// Locals("organization") — the namespace GetOrganization reads.
|
||||
// - PinBillingSubject — pins the caller's OWN account.Payer subject into BOTH query and
|
||||
// body AND fail-closes an unvalidated caller. It is the auth gate
|
||||
// on every one, and the IDOR control on the subject-scoped one.
|
||||
//
|
||||
// payment-methods (save a card-on-file / vault a Square nonce) is SUBJECT-scoped: commerce's
|
||||
// CreatePaymentMethod reads `customerId` from the BODY, so PinBillingSubject's body-pin is
|
||||
// load-bearing here — a member can only vault a card for their OWN subject, exactly the
|
||||
// boundary billingData's scopedBillingBody enforced. The Square nonce goes to Square; the
|
||||
// PAN never touches this binary.
|
||||
app.Post("/v1/billing/payment-methods",
|
||||
accountclient.RequireCSRF(),
|
||||
commercemid.RequestContext(),
|
||||
iammiddleware.IAMTokenRequired(),
|
||||
accountclient.PinBillingSubject(),
|
||||
commercebilling.CreatePaymentMethod,
|
||||
)
|
||||
|
||||
// subscriptions/:id/{cancel,reactivate} are org-NAMESPACE-scoped: commerce's handlers
|
||||
// resolve the subscription by `:id` WITHIN the caller's org namespace (a foreign org's id
|
||||
// is a 404 miss), so tenancy is the namespace IAMTokenRequired resolves and PinBillingSubject
|
||||
// is the fail-closed-anon auth gate — its pinned subject params are ignored by these
|
||||
// handlers (the SAME role it plays for the org-scoped payment-config read). The bridge's
|
||||
// subject-pin was likewise a no-op for these, so nothing is dropped.
|
||||
app.Post("/v1/billing/subscriptions/:id/cancel",
|
||||
accountclient.RequireCSRF(),
|
||||
commercemid.RequestContext(),
|
||||
iammiddleware.IAMTokenRequired(),
|
||||
accountclient.PinBillingSubject(),
|
||||
commercebilling.CancelBillingSubscription,
|
||||
)
|
||||
app.Post("/v1/billing/subscriptions/:id/reactivate",
|
||||
accountclient.RequireCSRF(),
|
||||
commercemid.RequestContext(),
|
||||
iammiddleware.IAMTokenRequired(),
|
||||
accountclient.PinBillingSubject(),
|
||||
commercebilling.ReactivateBillingSubscription,
|
||||
)
|
||||
|
||||
// In-process seams:
|
||||
// - commerceinproc routes the S2S billing byte-stream into the co-resident
|
||||
// app (the metering debit path) instead of a socket to a standalone pod.
|
||||
|
||||
@@ -7,13 +7,15 @@ import (
|
||||
)
|
||||
|
||||
// TestFrameworkContentModulesLinked guards the framework CONTENT modules
|
||||
// (cms/erp/help) against silent removal. They are not mount subsystems, so they
|
||||
// never appear in Wire(); they register their DocTypes and — for erp — the
|
||||
// ledger-posting lifecycle hooks into the framework engine from a package
|
||||
// init(), reached ONLY via the blank imports in apps.go. #248 dropped
|
||||
// those imports, which stripped the erp ledger hooks from the binary with no
|
||||
// mount change and no failing mount test. This asserts the engine's module
|
||||
// registry carries each lane, so that money-adjacent regression cannot recur.
|
||||
// (cms/erp/help) against silent removal. cms/erp are pure fixture lanes reached
|
||||
// ONLY via blank imports in apps.go; help is a real import + a Wire() spec (it also
|
||||
// mounts the /v1/help public plane) but still contributes its DocTypes the SAME way,
|
||||
// from a package init() (framework.RegisterModule). #248 dropped the blank imports,
|
||||
// which stripped the erp ledger hooks from the binary with no mount change and no
|
||||
// failing mount test. This asserts the engine's module registry carries each lane,
|
||||
// so that money-adjacent regression cannot recur — for help, dropping the import
|
||||
// would also drop its Wire() spec and fail TestWireOrderMatchesFrozen, but this keeps
|
||||
// the fixture-linkage guard uniform across all three lanes.
|
||||
func TestFrameworkContentModulesLinked(t *testing.T) {
|
||||
got := make(map[string]bool)
|
||||
for _, m := range framework.RegisteredModules() {
|
||||
|
||||
@@ -2,11 +2,19 @@ package apps
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/ads"
|
||||
"github.com/hanzoai/cloud/clients/automations"
|
||||
"github.com/hanzoai/cloud/clients/campaign"
|
||||
"github.com/hanzoai/cloud/clients/coding"
|
||||
"github.com/hanzoai/cloud/clients/experiments"
|
||||
"github.com/hanzoai/cloud/clients/framework"
|
||||
"github.com/hanzoai/cloud/clients/git"
|
||||
"github.com/hanzoai/cloud/clients/guide"
|
||||
"github.com/hanzoai/cloud/clients/integrations"
|
||||
"github.com/hanzoai/cloud/clients/principal"
|
||||
)
|
||||
|
||||
// wire_seams.go wires cross-subsystem in-process seams that cannot be a MountSpec
|
||||
@@ -24,6 +32,24 @@ import (
|
||||
func init() {
|
||||
integrations.SetCodingDispatcher(coding.NewDispatcher(git.CloneURL, git.VerifyRef, nil))
|
||||
|
||||
// Guide growth-OBSERVE seam: the /v1/guide/profile observe layer reads the org's
|
||||
// real platform truth through injected, org-scoped, honest-degrading probes.
|
||||
// clients/guide imports NONE of these subsystems (decomplected), so the
|
||||
// composition root — the ONE place that imports them all — binds the reads, the
|
||||
// same injected-function pattern the coding dispatcher above uses. Each probe is
|
||||
// PROVABLY org-scoped: framework.ModuleInstalled keys GetDocType on the org;
|
||||
// integrations.Connected keys store.Get on the org and never surfaces the token.
|
||||
// HasDeployment/RevenueCents/RecordCount are LEFT NIL: deploy is cluster/admin-
|
||||
// scoped (not per-org) and commerce-revenue-of-record + the crm record count have
|
||||
// no clean per-org in-process read yet — binding a read whose org-scoping we
|
||||
// cannot guarantee would be the bug. Until a provably-org-scoped read lands their
|
||||
// signals honest-degrade to not-present (the vocabulary is the contract; a nil
|
||||
// seam can never be a spurious true).
|
||||
guide.BindSignals(guide.Signals{
|
||||
ModuleInstalled: framework.ModuleInstalled,
|
||||
ConnectorPresent: integrations.Connected,
|
||||
})
|
||||
|
||||
// Inbound-event seam: a verified provider webhook (or chat channel) in
|
||||
// clients/integrations fires the automations engine's Deliver here — the ONE place
|
||||
// that imports both, so integrations never has to import automations (which imports
|
||||
@@ -34,4 +60,63 @@ func init() {
|
||||
Source: source, Name: name, DedupeKey: dedupeKey, Depth: depth, Payload: payload,
|
||||
})
|
||||
})
|
||||
|
||||
// GTM PAID channel: the /v1/campaign orchestrator fans out to executors that
|
||||
// satisfy campaign.Channel; the composition root is the ONE place that imports
|
||||
// both campaign and the ad plane, so it adapts ads' connector-consuming
|
||||
// execution funcs (provider.go — each resolves the org's ad token through
|
||||
// integrations.TokenFor and fails closed) onto the primitive-typed channel
|
||||
// seam. campaign never imports ads and ads never imports campaign — the same
|
||||
// injected-function decoupling the coding dispatcher above uses.
|
||||
campaign.RegisterChannel(campaign.NewChannel(campaign.KindPaid,
|
||||
func(ctx context.Context, org string, p campaign.Plan) (campaign.Ref, error) {
|
||||
r, err := ads.LaunchPaid(ctx, org, ads.PaidPlan{
|
||||
Platform: p.Platform, Account: p.Account, Name: p.Name,
|
||||
Objective: p.Objective, BudgetCents: p.BudgetCents, ScheduleAt: p.ScheduleAt,
|
||||
})
|
||||
return campaign.Ref{Platform: r.Platform, Account: r.Account, ExternalID: r.ExternalID, Status: r.Status, Detail: r.Detail}, err
|
||||
},
|
||||
func(ctx context.Context, org string, ref campaign.Ref) (int64, error) {
|
||||
return ads.PaidSpend(ctx, org, ads.PaidRef{Platform: ref.Platform, Account: ref.Account, ExternalID: ref.ExternalID})
|
||||
},
|
||||
func(ctx context.Context, org string, ref campaign.Ref) error {
|
||||
return ads.PausePaid(ctx, org, ads.PaidRef{Platform: ref.Platform, Account: ref.Account, ExternalID: ref.ExternalID})
|
||||
},
|
||||
))
|
||||
|
||||
// GTM creative A/B composes the merged EXPERIMENT primitive (clients/experiments)
|
||||
// — campaign never reinvents bucketing or measurement. Assign resolves the
|
||||
// subject's variant (creative) from the experiment's flag; Analyze is pull-model
|
||||
// (it reads the metric from analytics itself), returned as opaque JSON so
|
||||
// campaign stays decoupled from the analysis type. Campaign-linked experiments
|
||||
// live in the org's default project. Both are nil-safe upstream: an org that
|
||||
// never created a "campaign:<id>" experiment runs a single creative (Assign
|
||||
// errors → "" → Content[0]).
|
||||
campaign.SetExperiment(
|
||||
func(ctx context.Context, org, experimentID, subject string) (string, error) {
|
||||
a, err := experiments.Assign(ctx, org, principal.DefaultProject, experimentID, subject, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return a.Variant, nil
|
||||
},
|
||||
func(ctx context.Context, org, experimentID string, start, end time.Time) (json.RawMessage, error) {
|
||||
an, err := experiments.Analyze(ctx, org, principal.DefaultProject, experimentID, start, end, 0.05)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(an)
|
||||
},
|
||||
)
|
||||
|
||||
// GTM ORGANIC + EMAIL channels wire HERE the same way once their executors land
|
||||
// (designed follow-ons — the publish rename + marketing email-connector wiring):
|
||||
//
|
||||
// campaign.RegisterChannel(campaign.NewChannel(campaign.KindOrganic,
|
||||
// publish.Syndicate, publish.NoSpend, publish.Unpublish)) // social connectors
|
||||
// campaign.RegisterChannel(campaign.NewChannel(campaign.KindEmail,
|
||||
// marketing.Broadcast, marketing.NoSpend, marketing.Halt)) // email connectors
|
||||
//
|
||||
// Until wired, those channels record "unavailable" on a fan-out (honest) and a
|
||||
// campaign runs a single creative — never a fabricated launch or variant.
|
||||
}
|
||||
|
||||
+16
-2
@@ -45,8 +45,8 @@ var frozen = []struct {
|
||||
{"rollingcap", false, false}, // rolling spend-cap gate (after billing); golden drifted — refrozen
|
||||
{"account-bridge", false, false}, // was order 122
|
||||
{"do", false, false}, // was order 123
|
||||
{"platform", true, false}, // was order 124
|
||||
{"projects", false, false}, // was order 125
|
||||
{"platform", true, true}, // was order 124
|
||||
{"projects", false, true}, // was order 125
|
||||
{"dns", false, false}, // new: /v1/dns zone plane (after projects)
|
||||
{"domain", false, false}, // new: Hanzo Domains registrar (/v1/domain), after dns
|
||||
{"prompts", false, false}, // was order 126
|
||||
@@ -59,33 +59,41 @@ var frozen = []struct {
|
||||
{"functions", false, false}, // was order 128
|
||||
{"tracker", false, false}, // was order 129
|
||||
{"templates", false, false}, // was order 129
|
||||
{"blueprint", true, false}, // new: OSS-template compute-cost basis /v1/blueprint (after templates); owns health, embedded content → no Shutdown
|
||||
{"framework", false, true}, // was order 129
|
||||
{"knowledge", false, false}, // was order 130
|
||||
{"help", false, false}, // new: Hanzo Support public plane /v1/help (after knowledge; framework lane with a companion subsystem, no store → no Shutdown)
|
||||
{"content", false, true}, // new: marketing content loop (after knowledge)
|
||||
{"catalogsync", false, true}, // new: reverse loop (product.created → render) after content
|
||||
{"webhooks", false, true}, // new: platform-global /v1/webhooks registry + bus-driven dispatcher (after catalogsync); owns per-org stores + worker pool → Shutdown
|
||||
{"ml", true, false}, // was order 130
|
||||
{"usage", false, false}, // was order 131
|
||||
{"leaderboard", false, true}, // new: gamified usage analytics (after usage), owns opt-in SQLite (Shutdown)
|
||||
{"crm", false, false}, // was order 131
|
||||
{"marketing", false, true}, // new: marketing domain fold (after crm)
|
||||
{"ads", false, true}, // new: ads domain fold (after crm)
|
||||
{"campaign", false, true}, // new: /v1/campaign GTM orchestration (after ads); fans out to channels
|
||||
{"validators", false, true}, // new: NFT-gated node provisioning (after ads); golden refrozen
|
||||
{"social", false, true}, // new: /v1/social fold (after crm)
|
||||
{"analytics", true, false}, // was order 132
|
||||
{"git", false, false}, // was order 132
|
||||
{"sync", false, true}, // /v1/sync engine (owns per-org DB handles → Shutdown)
|
||||
{"visor", false, false}, // was order 133
|
||||
{"venue", false, false}, // new: /v1/cloud connect-a-cloud-account plane (after visor); folds discovered clusters into the fleet
|
||||
{"captable", false, true}, // was order 133
|
||||
{"code", false, true}, // was order 134
|
||||
{"zero-trust", false, false}, // was order 134
|
||||
{"share", false, false}, // ngrok-native public sharing (/v1/share/*)
|
||||
{"dataroom", true, true}, // was order 134
|
||||
{"graph", false, false}, // was order 135
|
||||
{"security", true, true}, // was order 136
|
||||
{"integrations", false, true}, // was order 137
|
||||
{"destinations", false, true}, // new: /v1/destinations CDP fan-out (after integrations, before cloudflare)
|
||||
{"cloudflare", false, false}, // new: /v1/cloudflare edge plane (after integrations)
|
||||
{"sbom", true, false}, // was order 137
|
||||
{"team", false, true}, // was order 138
|
||||
{"settings", false, true}, // was order 138
|
||||
{"prefs", false, true}, // new: per-user preference plane (after settings); Shutdown closes the store
|
||||
{"notify", true, false}, // was order 139
|
||||
{"channels", false, true}, // new: /v1/channels transport plane (after notify; must mount after integrations so RegisterIngress installs before webhooks emit)
|
||||
{"gateway", false, false}, // was order 139
|
||||
@@ -102,6 +110,9 @@ var frozen = []struct {
|
||||
{"product", false, false}, // was order 145
|
||||
{"evals", false, false}, // was order 145
|
||||
{"benchmark", false, false}, // benchmark plane (after evals, before treasury)
|
||||
{"research", false, true}, // R&D evidence plane + /research board (HIP-0512), arena sibling after benchmark; Shutdown closes per-org stores
|
||||
{"experiments", false, true}, // unified A/B EXPERIMENT primitive: composes flags(assign)+analytics(measure)+research(evidence); Shutdown closes the registry stores
|
||||
{"books", false, true}, // AI bookkeeper (/v1/books, per-org SQLite); Shutdown closes org stores
|
||||
{"treasury", false, true}, // was order 146
|
||||
{"admin", false, false}, // was order 146
|
||||
{"admission", false, true}, // launch-control gate: composes flags (registry+seed+mode route+Enforce); Shutdown closes the registry store
|
||||
@@ -112,7 +123,10 @@ var frozen = []struct {
|
||||
{"referrals", false, false}, // was order 149
|
||||
{"guide", false, true}, // new: Business AI Guide (after referrals, before ai)
|
||||
{"company", false, true}, // new: Hanzo Company formation state machine (after guide)
|
||||
{"compliance", true, true}, // new: Hanzo Compliance — KYC/KYB + accreditation + audit posture (after company)
|
||||
{"legal", true, true}, // new: Hanzo Legal — template + generation engine + e-sign/filing (after compliance)
|
||||
{"agent", false, false}, // new: /v1/agent tool-calling round (before zen/ai catch-all)
|
||||
{"ask", false, false}, // new: unified grounded advisor /v1/ask (before zen/ai catch-all)
|
||||
{"zen", false, false}, // zen* claim middleware before ai's catch-all (hip-00NN)
|
||||
{"ai", false, false}, // was order 150
|
||||
{"plugins", false, false}, // was order 900
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
// Copyright 2026 The Hanzo Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestJWTAudiences_AcceptsAdminGuard is the operator-cockpit keystone. The
|
||||
// admin.hanzo.ai forward-auth guard is the confidential client `hanzo-admin-guard`,
|
||||
// so IAM mints its access tokens with aud=hanzo-admin-guard (each app's aud is its
|
||||
// client_id). The guard forwards that bearer to cloud-api /v1/admin/*; the identity
|
||||
// sanitizer only grants SuperAdmin (owner==adminOrg) to a VALIDATED principal, and
|
||||
// validation enforces this audience allowlist. If hanzo-admin-guard is not accepted
|
||||
// the token resolves anonymous and the SuperAdmin gate reads false -> 403, even
|
||||
// though the token's owner IS admin. Pin the client_id into the baked default so the
|
||||
// forwarded bearer validates.
|
||||
func TestJWTAudiences_AcceptsAdminGuard(t *testing.T) {
|
||||
os.Unsetenv("CLOUD_JWT_AUDIENCES")
|
||||
os.Unsetenv("GATEWAY_ALLOWED_AUDIENCES")
|
||||
|
||||
has := func(list []string, v string) bool {
|
||||
for _, s := range list {
|
||||
if s == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
if !has(defaultJWTAudiences, "hanzo-admin-guard") {
|
||||
t.Fatalf("defaultJWTAudiences must include hanzo-admin-guard (the admin-cockpit guard client_id); got %v", defaultJWTAudiences)
|
||||
}
|
||||
if !has(jwtAudiencesFromEnv(), "hanzo-admin-guard") {
|
||||
t.Fatalf("resolved JWT audiences must include hanzo-admin-guard; got %v", jwtAudiencesFromEnv())
|
||||
}
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
// Copyright 2026 The Hanzo Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestJWTAudiences_AcceptsHanzoTeam pins the hanzo.team OIDC client. IAM mints
|
||||
// team's access tokens with aud=hanzo-team (each app's aud is its client_id);
|
||||
// the team OAuth callback sets that token as the hanzo_iam_token cookie, and
|
||||
// the usage/wallet page (/v1/team/billing/ui/) reads /v1/billing/balance +
|
||||
// /v1/usage/summary same-origin on it. If hanzo-team is not accepted the
|
||||
// cookie resolves anonymous and every wallet read 401s — and the callback's
|
||||
// own validator (NewTokenValidator shares this allowlist) refuses the login.
|
||||
func TestJWTAudiences_AcceptsHanzoTeam(t *testing.T) {
|
||||
os.Unsetenv("CLOUD_JWT_AUDIENCES")
|
||||
os.Unsetenv("GATEWAY_ALLOWED_AUDIENCES")
|
||||
|
||||
has := func(list []string, v string) bool {
|
||||
for _, s := range list {
|
||||
if s == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
if !has(defaultJWTAudiences, "hanzo-team") {
|
||||
t.Fatalf("defaultJWTAudiences must include hanzo-team (the hanzo.team client_id); got %v", defaultJWTAudiences)
|
||||
}
|
||||
if !has(jwtAudiencesFromEnv(), "hanzo-team") {
|
||||
t.Fatalf("resolved JWT audiences must include hanzo-team; got %v", jwtAudiencesFromEnv())
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
// Copyright 2026 The Hanzo Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestJWTAudiences_AcceptsHanzoWorld pins the world.hanzo.ai OIDC client. IAM
|
||||
// mints world's access tokens with aud=hanzo-world (each app's aud is its
|
||||
// client_id). Those bearers hit cloud-api; the identity sanitizer only trusts a
|
||||
// principal whose aud is in this allowlist. If hanzo-world is not accepted the
|
||||
// token resolves anonymous and the analyst's api.hanzo.ai calls 401. Pin the
|
||||
// client_id into the baked default so the forwarded bearer validates.
|
||||
func TestJWTAudiences_AcceptsHanzoWorld(t *testing.T) {
|
||||
os.Unsetenv("CLOUD_JWT_AUDIENCES")
|
||||
os.Unsetenv("GATEWAY_ALLOWED_AUDIENCES")
|
||||
|
||||
has := func(list []string, v string) bool {
|
||||
for _, s := range list {
|
||||
if s == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
if !has(defaultJWTAudiences, "hanzo-world") {
|
||||
t.Fatalf("defaultJWTAudiences must include hanzo-world (the world.hanzo.ai client_id); got %v", defaultJWTAudiences)
|
||||
}
|
||||
if !has(jwtAudiencesFromEnv(), "hanzo-world") {
|
||||
t.Fatalf("resolved JWT audiences must include hanzo-world; got %v", jwtAudiencesFromEnv())
|
||||
}
|
||||
}
|
||||
+141
-67
@@ -41,14 +41,21 @@ import (
|
||||
type idClaims struct {
|
||||
jwt.Claims
|
||||
|
||||
Owner string `json:"owner"` // org slug (the org)
|
||||
Project string `json:"project"` // org SUB-SCOPE within owner (empty ⟹ default project)
|
||||
BillingAccount string `json:"billing_account"` // WHO PAYS, stated by IAM (empty ⟹ pre-claim token)
|
||||
Name string `json:"name"` // display name (id fallback)
|
||||
PreferredUsername string `json:"preferred_username"` // id fallback
|
||||
Email string `json:"email"`
|
||||
IsAdmin bool `json:"isAdmin"`
|
||||
Orgs []model.OrgRef `json:"orgs"` // membership SET (home first); empty on legacy tokens
|
||||
Owner string `json:"owner"` // org slug (the org)
|
||||
Project string `json:"project"` // org SUB-SCOPE within owner (empty ⟹ default project)
|
||||
BillingAccount string `json:"billing_account"` // WHO PAYS, stated by IAM (empty ⟹ pre-claim token)
|
||||
Name string `json:"name"` // display name (id fallback)
|
||||
PreferredUsername string `json:"preferred_username"` // id fallback
|
||||
Email string `json:"email"`
|
||||
IsAdmin bool `json:"isAdmin"`
|
||||
// Type is IAM's account kind: "application" for a client_credentials MACHINE
|
||||
// identity (object/token_oauth.go stamps Type:"application"), else a human kind
|
||||
// ("normal-user", …). It is the discriminator that keeps a machine token — of ANY
|
||||
// app, not only the KMS-sync one — from ever being granted SuperAdmin. Empty on a
|
||||
// token that predates the claim ⟹ treated as non-machine (fail toward the KMS-aud
|
||||
// check below, never toward granting admin).
|
||||
Type string `json:"type"`
|
||||
Orgs []model.OrgRef `json:"orgs"` // membership SET (home first); empty on legacy tokens
|
||||
}
|
||||
|
||||
// mintedProject returns the project id to stamp into X-Project-Id, or "" when the
|
||||
@@ -125,43 +132,41 @@ var jwtSigAlgs = []gojose.SignatureAlgorithm{
|
||||
// (cert-hanzo/cert-lux/cert-zoo/...), keyed by the token kid, so a single
|
||||
// jwksURL verifies all brands. Only the issuer-string comparison had to widen.
|
||||
type identityValidator struct {
|
||||
issuers []string
|
||||
audiences []string
|
||||
cache *jwksCache
|
||||
keys keyResolver // resolves an opaque API key to a principal; nil ⟹ keys stay anonymous
|
||||
issuers []string
|
||||
cache *jwksCache
|
||||
keys keyResolver // resolves an opaque API key to a principal; nil ⟹ keys stay anonymous
|
||||
}
|
||||
|
||||
// newIdentityValidator builds a validator whose trusted-issuer set is the primary
|
||||
// issuer UNIONED with every white-label brand issuer (BrandIssuers) plus any
|
||||
// WHITELABEL_ISSUERS override. ttl<=0 uses the 15m JWKS default. The union is
|
||||
// fail-secure: it only ADDS the known-good brand issuers, never an arbitrary one.
|
||||
func newIdentityValidator(issuer, jwksURL string, audiences []string, ttl time.Duration) *identityValidator {
|
||||
//
|
||||
// Trust is IAM-native: signature (JWKS) + issuer (this set) + expiry. There is NO
|
||||
// per-app audience allowlist — the `aud` (a minting app's client_id) is IAM's to
|
||||
// assign, not cloud's to mirror, so a new first-party app needs zero cloud change.
|
||||
func newIdentityValidator(issuer, jwksURL string, ttl time.Duration) *identityValidator {
|
||||
return &identityValidator{
|
||||
issuers: trustedIssuers(issuer),
|
||||
audiences: audiences,
|
||||
cache: newJWKSCache(jwksURL, ttl),
|
||||
keys: sharedKeys(), // ONE resolver+cache, shared with OrgForKey (analytics capture)
|
||||
issuers: trustedIssuers(issuer),
|
||||
cache: newJWKSCache(jwksURL, ttl),
|
||||
keys: sharedKeys(), // ONE resolver+cache, shared with OrgForKey (analytics capture)
|
||||
}
|
||||
}
|
||||
|
||||
// kmsMachineAudSuffix is the fixed suffix of a per-org PaaS-KMS sync machine
|
||||
// identity's audience. Each org's KMS sync authenticates as a dedicated,
|
||||
// NON-shared IAM application named "<org>-platform-kms" (Organization=<org>,
|
||||
// client_credentials grant), so IAM stamps the token's aud == the app's own
|
||||
// clientId == "<org>-platform-kms" (a non-shared app's audience is its clientId,
|
||||
// object/token_jwt.go tokenAudience) and owner == <org>
|
||||
// (object/token_oauth.go GetClientCredentialsToken sets owner = app.Organization).
|
||||
// identity's audience. Each org's KMS sync authenticates as a dedicated, NON-shared
|
||||
// IAM application named "<org>-platform-kms" (Organization=<org>, client_credentials
|
||||
// grant), so IAM stamps the token's aud == the app's own clientId == "<org>-platform-kms"
|
||||
// (object/token_jwt.go tokenAudience) and owner == <org> (object/token_oauth.go
|
||||
// GetClientCredentialsToken sets owner = app.Organization).
|
||||
//
|
||||
// That audience is, by construction, absent from CLOUD_JWT_AUDIENCES — it is
|
||||
// per-org, not a fixed app — which is EXACTLY why the sync stayed pending: the
|
||||
// machine token failed the audience check below, SanitizeIdentity treated it as
|
||||
// anonymous, and the /v1/kms org-scope guard 403'd it before the store. The fix is
|
||||
// to accept this one audience, but ONLY when it equals the token's OWN owner claim
|
||||
// plus this suffix, so it certifies "the KMS sync identity for its own org" and
|
||||
// grants nothing wider. Org-scoping is still enforced downstream by owner at the guard
|
||||
// (owner == :org); this only lets a legitimately-minted, owner-scoped machine token
|
||||
// clear validation. A per-org application means a per-org clientSecret — never
|
||||
// a shared platform-wide reader, which would be a cross-org hole.
|
||||
// Validation no longer consults the audience at all (trust is signature + issuer +
|
||||
// expiry), so a machine token clears validate() like any other. This suffix survives
|
||||
// for the OPPOSITE reason: to RECOGNISE a machine principal (isKMSMachinePrincipal) so
|
||||
// SanitizeIdentity can DENY it SuperAdmin even when it carries owner==adminOrg — a
|
||||
// client_credentials machine identity must never wield platform-admin. The match is
|
||||
// bound to the token's OWN owner claim (<owner>-platform-kms), so it certifies "the
|
||||
// KMS sync identity for its own org" and grants nothing wider.
|
||||
const kmsMachineAudSuffix = "-platform-kms"
|
||||
|
||||
// kmsMachineAudience returns the audience an org org's PaaS-KMS sync identity
|
||||
@@ -197,6 +202,44 @@ func isKMSMachinePrincipal(claims *idClaims) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// isMachinePrincipal reports whether a validated token is a MACHINE (non-human)
|
||||
// identity — the predicate SanitizeIdentity uses to DENY SuperAdmin and the org-admin
|
||||
// signal. A client_credentials token carries IAM's `type` == "application"
|
||||
// (object/token_oauth.go), which catches EVERY machine app regardless of its audience;
|
||||
// this is the fix for the audience-decomplection widening SuperAdmin's reach — a
|
||||
// generic admin-org machine app must not become platform-admin just because it belongs
|
||||
// to the admin org. It UNIONS the owner-bound KMS-sync check so a machine token is
|
||||
// still excluded even on the (defensive) path where `type` is absent but the
|
||||
// <owner>-platform-kms audience is present. Fail-closed: unknown/empty type is treated
|
||||
// as NON-machine so a real human admin (who may carry no `type`) is never locked out —
|
||||
// the KMS-aud fallback still catches the one machine family we can identify audience-only.
|
||||
func isMachinePrincipal(claims *idClaims) bool {
|
||||
return claims.Type == "application" || isKMSMachinePrincipal(claims)
|
||||
}
|
||||
|
||||
// isMember reports whether org is in the token's signed membership set — the
|
||||
// `orgs` claim IAM mints for a USER token, home org first. It is the ONE test that
|
||||
// turns a client's org SELECTION into an effective org (SanitizeIdentity), and
|
||||
// therefore into the ledger that pays (principal.BillingOrg).
|
||||
//
|
||||
// The comparison is VERBATIM, no folding, for the same reason the owner claim is
|
||||
// taken verbatim: "acme" and "ACME" are DISTINCT orgs in IAM, and a fold would let
|
||||
// a member of one select the other. An empty org is never a member, so an absent
|
||||
// selection leaves the caller in their home org. An empty set (a legacy token, an
|
||||
// opaque key, a machine principal — IAM never mints `orgs` for a client_credentials
|
||||
// token) admits nothing, which is exactly the pre-claim behavior.
|
||||
func isMember(orgs []model.OrgRef, org string) bool {
|
||||
if org == "" {
|
||||
return false
|
||||
}
|
||||
for _, o := range orgs {
|
||||
if o.Org == org {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// validate parses raw, verifies its signature against the JWKS, and enforces
|
||||
// issuer/audience/expiry. Returns the claims on success, an error otherwise.
|
||||
func (v *identityValidator) validate(raw string) (*idClaims, error) {
|
||||
@@ -214,14 +257,13 @@ func (v *identityValidator) validate(raw string) (*idClaims, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Fail SECURE on a misconfigured (empty) trust set: an empty issuer OR audience
|
||||
// allowlist must REJECT every token, never silently disable that axis. In
|
||||
// production both are always resolved non-empty (BrandIssuers + the baked
|
||||
// audience defaults, unioned in config.go so they are "never empty"), so this
|
||||
// fires ONLY on an operator misconfiguration (CLOUD_JWT_AUDIENCES="" emptying the
|
||||
// resolved set, or an empty issuer set) — and then it denies, it never admits (I2).
|
||||
if len(v.issuers) == 0 || len(v.audiences) == 0 {
|
||||
return nil, fmt.Errorf("identity validator misconfigured: empty issuer or audience allowlist")
|
||||
// Fail SECURE on a misconfigured (empty) trust set: with no trusted issuer every
|
||||
// token must be REJECTED, never silently admitted. In production the set is always
|
||||
// non-empty (the primary issuer + BrandIssuers, unioned in config.go so it is
|
||||
// "never empty"), so this fires ONLY on an operator misconfiguration — and then it
|
||||
// denies, it never admits (I2).
|
||||
if len(v.issuers) == 0 {
|
||||
return nil, fmt.Errorf("identity validator misconfigured: empty issuer set")
|
||||
}
|
||||
|
||||
// Reject a missing issuer: an empty issuer must never pass the set check.
|
||||
@@ -234,28 +276,22 @@ func (v *identityValidator) validate(raw string) (*idClaims, error) {
|
||||
if claims.Expiry == nil {
|
||||
return nil, fmt.Errorf("missing expiry")
|
||||
}
|
||||
// Issuer must be one of the trusted brand issuers. go-jose's jwt.Expected
|
||||
// checks a SINGLE issuer, so the issuer is validated here against the set and
|
||||
// left out of Expected (audience + expiry stay with Expected).
|
||||
// Issuer must be one of the trusted brand issuers. go-jose's jwt.Expected checks a
|
||||
// SINGLE issuer, so the issuer is validated here against the set and left out of
|
||||
// Expected (only expiry/not-before stay with Expected).
|
||||
if !issuerAllowed(claims.Issuer, v.issuers) {
|
||||
return nil, fmt.Errorf("untrusted issuer %q", claims.Issuer)
|
||||
}
|
||||
// Audience: the static allowlist (CLOUD_JWT_AUDIENCES / brand app client_ids)
|
||||
// PLUS the per-org PaaS-KMS sync machine audience bound to THIS token's own
|
||||
// owner (<owner>-platform-kms). The machine audience is added only when the
|
||||
// allowlist is active (non-empty — always so in production) and only for the
|
||||
// token's own org, so accepting it never widens org-scoping: the /v1/kms guard still
|
||||
// gates on owner == :org. Without this, a real client_credentials machine token
|
||||
// (aud == its per-org clientId, never in the allowlist) fails here and the
|
||||
// sync silently stays pending — the activation blocker.
|
||||
// The audience allowlist is guaranteed non-empty (checked above), so the
|
||||
// audience axis is ALWAYS enforced — never silently skipped.
|
||||
auds := v.audiences
|
||||
if mach := kmsMachineAudience(claims.Owner); mach != "" {
|
||||
auds = append(append(make([]string, 0, len(v.audiences)+1), v.audiences...), mach)
|
||||
}
|
||||
expected := jwt.Expected{AnyAudience: jwt.Audience(auds)}
|
||||
if err := claims.Claims.ValidateWithLeeway(expected, 2*time.Minute); err != nil {
|
||||
// Audience is NOT an access gate. A valid signature from a trusted issuer (both
|
||||
// checked above) proves IAM minted this token for one of ITS OWN registered apps;
|
||||
// the `aud` merely names which app. Cloud does not keep a per-app allowlist to
|
||||
// mirror IAM's registry — that mirror drifted and silently 401'd every new
|
||||
// first-party app until hand-edited. Org scope is the `owner` claim, enforced by
|
||||
// every downstream guard; SuperAdmin is owner==adminOrg AND !isKMSMachinePrincipal
|
||||
// (SanitizeIdentity). Expiry + not-before are STILL enforced here: Expected{} with a
|
||||
// zero Time validates against time.Now(); an empty AnyAudience skips ONLY the
|
||||
// audience match (go-jose/v4 jwt/validation.go).
|
||||
if err := claims.Claims.ValidateWithLeeway(jwt.Expected{}, 2*time.Minute); err != nil {
|
||||
return nil, fmt.Errorf("claims: %w", err)
|
||||
}
|
||||
return &claims, nil
|
||||
@@ -375,14 +411,52 @@ func (c *jwksCache) fetch() (*gojose.JSONWebKeySet, error) {
|
||||
// Token extraction — mirrors iamauth's Bearer / Basic / API-key helpers.
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
// isAPIKey reports whether tok is an opaque, backend-validated key (hk-/sk-/…)
|
||||
// rather than a JWT, so the sanitizer skips JWT parsing for it.
|
||||
// APIKeyPrefixes are the Hanzo API key families. A published key (pk-) is
|
||||
// write-only and scoped, so it is safe in a public bundle; a secret key (sk-)
|
||||
// authenticates a server. Everything else is OAuth2, which is a JWT and not a key.
|
||||
//
|
||||
// hk- is sk- under an older name and is on its way out. It stays accepted here
|
||||
// because IAM mints it — cloud only validates (see account.go mintKey, which
|
||||
// delegates to iam.mintUserKey) — so dropping it here before IAM renames the
|
||||
// family would reject every key IAM hands out. Retire it in that order: IAM mints
|
||||
// sk-, holders re-key, then delete the entry below.
|
||||
//
|
||||
// fw_ and hz_ were listed here and never minted by anything: dead entries that
|
||||
// widened what counts as a credential for no reason. Gone.
|
||||
//
|
||||
// This is the ONE authority. Admission mirrors it rather than importing it (it
|
||||
// stays free of cloud-internal imports); if this list changes, that copy must too.
|
||||
var APIKeyPrefixes = []string{"pk-", "sk-", "hk-"}
|
||||
|
||||
// PublishablePrefix is the ONE publishable spelling: pk- is the key you may ship
|
||||
// in a browser bundle, sk- is the one you may not. Stripe's split, same reason.
|
||||
const PublishablePrefix = "pk-"
|
||||
|
||||
// IsPublishableKey reports whether tok is a publishable key.
|
||||
//
|
||||
// A publishable key is NOT a credential: it identifies a tenant so a public
|
||||
// surface can WRITE (ingest events), and it must never mint a principal that can
|
||||
// READ. Cloud resolved any isAPIKey token — pk- included — into "the same
|
||||
// principal a JWT yields", which made a key documented as "safe to show" into a
|
||||
// full bearer for the org that owns it. IdentityFromRequest now refuses it, so
|
||||
// publishable means publishable.
|
||||
//
|
||||
// It stays in APIKeyPrefixes on purpose: OrgForKey must still resolve a pk- to
|
||||
// its owning org, because that is exactly how the ingest door learns which tenant
|
||||
// a browser beacon belongs to. Resolvable, not authenticating.
|
||||
func IsPublishableKey(tok string) bool {
|
||||
return strings.HasPrefix(strings.TrimSpace(tok), PublishablePrefix)
|
||||
}
|
||||
|
||||
// isAPIKey reports whether tok is an opaque, backend-validated key rather than a
|
||||
// JWT, so the sanitizer skips JWT parsing for it.
|
||||
func isAPIKey(tok string) bool {
|
||||
return strings.HasPrefix(tok, "hk-") ||
|
||||
strings.HasPrefix(tok, "sk-") ||
|
||||
strings.HasPrefix(tok, "pk-") ||
|
||||
strings.HasPrefix(tok, "fw_") ||
|
||||
strings.HasPrefix(tok, "hz_")
|
||||
for _, p := range APIKeyPrefixes {
|
||||
if strings.HasPrefix(tok, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// bearerFromAuth extracts the token from a "Bearer <token>" header value.
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package cloud
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestValidate_AudienceIsNotAGate proves the IAM-native trust model: a token signed
|
||||
// by a trusted issuer validates REGARDLESS of its `aud` (the minting app's client_id).
|
||||
// Cloud keeps no per-app audience allowlist mirroring IAM's registry — so a brand-new
|
||||
// first-party app works with zero cloud change, and the specific app tokens the old
|
||||
// per-app tests pinned (admin-guard, world, team, commerce) are accepted by the SAME
|
||||
// rule as everything else. Trust is signature + issuer + expiry; org scope is the
|
||||
// owner claim, enforced downstream. Replaces the three audience_*_test.go files that
|
||||
// asserted a static allowlist which no longer exists.
|
||||
func TestValidate_AudienceIsNotAGate(t *testing.T) {
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("genkey: %v", err)
|
||||
}
|
||||
jwks := jwksServer(t, &key.PublicKey)
|
||||
v := newIdentityValidator(testIssuer, jwks.URL, 0)
|
||||
future := time.Now().Add(time.Hour)
|
||||
|
||||
// Every audience — the apps the deleted per-app tests pinned AND a never-registered
|
||||
// one — validates identically, because aud is not an access gate.
|
||||
for _, aud := range []string{
|
||||
"hanzo-admin-guard", // admin.hanzo.ai forward-auth cockpit
|
||||
"hanzo-world", // world.hanzo.ai analyst tokens
|
||||
"hanzo-team", // hanzo.team wallet page
|
||||
"hanzo-commerce", // commerce.hanzo.ai admin AI assistant
|
||||
"a-brand-new-first-party-app-never-listed-anywhere",
|
||||
} {
|
||||
tok := signWith(t, key, tokenClaims(aud, "acme", "", false, future))
|
||||
id, err := v.validate(tok)
|
||||
if err != nil {
|
||||
t.Fatalf("aud=%q from a trusted issuer must validate (no allowlist), got %v", aud, err)
|
||||
}
|
||||
if id.Owner != "acme" {
|
||||
t.Errorf("aud=%q: owner must be carried through, got %q", aud, id.Owner)
|
||||
}
|
||||
}
|
||||
|
||||
// Expiry is STILL enforced (dropping the aud gate must not disable time checks):
|
||||
// a token expired beyond the 2m leeway is rejected whatever its aud.
|
||||
expired := signWith(t, key, tokenClaims("hanzo-commerce", "acme", "", false, time.Now().Add(-time.Hour)))
|
||||
if _, err := v.validate(expired); err == nil {
|
||||
t.Error("expired token must be REJECTED even though audience is no longer gated")
|
||||
}
|
||||
}
|
||||
+49
-35
@@ -1,15 +1,15 @@
|
||||
package cloud
|
||||
|
||||
// V6 (the activation blocker) — the identity validator must accept a per-org
|
||||
// PaaS-KMS sync machine token: a client_credentials JWT whose aud is the org's
|
||||
// own IAM application clientId "<owner>-platform-kms" (a per-org value, NEVER in
|
||||
// CLOUD_JWT_AUDIENCES) — but ONLY when that audience is bound to the token's OWN
|
||||
// owner claim. Before the fix the machine token failed the audience check,
|
||||
// SanitizeIdentity resolved anonymous, and the /v1/kms guard 403'd it, so the sync
|
||||
// silently stayed pending. These are white-box unit tests of validate() itself;
|
||||
// the end-to-end proof through SanitizeIdentity + the real guard lives in
|
||||
// clients/kms (v6_aud_e2e_test.go). Reuses the jwksServer/signWith/tokenClaims
|
||||
// helpers from middleware_identity_test.go (same package).
|
||||
// The per-org PaaS-KMS sync identity authenticates as its own IAM application
|
||||
// "<owner>-platform-kms" (client_credentials), so its token carries owner=<org> and
|
||||
// aud=<owner>-platform-kms. Validation no longer gates on the audience at all (trust
|
||||
// is signature + issuer + expiry), so a machine token clears validate() like any
|
||||
// other. The owner-bound machine aud survives only to IDENTIFY such a principal
|
||||
// (isKMSMachinePrincipal) so SanitizeIdentity can DENY it SuperAdmin even in the admin
|
||||
// org — a client_credentials machine identity must never wield platform-admin. These
|
||||
// are white-box unit tests of that identification; the end-to-end proof through
|
||||
// SanitizeIdentity + the real guard lives in clients/kms (v6_aud_e2e_test.go). Reuses
|
||||
// the jwksServer/signWith/tokenClaims helpers from middleware_identity_test.go.
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
@@ -18,19 +18,16 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIdentityValidator_KMSMachineAudience(t *testing.T) {
|
||||
func TestIdentityValidator_KMSMachinePrincipal(t *testing.T) {
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatalf("genkey: %v", err)
|
||||
}
|
||||
jwks := jwksServer(t, &key.PublicKey)
|
||||
// The static allowlist deliberately contains NO *-platform-kms audience, so any
|
||||
// acceptance below can come ONLY from the owner-bound machine-aud rule, not the
|
||||
// allowlist — this is what makes it a fix and not a config workaround.
|
||||
v := newIdentityValidator(testIssuer, jwks.URL, []string{"hanzo-console"}, 0)
|
||||
v := newIdentityValidator(testIssuer, jwks.URL, 0)
|
||||
future := time.Now().Add(time.Hour)
|
||||
|
||||
t.Run("machine token for its own org is accepted", func(t *testing.T) {
|
||||
t.Run("own-org machine token validates and is recognised as a machine principal", func(t *testing.T) {
|
||||
c, err := v.validate(signWith(t, key, tokenClaims("maxpower-platform-kms", "maxpower", "", false, future)))
|
||||
if err != nil {
|
||||
t.Fatalf("machine token rejected: %v", err)
|
||||
@@ -38,34 +35,51 @@ func TestIdentityValidator_KMSMachineAudience(t *testing.T) {
|
||||
if c.Owner != "maxpower" {
|
||||
t.Fatalf("owner=%q, want maxpower", c.Owner)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("machine aud for a DIFFERENT org is rejected (owner-bound)", func(t *testing.T) {
|
||||
// owner=maxpower but aud=acme-platform-kms: the accepted machine aud is bound
|
||||
// to the token's OWN owner (maxpower-platform-kms), so this must fail — it is
|
||||
// not a blanket "*-platform-kms" wildcard.
|
||||
if _, err := v.validate(signWith(t, key, tokenClaims("acme-platform-kms", "maxpower", "", false, future))); err == nil {
|
||||
t.Fatal("cross-org machine audience must be rejected (owner-bound)")
|
||||
if !isKMSMachinePrincipal(c) {
|
||||
t.Fatal("aud==<owner>-platform-kms must be recognised as a machine principal")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("arbitrary audience still rejected (fix is scoped, not a disable)", func(t *testing.T) {
|
||||
if _, err := v.validate(signWith(t, key, tokenClaims("some-random-app", "maxpower", "", false, future))); err == nil {
|
||||
t.Fatal("an arbitrary audience must still be rejected")
|
||||
t.Run("admin-org machine token is recognised so SuperAdmin is denied", func(t *testing.T) {
|
||||
c, err := v.validate(signWith(t, key, tokenClaims("admin-platform-kms", "admin", "", true, future)))
|
||||
if err != nil {
|
||||
t.Fatalf("admin machine token rejected: %v", err)
|
||||
}
|
||||
if !isKMSMachinePrincipal(c) {
|
||||
t.Fatal("admin-org machine token must be recognised (SanitizeIdentity denies it SuperAdmin)")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("machine aud with empty owner is rejected (fail closed)", func(t *testing.T) {
|
||||
// aud="-platform-kms" with owner="": kmsMachineAudience("")=="" so no machine
|
||||
// audience is granted and the bare suffix is not in the allowlist.
|
||||
if _, err := v.validate(signWith(t, key, tokenClaims("-platform-kms", "", "", false, future))); err == nil {
|
||||
t.Fatal("machine aud with empty owner must be rejected")
|
||||
t.Run("machine aud bound to a DIFFERENT org is not this owner's machine principal", func(t *testing.T) {
|
||||
// owner=maxpower, aud=acme-platform-kms: the machine-principal match is bound to
|
||||
// the token's OWN owner (maxpower-platform-kms), not a "*-platform-kms" wildcard.
|
||||
// It validates (aud is not gated) and is owner-scoped to maxpower downstream.
|
||||
c, err := v.validate(signWith(t, key, tokenClaims("acme-platform-kms", "maxpower", "", false, future)))
|
||||
if err != nil {
|
||||
t.Fatalf("token rejected: %v", err)
|
||||
}
|
||||
if isKMSMachinePrincipal(c) {
|
||||
t.Fatal("a cross-org machine aud must not count as this owner's machine principal")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("normal static-allowlist token still accepted (regression)", func(t *testing.T) {
|
||||
if _, err := v.validate(signWith(t, key, tokenClaims("hanzo-console", "maxpower", "", false, future))); err != nil {
|
||||
t.Fatalf("static-allowlist token rejected: %v", err)
|
||||
t.Run("ordinary app token is not a machine principal", func(t *testing.T) {
|
||||
c, err := v.validate(signWith(t, key, tokenClaims("hanzo-console", "maxpower", "", false, future)))
|
||||
if err != nil {
|
||||
t.Fatalf("token rejected: %v", err)
|
||||
}
|
||||
if isKMSMachinePrincipal(c) {
|
||||
t.Fatal("an ordinary app token is not a machine principal")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty-owner token is never a machine principal (fail closed)", func(t *testing.T) {
|
||||
c, err := v.validate(signWith(t, key, tokenClaims("-platform-kms", "", "", false, future)))
|
||||
if err != nil {
|
||||
t.Fatalf("token rejected: %v", err)
|
||||
}
|
||||
if isKMSMachinePrincipal(c) {
|
||||
t.Fatal(`empty-owner token must never be a machine principal (kmsMachineAudience("")=="")`)
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@@ -9,10 +9,10 @@ import (
|
||||
)
|
||||
|
||||
// TestValidate_FailSecureOnEmptyTrustSet proves I2: a validator whose resolved
|
||||
// issuer OR audience allowlist is empty REJECTS an otherwise-valid, correctly
|
||||
// signed token — the axis is never silently disabled. Production always resolves
|
||||
// non-empty sets; this guards the misconfiguration path (CLOUD_JWT_AUDIENCES=""
|
||||
// or an empty issuer set), which must fail closed, not open.
|
||||
// issuer set is empty REJECTS an otherwise-valid, correctly signed token — the axis
|
||||
// is never silently disabled. Production always resolves a non-empty set; this
|
||||
// guards the misconfiguration path (an empty issuer set), which must fail closed,
|
||||
// not open.
|
||||
func TestValidate_FailSecureOnEmptyTrustSet(t *testing.T) {
|
||||
key, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
@@ -23,20 +23,15 @@ func TestValidate_FailSecureOnEmptyTrustSet(t *testing.T) {
|
||||
tok := signWith(t, key, tokenClaims("hanzo-console", "acme", "", false, future))
|
||||
|
||||
// Sanity: a properly configured validator accepts the token.
|
||||
if _, err := newIdentityValidator(testIssuer, jwks.URL, []string{"hanzo-console"}, 0).validate(tok); err != nil {
|
||||
if _, err := newIdentityValidator(testIssuer, jwks.URL, 0).validate(tok); err != nil {
|
||||
t.Fatalf("baseline valid token must be accepted, got %v", err)
|
||||
}
|
||||
|
||||
// Empty audience set → deny.
|
||||
if _, err := newIdentityValidator(testIssuer, jwks.URL, nil, 0).validate(tok); err == nil {
|
||||
t.Error("empty audience allowlist must REJECT (fail-secure), not accept")
|
||||
}
|
||||
|
||||
// Empty issuer set → deny (construct directly; trustedIssuers never yields empty
|
||||
// with a primary, so bypass it to exercise the guard).
|
||||
vEmptyIss := &identityValidator{issuers: nil, audiences: []string{"hanzo-console"}, cache: newJWKSCache(jwks.URL, 0), keys: newIAMKeys()}
|
||||
vEmptyIss := &identityValidator{issuers: nil, cache: newJWKSCache(jwks.URL, 0), keys: newIAMKeys()}
|
||||
if _, err := vEmptyIss.validate(tok); err == nil {
|
||||
t.Error("empty issuer allowlist must REJECT (fail-secure), not accept")
|
||||
t.Error("empty issuer set must REJECT (fail-secure), not accept")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,7 +107,7 @@ func TestBrandIssuers(t *testing.T) {
|
||||
// full brand set, so a lux token would pass the issuer gate on the hanzo binary.
|
||||
func TestNewIdentityValidator_MultiIssuer(t *testing.T) {
|
||||
os.Unsetenv("WHITELABEL_ISSUERS")
|
||||
v := newIdentityValidator("https://hanzo.id", "http://iam.hanzo.svc/v1/iam/.well-known/jwks", []string{"hanzo-cloud", "lux-cloud"}, 0)
|
||||
v := newIdentityValidator("https://hanzo.id", "http://iam.hanzo.svc/v1/iam/.well-known/jwks", 0)
|
||||
if !issuerAllowed("https://lux.id", v.issuers) {
|
||||
t.Fatalf("validator must trust the lux issuer, set=%v", v.issuers)
|
||||
}
|
||||
@@ -123,64 +118,3 @@ func TestNewIdentityValidator_MultiIssuer(t *testing.T) {
|
||||
t.Fatalf("validator must reject an untrusted issuer, set=%v", v.issuers)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBrandAudiences proves every brand's cloud audience (<brand>-cloud) is derived
|
||||
// from the brands registry — one source of truth, mirroring BrandIssuers.
|
||||
func TestBrandAudiences(t *testing.T) {
|
||||
got := BrandAudiences()
|
||||
for _, want := range []string{"hanzo-cloud", "lux-cloud", "zoo-cloud", "pars-cloud", "bootnode-cloud"} {
|
||||
found := false
|
||||
for _, g := range got {
|
||||
if g == want {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("BrandAudiences()=%v missing %q", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestJWTAudiencesFromEnv_BrandUnion proves the resolved audience allowlist ALWAYS
|
||||
// includes every brand's <brand>-cloud aud (so a lux token validates), whether the
|
||||
// list comes from the baked default or a hanzo-only env override — and that an
|
||||
// env-supplied entry is not duplicated.
|
||||
func TestJWTAudiencesFromEnv_BrandUnion(t *testing.T) {
|
||||
has := func(list []string, v string) bool {
|
||||
for _, s := range list {
|
||||
if s == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Baked default path (no env).
|
||||
os.Unsetenv("CLOUD_JWT_AUDIENCES")
|
||||
os.Unsetenv("GATEWAY_ALLOWED_AUDIENCES")
|
||||
def := jwtAudiencesFromEnv()
|
||||
for _, want := range []string{"hanzo-cloud", "lux-cloud", "zoo-cloud", "pars-cloud"} {
|
||||
if !has(def, want) {
|
||||
t.Errorf("baked audiences %v missing brand aud %q", def, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A legacy hanzo-only env override must STILL accept lux-cloud (brand union),
|
||||
// with no duplicate of the env-supplied hanzo-cloud.
|
||||
os.Setenv("GATEWAY_ALLOWED_AUDIENCES", "hanzo-app,hanzo-console,hanzo-cloud")
|
||||
defer os.Unsetenv("GATEWAY_ALLOWED_AUDIENCES")
|
||||
got := jwtAudiencesFromEnv()
|
||||
if !has(got, "lux-cloud") {
|
||||
t.Fatalf("hanzo-only env override must still accept lux-cloud, got %v", got)
|
||||
}
|
||||
n := 0
|
||||
for _, s := range got {
|
||||
if s == "hanzo-cloud" {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("hanzo-cloud must appear exactly once (no duplicate), got %d in %v", n, got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,19 +137,3 @@ func BrandIssuers() []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// BrandAudiences returns the OAuth `aud` (== IAM client_id == app name) of every
|
||||
// white-label brand's cloud login app: `<brand>-cloud` (hanzo-cloud, lux-cloud,
|
||||
// zoo-cloud, pars-cloud, bootnode-cloud). A brand's session token carries
|
||||
// aud=<brand>-cloud (HIP-0111: client_id == app == aud), so the audience allowlist
|
||||
// must include each to accept a lux/zoo/pars token on the ONE binary. Derived from
|
||||
// the same `brands` registry as BrandIssuers — one source of truth, no hand-listing.
|
||||
func BrandAudiences() []string {
|
||||
out := make([]string, 0, len(brands))
|
||||
for id := range brands {
|
||||
if id != "" {
|
||||
out = append(out, id+"-cloud")
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -2,13 +2,18 @@ package cloud
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
aiobject "github.com/hanzoai/ai/object"
|
||||
"github.com/hanzoai/cloud/cek"
|
||||
"github.com/hanzoai/cloud/clients/commerceinproc"
|
||||
"github.com/hanzoai/cloud/clients/metering"
|
||||
"github.com/hanzoai/cloud/internal/org"
|
||||
s3 "github.com/hanzoai/s3-go"
|
||||
luxlog "github.com/luxfi/log"
|
||||
"github.com/zap-proto/zip"
|
||||
|
||||
@@ -103,6 +108,7 @@ func BuildDeps(cfg *Config) Deps {
|
||||
deps.O11y = pick(cfg, logger, "o11y", "O11y", cfg.O11yZAPAddr, clients.O11yRPCAt, clients.DisabledO11y)
|
||||
deps.VFS = pickVFSClient(cfg, logger)
|
||||
deps.MQ = pick(cfg, logger, "mq", "MQ", cfg.MQZAPAddr, clients.MQRPCAt, clients.DisabledMQ)
|
||||
deps.Durable = buildDurability(cfg, logger)
|
||||
|
||||
// Payments and Vault never co-resident. Disabled stub when no
|
||||
// endpoint, otherwise RPC.
|
||||
@@ -713,6 +719,148 @@ func pickVFSClient(cfg *Config, log luxlog.Logger) VFSClient {
|
||||
return clients.DisabledVFS()
|
||||
}
|
||||
|
||||
// durableBucket holds every org's HA-SQLite snapshot (and its writer lease). One
|
||||
// bucket, keys laid out orgs/<slug>[/…]/<subsystem>.db per HIP-0302 — the durable
|
||||
// twin of the on-disk DataDir layout.
|
||||
//
|
||||
// OPERATIONAL REQUIREMENTS the fence depends on (enforce in the SeaweedFS deployment,
|
||||
// not in code):
|
||||
//
|
||||
// - Object versioning + a no-expiry / no-lifecycle-deletion policy on this prefix.
|
||||
// The writer lease (orgs/<slug>/.owner) is the round's system of record; if the
|
||||
// gateway silently drops or rolls back that object, a monotone round can reset and
|
||||
// un-fence a zombie writer (Red M4). A local high-water-round floor per pod is the
|
||||
// future in-process defense; the object lifecycle is the operational one.
|
||||
// - RWO, per-writer PVCs for DataDir — NEVER an RWX shared volume (Red M5). Two pods
|
||||
// on one DataDir corrupt SQLite regardless of this fence; single-writer here is the
|
||||
// durable-copy fence, and per-pod RWO is the local-file guarantee the shard router
|
||||
// already relies on (see shardrouter.go).
|
||||
const durableBucket = "org-db"
|
||||
|
||||
// buildDurability constructs the deployment's HA-durability factory, or nil when the
|
||||
// deployment has no object store to be durable against (dev/single-node — every
|
||||
// OrgStore then stays local-only). It composes the SeaweedFS S3 If-Match
|
||||
// ConditionalStore (the SAME s3admin identity deps.VFS uses), the writer membership
|
||||
// over CLOUD_PEERS (the SAME set the shard router elects on, so the store-layer owner
|
||||
// and the routed owner agree), and the per-org envelope Cipher rooted at the KMS
|
||||
// master. Any construction failure fails SAFE to nil (local-only) rather than crash
|
||||
// the boot; an encryption-capable build with no usable cipher is REFUSED — a build
|
||||
// that promises encryption never ships plaintext snapshots to the object store.
|
||||
func buildDurability(cfg *Config, log luxlog.Logger) *Durability {
|
||||
// A multi-replica deployment REQUIRES the durable plane: with >1 writer, a per-org
|
||||
// store that is not hydrate-on-open + fenced is the outage this exists to fix.
|
||||
// disabledDurability logs at the severity the replica count warrants, so a
|
||||
// misconfigured prod deployment is never SILENTLY non-durable (Red L2).
|
||||
multiReplica := len(parsePeers(cfg.ShardPeers)) > 1
|
||||
|
||||
// Explicit opt-in: the object-store fence rests on the deployed SeaweedFS enforcing
|
||||
// conditional-PUT (If-Match) atomically, which must be validated against the deployed
|
||||
// version before it fences real tenant data (the takeover-fence staging gate). Until
|
||||
// CLOUD_RESEARCH_DURABLE is set the store runs local-only — the shard router still
|
||||
// pins each org to one writer, so this is not the rolling-deploy outage; only the
|
||||
// cross-restart object-store snapshot waits for the opt-in.
|
||||
if !cfg.ResearchDurable {
|
||||
disabledDurability(log, multiReplica, "CLOUD_RESEARCH_DURABLE not set — HA object-store durability is opt-in pending the SeaweedFS conditional-PUT atomicity gate")
|
||||
return nil
|
||||
}
|
||||
|
||||
admin := s3admin.New()
|
||||
if !admin.Configured() {
|
||||
disabledDurability(log, multiReplica, "no S3 admin creds (S3_ADMIN_* unset)")
|
||||
return nil
|
||||
}
|
||||
client, err := admin.Client()
|
||||
if err != nil {
|
||||
disabledDurability(log, multiReplica, fmt.Sprintf("S3 client construction failed: %v", err))
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
if err := ensureDurableBucket(ctx, admin, client); err != nil {
|
||||
// Non-fatal: the bucket likely already exists; a later ship/hydrate retries.
|
||||
log.Warn("durability bucket ensure failed (continuing)", "bucket", durableBucket, "err", err)
|
||||
}
|
||||
cancel()
|
||||
|
||||
// Membership: CLOUD_PEERS (the shard router's set). A single-pod deployment with
|
||||
// no peers is its own sole writer — still hydrate-on-open + fenced ship across a
|
||||
// rolling restart.
|
||||
self := firstNonEmptyStr(strings.TrimSpace(cfg.ShardSelf), hostnameOr("cloud-0"))
|
||||
peers := parsePeers(cfg.ShardPeers)
|
||||
if len(peers) == 0 {
|
||||
peers = []org.Member{{ID: self, Addr: self}}
|
||||
}
|
||||
members := org.NewMembership(self, org.StaticSource(peers...), 5*time.Second)
|
||||
_ = members.Start(context.Background()) // static source: the initial refresh populates Members()
|
||||
|
||||
cipher := durableCipher(cfg, log)
|
||||
if cipher == nil && cek.Encrypting() {
|
||||
// The master that satisfied cek must decode here too, so this is a genuine
|
||||
// misconfig, not a dev path: never ship plaintext snapshots AND never silently
|
||||
// drop durability — fail closed and log LOUDLY for the replica count.
|
||||
disabledDurability(log, multiReplica, "encryption-capable build but no durable cipher (would ship plaintext snapshots)")
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Info("durability enabled", "bucket", durableBucket, "self", self, "peers", len(peers), "encrypted", cipher != nil)
|
||||
return org.NewDurability(org.NewS3ConditionalStore(client, durableBucket), members, cipher)
|
||||
}
|
||||
|
||||
// disabledDurability records that the durable plane is OFF, at ERROR when the
|
||||
// deployment is multi-replica (per-org stores then survive only via shard routing +
|
||||
// per-pod RWO PVC; a lost/rescheduled PVC loses committed data — the operator MUST
|
||||
// configure S3) and at INFO when single-replica/dev (local-only is the expected
|
||||
// posture). One place, so no disabled path is silent on a deployment that needs HA.
|
||||
func disabledDurability(log luxlog.Logger, multiReplica bool, why string) {
|
||||
if multiReplica {
|
||||
log.Error("DURABILITY DISABLED on a MULTI-REPLICA deployment — per-org stores survive only via shard routing + per-pod RWO PVC; a lost/rescheduled PVC loses committed data. Configure S3_ADMIN_* to enable the durable plane.", "reason", why)
|
||||
return
|
||||
}
|
||||
log.Info("durability disabled — single-replica/dev, per-org stores stay local-only", "reason", why)
|
||||
}
|
||||
|
||||
// ensureDurableBucket creates the durable bucket if absent (idempotent).
|
||||
func ensureDurableBucket(ctx context.Context, admin s3admin.Admin, client *s3.Client) error {
|
||||
ok, err := client.BucketExists(ctx, durableBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok {
|
||||
return nil
|
||||
}
|
||||
return client.MakeBucket(ctx, durableBucket, s3.MakeBucketOptions{Region: admin.Region()})
|
||||
}
|
||||
|
||||
// durableCipher builds the per-org envelope Cipher from the base64 KMS master
|
||||
// (CLOUD_KMS_MASTER_KEY_REF — the SAME key cek derives from). nil when no valid
|
||||
// 32-byte master is configured (pure-Go dev → plaintext durable object, matching the
|
||||
// plaintext local file).
|
||||
func durableCipher(cfg *Config, log luxlog.Logger) *org.Cipher {
|
||||
ref := strings.TrimSpace(cfg.KMSMasterKeyRef)
|
||||
if ref == "" {
|
||||
return nil
|
||||
}
|
||||
master, err := base64.StdEncoding.DecodeString(ref)
|
||||
if err != nil {
|
||||
log.Warn("durable cipher: KMS master key ref is not valid base64", "err", err)
|
||||
return nil
|
||||
}
|
||||
c, err := org.NewCipher(master)
|
||||
if err != nil {
|
||||
log.Warn("durable cipher: invalid KMS master key", "err", err)
|
||||
return nil
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// hostnameOr returns the OS hostname, or def when unavailable — a stable self id for
|
||||
// a single-pod deployment that sets no CLOUD_POD_NAME.
|
||||
func hostnameOr(def string) string {
|
||||
if h, err := os.Hostname(); err == nil && h != "" {
|
||||
return h
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func pickPaymentsClient(cfg *Config, log luxlog.Logger) PaymentsClient {
|
||||
if cfg.PaymentsZAPAddr != "" {
|
||||
log.Info("deps.Payments → ZAP RPC", "addr", cfg.PaymentsZAPAddr)
|
||||
|
||||
+2
-2
@@ -105,8 +105,8 @@ type Config struct {
|
||||
PlatformURL string `json:"platform_url,omitempty"`
|
||||
CloudURL string `json:"cloud_url,omitempty"`
|
||||
ClientID string `json:"client_id,omitempty"`
|
||||
APIKey string `json:"apiKey,omitempty"` // hk-… key; what `hanzo code` hands the agents
|
||||
CodeTool string `json:"code_tool,omitempty"` // default agent for bare `hanzo` / `hanzo code`: dev|claude|codex
|
||||
APIKey string `json:"apiKey,omitempty"` // hk-… key; what `hanzo code` hands the agents
|
||||
CodeTool string `json:"code_tool,omitempty"` // default agent for bare `hanzo` / `hanzo code`: dev|claude|codex
|
||||
CodeModel string `json:"code_model,omitempty"` // default model for `hanzo code` (else defaultCodeModel)
|
||||
}
|
||||
|
||||
|
||||
+18
-18
@@ -6,8 +6,8 @@ package cli
|
||||
// remember.
|
||||
//
|
||||
// hanzo code claude # Claude Code on the default model
|
||||
// hanzo code codex deepseek-v4-pro
|
||||
// hanzo code dev glm5.2 # ids are resolved fuzzily: glm5.2 -> glm-5.2
|
||||
// hanzo code codex enso-pro
|
||||
// hanzo code dev zen5 # ids are resolved fuzzily: zen5pro -> zen5-pro
|
||||
// hanzo code ls # what can I run?
|
||||
//
|
||||
// Two wire protocols cover all three agents: Claude Code speaks Anthropic,
|
||||
@@ -71,7 +71,7 @@ func (t zenTier) slotID() string {
|
||||
var zenTiers = []zenTier{
|
||||
{zen: "zen5-flash", env: "ANTHROPIC_DEFAULT_HAIKU_MODEL", name: "Zen5 Flash", desc: "Hanzo Zen5 Flash — fast, cheap tier"},
|
||||
{zen: "zen5", carrier: "claude-sonnet-4-6[1m]", env: "ANTHROPIC_DEFAULT_SONNET_MODEL", name: "Zen5", desc: "Hanzo Zen5 — frontier tier (1M context)"},
|
||||
{zen: "zen5-pro", carrier: "claude-opus-4-8[1m]", env: "ANTHROPIC_DEFAULT_OPUS_MODEL", name: "Zen5 Pro", desc: "Hanzo Zen5 Pro — DeepSeek-V4 class (1M context)"},
|
||||
{zen: "zen5-pro", carrier: "claude-opus-4-8[1m]", env: "ANTHROPIC_DEFAULT_OPUS_MODEL", name: "Zen5 Pro", desc: "Hanzo Zen5 Pro — deep reasoning (1M context)"},
|
||||
{zen: "zen5-pro", carrier: "claude-fable-5[1m]", env: "ANTHROPIC_DEFAULT_FABLE_MODEL", name: "Zen5 Pro (max effort)", desc: "Hanzo Zen5 Pro, top tier (1M context)"},
|
||||
{zen: "zen5-coder", carrier: "claude-sonnet-5", name: "Zen5 Coder", desc: "Hanzo Zen5 Coder — code-specialized (1M context)"},
|
||||
}
|
||||
@@ -144,19 +144,19 @@ func openaiWire(base, token, _ string) map[string]string {
|
||||
}
|
||||
|
||||
type codeAgent struct {
|
||||
bin string // executable to exec
|
||||
wire wire // how it finds the cloud
|
||||
fullAuto []string // flags that bypass approval prompts
|
||||
continueArgs []string // harness-native form of Hanzo -c/--continue
|
||||
modelArg []string // how the model is passed on argv (empty: via env)
|
||||
carrier func(model string) string // maps the resolved model to a client-recognized id (claude: zen→carrier); nil = pass through
|
||||
bin string // executable to exec
|
||||
wire wire // how it finds the cloud
|
||||
fullAuto []string // flags that bypass approval prompts
|
||||
continueArgs []string // harness-native form of Hanzo -c/--continue
|
||||
modelArg []string // how the model is passed on argv (empty: via env)
|
||||
carrier func(model string) string // maps the resolved model to a client-recognized id (claude: zen→carrier); nil = pass through
|
||||
provider func(base, model string) []string // agents that need the endpoint declared, not just env'd
|
||||
clear []string // env that would shadow the wire (a stale key in the shell)
|
||||
configHome string // env var that relocates the agent's config dir to ~/.hanzo ("" = share the user's own install)
|
||||
seed func(dir string) error // one-time defaults for the isolated config dir
|
||||
appendSystem []string // --append-system-prompt + text; ALWAYS applied (identity, not a permission bypass — present in --safe too)
|
||||
mcp bool // auto-wire the Hanzo MCP server (code/vector/web/vision tools) as an stdio server scoped to the cwd
|
||||
install string // hint when the binary is missing
|
||||
clear []string // env that would shadow the wire (a stale key in the shell)
|
||||
configHome string // env var that relocates the agent's config dir to ~/.hanzo ("" = share the user's own install)
|
||||
seed func(dir string) error // one-time defaults for the isolated config dir
|
||||
appendSystem []string // --append-system-prompt + text; ALWAYS applied (identity, not a permission bypass — present in --safe too)
|
||||
mcp bool // auto-wire the Hanzo MCP server (code/vector/web/vision tools) as an stdio server scoped to the cwd
|
||||
install string // hint when the binary is missing
|
||||
}
|
||||
|
||||
// codeContextWindow is the input context (tokens) the served coding model
|
||||
@@ -272,14 +272,14 @@ func newCodeCmd(envOf func() *Env, _ *globalFlags) *cobra.Command {
|
||||
Long: "Run @hanzo/dev, Claude Code, or Codex against api.hanzo.ai with the endpoint,\n" +
|
||||
"credential and model injected — no env vars to remember. `hanzo code` alone runs\n" +
|
||||
"dev (the Hanzo agent); name an agent to pick another. Model ids resolve fuzzily\n" +
|
||||
"(glm5.2 -> glm-5.2), -c resumes either harness, and agents run full-auto unless\n" +
|
||||
"(zen5pro -> zen5-pro), -c resumes either harness, and agents run full-auto unless\n" +
|
||||
"you pass --safe. Unknown options pass through; -- forces verbatim passthrough.",
|
||||
Example: " hanzo code # dev, the default agent\n" +
|
||||
" hanzo code claude\n" +
|
||||
" hanzo code claude -c\n" +
|
||||
" hanzo code codex -c\n" +
|
||||
" hanzo code codex deepseek-v4-pro\n" +
|
||||
" hanzo code dev glm5.2 -- --resume\n" +
|
||||
" hanzo code codex enso-pro\n" +
|
||||
" hanzo code dev zen5 -- --resume\n" +
|
||||
" hanzo code ls",
|
||||
// Bare `hanzo code` (or `hanzo code <model>/-- args` with no agent name) runs
|
||||
// the configured default agent (code_tool, else dev). A recognized agent name
|
||||
|
||||
+366
-36
@@ -18,7 +18,7 @@ package cli
|
||||
// --serve-engine adds the `engine.serve` capability: the worker probes a local
|
||||
// hanzo-engine (the OpenAI + Anthropic model server on :1234), advertises its model
|
||||
// endpoint in the presence record, and prints (or with --register-provider, POSTs)
|
||||
// the /v1/add-provider call that routes api.hanzo.ai model traffic to this GPU. One
|
||||
// the POST /v1/ai/providers call that routes api.hanzo.ai model traffic to this GPU. One
|
||||
// fleet, two job types: engine.serve (model serving) alongside studio.render.
|
||||
|
||||
import (
|
||||
@@ -42,8 +42,10 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
@@ -90,6 +92,7 @@ const (
|
||||
const (
|
||||
studioCap = "studio.render"
|
||||
engineCap = "engine.serve"
|
||||
fnCap = "fn.run" // ephemeral script execution (uv-run) on this node
|
||||
)
|
||||
|
||||
// The node-level command surface — `hanzo link | unlink | status` — is wired in
|
||||
@@ -152,6 +155,8 @@ type engineAdvertisement struct {
|
||||
type gpuInfo struct {
|
||||
Name string `json:"name"`
|
||||
MemoryTotal string `json:"memoryTotal,omitempty"`
|
||||
Arch string `json:"arch,omitempty"` // native target, e.g. "gfx1151" (AMD)
|
||||
Unified bool `json:"unified,omitempty"` // memory is a unified CPU/GPU pool (APU / SoC)
|
||||
}
|
||||
|
||||
// registration is the fleet presence activity's Input — the shape cloud's
|
||||
@@ -172,6 +177,10 @@ type registration struct {
|
||||
Memory int64 `json:"memory,omitempty"`
|
||||
Version string `json:"version"`
|
||||
JobQueue string `json:"jobQueue"`
|
||||
Rocm string `json:"rocm,omitempty"` // host ROCm version (AMD only)
|
||||
Hip string `json:"hip,omitempty"` // host HIP version (AMD only)
|
||||
Cuda string `json:"cuda,omitempty"` // host CUDA toolkit version (NVIDIA only)
|
||||
Driver string `json:"driver,omitempty"` // host NVIDIA driver version
|
||||
GPUs []gpuInfo `json:"gpus"`
|
||||
Capabilities []string `json:"capabilities,omitempty"`
|
||||
Engine *engineAdvertisement `json:"engine,omitempty"`
|
||||
@@ -276,6 +285,9 @@ func newWorker(env *Env, jobsNS string) (*worker, error) {
|
||||
"echo": echoHandler,
|
||||
"studio.render": w.studioRenderHandler,
|
||||
}
|
||||
if _, err := exec.LookPath("uv"); err == nil {
|
||||
w.handlers[fnCap] = fnRunHandler // functions-runner: needs uv, nothing else
|
||||
}
|
||||
return w, nil
|
||||
}
|
||||
|
||||
@@ -311,25 +323,74 @@ func detectGPUs() []gpuInfo {
|
||||
return nil
|
||||
}
|
||||
|
||||
// detectNvidiaGPUs reports NVIDIA accelerators via nvidia-smi (name + total VRAM).
|
||||
// detectNvidiaGPUs reports NVIDIA accelerators via nvidia-smi: name, memory,
|
||||
// and the sm arch (compute capability). A unified SoC (GB10 Grace-Blackwell)
|
||||
// reports memory.total as "[N/A]" — there is no dedicated VRAM counter — so it
|
||||
// reports the MACHINE's RAM snapped to hardware capacity, unified=true, the
|
||||
// same convention the AMD APU and Apple paths use.
|
||||
func detectNvidiaGPUs() []gpuInfo {
|
||||
out, err := exec.Command("nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader").Output()
|
||||
out, err := exec.Command("nvidia-smi", "--query-gpu=name,memory.total,compute_cap", "--format=csv,noheader").Output()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return parseNvidiaSmiCSV(out, detectMemTotal()/(1<<20))
|
||||
}
|
||||
|
||||
// parseNvidiaSmiCSV parses `nvidia-smi --query-gpu=name,memory.total,compute_cap`
|
||||
// output ("NVIDIA GB10, [N/A], 12.1"). Pure — hostMiB is injected for the
|
||||
// unified-SoC memory figure.
|
||||
func parseNvidiaSmiCSV(out []byte, hostMiB int64) []gpuInfo {
|
||||
var gpus []gpuInfo
|
||||
sc := bufio.NewScanner(bytes.NewReader(out))
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" {
|
||||
fields := strings.Split(sc.Text(), ",")
|
||||
if len(fields) < 2 || strings.TrimSpace(fields[0]) == "" {
|
||||
continue
|
||||
}
|
||||
name, mem, _ := strings.Cut(line, ",")
|
||||
gpus = append(gpus, gpuInfo{Name: strings.TrimSpace(name), MemoryTotal: strings.TrimSpace(mem)})
|
||||
g := gpuInfo{Name: strings.TrimSpace(fields[0]), MemoryTotal: strings.TrimSpace(fields[1])}
|
||||
var memMiB int64
|
||||
if _, err := fmt.Sscanf(g.MemoryTotal, "%d MiB", &memMiB); err != nil || memMiB <= 0 {
|
||||
// No dedicated VRAM counter — a unified SoC. The machine's RAM is
|
||||
// the GPU's RAM.
|
||||
if hostMiB > 0 {
|
||||
g.MemoryTotal = fmt.Sprintf("%d MiB", snapUnified(hostMiB))
|
||||
g.Unified = true
|
||||
} else {
|
||||
g.MemoryTotal = ""
|
||||
}
|
||||
}
|
||||
if len(fields) >= 3 {
|
||||
if cap := strings.TrimSpace(fields[2]); cap != "" && cap != "[N/A]" {
|
||||
g.Arch = "sm_" + strings.ReplaceAll(cap, ".", "")
|
||||
}
|
||||
}
|
||||
gpus = append(gpus, g)
|
||||
}
|
||||
return gpus
|
||||
}
|
||||
|
||||
// nvidiaSoft is the host CUDA/driver inventory, detected once — the NVIDIA
|
||||
// mirror of amdSoftware. Empty on non-NVIDIA hosts.
|
||||
type nvidiaSoft struct{ cuda, driver string }
|
||||
|
||||
var nvidiaSoftware = sync.OnceValue(func() nvidiaSoft {
|
||||
var v nvidiaSoft
|
||||
if out, err := exec.Command("nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader").Output(); err == nil {
|
||||
v.driver = strings.TrimSpace(strings.SplitN(string(out), "\n", 2)[0])
|
||||
}
|
||||
if b, err := os.ReadFile("/usr/local/cuda/version.json"); err == nil {
|
||||
var m struct {
|
||||
Cuda struct {
|
||||
Version string `json:"version"`
|
||||
} `json:"cuda"`
|
||||
}
|
||||
if json.Unmarshal(b, &m) == nil {
|
||||
v.cuda = m.Cuda.Version
|
||||
}
|
||||
}
|
||||
return v
|
||||
})
|
||||
|
||||
// detectAmdGPUs reports AMD accelerators — discrete Radeon cards and gfx APUs alike
|
||||
// (e.g. evo's gfx1151 Radeon 8060S on the RYZEN AI MAX+ 395). Resolution order:
|
||||
// rocm-smi (marketing name + gfx target), then the kfd topology under /sys (gfx
|
||||
@@ -338,11 +399,11 @@ func detectNvidiaGPUs() []gpuInfo {
|
||||
func detectAmdGPUs() []gpuInfo {
|
||||
if out, err := exec.Command("rocm-smi", "--showproductname", "--csv").Output(); err == nil {
|
||||
if gpus := parseRocmSmiCSV(out); len(gpus) > 0 {
|
||||
return fillAmdVRAM(gpus, amdVRAMTotals(sysfsDRM))
|
||||
return apuProcessorNames(fillAmdVRAM(gpus, amdVRAMTotals(sysfsDRM)), detectCPUModel())
|
||||
}
|
||||
}
|
||||
if gpus := parseKfdTopology(sysfsKfdNodes); len(gpus) > 0 {
|
||||
return fillAmdVRAM(gpus, amdVRAMTotals(sysfsDRM))
|
||||
return apuProcessorNames(fillAmdVRAM(gpus, amdVRAMTotals(sysfsDRM)), detectCPUModel())
|
||||
}
|
||||
if out, err := exec.Command("vulkaninfo", "--summary").Output(); err == nil {
|
||||
if gpus := parseVulkaninfoSummary(out); len(gpus) > 0 {
|
||||
@@ -388,12 +449,17 @@ func parseRocmSmiCSV(out []byte) []gpuInfo {
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
arch := ""
|
||||
if gfx >= 0 && gfx < len(fields) {
|
||||
if v := strings.TrimSpace(fields[gfx]); v != "" {
|
||||
arch = v
|
||||
name += " (" + v + ")"
|
||||
}
|
||||
}
|
||||
gpus = append(gpus, gpuInfo{Name: name})
|
||||
if fam := amdFamily(arch); fam != "" {
|
||||
name = fam + " \u00b7 " + name
|
||||
}
|
||||
gpus = append(gpus, gpuInfo{Name: name, Arch: arch})
|
||||
}
|
||||
return gpus
|
||||
}
|
||||
@@ -422,7 +488,12 @@ func parseKfdTopology(nodesDir string) []gpuInfo {
|
||||
if simd <= 0 || gfxVer <= 0 {
|
||||
continue // CPU node or non-GPU
|
||||
}
|
||||
gpus = append(gpus, gpuInfo{Name: "AMD GPU (" + gfxName(gfxVer) + ")"})
|
||||
arch := gfxName(gfxVer)
|
||||
name := "AMD GPU (" + arch + ")"
|
||||
if fam := amdFamily(arch); fam != "" {
|
||||
name = fam + " (" + arch + ")"
|
||||
}
|
||||
gpus = append(gpus, gpuInfo{Name: name, Arch: arch})
|
||||
}
|
||||
return gpus
|
||||
}
|
||||
@@ -452,9 +523,119 @@ func gfxName(v int) string {
|
||||
return fmt.Sprintf("gfx%d%d%d", v/10000, (v/100)%100, v%100)
|
||||
}
|
||||
|
||||
// amdVRAMTotals returns each amdgpu card's total VRAM in MiB, in DRM card order,
|
||||
// read from /sys/class/drm/card*/device/mem_info_vram_total (vendor 0x1002).
|
||||
func amdVRAMTotals(drmDir string) []int64 {
|
||||
// amdFamily maps a gfx target to the APU/SoC family marketing name, so the board
|
||||
// says what the silicon IS ("AMD Strix Halo"), not just the iGPU series. Discrete
|
||||
// cards return "" and keep their own name.
|
||||
func amdFamily(arch string) string {
|
||||
switch arch {
|
||||
case "gfx1151":
|
||||
return "AMD Strix Halo"
|
||||
case "gfx1150":
|
||||
return "AMD Strix Point"
|
||||
case "gfx1103":
|
||||
return "AMD Phoenix"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// apuProcessorNames renders a unified-memory APU as the PROCESSOR it is — the
|
||||
// cpuinfo marketing name ("AMD Ryzen AI Max+ 395 w/ Radeon 8060S") beats the
|
||||
// iGPU series, because that is the product the owner bought. Discrete cards and
|
||||
// non-Ryzen hosts keep their existing names.
|
||||
func apuProcessorNames(gpus []gpuInfo, cpuModel string) []gpuInfo {
|
||||
name := amdAPUName(cpuModel)
|
||||
if name == "" {
|
||||
return gpus
|
||||
}
|
||||
for i, g := range gpus {
|
||||
if !g.Unified {
|
||||
continue
|
||||
}
|
||||
if g.Arch != "" {
|
||||
gpus[i].Name = name + " (" + g.Arch + ")"
|
||||
} else {
|
||||
gpus[i].Name = name
|
||||
}
|
||||
}
|
||||
return gpus
|
||||
}
|
||||
|
||||
// amdAPUName normalizes an AMD APU cpuinfo model into its marketing name:
|
||||
// "AMD RYZEN AI MAX+ 395 w/ Radeon 8060S" -> "AMD Ryzen AI Max+ 395 w/ Radeon
|
||||
// 8060S". BIOS strings shout; the board should not. Non-Ryzen models return "".
|
||||
func amdAPUName(cpuModel string) string {
|
||||
m := strings.Join(strings.Fields(cpuModel), " ")
|
||||
if !strings.Contains(strings.ToLower(m), "ryzen") {
|
||||
return ""
|
||||
}
|
||||
words := strings.Split(m, " ")
|
||||
for i, w := range words {
|
||||
if w == "AMD" || w == "AI" || w != strings.ToUpper(w) {
|
||||
continue // brand/initialism stays; mixed-case is already right
|
||||
}
|
||||
if strings.ContainsFunc(w, unicode.IsDigit) {
|
||||
continue // model numbers ("8060S", "395") keep their casing
|
||||
}
|
||||
if r := []rune(w); len(r) > 1 && strings.ContainsFunc(w, unicode.IsLetter) {
|
||||
words[i] = string(r[0]) + strings.ToLower(string(r[1:]))
|
||||
}
|
||||
}
|
||||
return strings.Join(words, " ")
|
||||
}
|
||||
|
||||
// amdSoft is the host ROCm/HIP toolchain inventory, detected once: ROCm from
|
||||
// /opt/rocm/.info/version, HIP from `hipconfig --version` (which exists even on
|
||||
// TheRock-style installs that lack the .info file). Empty on non-AMD hosts.
|
||||
type amdSoft struct{ rocm, hip string }
|
||||
|
||||
var amdSoftware = sync.OnceValue(func() amdSoft {
|
||||
var v amdSoft
|
||||
if b, err := os.ReadFile("/opt/rocm/.info/version"); err == nil {
|
||||
v.rocm = strings.TrimSpace(string(b))
|
||||
}
|
||||
if out, err := exec.Command("hipconfig", "--version").Output(); err == nil {
|
||||
v.hip = strings.TrimSpace(string(out))
|
||||
}
|
||||
return v
|
||||
})
|
||||
|
||||
// pickAmdMem chooses the honest memory figure for a card: dedicated VRAM for a
|
||||
// discrete GPU; for an APU whose "VRAM" is a token carve-out (Strix Halo: 1 GiB
|
||||
// VRAM beside a ~118 GiB GTT pool) the figure is the MACHINE's unified RAM —
|
||||
// the same convention Apple silicon reports (an M4 Max says 128 GiB, not the
|
||||
// wired-down remainder) — snapped to hardware capacity. Unified when GTT wins.
|
||||
func pickAmdMem(vramMiB, gttMiB, hostMiB int64) (miB int64, unified bool) {
|
||||
if gttMiB > vramMiB*4 {
|
||||
m := gttMiB
|
||||
if hostMiB > m {
|
||||
m = hostMiB
|
||||
}
|
||||
return snapUnified(m), true
|
||||
}
|
||||
return vramMiB, false
|
||||
}
|
||||
|
||||
// snapUnified rounds a kernel-visible unified-memory figure up to the hardware
|
||||
// DIMM capacity (the next 16 GiB multiple) when the gap is a plausible firmware
|
||||
// reservation (≤ 6 GiB): a 128 GiB Strix Halo shows 124.4 GiB to Linux because
|
||||
// BIOS + the VRAM carve-out are invisible to the OS. A larger gap (say a 16 GiB
|
||||
// carve-out) is real capacity the pool lost — reported as-is, never invented.
|
||||
func snapUnified(miB int64) int64 {
|
||||
const step = 16 << 10 // 16 GiB in MiB
|
||||
next := ((miB + step - 1) / step) * step
|
||||
if next-miB <= 8<<10 {
|
||||
return next
|
||||
}
|
||||
return miB
|
||||
}
|
||||
|
||||
// amdMem is one card's memory inventory in MiB: dedicated VRAM plus the GTT
|
||||
// (system-memory) pool an APU actually computes in.
|
||||
type amdMem struct{ vramMiB, gttMiB int64 }
|
||||
|
||||
// amdVRAMTotals returns each amdgpu card's memory totals in MiB, in DRM card order,
|
||||
// read from /sys/class/drm/card*/device/mem_info_{vram,gtt}_total (vendor 0x1002).
|
||||
func amdVRAMTotals(drmDir string) []amdMem {
|
||||
entries, err := os.ReadDir(drmDir)
|
||||
if err != nil {
|
||||
return nil
|
||||
@@ -469,32 +650,44 @@ func amdVRAMTotals(drmDir string) []int64 {
|
||||
sort.Slice(cards, func(i, j int) bool {
|
||||
return atoiSafe(strings.TrimPrefix(cards[i], "card")) < atoiSafe(strings.TrimPrefix(cards[j], "card"))
|
||||
})
|
||||
var mems []int64
|
||||
readMiB := func(path string) int64 {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
var n int64
|
||||
if _, err := fmt.Sscan(strings.TrimSpace(string(b)), &n); err != nil || n <= 0 {
|
||||
return 0
|
||||
}
|
||||
return n / (1024 * 1024)
|
||||
}
|
||||
var mems []amdMem
|
||||
for _, c := range cards {
|
||||
dev := filepath.Join(drmDir, c, "device")
|
||||
if vendor, _ := os.ReadFile(filepath.Join(dev, "vendor")); strings.TrimSpace(string(vendor)) != "0x1002" {
|
||||
continue
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(dev, "mem_info_vram_total"))
|
||||
if err != nil {
|
||||
vram := readMiB(filepath.Join(dev, "mem_info_vram_total"))
|
||||
if vram == 0 {
|
||||
continue
|
||||
}
|
||||
var bytesTotal int64
|
||||
if _, err := fmt.Sscan(strings.TrimSpace(string(b)), &bytesTotal); err == nil && bytesTotal > 0 {
|
||||
mems = append(mems, bytesTotal/(1024*1024))
|
||||
}
|
||||
mems = append(mems, amdMem{vramMiB: vram, gttMiB: readMiB(filepath.Join(dev, "mem_info_gtt_total"))})
|
||||
}
|
||||
return mems
|
||||
}
|
||||
|
||||
// fillAmdVRAM attaches VRAM totals to the GPU list positionally when the counts
|
||||
// fillAmdVRAM attaches memory totals to the GPU list positionally when the counts
|
||||
// match (the common single-GPU case always does); otherwise the names stand alone.
|
||||
func fillAmdVRAM(gpus []gpuInfo, memsMiB []int64) []gpuInfo {
|
||||
if len(memsMiB) != len(gpus) {
|
||||
// An APU reports its unified GTT pool, not the token VRAM carve-out.
|
||||
func fillAmdVRAM(gpus []gpuInfo, mems []amdMem) []gpuInfo {
|
||||
if len(mems) != len(gpus) {
|
||||
return gpus
|
||||
}
|
||||
hostMiB := detectMemTotal() / (1 << 20)
|
||||
for i := range gpus {
|
||||
gpus[i].MemoryTotal = fmt.Sprintf("%d MiB", memsMiB[i])
|
||||
miB, unified := pickAmdMem(mems[i].vramMiB, mems[i].gttMiB, hostMiB)
|
||||
gpus[i].MemoryTotal = fmt.Sprintf("%d MiB", miB)
|
||||
gpus[i].Unified = unified
|
||||
}
|
||||
return gpus
|
||||
}
|
||||
@@ -742,7 +935,7 @@ type connectOpts struct {
|
||||
serveEngine bool
|
||||
engineURL string // local URL to probe hanzo-engine
|
||||
engineEndpoint string // public URL to advertise (defaults to engineURL)
|
||||
registerProvider bool // auto POST /v1/add-provider for the engine
|
||||
registerProvider bool // auto POST /v1/ai/providers for the engine
|
||||
studioDir string // local Studio checkout to launch + supervise on :8188
|
||||
studioURL string // studio base the render mirror uploads finished images to
|
||||
mirror bool // sweep local renders into the org studio library (default on)
|
||||
@@ -909,6 +1102,10 @@ func (w *worker) buildRegistration() registration {
|
||||
Memory: w.memory,
|
||||
Version: Version,
|
||||
JobQueue: w.jobsNS,
|
||||
Rocm: amdSoftware().rocm,
|
||||
Hip: amdSoftware().hip,
|
||||
Cuda: nvidiaSoftware().cuda,
|
||||
Driver: nvidiaSoftware().driver,
|
||||
GPUs: w.gpus,
|
||||
Capabilities: w.capabilities(),
|
||||
Engine: w.engine,
|
||||
@@ -927,9 +1124,23 @@ func (w *worker) capabilities() []string {
|
||||
if w.serveEngine {
|
||||
caps = append(caps, engineCap)
|
||||
}
|
||||
if _, ok := w.handlers[fnCap]; ok {
|
||||
caps = append(caps, fnCap)
|
||||
}
|
||||
return caps
|
||||
}
|
||||
|
||||
// hasNonRenderLane reports whether this worker serves any lane beyond the render
|
||||
// path (echo is a smoke type, not a lane).
|
||||
func (w *worker) hasNonRenderLane() bool {
|
||||
for name := range w.handlers {
|
||||
if name != "echo" && name != studioCap {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// studioReachable probes the local studio's /queue (bounded), authenticated. A node
|
||||
// that can answer it is up enough to accept a render.
|
||||
func (w *worker) studioReachable(ctx context.Context) bool {
|
||||
@@ -1024,10 +1235,12 @@ func (w *worker) claimFrom(ctx context.Context, taskQueue string) (claimedActivi
|
||||
// queue name, so one worker never steals another GPU's targeted job (an empty
|
||||
// taskQueue on claim would match any lane and do exactly that).
|
||||
func (w *worker) claimAndRun(ctx context.Context, out io.Writer) error {
|
||||
// Poison-loop guard: a node that can't serve renders must not claim render lanes —
|
||||
// it would only fail every claimed job on the gated execute seam. It still
|
||||
// heartbeats presence (shows in the fleet), just idle, until it becomes ready.
|
||||
if !w.studioReady {
|
||||
// Poison-loop guard, per-LANE: a node that can't serve renders must not sit on
|
||||
// render jobs — but a node with other lanes (fn.run) still claims. When renders
|
||||
// are this worker's only real lane and the studio isn't ready, stay idle; a
|
||||
// claimed render on a non-ready node is declined below so an eligible worker
|
||||
// takes it.
|
||||
if !w.studioReady && !w.hasNonRenderLane() {
|
||||
return nil
|
||||
}
|
||||
act, claimed, err := w.claimFrom(ctx, w.gpuQueue())
|
||||
@@ -1053,6 +1266,12 @@ func (w *worker) claimAndRun(ctx context.Context, out io.Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if act.Type.Name == studioCap && !w.studioReady {
|
||||
cause := "studio not ready on this node — declined so a render-capable worker takes it"
|
||||
_, _ = w.call(ctx, http.MethodPost, w.actPath(wf, run, "fail"), map[string]any{"cause": cause, "identity": w.identity}, nil)
|
||||
fmt.Fprintf(out, " → declined (%s)\n", cause)
|
||||
return nil
|
||||
}
|
||||
h, ok := w.handlers[act.Type.Name]
|
||||
if !ok {
|
||||
cause := fmt.Sprintf("no handler for job type %q", act.Type.Name)
|
||||
@@ -1371,6 +1590,117 @@ func echoHandler(_ context.Context, input json.RawMessage) (any, error) {
|
||||
return map[string]any{"echo": v}, nil
|
||||
}
|
||||
|
||||
// fnRunInput is the fn.run job payload: an inline script executed on this node in
|
||||
// an ephemeral `uv run` environment. The queue is org-scoped, so the submitter is
|
||||
// running code on their OWN fleet — same trust domain as a CI runner.
|
||||
type fnRunInput struct {
|
||||
Name string `json:"name,omitempty"` // label for logs
|
||||
Script string `json:"script"` // Python source (required)
|
||||
Requirements []string `json:"requirements,omitempty"` // uv --with deps, e.g. ["numpy", "torch==2.5.*"]
|
||||
Env map[string]string `json:"env,omitempty"` // extra environment
|
||||
TimeoutSeconds int `json:"timeoutSeconds,omitempty"` // default 3600, cap 21600
|
||||
}
|
||||
|
||||
const (
|
||||
fnDefaultTimeout = time.Hour
|
||||
fnMaxTimeout = 6 * time.Hour
|
||||
fnOutputTail = 128 << 10 // keep the LAST 128 KiB of combined output
|
||||
)
|
||||
|
||||
// fnValidate parses and bounds an fn.run payload. Requirements must look like
|
||||
// package specs — a leading dash would inject uv flags.
|
||||
func fnValidate(input json.RawMessage) (fnRunInput, error) {
|
||||
var in fnRunInput
|
||||
if err := json.Unmarshal(input, &in); err != nil {
|
||||
return in, fmt.Errorf("fn.run: bad input: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(in.Script) == "" {
|
||||
return in, fmt.Errorf("fn.run: input needs a `script`")
|
||||
}
|
||||
for _, r := range in.Requirements {
|
||||
if r == "" || strings.HasPrefix(r, "-") {
|
||||
return in, fmt.Errorf("fn.run: bad requirement %q", r)
|
||||
}
|
||||
}
|
||||
if in.TimeoutSeconds <= 0 {
|
||||
in.TimeoutSeconds = int(fnDefaultTimeout / time.Second)
|
||||
}
|
||||
if in.TimeoutSeconds > int(fnMaxTimeout/time.Second) {
|
||||
in.TimeoutSeconds = int(fnMaxTimeout / time.Second)
|
||||
}
|
||||
return in, nil
|
||||
}
|
||||
|
||||
// tailBuffer keeps the last cap bytes written — a training loop can log gigabytes;
|
||||
// the activity result carries the end, where the outcome lives.
|
||||
type tailBuffer struct {
|
||||
cap int
|
||||
buf []byte
|
||||
truncated bool
|
||||
}
|
||||
|
||||
func (t *tailBuffer) Write(p []byte) (int, error) {
|
||||
t.buf = append(t.buf, p...)
|
||||
if len(t.buf) > t.cap {
|
||||
t.buf = t.buf[len(t.buf)-t.cap:]
|
||||
t.truncated = true
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// fnRunHandler executes an fn.run job: write the script to an ephemeral dir, run
|
||||
// it under `uv run` (which resolves requirements into a throwaway env — ROCm/CUDA
|
||||
// wheels included), and return the output tail + exit code. Nonzero exit fails
|
||||
// the activity with the tail as the cause, so the submitter sees the traceback.
|
||||
func fnRunHandler(ctx context.Context, input json.RawMessage) (any, error) {
|
||||
in, err := fnValidate(input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dir, err := os.MkdirTemp("", "fnrun-*")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fn.run: %w", err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
script := filepath.Join(dir, "main.py")
|
||||
if err := os.WriteFile(script, []byte(in.Script), 0o600); err != nil {
|
||||
return nil, fmt.Errorf("fn.run: %w", err)
|
||||
}
|
||||
runCtx, cancel := context.WithTimeout(ctx, time.Duration(in.TimeoutSeconds)*time.Second)
|
||||
defer cancel()
|
||||
args := []string{"run", "--no-project", "--quiet"}
|
||||
for _, r := range in.Requirements {
|
||||
args = append(args, "--with", r)
|
||||
}
|
||||
args = append(args, script)
|
||||
cmd := exec.CommandContext(runCtx, "uv", args...)
|
||||
cmd.Dir = dir
|
||||
cmd.Env = os.Environ()
|
||||
for k, v := range in.Env {
|
||||
cmd.Env = append(cmd.Env, k+"="+v)
|
||||
}
|
||||
tail := &tailBuffer{cap: fnOutputTail}
|
||||
cmd.Stdout = tail
|
||||
cmd.Stderr = tail
|
||||
start := time.Now()
|
||||
runErr := cmd.Run()
|
||||
dur := time.Since(start)
|
||||
out := string(tail.buf)
|
||||
if runCtx.Err() == context.DeadlineExceeded {
|
||||
return nil, fmt.Errorf("fn.run: timed out after %ds\n%s", in.TimeoutSeconds, out)
|
||||
}
|
||||
if runErr != nil {
|
||||
return nil, fmt.Errorf("fn.run: %v\n%s", runErr, out)
|
||||
}
|
||||
return map[string]any{
|
||||
"name": in.Name,
|
||||
"exitCode": 0,
|
||||
"durationMs": dur.Milliseconds(),
|
||||
"output": out,
|
||||
"truncated": tail.truncated,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// workerToken is the KMS-sourced STUDIO_WORKER_TOKEN this box holds — the credential
|
||||
// the local studio's --worker-mode gate checks. Empty on an unconfigured box (the
|
||||
// render preflight refuses such a node before it ever claims a render lane).
|
||||
@@ -1898,7 +2228,7 @@ func describeEngine(adv *engineAdvertisement) string {
|
||||
return fmt.Sprintf("ready · %d models", n)
|
||||
}
|
||||
|
||||
// providerBody is the POST /v1/add-provider payload registering this node's engine
|
||||
// providerBody is the POST /v1/ai/providers payload registering this node's engine
|
||||
// as an org model provider. hanzo-engine is OpenAI-compatible, so Type=Local: the
|
||||
// gateway speaks the OpenAI wire format to it and auto-appends /v1 to providerUrl.
|
||||
func (w *worker) providerBody() map[string]any {
|
||||
@@ -1930,23 +2260,23 @@ func (w *worker) printEngineHint(out io.Writer) {
|
||||
fmt.Fprintf(out, "serving hanzo-engine (OpenAI + Anthropic) at %s — %s\n", adv.URL, describeEngine(adv))
|
||||
body, _ := json.Marshal(w.providerBody())
|
||||
fmt.Fprintln(out, " route api.hanzo.ai model calls to this GPU by registering it as an org provider:")
|
||||
fmt.Fprintf(out, " curl -sS %s/v1/add-provider -H \"Authorization: Bearer $HANZO_TOKEN\" \\\n", w.baseURL)
|
||||
fmt.Fprintf(out, " curl -sS %s/v1/ai/providers -H \"Authorization: Bearer $HANZO_TOKEN\" \\\n", w.baseURL)
|
||||
fmt.Fprintf(out, " -H 'Content-Type: application/json' -d '%s'\n", body)
|
||||
fmt.Fprintln(out, " (or pass --register-provider. The endpoint must be reachable from api.hanzo.ai —")
|
||||
fmt.Fprintln(out, " a cloud GPU is in-cluster; a BYO node needs a public URL/tunnel. add-provider needs a platform-admin token today.)")
|
||||
fmt.Fprintln(out, " a cloud GPU is in-cluster; a BYO node needs a public URL/tunnel. registering a provider needs a platform-admin token today.)")
|
||||
}
|
||||
|
||||
// registerProvider POSTs /v1/add-provider so the gateway routes model calls to this
|
||||
// registerProvider POSTs /v1/ai/providers so the gateway routes model calls to this
|
||||
// node's engine. Requires the engine to be reachable and (today) a platform-admin
|
||||
// token; both failures are reported clearly rather than swallowed.
|
||||
func (w *worker) registerProvider(ctx context.Context, adv *engineAdvertisement) error {
|
||||
if adv == nil || adv.Status != "ready" {
|
||||
return fmt.Errorf("engine not ready at %s — start hanzo-engine, then retry", w.engineURL)
|
||||
}
|
||||
code, err := w.call(ctx, http.MethodPost, "/v1/add-provider", w.providerBody(), nil)
|
||||
code, err := w.call(ctx, http.MethodPost, "/v1/ai/providers", w.providerBody(), nil)
|
||||
if err != nil {
|
||||
if code == http.StatusForbidden {
|
||||
return fmt.Errorf("add-provider is gated to a platform-admin token today; register from the console or with an admin token: %w", err)
|
||||
return fmt.Errorf("registering a provider is gated to a platform-admin token today; register from the console or with an admin token: %w", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
+79
-2
@@ -17,7 +17,7 @@ func TestParseRocmSmiCSV(t *testing.T) {
|
||||
if len(gpus) != 1 {
|
||||
t.Fatalf("want 1 AMD GPU, got %d (%+v)", len(gpus), gpus)
|
||||
}
|
||||
if got, want := gpus[0].Name, "Radeon 8060S Graphics (gfx1151)"; got != want {
|
||||
if got, want := gpus[0].Name, "AMD Strix Halo \u00b7 Radeon 8060S Graphics (gfx1151)"; got != want {
|
||||
t.Errorf("name = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
@@ -49,9 +49,12 @@ func TestParseKfdTopology(t *testing.T) {
|
||||
if len(gpus) != 1 {
|
||||
t.Fatalf("want 1 GPU node (CPU node 0 skipped), got %d (%+v)", len(gpus), gpus)
|
||||
}
|
||||
if got, want := gpus[0].Name, "AMD GPU (gfx1151)"; got != want {
|
||||
if got, want := gpus[0].Name, "AMD Strix Halo (gfx1151)"; got != want {
|
||||
t.Errorf("name = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := gpus[0].Arch, "gfx1151"; got != want {
|
||||
t.Errorf("arch = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseVulkaninfoSummary — the last resort keeps only AMD/Radeon devices so it
|
||||
@@ -78,3 +81,77 @@ func writeNode(t *testing.T, root, id, props string) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPickAmdMemUnified — an APU whose VRAM is a token carve-out reports the
|
||||
// machine's unified RAM snapped to hardware capacity (evo: 1 GiB VRAM, 118 GiB
|
||||
// GTT, 124.4 GiB kernel-visible of 128 GiB physical → 128 GiB); a discrete card
|
||||
// keeps its VRAM.
|
||||
func TestPickAmdMemUnified(t *testing.T) {
|
||||
if miB, unified := pickAmdMem(1024, 120832, 127411); !unified || miB != 131072 {
|
||||
t.Errorf("APU: got (%d, %v), want (131072, true)", miB, unified)
|
||||
}
|
||||
if miB, unified := pickAmdMem(24576, 8192, 127411); unified || miB != 24576 {
|
||||
t.Errorf("discrete: got (%d, %v), want (24576, false)", miB, unified)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapUnified — a small firmware gap snaps up to the DIMM capacity; a big
|
||||
// carve-out is real lost capacity and stays as-is; exact multiples stand.
|
||||
func TestSnapUnified(t *testing.T) {
|
||||
for _, c := range []struct{ in, want int64 }{
|
||||
{127411, 131072}, // evo: 124.4 GiB visible of 128 GiB physical
|
||||
{124610, 131072}, // spark GB10: 121.7 GiB visible of 128 GiB physical (~6.3 GiB firmware)
|
||||
{131072, 131072}, // exact 128 GiB
|
||||
{119194, 119194}, // ~116 GiB visible (16 GiB BIOS carve) — 11.6 GiB gap, keep honest
|
||||
{63488, 65536}, // 62 GiB visible of 64 GiB
|
||||
} {
|
||||
if got := snapUnified(c.in); got != c.want {
|
||||
t.Errorf("snapUnified(%d) = %d, want %d", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAmdAPUName — the board renders the APU as the processor the owner bought,
|
||||
// normalized from the BIOS shouting; non-Ryzen hosts opt out.
|
||||
func TestAmdAPUName(t *testing.T) {
|
||||
if got, want := amdAPUName("AMD RYZEN AI MAX+ 395 w/ Radeon 8060S"), "AMD Ryzen AI Max+ 395 w/ Radeon 8060S"; got != want {
|
||||
t.Errorf("amdAPUName = %q, want %q", got, want)
|
||||
}
|
||||
if got := amdAPUName("Intel(R) Core(TM) i9-14900K"); got != "" {
|
||||
t.Errorf("non-Ryzen host must opt out, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApuProcessorNames — only unified-memory cards are renamed; a discrete
|
||||
// card on the same host keeps its own identity.
|
||||
func TestApuProcessorNames(t *testing.T) {
|
||||
gpus := []gpuInfo{
|
||||
{Name: "AMD Strix Halo \u00b7 Radeon 8060S Graphics (gfx1151)", Arch: "gfx1151", Unified: true},
|
||||
{Name: "Radeon RX 7900 XTX (gfx1100)", Arch: "gfx1100"},
|
||||
}
|
||||
out := apuProcessorNames(gpus, "AMD RYZEN AI MAX+ 395 w/ Radeon 8060S")
|
||||
if got, want := out[0].Name, "AMD Ryzen AI Max+ 395 w/ Radeon 8060S (gfx1151)"; got != want {
|
||||
t.Errorf("APU name = %q, want %q", got, want)
|
||||
}
|
||||
if got, want := out[1].Name, "Radeon RX 7900 XTX (gfx1100)"; got != want {
|
||||
t.Errorf("discrete name = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseNvidiaSmiCSV — a GB10-class unified SoC ("[N/A]" VRAM) reports the
|
||||
// machine RAM snapped to capacity + its sm arch; a discrete card keeps its VRAM.
|
||||
func TestParseNvidiaSmiCSV(t *testing.T) {
|
||||
out := []byte("NVIDIA GB10, [N/A], 12.1\n")
|
||||
gpus := parseNvidiaSmiCSV(out, 124610)
|
||||
if len(gpus) != 1 {
|
||||
t.Fatalf("want 1 GPU, got %d", len(gpus))
|
||||
}
|
||||
g := gpus[0]
|
||||
if g.Name != "NVIDIA GB10" || g.MemoryTotal != "131072 MiB" || !g.Unified || g.Arch != "sm_121" {
|
||||
t.Errorf("GB10 = %+v", g)
|
||||
}
|
||||
disc := parseNvidiaSmiCSV([]byte("NVIDIA GeForce RTX 4090, 24564 MiB, 8.9\n"), 124610)
|
||||
if d := disc[0]; d.MemoryTotal != "24564 MiB" || d.Unified || d.Arch != "sm_89" {
|
||||
t.Errorf("discrete = %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestFnValidate — payload bounds: script required, flag-injection via
|
||||
// requirements refused, timeout defaulted and capped.
|
||||
func TestFnValidate(t *testing.T) {
|
||||
if _, err := fnValidate(json.RawMessage(`{}`)); err == nil {
|
||||
t.Error("empty script must be refused")
|
||||
}
|
||||
if _, err := fnValidate(json.RawMessage(`{"script":"print(1)","requirements":["--index-url=evil"]}`)); err == nil {
|
||||
t.Error("flag-shaped requirement must be refused")
|
||||
}
|
||||
in, err := fnValidate(json.RawMessage(`{"script":"print(1)"}`))
|
||||
if err != nil || in.TimeoutSeconds != 3600 {
|
||||
t.Errorf("default timeout: got %d, err %v", in.TimeoutSeconds, err)
|
||||
}
|
||||
in, _ = fnValidate(json.RawMessage(`{"script":"print(1)","timeoutSeconds":999999}`))
|
||||
if in.TimeoutSeconds != 21600 {
|
||||
t.Errorf("timeout cap: got %d, want 21600", in.TimeoutSeconds)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTailBuffer — a chatty training loop keeps only the LAST bytes, flagged.
|
||||
func TestTailBuffer(t *testing.T) {
|
||||
tb := &tailBuffer{cap: 8}
|
||||
tb.Write([]byte("0123456789"))
|
||||
if string(tb.buf) != "23456789" || !tb.truncated {
|
||||
t.Errorf("tail = %q truncated=%v", tb.buf, tb.truncated)
|
||||
}
|
||||
tb2 := &tailBuffer{cap: 8}
|
||||
tb2.Write([]byte("abc"))
|
||||
if string(tb2.buf) != "abc" || tb2.truncated {
|
||||
t.Errorf("small write mangled: %q %v", tb2.buf, tb2.truncated)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHasNonRenderLane — echo and studio.render alone keep the render-only
|
||||
// poison guard; a registered fn.run lane lifts it.
|
||||
func TestHasNonRenderLane(t *testing.T) {
|
||||
w := &worker{handlers: map[string]jobHandler{"echo": echoHandler, studioCap: nil}}
|
||||
if w.hasNonRenderLane() {
|
||||
t.Error("render-only worker must report no extra lane")
|
||||
}
|
||||
w.handlers[fnCap] = fnRunHandler
|
||||
if !w.hasNonRenderLane() {
|
||||
t.Error("fn.run lane must lift the guard")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFnRunSmoke — the real handler end-to-end against the local uv: a script
|
||||
// that prints and exits 0. Skipped where uv is absent.
|
||||
func TestFnRunSmoke(t *testing.T) {
|
||||
if !uvPresent() {
|
||||
t.Skip("uv not installed")
|
||||
}
|
||||
res, err := fnRunHandler(t.Context(), json.RawMessage(`{"script":"print(2+2)","timeoutSeconds":120}`))
|
||||
if err != nil {
|
||||
t.Fatalf("fn.run: %v", err)
|
||||
}
|
||||
m := res.(map[string]any)
|
||||
if !strings.Contains(m["output"].(string), "4") {
|
||||
t.Errorf("output = %q, want it to contain 4", m["output"])
|
||||
}
|
||||
}
|
||||
|
||||
func uvPresent() bool {
|
||||
_, err := exec.LookPath("uv")
|
||||
return err == nil
|
||||
}
|
||||
+1
-1
@@ -72,7 +72,7 @@ func newLinkCmd(envOf func() *Env, _ *globalFlags) *cobra.Command {
|
||||
f.BoolVar(&opts.serveEngine, "serve-engine", false, "also advertise a hanzo-engine model server (OpenAI + Anthropic) running on this node")
|
||||
f.StringVar(&opts.engineURL, "engine-url", defaultEngineURL, "local URL where hanzo-engine is probed (GET /v1/models)")
|
||||
f.StringVar(&opts.engineEndpoint, "engine-endpoint", "", "public URL to advertise for gateway routing (defaults to --engine-url; a node behind NAT needs a reachable URL/tunnel)")
|
||||
f.BoolVar(&opts.registerProvider, "register-provider", false, "auto-register the engine endpoint as an org model provider (POST /v1/add-provider)")
|
||||
f.BoolVar(&opts.registerProvider, "register-provider", false, "auto-register the engine endpoint as an org model provider (POST /v1/ai/providers)")
|
||||
f.StringVar(&opts.studioDir, "studio-dir", os.Getenv("HANZO_STUDIO_DIR"), "local Hanzo Studio checkout; when set, link launches and supervises the render backend on 127.0.0.1:8188")
|
||||
f.StringVar(&opts.studioURL, "studio-url", firstNonEmpty(os.Getenv("HANZO_STUDIO_UPLOAD_URL"), defaultStudioUploadURL), "studio base URL the render mirror uploads finished images to (POST /v1/library/upload)")
|
||||
f.BoolVar(&opts.mirror, "mirror", true, "sweep local renders into the org studio library; --mirror=false serves jobs only")
|
||||
|
||||
@@ -50,6 +50,7 @@
|
||||
package account
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -153,6 +154,11 @@ func routesAccount(s *cloud.Service[state], app *zip.App) {
|
||||
// that must beat the /v1/commerce/* bridge (122) AND the commerce embed (100) — so it
|
||||
// mounts here at 48, ahead of both.
|
||||
app.Post("/v1/commerce/topup/wallet", rateLimit(s, s.State.writesRL, requireCSRF(s, cloud.Handle(s, walletTopup))))
|
||||
// The accepted rails are public on-chain data (chain, token, treasury), read by
|
||||
// the browser to render the send UI. A GET with no side effects and no secret,
|
||||
// so it needs neither CSRF nor the write limiter — but it MUST sit beside the
|
||||
// POST at this priority, or the /v1/commerce/* bridge swallows it.
|
||||
app.Get("/v1/commerce/topup/rails", cloud.Handle(s, topupRails))
|
||||
}
|
||||
|
||||
// routesBridge wires the per-tenant catch-all data bridges (order 122).
|
||||
@@ -367,18 +373,58 @@ func onboard(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
return herr
|
||||
}
|
||||
|
||||
// Create the org (cloning the caller's current org for password/locale
|
||||
// compatibility), then — first-run only — move the zero-org user in as admin.
|
||||
// ADDITIONAL org (caller already has a home): create it WITHOUT moving them —
|
||||
// they reach it via the OrgSwitcher (a move would strip their SuperAdmin / orphan
|
||||
// their current org).
|
||||
if additional {
|
||||
org := buildOrg(s, c, slug, displayName, body.Personal, cr.owner)
|
||||
if err := s.State.iam.createOrganization(c.Context(), org); err != nil {
|
||||
return zip.Errorf(http.StatusBadGateway, "could not create the organization: %v", err)
|
||||
}
|
||||
return c.JSON(http.StatusOK, onboardResp{Org: slug, DisplayName: displayName, Additional: true})
|
||||
}
|
||||
|
||||
// FIRST-RUN: drive the ONE atomic IAM provision (org + admin move + hashed
|
||||
// org-scoped credential), replacing the create-org + move-user pair — a mid-flight
|
||||
// retry now converges on the founder's own org instead of orphaning it. The org
|
||||
// starts at a zero balance (usage is pre-paid). Prefer it whenever the
|
||||
// service-token path is wired; fall back to the legacy pair only when it is not,
|
||||
// so a partial deploy still onboards.
|
||||
if s.State.iam.provisionReady() {
|
||||
resp, err := onboardFirstRun(c.Context(), s.State.iam, cr.id, slug, displayName, body.Personal)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.JSON(http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// Legacy fallback (service token unset): create then move — the non-atomic pair.
|
||||
org := buildOrg(s, c, slug, displayName, body.Personal, cr.owner)
|
||||
if err := s.State.iam.createOrganization(c.Context(), org); err != nil {
|
||||
return zip.Errorf(http.StatusBadGateway, "could not create the organization: %v", err)
|
||||
}
|
||||
if !additional {
|
||||
if err := s.State.iam.moveUserToOrg(c.Context(), cr.id, slug); err != nil {
|
||||
return zip.Errorf(http.StatusBadGateway, "org created but could not assign you to it: %v", err)
|
||||
}
|
||||
if err := s.State.iam.moveUserToOrg(c.Context(), cr.id, slug); err != nil {
|
||||
return zip.Errorf(http.StatusBadGateway, "org created but could not assign you to it: %v", err)
|
||||
}
|
||||
return c.JSON(http.StatusOK, onboardResp{Org: slug, DisplayName: displayName, Additional: additional})
|
||||
return c.JSON(http.StatusOK, onboardResp{Org: slug, DisplayName: displayName, Additional: false})
|
||||
}
|
||||
|
||||
// onboardFirstRun drives the ONE atomic IAM provision for a zero-org caller (create
|
||||
// org + move them in as admin + mint the hashed org-scoped credential), replacing
|
||||
// the create-org + move-user pair so a mid-flight retry converges on the founder's
|
||||
// own org instead of orphaning it. The org starts at a ZERO balance — usage is
|
||||
// pre-paid, so there is no signup grant. Split out so the provisioning glue is
|
||||
// unit-tested against mock IAM without the CSRF/routing/principal shell.
|
||||
func onboardFirstRun(ctx context.Context, iam *iamClient, callerID, slug, displayName string, personal bool) (onboardResp, error) {
|
||||
row, err := iam.getUserRow(ctx, callerID)
|
||||
if err != nil {
|
||||
return onboardResp{}, zip.Errorf(http.StatusBadGateway, "could not resolve the user: %v", err)
|
||||
}
|
||||
res, err := iam.provision(ctx, row.Owner, row.Name, slug, personal)
|
||||
if err != nil {
|
||||
return onboardResp{}, zip.Errorf(http.StatusBadGateway, "could not provision the organization: %v", err)
|
||||
}
|
||||
return onboardResp{Org: res.Org, DisplayName: displayName, Additional: false}, nil
|
||||
}
|
||||
|
||||
// resolveOnboardName derives the base slug + display name from the request, or a
|
||||
|
||||
@@ -46,8 +46,12 @@ import (
|
||||
// - Neither: refuse. A bearer-less request with a forged X-Org-Id has no validated
|
||||
// principal and is fail-closed here, before the read handler runs.
|
||||
//
|
||||
// The pin rewrites the request URI's query string in place; fasthttp's SetQueryString
|
||||
// resets the parsed-args cache, so the handler's later c.Query() reads the pinned values.
|
||||
// The pin rewrites the request URI's query string AND (on a write) the JSON body in place;
|
||||
// fasthttp's SetQueryString resets the parsed-args cache and SetBody replaces the body
|
||||
// bytes, so the handler's later c.Query() / c.Bind() read the pinned values. Pinning the
|
||||
// body is what keeps a co-resident WRITE handler that reads its subject from the body
|
||||
// (commerce's CreatePaymentMethod reads customerId from the JSON body) IDOR-safe — query-only
|
||||
// pinning would leave a client-named customerId/userId in the body untouched.
|
||||
func PinBillingSubject() zip.Handler {
|
||||
return func(c *zip.Ctx) error {
|
||||
inQuery, _ := url.ParseQuery(string(c.Fiber().Request().URI().QueryString()))
|
||||
@@ -69,7 +73,17 @@ func PinBillingSubject() zip.Handler {
|
||||
Account: principal.BillingAccount(c),
|
||||
}).Subject()
|
||||
|
||||
// Pin the subject on BOTH the query AND the write body — the SAME two-helper
|
||||
// scoping billingData applies (scopedBillingSearch + scopedBillingBody). A
|
||||
// co-resident WRITE handler that reads its subject from the body (commerce's
|
||||
// CreatePaymentMethod → customerId) is only IDOR-safe if the body is pinned too.
|
||||
// scopedBillingBody overwrites the subject keys and preserves every other field
|
||||
// (card, type, sourceId, …); a non-JSON / empty body is returned unchanged, so a
|
||||
// GET read carries no body and is unaffected.
|
||||
c.Fiber().Request().URI().SetQueryString(scopedBillingSearch(inQuery, subject).Encode())
|
||||
if len(c.Body()) > 0 {
|
||||
c.Fiber().Request().SetBody(scopedBillingBody(c.Body(), subject))
|
||||
}
|
||||
return c.Next()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,17 @@ func echoQuery(c *zip.Ctx) error {
|
||||
})
|
||||
}
|
||||
|
||||
// echoBody is the downstream stand-in for a commerce WRITE handler (e.g. CreatePaymentMethod)
|
||||
// that reads its subject from the JSON BODY: it reports the body it observes AFTER the pin, so
|
||||
// a test can assert the subject the handler would persist and that non-subject fields survive.
|
||||
func echoBody(c *zip.Ctx) error {
|
||||
var got map[string]any
|
||||
if len(c.Body()) > 0 {
|
||||
_ = json.Unmarshal(c.Body(), &got)
|
||||
}
|
||||
return c.JSON(200, got)
|
||||
}
|
||||
|
||||
func pinApp(t *testing.T) *zip.App {
|
||||
t.Helper()
|
||||
app := zip.New(zip.Config{Logger: luxlog.New("test")})
|
||||
@@ -38,6 +49,7 @@ func pinApp(t *testing.T) *zip.App {
|
||||
t.Fatalf("MountAccount: %v", err)
|
||||
}
|
||||
app.Get("/probe", PinBillingSubject(), echoQuery)
|
||||
app.Post("/probe", PinBillingSubject(), echoBody)
|
||||
return app
|
||||
}
|
||||
|
||||
@@ -67,6 +79,33 @@ func TestPinBillingSubject_PinsCallerAndDropsOrg(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestPinBillingSubject_PinsBodyForWrites — a POST whose JSON body names a FOREIGN subject
|
||||
// (customerId/userId/user) has every subject key overwritten with the caller's OWN subject
|
||||
// before the handler binds it, while non-subject fields survive. This is the boundary
|
||||
// commerce's CreatePaymentMethod (which reads customerId from the body) relies on to be
|
||||
// IDOR-safe co-resident — byte-identical to billingData's scopedBillingBody on the bridge.
|
||||
func TestPinBillingSubject_PinsBodyForWrites(t *testing.T) {
|
||||
app := pinApp(t)
|
||||
code, body := callH(t, app, http.MethodPost, "/probe", alice,
|
||||
`{"customerId":"victim","userId":"victim","user":"victim","card":{"last4":"4242"}}`)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("want 200, got %d (%s)", code, body)
|
||||
}
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatalf("bad body: %s", body)
|
||||
}
|
||||
for _, k := range billingSubjectKeys {
|
||||
if got[k] != "acme" { // alice/acme resolves to the org subject "acme"
|
||||
t.Fatalf("handler must see body %s=acme (caller's own subject), got %v", k, got[k])
|
||||
}
|
||||
}
|
||||
card, ok := got["card"].(map[string]any)
|
||||
if !ok || card["last4"] != "4242" {
|
||||
t.Fatalf("non-subject body field must survive the pin, got card=%v", got["card"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestPinBillingSubject_RefusesUnvalidated — a forged X-Org-Id with NO validated
|
||||
// X-User-Id (and no service token) is refused before the read handler runs: no
|
||||
// cross-tenant billing read is possible.
|
||||
|
||||
@@ -173,6 +173,23 @@ func requireCSRF(s *cloud.Service[state], next zip.Handler) zip.Handler {
|
||||
}
|
||||
}
|
||||
|
||||
// RequireCSRF exposes the ambient-cookie anti-CSRF gate as a STANDALONE middleware for a
|
||||
// co-resident money-WRITE route registered OUTSIDE this package — specifically
|
||||
// apps/commerce.go's POST /v1/billing/topup/token, which shadows the account-bridge's
|
||||
// POST /v1/billing/* wildcard (order 100 < 122) that would otherwise have wrapped the
|
||||
// write in requireCSRF. Moving the route co-resident to break the commerceinproc
|
||||
// self-dispatch loop must NOT silently drop that anti-CSRF gate, so the identical
|
||||
// enforcement rides along as its own handler. It binds to the SAME process-wide key
|
||||
// (sharedCSRFKey) the GET /v1/csrf issuer and the bridge verifier use, so a token minted
|
||||
// at /v1/csrf verifies here byte-identically. Enforces ONLY on the ambient-cookie path (a
|
||||
// Bearer/gateway/API caller is not CSRF-able); on success it c.Next()s into the rest of
|
||||
// the chain. The minimal Service carries only the shared key — requireCSRF/verifyCSRF
|
||||
// read nothing else off it.
|
||||
func RequireCSRF() zip.Handler {
|
||||
s := &cloud.Service[state]{State: state{csrfKey: sharedCSRFKey(nil)}}
|
||||
return requireCSRF(s, func(c *zip.Ctx) error { return c.Next() })
|
||||
}
|
||||
|
||||
// issueCSRFToken serves GET /v1/csrf: for a VALIDATED caller, a fresh token
|
||||
// bound to their identity. no-store so it is never cached by a shared proxy. This is
|
||||
// the same-origin endpoint the embedded SPA reads (its response body is unreadable to
|
||||
|
||||
@@ -44,6 +44,7 @@ const iamMaxBody = 4 << 20
|
||||
// iamClient is the confidential-client caller. clientID/clientSecret authenticate
|
||||
// as the `hanzo-console` app; an empty pair means "not configured" (handlers 501).
|
||||
type iamClient struct {
|
||||
serviceToken string // IAM_SERVICE_TOKEN — the Bearer for the admin provision endpoint
|
||||
base string
|
||||
clientID string
|
||||
clientSecret string
|
||||
@@ -56,10 +57,91 @@ func newIAMClient() *iamClient {
|
||||
base: base,
|
||||
clientID: strings.TrimSpace(os.Getenv("IAM_MINT_CLIENT_ID")),
|
||||
clientSecret: strings.TrimSpace(os.Getenv("IAM_MINT_CLIENT_SECRET")),
|
||||
serviceToken: strings.TrimSpace(os.Getenv("IAM_SERVICE_TOKEN")),
|
||||
http: &http.Client{Timeout: 15 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
// provisionResult is the /v1/iam/admin/provision response: the converged org and its
|
||||
// hashed credential (accessSecret shown ONCE on first mint). The org starts at a zero
|
||||
// balance — usage is pre-paid, no signup grant.
|
||||
type provisionResult struct {
|
||||
Org string `json:"org"`
|
||||
AccessKey string `json:"accessKey"`
|
||||
AccessSecret string `json:"accessSecret"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
// provisionReady reports whether the service-token provisioning path is wired.
|
||||
func (c *iamClient) provisionReady() bool { return c != nil && c.serviceToken != "" }
|
||||
|
||||
// provision drives the ONE atomic IAM onboarding op: create the org, move the named
|
||||
// user in as its admin, and mint its hashed org-scoped credential — the service-token
|
||||
// endpoint that replaces the create-org + move-user pair, so there is no orphan
|
||||
// between two writes and a mid-flight retry converges. orgSlug is the caller's
|
||||
// already-resolved slug (IAM honors it verbatim). The org starts at a zero balance.
|
||||
func (c *iamClient) provision(ctx context.Context, owner, name, orgSlug string, personal bool) (provisionResult, error) {
|
||||
if !c.provisionReady() {
|
||||
return provisionResult{}, errNotConfigured
|
||||
}
|
||||
body, err := json.Marshal(map[string]any{
|
||||
"owner": owner, "name": name, "orgSlug": orgSlug, "personal": personal,
|
||||
})
|
||||
if err != nil {
|
||||
return provisionResult{}, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+"/v1/iam/admin/provision", strings.NewReader(string(body)))
|
||||
if err != nil {
|
||||
return provisionResult{}, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+c.serviceToken)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return provisionResult{}, fmt.Errorf("iam unreachable: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
raw, err := io.ReadAll(io.LimitReader(resp.Body, iamMaxBody))
|
||||
if err != nil {
|
||||
return provisionResult{}, err
|
||||
}
|
||||
var out provisionResult
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return provisionResult{}, fmt.Errorf("iam provision non-json response (%d)", resp.StatusCode)
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 || out.Error != "" {
|
||||
msg := out.Error
|
||||
if msg == "" {
|
||||
msg = fmt.Sprintf("iam status %d", resp.StatusCode)
|
||||
}
|
||||
return provisionResult{}, fmt.Errorf("iam provision: %s", msg)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// userRow is the subset of an IAM user the onboarding path reads to resolve the
|
||||
// caller's authoritative (owner, name) — a zero-org caller's owner is not on its
|
||||
// token, so provision needs it from the row.
|
||||
type userRow struct {
|
||||
Owner string `json:"owner"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// getUserRow resolves the user by the caller's id (the same read the move did) into
|
||||
// its authoritative (owner, name).
|
||||
func (c *iamClient) getUserRow(ctx context.Context, id string) (userRow, error) {
|
||||
raw, err := c.getUser(ctx, id)
|
||||
if err != nil {
|
||||
return userRow{}, err
|
||||
}
|
||||
var row userRow
|
||||
if err := json.Unmarshal(raw, &row); err != nil {
|
||||
return userRow{}, fmt.Errorf("iam get-user: decode: %w", err)
|
||||
}
|
||||
return row, nil
|
||||
}
|
||||
|
||||
// configured reports whether the confidential client is wired. Handlers 501 when
|
||||
// false — the deployment simply lacks the `hanzo-console` credential (the honest
|
||||
// "not configured on this deployment" state, never a fabricated result).
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package account
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestOnboardFirstRun_ProvisionsOnce proves the production signup path drives ONE
|
||||
// atomic IAM provision (not the old create-org + move-user pair): it resolves the
|
||||
// zero-org caller's authoritative (owner, name) and provisions the org + hashed
|
||||
// credential in a single call. No trial credit is granted — the org starts at a zero
|
||||
// balance and usage is pre-paid.
|
||||
func TestOnboardFirstRun_ProvisionsOnce(t *testing.T) {
|
||||
var provisionCalls int
|
||||
var provBody map[string]any
|
||||
|
||||
iamSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/v1/iam/get-user":
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"status": "ok",
|
||||
"data": map[string]any{"owner": "landing", "name": "dave"},
|
||||
})
|
||||
case "/v1/iam/admin/provision":
|
||||
provisionCalls++
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
_ = json.Unmarshal(b, &provBody)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"org": "dave", "accessKey": "hk-x", "accessSecret": "sk-x",
|
||||
})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer iamSrv.Close()
|
||||
|
||||
iam := &iamClient{base: iamSrv.URL, clientID: "c", clientSecret: "s", serviceToken: "svc", http: &http.Client{}}
|
||||
|
||||
resp, err := onboardFirstRun(context.Background(), iam, "dave", "dave", "Dave", true)
|
||||
if err != nil {
|
||||
t.Fatalf("onboardFirstRun: %v", err)
|
||||
}
|
||||
if resp.Org != "dave" || resp.Additional {
|
||||
t.Fatalf("resp = %+v, want org=dave additional=false", resp)
|
||||
}
|
||||
if provisionCalls != 1 {
|
||||
t.Fatalf("provision calls = %d, want 1 (ONE atomic op, not create+move)", provisionCalls)
|
||||
}
|
||||
if provBody["owner"] != "landing" || provBody["name"] != "dave" || provBody["orgSlug"] != "dave" {
|
||||
t.Fatalf("provision body = %v, want owner=landing name=dave orgSlug=dave", provBody)
|
||||
}
|
||||
|
||||
// Retry converges: provision is idempotent (same org), no orphan.
|
||||
if _, err := onboardFirstRun(context.Background(), iam, "dave", "dave", "Dave", true); err != nil {
|
||||
t.Fatalf("retry: %v", err)
|
||||
}
|
||||
if provisionCalls != 2 {
|
||||
t.Fatalf("provision calls = %d, want 2 (retried, converges)", provisionCalls)
|
||||
}
|
||||
}
|
||||
+187
-60
@@ -1,28 +1,36 @@
|
||||
// topup.go ports console's app/billing/v1/topup/wallet/route.ts into the unified
|
||||
// binary at POST /v1/commerce/topup/wallet (task #41). It is the verify-and-record
|
||||
// seam for an HUSD wallet top-up: the browser sends an HUSD ERC-20 transfer to the
|
||||
// treasury and posts the tx hash here; this handler reads the receipt from the Hanzo
|
||||
// EVM, confirms it is a mined, successful HUSD Transfer(from → treasury, value),
|
||||
// derives USD cents from the (18-decimal, USD-pegged) on-chain value, records it to
|
||||
// commerce as an `husd` crypto payment, and returns the credited amount + balance.
|
||||
// topup.go is the verify-and-record seam for a crypto wallet top-up: the browser
|
||||
// sends a USD-pegged ERC-20 transfer to our treasury and posts the tx hash here;
|
||||
// this handler reads the receipt from that chain, confirms it is a mined, successful
|
||||
// Transfer(from → treasury, value), derives USD cents from the on-chain value using
|
||||
// the TOKEN'S OWN decimals, records it to commerce, and returns the credited amount
|
||||
// plus the new balance.
|
||||
//
|
||||
// A rail is one accepted (chain, token, treasury) triple, configured as data in
|
||||
// TOPUP_RAILS and discoverable at GET /v1/commerce/topup/rails. This replaced a
|
||||
// single hardcoded HUSD-on-Hanzo-Mainnet pair: HUSD is not deployed, so the surface
|
||||
// was permanently 501 — complete, correct and unable to take a cent. Customers
|
||||
// already hold USDC on Base/Ethereum/Polygon, so accepting the assets they have is
|
||||
// what makes this earn.
|
||||
//
|
||||
// THE CREDITED AMOUNT IS THE ON-CHAIN VALUE, never a client number — which is exactly
|
||||
// why this MUST be a server handler and cannot collapse to a same-origin call. Two
|
||||
// hardenings over the Node route:
|
||||
// why this MUST be a server handler and cannot collapse to a same-origin call. Three
|
||||
// properties worth keeping:
|
||||
//
|
||||
// - IDOR-safe: the credit lands on the VALIDATED caller's own org/user (the
|
||||
// gateway-verified X-Org-Id/X-User-Id), never a client-supplied `userId`.
|
||||
// - S2S to commerce: recorded with the admin COMMERCE_SERVICE_TOKEN + the caller's
|
||||
// X-Org-Id (the same service-to-service pattern clients/admin reads balances on),
|
||||
// not by forwarding a browser cookie.
|
||||
// - Per-rail decimals: cents come from 10^(decimals-2), so a 6-decimal USDC and an
|
||||
// 18-decimal token cannot be priced with one another's divisor.
|
||||
//
|
||||
// The EVM receipt is read over plain JSON-RPC (eth_getTransactionReceipt) — one
|
||||
// well-known call + one well-known event, so the stdlib is sufficient and no EVM
|
||||
// client dependency is pulled in.
|
||||
// client dependency is pulled in. That also means a new chain costs no new code.
|
||||
//
|
||||
// Honest failure (no fabricated credit, ever): HUSD/treasury unconfigured
|
||||
// (greenfield — HUSD not yet deployed) → 501; a missing/failed/non-HUSD-to-treasury
|
||||
// tx → 400; the chain or commerce unreachable → 502.
|
||||
// Honest failure (no fabricated credit, ever): no rail configured → 501; an unknown
|
||||
// rail, or a missing/failed/non-matching tx → 400; the chain or commerce unreachable
|
||||
// → 502.
|
||||
package account
|
||||
|
||||
import (
|
||||
@@ -63,9 +71,15 @@ var commerceHTTP = commerceinproc.Client(15 * time.Second)
|
||||
// constant (no need to hash at runtime).
|
||||
const transferTopic = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"
|
||||
|
||||
// husdCentDivisor: HUSD is an 18-decimal, USD-pegged stablecoin, so 1e16 base units
|
||||
// = 1 cent. Mirrors route.ts's `value / 10n ** 16n`.
|
||||
var husdCentDivisor = new(big.Int).Exp(big.NewInt(10), big.NewInt(16), nil)
|
||||
// centDivisor: base units per cent for a `decimals`-place USD-pegged token, i.e.
|
||||
// 10^(decimals-2). This MUST be per-token, not a constant: HUSD has 18 decimals
|
||||
// (1e16 per cent) while USDC has 6 (1e4 per cent). Sharing one divisor across both
|
||||
// would misprice a credit by 10^12 — the difference between a $10 top-up and a
|
||||
// $10,000,000,000 one — so the token's own decimals are carried on the rail and
|
||||
// used here.
|
||||
func centDivisor(decimals int) *big.Int {
|
||||
return new(big.Int).Exp(big.NewInt(10), big.NewInt(int64(decimals-2)), nil)
|
||||
}
|
||||
|
||||
var (
|
||||
addrRe = regexp.MustCompile(`^0x[0-9a-fA-F]{40}$`)
|
||||
@@ -74,43 +88,101 @@ var (
|
||||
|
||||
func isAddr(a string) bool { return addrRe.MatchString(a) }
|
||||
|
||||
// topupConfig is the deployment's HUSD + commerce wiring, resolved from server-only
|
||||
// env (sourced from KMS by the deployment, never a browser value).
|
||||
// rail is ONE accepted way to pay: a USD-pegged ERC-20 on a specific chain, sent to
|
||||
// a treasury address we control there. Everything a receipt check needs is on the
|
||||
// rail, so accepting a new chain or token is data, never code.
|
||||
//
|
||||
// This replaced a single hardcoded (HUSD, treasury) pair. That pair could only ever
|
||||
// describe HUSD on Hanzo Mainnet, and since HUSD is not deployed the whole surface
|
||||
// was permanently 501 — architecturally complete and earning nothing. Customers
|
||||
// already hold USDC on Base/Ethereum/Polygon, so the rail set is what makes the
|
||||
// path able to take money at all.
|
||||
type rail struct {
|
||||
// Stable id the client names when submitting, e.g. "base-usdc".
|
||||
ID string `json:"id"`
|
||||
// Human chain name for the UI, e.g. "Base".
|
||||
Chain string `json:"chain"`
|
||||
// EIP-155 chain id — the wallet must be on this chain.
|
||||
ChainID int64 `json:"chainId"`
|
||||
// JSON-RPC endpoint used to read the receipt. Carried in the CONFIG json (this
|
||||
// struct is what TOPUP_RAILS decodes into) but never published — the public
|
||||
// listing is a separate view type, because an input model and an output model
|
||||
// are different things and collapsing them once made this field unsettable.
|
||||
RPCURL string `json:"rpcUrl"`
|
||||
// The ERC-20 contract.
|
||||
Token string `json:"token"`
|
||||
// Display symbol, e.g. "USDC".
|
||||
Symbol string `json:"symbol"`
|
||||
// Token decimals. USDC is 6; an 18-decimal token must say 18.
|
||||
Decimals int `json:"decimals"`
|
||||
// Where the customer sends funds on this chain. Public by nature.
|
||||
Treasury string `json:"treasury"`
|
||||
}
|
||||
|
||||
// ok reports whether a rail is usable. A malformed rail is DROPPED rather than
|
||||
// rejected at request time, so one bad entry cannot take the whole surface down —
|
||||
// and a rail with impossible decimals cannot silently misprice a credit.
|
||||
func (r rail) ok() bool {
|
||||
return r.ID != "" && isAddr(r.Token) && isAddr(r.Treasury) && r.RPCURL != "" &&
|
||||
r.Decimals >= 2 && r.Decimals <= 36
|
||||
}
|
||||
|
||||
// topupConfig is the deployment's accepted rails + commerce wiring, resolved from
|
||||
// server-only env (sourced from KMS by the deployment, never a browser value).
|
||||
type topupConfig struct {
|
||||
husd string // HUSD ERC-20 contract address
|
||||
treasury string // the top-up treasury address
|
||||
rpcURL string // Hanzo EVM JSON-RPC endpoint
|
||||
chainID int64
|
||||
rails []rail
|
||||
commerce string // commerce base (e.g. http://commerce.hanzo.svc.cluster.local:8001)
|
||||
token string // admin S2S bearer for commerce (COMMERCE_SERVICE_TOKEN; never logged)
|
||||
}
|
||||
|
||||
// TOPUP_RAILS is a JSON array of rails — ONE variable describing the whole accepted
|
||||
// set, rather than a family of per-token env names that would have to be invented
|
||||
// again for every chain. Unparseable or malformed entries are dropped.
|
||||
func loadTopupConfig() topupConfig {
|
||||
chainID := int64(36963)
|
||||
if v := strings.TrimSpace(os.Getenv("HANZO_CHAIN_ID")); v != "" {
|
||||
if n, ok := new(big.Int).SetString(v, 10); ok {
|
||||
chainID = n.Int64()
|
||||
var rails []rail
|
||||
if raw := strings.TrimSpace(os.Getenv("TOPUP_RAILS")); raw != "" {
|
||||
var parsed []rail
|
||||
if err := json.Unmarshal([]byte(raw), &parsed); err == nil {
|
||||
for _, r := range parsed {
|
||||
r.RPCURL = strings.TrimRight(strings.TrimSpace(r.RPCURL), "/")
|
||||
if r.ok() {
|
||||
rails = append(rails, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return topupConfig{
|
||||
husd: strings.TrimSpace(os.Getenv("HANZO_HUSD_ADDRESS")),
|
||||
treasury: strings.TrimSpace(os.Getenv("HANZO_HUSD_TREASURY")),
|
||||
rpcURL: strings.TrimRight(getenv("HANZO_RPC_URL", "https://rpc.hanzo.network"), "/"),
|
||||
chainID: chainID,
|
||||
rails: rails,
|
||||
commerce: strings.TrimRight(getenv("COMMERCE_URL", "https://api.hanzo.ai"), "/"),
|
||||
token: strings.TrimSpace(os.Getenv("COMMERCE_SERVICE_TOKEN")),
|
||||
}
|
||||
}
|
||||
|
||||
// configured reports whether the greenfield gate is satisfied — a valid HUSD contract
|
||||
// AND treasury. Until HUSD is deployed both are unset and the surface is honestly 501.
|
||||
func (t topupConfig) configured() bool { return isAddr(t.husd) && isAddr(t.treasury) }
|
||||
// configured reports whether ANY rail is accepted. With none the surface is honestly
|
||||
// 501 rather than pretending to take money it cannot verify.
|
||||
func (t topupConfig) configured() bool { return len(t.rails) > 0 }
|
||||
|
||||
// find returns the named rail. An unknown id is a client error, not a server one.
|
||||
func (t topupConfig) find(id string) (rail, bool) {
|
||||
for _, r := range t.rails {
|
||||
if strings.EqualFold(r.ID, id) {
|
||||
return r, true
|
||||
}
|
||||
}
|
||||
return rail{}, false
|
||||
}
|
||||
|
||||
type walletTopupReq struct {
|
||||
// Which accepted rail the transfer was sent on, e.g. "base-usdc". The client
|
||||
// names it rather than the server guessing from the tx: the same address can
|
||||
// exist on several chains, so inferring would risk crediting against the wrong
|
||||
// treasury.
|
||||
Rail string `json:"rail"`
|
||||
TxHash string `json:"txHash"`
|
||||
FromAddress string `json:"fromAddress"`
|
||||
// A client-supplied `userId` is intentionally NOT read — the credit lands on the
|
||||
// validated caller (no IDOR).
|
||||
// validated caller (no IDOR). Neither is any amount: the credit is the ON-CHAIN
|
||||
// value, so a client number could never inflate it.
|
||||
}
|
||||
|
||||
type walletTopupResp struct {
|
||||
@@ -120,13 +192,52 @@ type walletTopupResp struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
// walletTopup verifies a sent HUSD transfer on-chain and credits the caller's org.
|
||||
// Mirrors POST app/billing/v1/topup/wallet/route.ts.
|
||||
// topupRails answers GET /v1/commerce/topup/rails: the accepted (chain, token,
|
||||
// treasury) set, so the browser can render "send USDC here" WITHOUT the addresses
|
||||
// being baked into its bundle.
|
||||
//
|
||||
// This exists because the console previously gated its top-up UI on
|
||||
// NEXT_PUBLIC_HANZO_HUSD_ADDRESS/_TREASURY — build-time constants. Enabling a rail
|
||||
// therefore meant rebuilding and redeploying the frontend, and with them unset the
|
||||
// UI reported "not available yet" no matter what the server could actually accept.
|
||||
// Serving the set at runtime keeps ONE source of truth (the server's config) and
|
||||
// lets a rail be switched on without shipping a bundle.
|
||||
//
|
||||
// Everything here is public on-chain data; no secret is exposed.
|
||||
func topupRails(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
cfg := loadTopupConfig()
|
||||
// Encode as [] rather than null, so clients can just read .length.
|
||||
view := make([]railView, 0, len(cfg.rails))
|
||||
for _, r := range cfg.rails {
|
||||
view = append(view, railView{
|
||||
ID: r.ID, Chain: r.Chain, ChainID: r.ChainID,
|
||||
Token: r.Token, Symbol: r.Symbol, Decimals: r.Decimals, Treasury: r.Treasury,
|
||||
})
|
||||
}
|
||||
return c.JSON(http.StatusOK, map[string]any{"rails": view})
|
||||
}
|
||||
|
||||
// railView is what a browser is told about a rail: everything needed to send funds
|
||||
// and nothing else. Distinct from `rail` so that adding an operational field to the
|
||||
// config (an RPC URL, a key reference, a provider credential) cannot leak by merely
|
||||
// existing — a new field is published only if it is added here on purpose.
|
||||
type railView struct {
|
||||
ID string `json:"id"`
|
||||
Chain string `json:"chain"`
|
||||
ChainID int64 `json:"chainId"`
|
||||
Token string `json:"token"`
|
||||
Symbol string `json:"symbol"`
|
||||
Decimals int `json:"decimals"`
|
||||
Treasury string `json:"treasury"`
|
||||
}
|
||||
|
||||
// walletTopup verifies a sent stablecoin transfer on-chain and credits the caller's
|
||||
// org. The credited amount is the ON-CHAIN value, never a client number.
|
||||
func walletTopup(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
cfg := loadTopupConfig()
|
||||
// Greenfield gate: no HUSD contract / treasury ⇒ honest "not configured yet".
|
||||
// No accepted rail ⇒ honest "not configured yet" rather than a fake credit.
|
||||
if !cfg.configured() {
|
||||
return zip.Errorf(http.StatusNotImplemented, "HUSD top-up is not configured yet (HUSD is not deployed on Hanzo Mainnet)")
|
||||
return zip.Errorf(http.StatusNotImplemented, "crypto top-up is not configured yet (no payment rail is enabled)")
|
||||
}
|
||||
|
||||
// The credit lands on the VALIDATED caller's own org (X-Org-Id) — require it.
|
||||
@@ -143,15 +254,26 @@ func walletTopup(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
if !txHashRe.MatchString(txHash) {
|
||||
return zip.ErrBadRequest("a valid transaction hash is required")
|
||||
}
|
||||
// With exactly one rail the client may omit it; naming it is required as soon as
|
||||
// there is a choice, so a transfer can never be checked against another chain's
|
||||
// treasury by default.
|
||||
railID := strings.TrimSpace(body.Rail)
|
||||
if railID == "" && len(cfg.rails) == 1 {
|
||||
railID = cfg.rails[0].ID
|
||||
}
|
||||
rl, ok := cfg.find(railID)
|
||||
if !ok {
|
||||
return zip.ErrBadRequest("unknown payment rail: name one from GET /v1/commerce/topup/rails")
|
||||
}
|
||||
|
||||
// ── 1. Verify the HUSD transfer on-chain ─────────────────────────────────────
|
||||
cents, verifiedFrom, herr := verifyHusdTransfer(c.Context(), cfg, txHash, strings.TrimSpace(body.FromAddress))
|
||||
// ── 1. Verify the transfer on-chain ──────────────────────────────────────────
|
||||
cents, verifiedFrom, herr := verifyTransfer(c.Context(), rl, txHash, strings.TrimSpace(body.FromAddress))
|
||||
if herr != nil {
|
||||
return herr
|
||||
}
|
||||
|
||||
// ── 2. Record to commerce as an HUSD crypto payment (S2S) ────────────────────
|
||||
status, herr := recordHusdPayment(c.Context(), cfg, cr, txHash, verifiedFrom, cents)
|
||||
// ── 2. Record to commerce as a crypto payment on this rail (S2S) ─────────────
|
||||
status, herr := recordCryptoPayment(c.Context(), cfg, rl, cr, txHash, verifiedFrom, cents)
|
||||
if herr != nil {
|
||||
return herr
|
||||
}
|
||||
@@ -176,13 +298,14 @@ type rpcLog struct {
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
// verifyHusdTransfer reads the receipt and confirms exactly one mined, successful
|
||||
// HUSD Transfer to the treasury, returning the credited cents and the sender. Any
|
||||
// non-conforming tx is an honest 400; an unreachable chain is a 502.
|
||||
func verifyHusdTransfer(ctx context.Context, cfg topupConfig, txHash, wantFrom string) (int64, string, error) {
|
||||
rcpt, err := getReceipt(ctx, cfg.rpcURL, txHash)
|
||||
// verifyTransfer reads the receipt on the rail's chain and confirms a mined,
|
||||
// successful Transfer of the rail's token to the rail's treasury, returning the
|
||||
// credited cents and the sender. Any non-conforming tx is an honest 400; an
|
||||
// unreachable chain is a 502. Nothing is credited that the chain did not confirm.
|
||||
func verifyTransfer(ctx context.Context, rl rail, txHash, wantFrom string) (int64, string, error) {
|
||||
rcpt, err := getReceipt(ctx, rl.RPCURL, txHash)
|
||||
if err != nil {
|
||||
return 0, "", zip.Errorf(http.StatusBadGateway, "could not verify the transaction on Hanzo Mainnet: %v", err)
|
||||
return 0, "", zip.Errorf(http.StatusBadGateway, "could not verify the transaction on %s: %v", rl.Chain, err)
|
||||
}
|
||||
if rcpt == nil {
|
||||
return 0, "", zip.ErrBadRequest("transaction not found or not yet mined")
|
||||
@@ -191,10 +314,11 @@ func verifyHusdTransfer(ctx context.Context, cfg topupConfig, txHash, wantFrom s
|
||||
return 0, "", zip.ErrBadRequest("transaction failed on-chain")
|
||||
}
|
||||
|
||||
husd := strings.ToLower(cfg.husd)
|
||||
treasuryTopic := addrToTopic(cfg.treasury)
|
||||
token := strings.ToLower(rl.Token)
|
||||
treasuryTopic := addrToTopic(rl.Treasury)
|
||||
div := centDivisor(rl.Decimals)
|
||||
for _, lg := range rcpt.Logs {
|
||||
if strings.ToLower(lg.Address) != husd {
|
||||
if strings.ToLower(lg.Address) != token {
|
||||
continue
|
||||
}
|
||||
if len(lg.Topics) < 3 || strings.ToLower(lg.Topics[0]) != transferTopic {
|
||||
@@ -207,7 +331,7 @@ func verifyHusdTransfer(ctx context.Context, cfg topupConfig, txHash, wantFrom s
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
cents := new(big.Int).Div(value, husdCentDivisor)
|
||||
cents := new(big.Int).Div(value, div)
|
||||
if cents.Sign() <= 0 || !cents.IsInt64() {
|
||||
return 0, "", zip.ErrBadRequest("transferred amount is below the minimum (1 cent)")
|
||||
}
|
||||
@@ -217,7 +341,7 @@ func verifyHusdTransfer(ctx context.Context, cfg topupConfig, txHash, wantFrom s
|
||||
}
|
||||
return cents.Int64(), from, nil
|
||||
}
|
||||
return 0, "", zip.ErrBadRequest("no HUSD transfer to the treasury was found in this transaction")
|
||||
return 0, "", zip.ErrBadRequest("no " + rl.Symbol + " transfer to the treasury was found in this transaction")
|
||||
}
|
||||
|
||||
// getReceipt calls eth_getTransactionReceipt over JSON-RPC. A null result (not mined)
|
||||
@@ -277,18 +401,21 @@ func topicToAddr(topic string) string {
|
||||
|
||||
// ── commerce (S2S) ───────────────────────────────────────────────────────────────
|
||||
|
||||
// recordHusdPayment records the verified credit to commerce as an `husd` crypto
|
||||
// payment, scoped to the caller's org via the S2S service token + X-Org-Id.
|
||||
func recordHusdPayment(ctx context.Context, cfg topupConfig, cr caller, txHash, from string, cents int64) (string, error) {
|
||||
// recordCryptoPayment records the verified credit to commerce, scoped to the
|
||||
// caller's org via the S2S service token + X-Org-Id. Network, chain, currency and
|
||||
// destination all come from the RAIL the transfer was verified against, so the
|
||||
// ledger row describes the payment that actually happened rather than a fixed
|
||||
// assumption about which chain and token it was.
|
||||
func recordCryptoPayment(ctx context.Context, cfg topupConfig, rl rail, cr caller, txHash, from string, cents int64) (string, error) {
|
||||
payload, _ := json.Marshal(map[string]any{
|
||||
"method": "crypto",
|
||||
"network": "hanzo",
|
||||
"chainId": cfg.chainID,
|
||||
"currency": "husd",
|
||||
"network": rl.Chain,
|
||||
"chainId": rl.ChainID,
|
||||
"currency": strings.ToLower(rl.Symbol),
|
||||
"amount": cents,
|
||||
"txHash": txHash,
|
||||
"fromAddress": from,
|
||||
"toAddress": cfg.treasury,
|
||||
"toAddress": rl.Treasury,
|
||||
"userId": cr.id, // the VALIDATED caller — never a client-supplied id
|
||||
})
|
||||
raw, status, err := commerceDo(ctx, cfg.commerce, cfg.token, http.MethodPost, "/v1/billing/payment", nil, cr.owner, payload)
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"math/big"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
@@ -81,24 +82,46 @@ func (f *fakeCommerce) server(t *testing.T) *httptest.Server {
|
||||
return srv
|
||||
}
|
||||
|
||||
// husdReceipt builds a receipt carrying one HUSD Transfer(from→to, cents*1e16).
|
||||
// husdReceipt builds a receipt carrying one 18-decimal Transfer(from→to, cents*1e16).
|
||||
func husdReceipt(status, husd, from, to string, cents int64) map[string]any {
|
||||
value := new(big.Int).Mul(big.NewInt(cents), husdCentDivisor)
|
||||
return tokenReceipt(status, husd, from, to, cents, 18)
|
||||
}
|
||||
|
||||
// tokenReceipt builds a receipt for a Transfer of `cents` worth of a token with the
|
||||
// given decimals — the knob that proves a 6-decimal USDC is not priced with an
|
||||
// 18-decimal divisor.
|
||||
func tokenReceipt(status, token, from, to string, cents int64, decimals int) map[string]any {
|
||||
value := new(big.Int).Mul(big.NewInt(cents), centDivisor(decimals))
|
||||
return map[string]any{
|
||||
"status": status,
|
||||
"logs": []any{map[string]any{
|
||||
"address": husd,
|
||||
"address": token,
|
||||
"topics": []any{transferTopic, addrToTopic(from), addrToTopic(to)},
|
||||
"data": "0x" + fmt.Sprintf("%064x", value),
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func setTopupEnv(t *testing.T, husd, treasury, rpcURL, commerceURL string) {
|
||||
// setTopupEnv configures ONE 18-decimal rail, preserving these tests' original
|
||||
// arithmetic (18 decimals ⇒ 1e16 per cent) so they still assert the same cents.
|
||||
// An empty token/treasury yields no rail at all — the "not configured" case.
|
||||
func setTopupEnv(t *testing.T, token, treasury, rpcURL, commerceURL string) {
|
||||
t.Helper()
|
||||
t.Setenv("HANZO_HUSD_ADDRESS", husd)
|
||||
t.Setenv("HANZO_HUSD_TREASURY", treasury)
|
||||
t.Setenv("HANZO_RPC_URL", rpcURL)
|
||||
setTopupRails(t, commerceURL, rail{
|
||||
ID: "hanzo-husd", Chain: "Hanzo", ChainID: 36963, RPCURL: rpcURL,
|
||||
Token: token, Symbol: "HUSD", Decimals: 18, Treasury: treasury,
|
||||
})
|
||||
}
|
||||
|
||||
// setTopupRails installs an explicit rail set. Malformed rails are dropped by
|
||||
// loadTopupConfig, which is how the not-configured cases above stay 501.
|
||||
func setTopupRails(t *testing.T, commerceURL string, rails ...rail) {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(rails)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal rails: %v", err)
|
||||
}
|
||||
t.Setenv("TOPUP_RAILS", string(raw))
|
||||
t.Setenv("COMMERCE_URL", commerceURL)
|
||||
t.Setenv("COMMERCE_SERVICE_TOKEN", "svc-token")
|
||||
}
|
||||
@@ -239,3 +262,117 @@ func TestTopup_SenderMismatch_400(t *testing.T) {
|
||||
t.Fatalf("sender mismatch: want 400, got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// ── rails: the multi-chain generalisation ────────────────────────────────────────
|
||||
|
||||
const (
|
||||
tUSDC = "0x5555555555555555555555555555555555555555"
|
||||
tTreasBase = "0x6666666666666666666666666666666666666666"
|
||||
)
|
||||
|
||||
func usdcRail(rpcURL string) rail {
|
||||
return rail{
|
||||
ID: "base-usdc", Chain: "Base", ChainID: 8453, RPCURL: rpcURL,
|
||||
Token: tUSDC, Symbol: "USDC", Decimals: 6, Treasury: tTreasBase,
|
||||
}
|
||||
}
|
||||
|
||||
// The decimals bug this design exists to prevent: USDC has 6 decimals, so 500 cents
|
||||
// is 5_000_000 base units. Priced with an 18-decimal divisor it would round to ZERO
|
||||
// and silently credit nothing; priced the other way it would credit 10^12 times too
|
||||
// much. The rail's own decimals must be what is used.
|
||||
func TestTopup_USDC_SixDecimals_PricedByRail(t *testing.T) {
|
||||
rpc := &fakeRPC{result: tokenReceipt("0x1", tUSDC, tSender, tTreasBase, 500, 6)}
|
||||
com := &fakeCommerce{balance: 500}
|
||||
setTopupRails(t, com.server(t).URL, usdcRail(rpc.server(t).URL))
|
||||
app := mountBrand(t, "hanzo")
|
||||
|
||||
code, body := callH(t, app, http.MethodPost, "/v1/commerce/topup/wallet", alice,
|
||||
`{"rail":"base-usdc","txHash":"`+tTxHash+`","fromAddress":"`+tSender+`"}`)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("usdc topup: want 200, got %d (%s)", code, body)
|
||||
}
|
||||
var r walletTopupResp
|
||||
mustJSON(t, body, &r)
|
||||
if r.CreditedCents != 500 {
|
||||
t.Fatalf("6-decimal USDC must credit 500 cents, got %d", r.CreditedCents)
|
||||
}
|
||||
// The ledger row must describe the rail that was actually verified.
|
||||
if com.payment["currency"] != "usdc" || com.payment["network"] != "Base" {
|
||||
t.Fatalf("payment must record the real rail, got currency=%v network=%v",
|
||||
com.payment["currency"], com.payment["network"])
|
||||
}
|
||||
if id, _ := com.payment["chainId"].(float64); id != 8453 {
|
||||
t.Fatalf("chainId must be the rail's 8453, got %v", com.payment["chainId"])
|
||||
}
|
||||
}
|
||||
|
||||
// With several rails configured the client MUST name one: silently picking a default
|
||||
// could verify a transfer against another chain's treasury.
|
||||
func TestTopup_MultipleRails_RequiresNamingOne(t *testing.T) {
|
||||
rpc := &fakeRPC{result: tokenReceipt("0x1", tUSDC, tSender, tTreasBase, 500, 6)}
|
||||
com := &fakeCommerce{}
|
||||
setTopupRails(t, com.server(t).URL,
|
||||
usdcRail(rpc.server(t).URL),
|
||||
rail{ID: "hanzo-husd", Chain: "Hanzo", ChainID: 36963, RPCURL: rpc.server(t).URL,
|
||||
Token: tHusd, Symbol: "HUSD", Decimals: 18, Treasury: tTreasury},
|
||||
)
|
||||
app := mountBrand(t, "hanzo")
|
||||
|
||||
code, _ := callH(t, app, http.MethodPost, "/v1/commerce/topup/wallet", alice,
|
||||
`{"txHash":"`+tTxHash+`"}`)
|
||||
if code != http.StatusBadRequest {
|
||||
t.Fatalf("omitting the rail with >1 configured: want 400, got %d", code)
|
||||
}
|
||||
code, _ = callH(t, app, http.MethodPost, "/v1/commerce/topup/wallet", alice,
|
||||
`{"rail":"nope","txHash":"`+tTxHash+`"}`)
|
||||
if code != http.StatusBadRequest {
|
||||
t.Fatalf("unknown rail: want 400, got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// A transfer on the right token but to ANOTHER rail's treasury must not credit.
|
||||
func TestTopup_WrongRailTreasury_400(t *testing.T) {
|
||||
rpc := &fakeRPC{result: tokenReceipt("0x1", tUSDC, tSender, tTreasury, 500, 6)} // hanzo treasury
|
||||
com := &fakeCommerce{}
|
||||
setTopupRails(t, com.server(t).URL, usdcRail(rpc.server(t).URL))
|
||||
app := mountBrand(t, "hanzo")
|
||||
|
||||
code, _ := callH(t, app, http.MethodPost, "/v1/commerce/topup/wallet", alice,
|
||||
`{"rail":"base-usdc","txHash":"`+tTxHash+`"}`)
|
||||
if code != http.StatusBadRequest {
|
||||
t.Fatalf("transfer to a different treasury: want 400, got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// The public listing gives a browser what it needs to send funds — and must not leak
|
||||
// the operational RPC endpoint, which lives on the same config struct.
|
||||
func TestTopupRails_PublishesSendInfoWithoutRPC(t *testing.T) {
|
||||
com := &fakeCommerce{}
|
||||
setTopupRails(t, com.server(t).URL, usdcRail("http://secret-rpc.internal"))
|
||||
app := mountBrand(t, "hanzo")
|
||||
|
||||
code, body := callH(t, app, http.MethodGet, "/v1/commerce/topup/rails", alice, "")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("rails: want 200, got %d (%s)", code, body)
|
||||
}
|
||||
var got struct{ Rails []map[string]any }
|
||||
mustJSON(t, body, &got)
|
||||
if len(got.Rails) != 1 || got.Rails[0]["treasury"] != tTreasBase || got.Rails[0]["decimals"].(float64) != 6 {
|
||||
t.Fatalf("rails listing wrong: %s", body)
|
||||
}
|
||||
if _, leaked := got.Rails[0]["rpcUrl"]; leaked {
|
||||
t.Fatalf("rails listing must not publish the RPC endpoint: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// No rail configured ⇒ the listing is an empty array, not null, so a client can read
|
||||
// .length without a nil check — and the POST is an honest 501.
|
||||
func TestTopupRails_EmptyWhenUnconfigured(t *testing.T) {
|
||||
t.Setenv("TOPUP_RAILS", "")
|
||||
app := mountBrand(t, "hanzo")
|
||||
code, body := callH(t, app, http.MethodGet, "/v1/commerce/topup/rails", alice, "")
|
||||
if code != http.StatusOK || !strings.Contains(string(body), `"rails":[]`) {
|
||||
t.Fatalf("unconfigured rails: want 200 with [], got %d (%s)", code, body)
|
||||
}
|
||||
}
|
||||
|
||||
+77
-19
@@ -27,6 +27,7 @@ import (
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
@@ -38,6 +39,7 @@ import (
|
||||
"github.com/hanzoai/cloud/clients/admin/finance"
|
||||
"github.com/hanzoai/cloud/clients/admin/health"
|
||||
"github.com/hanzoai/cloud/clients/admin/iam"
|
||||
"github.com/hanzoai/cloud/clients/admin/infra"
|
||||
"github.com/hanzoai/cloud/clients/admin/invoices"
|
||||
"github.com/hanzoai/cloud/clients/admin/metrics"
|
||||
"github.com/hanzoai/cloud/clients/admin/revenue"
|
||||
@@ -70,6 +72,7 @@ func Mount(app *zip.App, deps cloud.Deps) error {
|
||||
DO: digitalocean.New(doTokenFromEnv()),
|
||||
AdminOrg: adminOrgOf(deps),
|
||||
AuditStore: deps.Audit,
|
||||
WLTenants: wlTenantsFromEnv(),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -103,6 +106,7 @@ func routes(app *zip.App, s *cloud.Service[core.State]) {
|
||||
g.Get("/applications", core.Guard(s, applications))
|
||||
g.Get("/products", core.Guard(s, products))
|
||||
g.Get("/compute", core.Guard(s, compute))
|
||||
g.Get("/block-storage", core.Guard(s, blockStorage))
|
||||
g.Get("/o11y", core.Guard(s, o11y))
|
||||
g.Get("/aimetrics", core.Guard(s, aimetrics))
|
||||
g.Post("/sync", core.Guard(s, syncNow))
|
||||
@@ -138,6 +142,7 @@ func routes(app *zip.App, s *cloud.Service[core.State]) {
|
||||
revenue.Routes(app, s)
|
||||
finance.Routes(app, s)
|
||||
metrics.Routes(app, s)
|
||||
infra.Routes(app, s)
|
||||
invoices.Routes(app, s)
|
||||
subscriptions.Routes(app, s)
|
||||
}
|
||||
@@ -160,6 +165,11 @@ func me(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
Email: strings.TrimSpace(c.UserEmail()),
|
||||
DisplayName: name,
|
||||
IsSuperAdmin: sc.Super,
|
||||
// The gate (GuardScoped) already proved this caller is either a SuperAdmin or an
|
||||
// admin of an ENABLED WL tenant, so an admitted non-super IS the WL tier — no
|
||||
// separate lookup needed. ScopeOrgs is the resolved subtree (empty ⇒ all, for super).
|
||||
IsWhiteLabel: !sc.Super,
|
||||
ScopeOrgs: sc.Orgs,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -332,13 +342,8 @@ func usage(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
}
|
||||
|
||||
// ── /v1/admin/products — workload registry (ProductRow[]) ────────────────────
|
||||
|
||||
// products is the workload/drift registry. That inventory is the platform.hanzo.ai apps
|
||||
// table / operator reconcile state, NOT an in-binary source. admin exposes the gated
|
||||
// endpoint and returns the real empty registry until that feed is wired.
|
||||
func products(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
return core.OKList(c, []productRow{}, 0)
|
||||
}
|
||||
// The handler + the fleet projection live in products.go: it reads the operator App-CR +
|
||||
// drift observation through the in-process paas.CurrentFleet seam (reuse, never fork).
|
||||
|
||||
// ── /v1/admin/overview — Platform Overview tiles (OverviewData) ───────────────
|
||||
|
||||
@@ -355,17 +360,40 @@ func overview(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
commercePartial := false
|
||||
if orgErr == nil {
|
||||
orgCount = len(orgs)
|
||||
// FAN OUT. Each org costs two independent reads (users, money), so doing this
|
||||
// serially made the dashboard's latency O(orgs): at 122 orgs that is ~244
|
||||
// blocking round-trips before a single tile renders, and it grows every time a
|
||||
// tenant signs up. The reads do not depend on each other, so they run
|
||||
// concurrently under a fixed ceiling — bounded so a large fleet cannot stampede
|
||||
// the finance ledger or the IAM store.
|
||||
const maxParallelOrgReads = 12
|
||||
var (
|
||||
mu sync.Mutex
|
||||
wg sync.WaitGroup
|
||||
sem = make(chan struct{}, maxParallelOrgReads)
|
||||
)
|
||||
for _, o := range orgs {
|
||||
userCount += orgUserCount(s, ctx, cr, o.Name)
|
||||
sp, cr2, ok := core.OrgMoney(s, ctx, o.Name)
|
||||
spend += sp
|
||||
credits += cr2
|
||||
if !ok {
|
||||
// This org's money did not read — the fleet spend/credits totals are now
|
||||
// an UNDERCOUNT, so the commerce source must report degraded, not healthy.
|
||||
commercePartial = true
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(org string) {
|
||||
defer wg.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
uc := orgUserCount(s, ctx, cr, org)
|
||||
sp, cr2, ok := core.OrgMoney(s, ctx, org)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
userCount += uc
|
||||
spend += sp
|
||||
credits += cr2
|
||||
if !ok {
|
||||
// This org's money did not read — the fleet spend/credits totals are
|
||||
// now an UNDERCOUNT, so the commerce source must report degraded,
|
||||
// not healthy.
|
||||
commercePartial = true
|
||||
}
|
||||
}(o.Name)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// Commerce freshness derives from the SAME per-org reads the totals fold — NOT a
|
||||
@@ -393,12 +421,18 @@ func overview(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
}
|
||||
sources = append(sources, core.SrcOf("o11y", oErr, o11yRows, now))
|
||||
|
||||
// Fleet workload registry — the operator App-CR + drift observation via the paas seam
|
||||
// (products.go). A nil/unready seam degrades to an honest-empty rollup (zeros, no error);
|
||||
// a hard observation error marks the "fleet" source degraded without failing the overview.
|
||||
fleetRows, fleetRoll, fleetErr := fleetProducts(ctx)
|
||||
sources = append(sources, core.SrcOf("fleet", fleetErr, len(fleetRows), now))
|
||||
|
||||
return core.OK(c, overviewData{
|
||||
Orgs: orgCount,
|
||||
Users: userCount,
|
||||
Products: 0, // workload registry feed pending (platform apps table)
|
||||
ActiveProducts: 0,
|
||||
Drift: 0,
|
||||
Products: fleetRoll.Total,
|
||||
ActiveProducts: fleetRoll.Active,
|
||||
Drift: fleetRoll.Drift,
|
||||
SpendCents30d: spend,
|
||||
Tokens30d: 0, // fleet token counters pending (insights/datastore)
|
||||
CreditsCents: credits,
|
||||
@@ -461,6 +495,30 @@ func adminOrgOf(_ cloud.Deps) string {
|
||||
return "admin"
|
||||
}
|
||||
|
||||
// wlTenantsFromEnv resolves the enabled white-label tenant allowlist from
|
||||
// ADMIN_WL_TENANT_ORGS (comma-separated org slugs). It is the ONE seed of
|
||||
// State.WLTenants — the fail-closed second admission tier: EMPTY/unset ⇒ no customer
|
||||
// org-admin is admitted (SuperAdmins only), so an absent/mis-set env fails CLOSED.
|
||||
// Each entry is trimmed and matched verbatim against principal.Org (the validated
|
||||
// owner), never folded; blank entries are dropped. Onboarding a reseller is a
|
||||
// deliberate, KMS-/git-auditable edit to this env, not a runtime self-service flip.
|
||||
func wlTenantsFromEnv() map[string]bool {
|
||||
raw := strings.TrimSpace(os.Getenv("ADMIN_WL_TENANT_ORGS"))
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
set := map[string]bool{}
|
||||
for _, part := range strings.Split(raw, ",") {
|
||||
if org := strings.TrimSpace(part); org != "" {
|
||||
set[org] = true
|
||||
}
|
||||
}
|
||||
if len(set) == 0 {
|
||||
return nil
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
// doTokenFromEnv reads the DigitalOcean token from the environment. Sourced from a
|
||||
// KMSSecret on the cloud deployment (DO_API_TOKEN) — never hard-coded.
|
||||
func doTokenFromEnv() string {
|
||||
|
||||
@@ -42,6 +42,14 @@ func mountService(t *testing.T, iamURL, commerceURL, healthURL string) (func(met
|
||||
Health: health.New(healthURL),
|
||||
DO: digitalocean.New(""), // no token → honest not-configured unless a test overrides s.State.DO
|
||||
AdminOrg: "admin",
|
||||
// The harness enables ONE white-label tenant — "maxpower" (the org orgAdminHdr
|
||||
// belongs to) — so the scoped-panel tests exercise the ADMITTED WL tier. The
|
||||
// gate now requires WL enablement for any non-super caller, so the deny tests use
|
||||
// a DIFFERENT org (not in this set) to prove a non-enabled org-admin is refused,
|
||||
// and the fail-closed default (empty set ⇒ deny) is covered by a dedicated unit
|
||||
// test on State.IsWhiteLabelTenant. A test that needs the fleet-only default
|
||||
// clears s.State.WLTenants after mount.
|
||||
WLTenants: map[string]bool{"maxpower": true},
|
||||
}}
|
||||
// Mirror the REAL Mount EXACTLY by registering the same routes() the subsystem uses
|
||||
// (org-scoped panels behind GuardScoped, the platform control plane behind Guard,
|
||||
@@ -99,6 +107,10 @@ var platformAdminRoutes = []adminRoute{
|
||||
{"GET", "/v1/admin/flags"},
|
||||
{"GET", "/v1/admin/waitlist"},
|
||||
{"POST", "/v1/admin/waitlist/boost"},
|
||||
{"GET", "/v1/admin/infra"},
|
||||
{"POST", "/v1/admin/infra/volumes/v1/snapshot"},
|
||||
{"DELETE", "/v1/admin/infra/volumes/v1"},
|
||||
{"POST", "/v1/admin/infra/nodes/1/cordon"},
|
||||
}
|
||||
|
||||
// adminRoutes is the full surface (both tiers) — the fail-closed gate test denies an
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package admin
|
||||
|
||||
// blockStorage — GET /v1/admin/block-storage, the realtime DO block-storage fleet the
|
||||
// operator's Block Storage board (admin.hanzo.ai) watches to scale DO before it runs out.
|
||||
// (Named `block-storage`, not `storage`, so the operator's separate S3 object-buckets
|
||||
// view keeps /v1/admin/storage — two distinct storage concerns, two endpoints.)
|
||||
// Two REAL sources, honest by construction:
|
||||
// - The FLEET inventory (count · total capacity · monthly cost · per-volume region +
|
||||
// attachment) from the DigitalOcean API (the same DO_API_TOKEN client the finance
|
||||
// dashboard already uses). DO exposes capacity + attachment but NOT fill %, so a
|
||||
// volume's used/pct stay ABSENT (the console renders an honest "—", never a
|
||||
// fabricated number).
|
||||
// - The analytics DATASTORE's own fill from Datastore `system.disks` (the 200Gi PVC
|
||||
// the datastore fork mounts) — total/free space over the SAME shared client
|
||||
// (aiobject.DatastoreQuery) the analytics + compute lenses read, no second
|
||||
// connection. This is THE number the operator scales on.
|
||||
//
|
||||
// SUPERADMIN ONLY (the s.guard wrap in admin.go): a cross-tenant infra read, all-orgs.
|
||||
// admin holds NO storage state — it only reads DO + the datastore. DO unconfigured →
|
||||
// empty fleet; datastore not connected → no datastore card. Never a fabricated fleet.
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
aiobject "github.com/hanzoai/ai/object"
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/clients/admin/core"
|
||||
"github.com/hanzoai/cloud/clients/admin/digitalocean"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// doBlockUsdPerGiB is DO block storage's list price ($0.10/GiB/mo) — the fleet cost
|
||||
// line for the ops budget when DO doesn't itemize per-volume spend.
|
||||
const doBlockUsdPerGiB = 0.10
|
||||
|
||||
// bytesPerGiB converts system.disks bytes (UInt64) → GiB.
|
||||
const bytesPerGiB = 1024 * 1024 * 1024
|
||||
|
||||
// storageVolume mirrors the console StorageVolume. UsedGiB/Pct are POINTERS so an
|
||||
// absent fill serializes as JSON null (→ the console's honest "—"), distinct from a
|
||||
// real 0%.
|
||||
type storageVolume struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Region string `json:"region"`
|
||||
SizeGiB int `json:"sizeGiB"`
|
||||
UsedGiB *float64 `json:"usedGiB"`
|
||||
Pct *float64 `json:"pct"`
|
||||
Attached bool `json:"attached"`
|
||||
Service string `json:"service"`
|
||||
}
|
||||
|
||||
// storageFleet is the roll-up: real count/capacity/cost; fleet fill absent (DO gives
|
||||
// no per-volume fill, so there is no honest fleet-wide used total to report).
|
||||
type storageFleet struct {
|
||||
Count int `json:"count"`
|
||||
TotalGiB int `json:"totalGiB"`
|
||||
UsedGiB *float64 `json:"usedGiB"`
|
||||
Pct *float64 `json:"pct"`
|
||||
MonthlyUsd int `json:"monthlyUsd"`
|
||||
}
|
||||
|
||||
// datastoreVolume is the analytics backend's own volume, fill REAL from system.disks.
|
||||
type datastoreVolume struct {
|
||||
Name string `json:"name"`
|
||||
Mount string `json:"mount"`
|
||||
SizeGiB int `json:"sizeGiB"`
|
||||
UsedGiB float64 `json:"usedGiB"`
|
||||
Pct float64 `json:"pct"`
|
||||
}
|
||||
|
||||
// storageAlert flags a near-full volume (only the datastore carries a real fill today,
|
||||
// so alerts are datastore-derived until a per-volume filesystem source is wired).
|
||||
type storageAlert struct {
|
||||
Volume string `json:"volume"`
|
||||
Pct float64 `json:"pct"`
|
||||
Level string `json:"level"`
|
||||
}
|
||||
|
||||
// storageSnapshot is the whole board payload the console normalizes.
|
||||
type storageSnapshot struct {
|
||||
Fleet storageFleet `json:"fleet"`
|
||||
Datastore *datastoreVolume `json:"datastore"`
|
||||
Volumes []storageVolume `json:"volumes"`
|
||||
Alerts []storageAlert `json:"alerts"`
|
||||
}
|
||||
|
||||
// blockStorage answers GET /v1/admin/block-storage. SuperAdmin only. Each source
|
||||
// degrades independently — a DO outage still returns the real datastore fill, and v.v.
|
||||
func blockStorage(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
ctx := c.Context()
|
||||
vols, _ := s.State.DO.Volumes(ctx) // honest empty on not-configured / unreachable
|
||||
fill := datastoreFill(ctx) // nil unless system.disks answered
|
||||
return core.OK(c, buildStorageSnapshot(vols, fill))
|
||||
}
|
||||
|
||||
// buildStorageSnapshot assembles the board payload (PURE — unit-tested). It folds the
|
||||
// DO inventory into the fleet totals + per-volume rows (fill absent), attaches the real
|
||||
// datastore card, and derives a near-full alert from the datastore fill. No fabrication:
|
||||
// a volume's fill is left nil (DO gives none), and the datastore card is present only
|
||||
// when system.disks actually answered.
|
||||
func buildStorageSnapshot(vols []digitalocean.Volume, fill *datastoreVolume) storageSnapshot {
|
||||
out := make([]storageVolume, 0, len(vols))
|
||||
totalGiB := 0
|
||||
for _, v := range vols {
|
||||
out = append(out, storageVolume{
|
||||
ID: v.ID,
|
||||
Name: v.Name,
|
||||
Region: v.Region,
|
||||
SizeGiB: v.SizeGiB,
|
||||
Attached: len(v.DropletIDs) > 0,
|
||||
})
|
||||
totalGiB += v.SizeGiB
|
||||
}
|
||||
alerts := make([]storageAlert, 0, 1)
|
||||
if fill != nil {
|
||||
if lvl := alertLevel(fill.Pct); lvl != "" {
|
||||
alerts = append(alerts, storageAlert{Volume: fill.Name, Pct: fill.Pct, Level: lvl})
|
||||
}
|
||||
}
|
||||
return storageSnapshot{
|
||||
Fleet: storageFleet{
|
||||
Count: len(vols),
|
||||
TotalGiB: totalGiB,
|
||||
MonthlyUsd: int(float64(totalGiB)*doBlockUsdPerGiB + 0.5),
|
||||
},
|
||||
Datastore: fill,
|
||||
Volumes: out,
|
||||
Alerts: alerts,
|
||||
}
|
||||
}
|
||||
|
||||
// alertLevel thresholds a fill %: critical ≥ 90, warn ≥ 80, else none (PURE, tested).
|
||||
func alertLevel(pct float64) string {
|
||||
switch {
|
||||
case pct >= 90:
|
||||
return "critical"
|
||||
case pct >= 80:
|
||||
return "warn"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// datastoreFill reads the analytics datastore's own volume usage from Datastore
|
||||
// `system.disks` (the largest disk by capacity is the data volume — the 200Gi PVC).
|
||||
// Returns nil when the datastore isn't connected or the query fails (honest — the
|
||||
// console shows no datastore card, never a fabricated fill).
|
||||
func datastoreFill(ctx context.Context) *datastoreVolume {
|
||||
if !aiobject.DatastoreEnabled() {
|
||||
return nil
|
||||
}
|
||||
rows, err := aiobject.DatastoreQuery(ctx,
|
||||
"SELECT name, path, total_space, free_space FROM system.disks ORDER BY total_space DESC LIMIT 1")
|
||||
if err != nil || len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
return datastoreFillFromRow(rows[0])
|
||||
}
|
||||
|
||||
// datastoreFillFromRow maps a system.disks row → the datastore card (PURE, tested).
|
||||
// nil when the disk reports no capacity (an unusable read, not a fabricated 0%).
|
||||
func datastoreFillFromRow(r map[string]any) *datastoreVolume {
|
||||
total := float64(chInt64(r["total_space"]))
|
||||
if total <= 0 {
|
||||
return nil
|
||||
}
|
||||
free := float64(chInt64(r["free_space"]))
|
||||
used := total - free
|
||||
if used < 0 {
|
||||
used = 0
|
||||
}
|
||||
return &datastoreVolume{
|
||||
Name: chStr(r["name"]),
|
||||
Mount: chStr(r["path"]),
|
||||
SizeGiB: int(total / bytesPerGiB),
|
||||
UsedGiB: round1(used / bytesPerGiB),
|
||||
Pct: round1(used / total * 100),
|
||||
}
|
||||
}
|
||||
|
||||
// round1 rounds to one decimal (used/pct read cleanly; not a scientific quantity).
|
||||
func round1(x float64) float64 { return float64(int(x*10+0.5)) / 10 }
|
||||
@@ -0,0 +1,130 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package admin
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/admin/digitalocean"
|
||||
)
|
||||
|
||||
// The DO inventory folds into fleet totals + rows, and per-volume fill stays ABSENT
|
||||
// (DO exposes no fill) so the console renders an honest "—", never a fabricated 0%.
|
||||
func TestBuildStorageSnapshotFleet(t *testing.T) {
|
||||
vols := []digitalocean.Volume{
|
||||
{ID: "a", Name: "pvc-a", Region: "sfo3", SizeGiB: 200, DropletIDs: []int{1}},
|
||||
{ID: "b", Name: "pvc-b", Region: "nyc1", SizeGiB: 100, DropletIDs: nil},
|
||||
{ID: "c", Name: "pvc-c", Region: "sfo3", SizeGiB: 50, DropletIDs: []int{2, 3}},
|
||||
}
|
||||
snap := buildStorageSnapshot(vols, nil)
|
||||
|
||||
if snap.Fleet.Count != 3 {
|
||||
t.Fatalf("count = %d, want 3", snap.Fleet.Count)
|
||||
}
|
||||
if snap.Fleet.TotalGiB != 350 {
|
||||
t.Fatalf("totalGiB = %d, want 350", snap.Fleet.TotalGiB)
|
||||
}
|
||||
// 350 GiB * $0.10 = $35.
|
||||
if snap.Fleet.MonthlyUsd != 35 {
|
||||
t.Fatalf("monthlyUsd = %d, want 35", snap.Fleet.MonthlyUsd)
|
||||
}
|
||||
if snap.Fleet.UsedGiB != nil || snap.Fleet.Pct != nil {
|
||||
t.Fatalf("fleet fill must be absent (DO gives none); got used=%v pct=%v", snap.Fleet.UsedGiB, snap.Fleet.Pct)
|
||||
}
|
||||
if len(snap.Volumes) != 3 {
|
||||
t.Fatalf("volumes = %d, want 3", len(snap.Volumes))
|
||||
}
|
||||
// Attachment derives from droplet_ids; fill is absent per row.
|
||||
if !snap.Volumes[0].Attached || snap.Volumes[1].Attached || !snap.Volumes[2].Attached {
|
||||
t.Fatalf("attachment wrong: %+v", snap.Volumes)
|
||||
}
|
||||
for _, v := range snap.Volumes {
|
||||
if v.UsedGiB != nil || v.Pct != nil {
|
||||
t.Fatalf("volume %s fill must be absent, got used=%v pct=%v", v.ID, v.UsedGiB, v.Pct)
|
||||
}
|
||||
}
|
||||
if snap.Datastore != nil {
|
||||
t.Fatalf("datastore must be nil when no fill was read; got %+v", snap.Datastore)
|
||||
}
|
||||
if len(snap.Alerts) != 0 {
|
||||
t.Fatalf("no alerts without a datastore fill; got %v", snap.Alerts)
|
||||
}
|
||||
}
|
||||
|
||||
// A near-full datastore raises exactly one alert; a healthy one raises none.
|
||||
func TestBuildStorageSnapshotDatastoreAlert(t *testing.T) {
|
||||
full := &datastoreVolume{Name: "default", Mount: "/var/lib/hanzo-datastore", SizeGiB: 196, UsedGiB: 178, Pct: 91}
|
||||
snap := buildStorageSnapshot(nil, full)
|
||||
if snap.Datastore == nil || snap.Datastore.Pct != 91 {
|
||||
t.Fatalf("datastore card missing/wrong: %+v", snap.Datastore)
|
||||
}
|
||||
if len(snap.Alerts) != 1 || snap.Alerts[0].Level != "critical" || snap.Alerts[0].Volume != "default" {
|
||||
t.Fatalf("expected one critical alert, got %+v", snap.Alerts)
|
||||
}
|
||||
|
||||
healthy := &datastoreVolume{Name: "default", SizeGiB: 196, UsedGiB: 13, Pct: 7}
|
||||
if a := buildStorageSnapshot(nil, healthy).Alerts; len(a) != 0 {
|
||||
t.Fatalf("a 7%%-full datastore must raise no alert, got %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertLevel(t *testing.T) {
|
||||
cases := []struct {
|
||||
pct float64
|
||||
want string
|
||||
}{
|
||||
{0, ""}, {7, ""}, {79.9, ""}, {80, "warn"}, {89.9, "warn"}, {90, "critical"}, {99, "critical"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := alertLevel(c.pct); got != c.want {
|
||||
t.Fatalf("alertLevel(%v) = %q, want %q", c.pct, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// system.disks bytes → GiB + pct, matching the live datastore (13.5G used of ~196G ≈ 7%).
|
||||
func TestDatastoreFillFromRow(t *testing.T) {
|
||||
// total ≈ 196.6 GiB, free ≈ 183.1 GiB → used ≈ 13.5 GiB → ~6.9%.
|
||||
gib := float64(bytesPerGiB) // a variable → runtime math (a float→int const conversion won't compile)
|
||||
total := int64(196.6 * gib)
|
||||
used := int64(13.5 * gib)
|
||||
row := map[string]any{
|
||||
"name": "default",
|
||||
"path": "/var/lib/hanzo-datastore",
|
||||
"total_space": uint64(total),
|
||||
"free_space": uint64(total - used),
|
||||
}
|
||||
d := datastoreFillFromRow(row)
|
||||
if d == nil {
|
||||
t.Fatal("expected a datastore fill, got nil")
|
||||
}
|
||||
if d.Name != "default" || d.Mount != "/var/lib/hanzo-datastore" {
|
||||
t.Fatalf("name/mount wrong: %+v", d)
|
||||
}
|
||||
if d.SizeGiB != 196 {
|
||||
t.Fatalf("sizeGiB = %d, want 196", d.SizeGiB)
|
||||
}
|
||||
if d.UsedGiB < 13.4 || d.UsedGiB > 13.6 {
|
||||
t.Fatalf("usedGiB = %v, want ~13.5", d.UsedGiB)
|
||||
}
|
||||
if d.Pct < 6.5 || d.Pct > 7.5 {
|
||||
t.Fatalf("pct = %v, want ~7", d.Pct)
|
||||
}
|
||||
|
||||
// A disk that reports no capacity is an unusable read → nil (never a fake 0%).
|
||||
if datastoreFillFromRow(map[string]any{"total_space": uint64(0)}) != nil {
|
||||
t.Fatal("zero-capacity disk must yield nil, not a fabricated 0%")
|
||||
}
|
||||
}
|
||||
@@ -6,9 +6,11 @@ import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/clients/admin/iam"
|
||||
"github.com/hanzoai/cloud/clients/finance"
|
||||
)
|
||||
|
||||
// MaxCustomerConcurrency bounds the per-org enrichment fan-out so a large fleet does
|
||||
@@ -34,13 +36,25 @@ func ListOrgs(s *cloud.Service[State], ctx context.Context, cr iam.Creds) ([]iam
|
||||
return orgs, nil
|
||||
}
|
||||
|
||||
// OrgMoney returns (spendCents, creditsCents, ok) for one org from commerce. ok is
|
||||
// false when the spend OR credits read FAILED — so a fleet aggregator can fold the
|
||||
// per-org failure into a PARTIAL/degraded source rather than presenting the resulting
|
||||
// undercount as authoritative (the SAME (row, ok) contract revenue.revenueOf uses). An
|
||||
// unwired commerce is NOT a failure: Spend/Credits return (0, nil) when unconfigured, so
|
||||
// ok stays true and the caller distinguishes "not configured" via Commerce.Ready().
|
||||
// OrgMoney returns (spendCents, creditsCents, ok) for one org — the ONE per-org money read
|
||||
// every fleet aggregator (overview, orgs, revenue) folds over. ok is false ONLY when a read
|
||||
// FAILED, so a caller folds the per-org failure into a PARTIAL/degraded source rather than
|
||||
// presenting the resulting undercount as authoritative. An org that simply has no money yet
|
||||
// reads a clean (0, 0, true), never a failure.
|
||||
//
|
||||
// CO-RESIDENCE (the money-plane trap). Commerce's own /v1/billing/* routes are behind
|
||||
// `//go:build cloud` and are NOT compiled into this binary, so the admin commerce client's
|
||||
// S2S reads self-dispatch by PATH into cloud's OWN handlers: GET /v1/billing/balance
|
||||
// re-enters the customer balance handler with no principal (401) and GET
|
||||
// /v1/billing/usage-rollup is unrouted (404). Reading commerce over HTTP would therefore
|
||||
// fail for EVERY org and falsely mark the money source DOWN while real money sits in the
|
||||
// co-resident ledger. So — exactly as clients/billing.balance()/usage() and
|
||||
// core.grantDeposit already resolve it — prefer the co-resident finance ledger
|
||||
// (finance.Current()); the commerce S2S read stays only as the split-deploy fallback.
|
||||
func OrgMoney(s *cloud.Service[State], ctx context.Context, org string) (spend, credits int64, ok bool) {
|
||||
if fin := finance.Current(); fin != nil {
|
||||
return orgMoneyFromFinance(ctx, fin, org)
|
||||
}
|
||||
ok = true
|
||||
if sp, err := s.State.Commerce.Spend(ctx, org); err == nil {
|
||||
spend = int64(sp.Consumed)
|
||||
@@ -55,6 +69,28 @@ func OrgMoney(s *cloud.Service[State], ctx context.Context, org string) (spend,
|
||||
return spend, credits, ok
|
||||
}
|
||||
|
||||
// orgMoneyFromFinance reads (spend30d, credits, ok) for one org from the co-resident finance
|
||||
// ledger — the SAME wallet the ai prepaid gate debits and clients/billing shows. credits =
|
||||
// the org-pool AVAILABLE prepaid balance; spend = the org's metered usage over the trailing
|
||||
// 30 days (SumUsageSince — the windowed usage source the rolling AI-spend cap already reads,
|
||||
// matching the SpendCents30d field the overview + orgs surfaces render). ok is false only on
|
||||
// a REAL read failure, so a no-activity org reads (0, 0, true) and never marks the money
|
||||
// source degraded.
|
||||
func orgMoneyFromFinance(ctx context.Context, fin finance.Client, org string) (spend, credits int64, ok bool) {
|
||||
ok = true
|
||||
if bal, err := fin.Balance(ctx, org, org, "usd", false); err == nil {
|
||||
credits = bal.Cents()
|
||||
} else {
|
||||
ok = false
|
||||
}
|
||||
if sum, err := fin.SumUsageSince(ctx, org, false, time.Now().AddDate(0, 0, -30).Unix()); err == nil {
|
||||
spend = sum
|
||||
} else {
|
||||
ok = false
|
||||
}
|
||||
return spend, credits, ok
|
||||
}
|
||||
|
||||
// FindOrg returns the IAM org by slug (nil, nil when it does not exist) so a management
|
||||
// action can validate its target before acting — never credit or suspend an org that
|
||||
// isn't real.
|
||||
|
||||
+21
-15
@@ -25,27 +25,33 @@ func Guard(s *cloud.Service[State], h Handler) zip.Handler {
|
||||
}
|
||||
|
||||
// GuardScoped is the gate for the ORG-SCOPED panels (me/overview/orgs/users/usage/
|
||||
// analytics/bases). It admits a SuperAdmin (principal.IsSuperAdmin) OR an ORG admin
|
||||
// (an admin of their OWN org — principal.IsOrgAdmin) pinned to a validated org, and the
|
||||
// handler then scopes every read to ResolveScope(c) — so a non-super caller passes the
|
||||
// gate but the DATA layer hard-limits them to their own org subtree. Cross-tenant reads
|
||||
// are impossible for a non-super caller regardless of input.
|
||||
// analytics/bases + spend-caps). It admits a SuperAdmin (principal.IsSuperAdmin) OR an
|
||||
// admin of an ENABLED WHITE-LABEL TENANT org (principal.IsOrgAdmin pinned to its
|
||||
// validated org AND that org ∈ State.WLTenants), and the handler then scopes every read
|
||||
// to ResolveScope(c) — so a non-super caller passes the gate but the DATA layer
|
||||
// hard-limits them to their own org subtree. Cross-tenant reads are impossible for a
|
||||
// non-super caller regardless of input.
|
||||
//
|
||||
// The non-super admission requires BOTH the sanitizer-minted X-User-IsOrgAdmin
|
||||
// (principal.IsOrgAdmin — the "admin of my own org" bit, unforgeable because the boundary
|
||||
// strips it on ingress) AND a validated principal pinned to its own org (principal.Org,
|
||||
// the ONE org accessor: validated X-User-Id + non-empty in-bounds X-Org-Id). So a
|
||||
// validated but NON-admin MEMBER of an org is
|
||||
// REFUSED here — the same denial an anonymous caller who forged X-Org-Id gets — closing
|
||||
// the same-tenant over-visibility gap where any org member could read their org's admin
|
||||
// panels. A validated non-super principal's X-Org-Id is PINNED by the boundary to their
|
||||
// own owner, never client-chosen.
|
||||
// THREE admission facts, ALL required for the non-super tier — the escalation line:
|
||||
// - X-User-IsOrgAdmin (principal.IsOrgAdmin — "admin of my own org", unforgeable
|
||||
// because SanitizeIdentity strips it on ingress and re-mints only from a validated
|
||||
// isAdmin claim), AND
|
||||
// - a validated principal pinned to its own org (principal.Org: validated X-User-Id +
|
||||
// non-empty in-bounds X-Org-Id — the boundary PINS a non-super caller's X-Org-Id to
|
||||
// their own owner, never client-chosen), AND
|
||||
// - that org is an ENABLED WL tenant (State.IsWhiteLabelTenant — the fail-closed
|
||||
// allowlist). This is the tier the old gate was MISSING: it admitted ANY org-admin,
|
||||
// so any customer's own-org admin could open the cockpit. Now an org-admin whose org
|
||||
// is not an enabled WL tenant is REFUSED — the SAME 403 a non-admin member or a
|
||||
// forged-X-Org-Id anonymous caller gets. A SuperAdmin never consults the allowlist.
|
||||
//
|
||||
// Fail-closed everywhere: nil/empty WLTenants ⇒ ONLY SuperAdmins reach these panels.
|
||||
func GuardScoped(s *cloud.Service[State], h Handler) zip.Handler {
|
||||
return func(c *zip.Ctx) error {
|
||||
if principal.IsSuperAdmin(c) {
|
||||
return h(s, c) // SuperAdmin: cross-tenant, admitted regardless of org pin.
|
||||
}
|
||||
if _, ok := principal.Org(c); ok && principal.IsOrgAdmin(c) {
|
||||
if org, ok := principal.Org(c); ok && principal.IsOrgAdmin(c) && s.State.IsWhiteLabelTenant(org) {
|
||||
return h(s, c)
|
||||
}
|
||||
return zip.ErrForbidden("admin required")
|
||||
|
||||
@@ -8,6 +8,8 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/hanzoai/cloud/audit"
|
||||
"github.com/hanzoai/cloud/clients/admin/commerce"
|
||||
"github.com/hanzoai/cloud/clients/admin/digitalocean"
|
||||
@@ -29,4 +31,28 @@ type State struct {
|
||||
DO *digitalocean.Client
|
||||
AdminOrg string
|
||||
AuditStore *audit.Recorder
|
||||
|
||||
// WLTenants is the fail-closed allowlist of enabled WHITE-LABEL TENANT orgs — the
|
||||
// resellers/brands whose OWN-org admins may reach the org-scoped cockpit panels
|
||||
// (GuardScoped) at admin.<brand>. It is the second admission tier next to the admin
|
||||
// org: a SuperAdmin (owner == AdminOrg) is cross-tenant and never consults this set;
|
||||
// EVERY other caller must be an admin of an org that is IN this set, and is then
|
||||
// hard-scoped to that org's subtree. Seeded ONCE from ADMIN_WL_TENANT_ORGS at Mount
|
||||
// and otherwise immutable, so the decision is a deliberate, git-/KMS-auditable
|
||||
// onboarding — never self-service. EMPTY by default: with no entry, NO customer
|
||||
// org-admin is admitted (only SuperAdmins reach the cockpit), so a mis-set/absent
|
||||
// env fails CLOSED, never open.
|
||||
WLTenants map[string]bool
|
||||
}
|
||||
|
||||
// IsWhiteLabelTenant reports whether org is an enabled white-label tenant — the ONE
|
||||
// place the WL admission decision is read. Nil/empty set ⇒ always false (fail-closed):
|
||||
// no org is a WL tenant until explicitly enabled. The org is matched VERBATIM (only
|
||||
// trimmed), the same owner key principal.Org yields, so folding can never collapse a
|
||||
// distinct owner into an enabled one.
|
||||
func (st State) IsWhiteLabelTenant(org string) bool {
|
||||
if len(st.WLTenants) == 0 {
|
||||
return false
|
||||
}
|
||||
return st.WLTenants[strings.TrimSpace(org)]
|
||||
}
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
// Package do reads DigitalOcean's billing API for the finance dashboard's cost
|
||||
// side. DO is our PRIMARY venue (a large promotional credit); this client turns
|
||||
// the customer balance + billing history into money.Cents the finance aggregator
|
||||
// folds into gross margin and runway.
|
||||
// Package digitalocean reads DigitalOcean's billing and infrastructure APIs. DO is
|
||||
// our PRIMARY venue (a large promotional credit); this client turns the customer
|
||||
// balance + billing history into money.Cents the finance aggregator folds into gross
|
||||
// margin and runway, and exposes the account's physical inventory — droplets,
|
||||
// block-storage volumes, DOKS clusters, load balancers — that the /v1/admin/infra
|
||||
// board reads.
|
||||
//
|
||||
// This is the ONE DigitalOcean client the admin plane uses. A new DO read is a
|
||||
// method here calling the shared get/send primitive, never a second client.
|
||||
//
|
||||
// Auth is a single personal-access token, DO_API_TOKEN, sourced from a KMSSecret on
|
||||
// the cloud env — NEVER hard-coded. When the token is unset the client is not Ready
|
||||
@@ -15,12 +20,14 @@
|
||||
package digitalocean
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -142,24 +149,402 @@ func (c *Client) History(ctx context.Context, perPage int) ([]Entry, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Volume is one DO block-storage volume: capacity + attachment + region. DO's API
|
||||
// gives capacity and which droplets a volume is attached to, but NOT fill % — the
|
||||
// caller enriches fill only where a filesystem source (the datastore's own
|
||||
// system.disks) reports it, and renders an honest "—" everywhere else.
|
||||
type Volume struct {
|
||||
ID string
|
||||
Name string
|
||||
Region string
|
||||
SizeGiB int
|
||||
DropletIDs []int
|
||||
// Tags carries DO's resource tags. DOKS stamps `k8s:<cluster-uuid>` on the volumes
|
||||
// it provisions, but that tag is ADVISORY ONLY — it survives cluster deletion and
|
||||
// is wrong often enough that it must never decide whether a volume is garbage. The
|
||||
// only sound liveness test is a PV cross-reference (see clients/admin/infra).
|
||||
Tags []string
|
||||
CreatedAt string
|
||||
}
|
||||
|
||||
// volumeWire is the raw DO /v2/volumes row.
|
||||
type volumeWire struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
SizeGigabytes int `json:"size_gigabytes"`
|
||||
Region struct {
|
||||
Slug string `json:"slug"`
|
||||
} `json:"region"`
|
||||
DropletIDs []int `json:"droplet_ids"`
|
||||
Tags []string `json:"tags"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
}
|
||||
|
||||
// Volumes lists ALL block-storage volumes across the account. Capacity and attachment
|
||||
// are real; per-volume fill is NOT exposed by DO and stays absent (honest) until a
|
||||
// filesystem source reports it.
|
||||
func (c *Client) Volumes(ctx context.Context) ([]Volume, error) {
|
||||
rows, err := listAll[volumeWire](ctx, c, "/v2/volumes", "volumes")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Volume, len(rows))
|
||||
for i, v := range rows {
|
||||
out[i] = Volume{
|
||||
ID: v.ID,
|
||||
Name: v.Name,
|
||||
Region: v.Region.Slug,
|
||||
SizeGiB: v.SizeGigabytes,
|
||||
DropletIDs: v.DropletIDs,
|
||||
Tags: v.Tags,
|
||||
CreatedAt: v.CreatedAt,
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Droplet is one DO droplet. LocalDiskGiB is the droplet's own disk, which is
|
||||
// INCLUDED in MonthlyCents — it is NOT separately billed, and conflating it with
|
||||
// block storage is how a fleet appears to hold terabytes it never pays for.
|
||||
type Droplet struct {
|
||||
ID int
|
||||
Name string
|
||||
Region string
|
||||
Status string
|
||||
SizeSlug string
|
||||
VCPUs int
|
||||
MemoryMiB int
|
||||
LocalDiskGiB int
|
||||
MonthlyCents money.Cents
|
||||
CreatedAt string
|
||||
PrivateIP string
|
||||
PublicIP string
|
||||
Tags []string
|
||||
VolumeIDs []string
|
||||
}
|
||||
|
||||
// dropletWire is the raw DO /v2/droplets row.
|
||||
type dropletWire struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
SizeSlug string `json:"size_slug"`
|
||||
VCPUs int `json:"vcpus"`
|
||||
Memory int `json:"memory"`
|
||||
Disk int `json:"disk"`
|
||||
Size struct {
|
||||
PriceMonthly float64 `json:"price_monthly"`
|
||||
} `json:"size"`
|
||||
Region struct {
|
||||
Slug string `json:"slug"`
|
||||
} `json:"region"`
|
||||
Networks struct {
|
||||
V4 []struct {
|
||||
Type string `json:"type"`
|
||||
IPAddress string `json:"ip_address"`
|
||||
} `json:"v4"`
|
||||
} `json:"networks"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
Tags []string `json:"tags"`
|
||||
VolumeIDs []string `json:"volume_ids"`
|
||||
}
|
||||
|
||||
// Droplets lists ALL droplets across the account.
|
||||
func (c *Client) Droplets(ctx context.Context) ([]Droplet, error) {
|
||||
rows, err := listAll[dropletWire](ctx, c, "/v2/droplets", "droplets")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Droplet, len(rows))
|
||||
for i, d := range rows {
|
||||
dr := Droplet{
|
||||
ID: d.ID,
|
||||
Name: d.Name,
|
||||
Region: d.Region.Slug,
|
||||
Status: d.Status,
|
||||
SizeSlug: d.SizeSlug,
|
||||
VCPUs: d.VCPUs,
|
||||
MemoryMiB: d.Memory,
|
||||
LocalDiskGiB: d.Disk,
|
||||
MonthlyCents: centsOf(d.Size.PriceMonthly),
|
||||
CreatedAt: d.CreatedAt,
|
||||
Tags: d.Tags,
|
||||
VolumeIDs: d.VolumeIDs,
|
||||
}
|
||||
for _, n := range d.Networks.V4 {
|
||||
switch n.Type {
|
||||
case "private":
|
||||
dr.PrivateIP = n.IPAddress
|
||||
case "public":
|
||||
dr.PublicIP = n.IPAddress
|
||||
}
|
||||
}
|
||||
out[i] = dr
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Cluster is one DOKS cluster.
|
||||
type Cluster struct {
|
||||
ID string
|
||||
Name string
|
||||
Region string
|
||||
Version string
|
||||
Status string
|
||||
NodePools int
|
||||
CreatedAt string
|
||||
}
|
||||
|
||||
// clusterWire is the raw DO /v2/kubernetes/clusters row.
|
||||
type clusterWire struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Region string `json:"region"`
|
||||
Version string `json:"version"`
|
||||
Status struct {
|
||||
State string `json:"state"`
|
||||
} `json:"status"`
|
||||
NodePools []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"node_pools"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
}
|
||||
|
||||
// Clusters lists ALL DOKS clusters. This is the authoritative denominator for the
|
||||
// orphan analysis: a volume may only be called unreferenced once EVERY cluster here
|
||||
// has been searched for a PV that claims it.
|
||||
func (c *Client) Clusters(ctx context.Context) ([]Cluster, error) {
|
||||
rows, err := listAll[clusterWire](ctx, c, "/v2/kubernetes/clusters", "kubernetes_clusters")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Cluster, len(rows))
|
||||
for i, k := range rows {
|
||||
out[i] = Cluster{
|
||||
ID: k.ID,
|
||||
Name: k.Name,
|
||||
Region: k.Region,
|
||||
Version: k.Version,
|
||||
Status: k.Status.State,
|
||||
NodePools: len(k.NodePools),
|
||||
CreatedAt: k.CreatedAt,
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Kubeconfig fetches a cluster's admin kubeconfig. DO returns a token-based config
|
||||
// against the cluster's public https endpoint (never an exec plugin), which the
|
||||
// caller must still funnel through fleet.SafeRESTConfig before dialing.
|
||||
func (c *Client) Kubeconfig(ctx context.Context, clusterID string) ([]byte, error) {
|
||||
if !c.Ready() {
|
||||
return nil, fmt.Errorf("DO_API_TOKEN not configured")
|
||||
}
|
||||
if strings.TrimSpace(clusterID) == "" {
|
||||
return nil, fmt.Errorf("cluster id required")
|
||||
}
|
||||
return c.get(ctx, "/v2/kubernetes/clusters/"+url.PathEscape(clusterID)+"/kubeconfig")
|
||||
}
|
||||
|
||||
// LoadBalancer is one DO load balancer. DO does not price LBs in the API, so cost is
|
||||
// derived from the billed unit count (see lbUnitCents).
|
||||
type LoadBalancer struct {
|
||||
ID string
|
||||
Name string
|
||||
Region string
|
||||
Status string
|
||||
IP string
|
||||
SizeUnit int
|
||||
MonthlyCents money.Cents
|
||||
DropletIDs []int
|
||||
}
|
||||
|
||||
// lbWire is the raw DO /v2/load_balancers row.
|
||||
type lbWire struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
IP string `json:"ip"`
|
||||
Region struct {
|
||||
Slug string `json:"slug"`
|
||||
} `json:"region"`
|
||||
SizeUnit int `json:"size_unit"`
|
||||
DropletIDs []int `json:"droplet_ids"`
|
||||
}
|
||||
|
||||
// LoadBalancers lists ALL load balancers across the account.
|
||||
func (c *Client) LoadBalancers(ctx context.Context) ([]LoadBalancer, error) {
|
||||
rows, err := listAll[lbWire](ctx, c, "/v2/load_balancers", "load_balancers")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]LoadBalancer, len(rows))
|
||||
for i, l := range rows {
|
||||
units := l.SizeUnit
|
||||
if units <= 0 {
|
||||
units = 1
|
||||
}
|
||||
out[i] = LoadBalancer{
|
||||
ID: l.ID,
|
||||
Name: l.Name,
|
||||
Region: l.Region.Slug,
|
||||
Status: l.Status,
|
||||
IP: l.IP,
|
||||
SizeUnit: units,
|
||||
MonthlyCents: money.Cents(units) * lbUnitCents,
|
||||
DropletIDs: l.DropletIDs,
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Snapshot is a created block-storage snapshot.
|
||||
type Snapshot struct {
|
||||
ID string
|
||||
Name string
|
||||
SizeGiB int
|
||||
}
|
||||
|
||||
// SnapshotVolume takes a point-in-time snapshot of a volume. This is the "undo" that
|
||||
// makes a delete recoverable, so the delete path takes one FIRST by default.
|
||||
func (c *Client) SnapshotVolume(ctx context.Context, volumeID, name string) (Snapshot, error) {
|
||||
var out Snapshot
|
||||
if !c.Ready() {
|
||||
return out, fmt.Errorf("DO_API_TOKEN not configured")
|
||||
}
|
||||
if strings.TrimSpace(volumeID) == "" {
|
||||
return out, fmt.Errorf("volume id required")
|
||||
}
|
||||
body, err := c.send(ctx, http.MethodPost, "/v2/volumes/"+url.PathEscape(volumeID)+"/snapshots",
|
||||
map[string]string{"name": name})
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
var w struct {
|
||||
Snapshot struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
SizeGigabytes int `json:"size_gigabytes"`
|
||||
} `json:"snapshot"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &w); err != nil {
|
||||
return out, fmt.Errorf("do snapshot decode: %w", err)
|
||||
}
|
||||
return Snapshot{ID: w.Snapshot.ID, Name: w.Snapshot.Name, SizeGiB: w.Snapshot.SizeGigabytes}, nil
|
||||
}
|
||||
|
||||
// DeleteVolume destroys a block-storage volume. Irreversible: callers MUST have
|
||||
// proven the volume is referenced by no PV in any cluster first.
|
||||
func (c *Client) DeleteVolume(ctx context.Context, volumeID string) error {
|
||||
if !c.Ready() {
|
||||
return fmt.Errorf("DO_API_TOKEN not configured")
|
||||
}
|
||||
if strings.TrimSpace(volumeID) == "" {
|
||||
return fmt.Errorf("volume id required")
|
||||
}
|
||||
_, err := c.send(ctx, http.MethodDelete, "/v2/volumes/"+url.PathEscape(volumeID), nil)
|
||||
return err
|
||||
}
|
||||
|
||||
// Pagination bounds for every DO collection read: 200 rows a page, a hard 25-page
|
||||
// cap so a runaway can never loop, and the 8 MiB body ceiling a full droplet page
|
||||
// needs (a volume page fits in far less).
|
||||
const (
|
||||
perPage = 200
|
||||
maxPages = 25
|
||||
maxBody = 8 << 20
|
||||
maxRespLen = maxBody
|
||||
)
|
||||
|
||||
// lbUnitCents is DO's published price for one load-balancer node ($12/mo). DO does
|
||||
// not return LB pricing in the API, so this is the one place the rate is written.
|
||||
const lbUnitCents = money.Cents(1200)
|
||||
|
||||
// listAll follows DO's page-number pagination for a collection endpoint, decoding
|
||||
// rows out of the response's named key. It is the ONE pagination loop in this client
|
||||
// — every collection read goes through it, so "stop on the short page or the reported
|
||||
// total" is stated once and cannot drift between endpoints.
|
||||
func listAll[T any](ctx context.Context, c *Client, path, key string) ([]T, error) {
|
||||
if !c.Ready() {
|
||||
return nil, fmt.Errorf("DO_API_TOKEN not configured")
|
||||
}
|
||||
var out []T
|
||||
for page := 1; page <= maxPages; page++ {
|
||||
body, err := c.get(ctx, fmt.Sprintf("%s?per_page=%d&page=%d", path, perPage, page))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var w struct {
|
||||
Meta struct {
|
||||
Total int `json:"total"`
|
||||
} `json:"meta"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &w); err != nil {
|
||||
return nil, fmt.Errorf("do %s decode: %w", key, err)
|
||||
}
|
||||
var keyed map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &keyed); err != nil {
|
||||
return nil, fmt.Errorf("do %s decode: %w", key, err)
|
||||
}
|
||||
var rows []T
|
||||
if raw, ok := keyed[key]; ok && len(raw) > 0 {
|
||||
if err := json.Unmarshal(raw, &rows); err != nil {
|
||||
return nil, fmt.Errorf("do %s decode: %w", key, err)
|
||||
}
|
||||
}
|
||||
out = append(out, rows...)
|
||||
// Stop on the last (short) page, or once we've collected the reported total.
|
||||
if len(rows) < perPage || (w.Meta.Total > 0 && len(out) >= w.Meta.Total) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// get performs one token-authenticated DO GET and returns the raw body.
|
||||
func (c *Client) get(ctx context.Context, path string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil)
|
||||
return c.send(ctx, http.MethodGet, path, nil)
|
||||
}
|
||||
|
||||
// send performs one token-authenticated DO request and returns the raw body. It is
|
||||
// the single HTTP primitive of this client: every read and every mutation funnels
|
||||
// through it, so auth, timeouts, the body ceiling and status handling exist once.
|
||||
func (c *Client) send(ctx context.Context, method, path string, payload any) ([]byte, error) {
|
||||
var rdr io.Reader
|
||||
if payload != nil {
|
||||
enc, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rdr = bytes.NewReader(enc)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.base+path, rdr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||
if payload != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("digitalocean unreachable: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxRespLen))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
// DO returns {"id":"...","message":"..."} on error — surface the message so a
|
||||
// failed mutation says WHY, not just a bare status.
|
||||
var e struct {
|
||||
Message string `json:"message"`
|
||||
}
|
||||
if json.Unmarshal(body, &e) == nil && strings.TrimSpace(e.Message) != "" {
|
||||
return nil, fmt.Errorf("digitalocean status %d: %s", resp.StatusCode, e.Message)
|
||||
}
|
||||
return nil, fmt.Errorf("digitalocean status %d", resp.StatusCode)
|
||||
}
|
||||
return body, nil
|
||||
@@ -178,5 +563,8 @@ func dollarsToCents(s string) money.Cents {
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return money.Cents(math.Round(f * 100))
|
||||
return centsOf(f)
|
||||
}
|
||||
|
||||
// centsOf rounds decimal dollars to integer cents.
|
||||
func centsOf(f float64) money.Cents { return money.Cents(math.Round(f * 100)) }
|
||||
|
||||
@@ -0,0 +1,768 @@
|
||||
// Package infra is the platform's DigitalOcean fleet board: the physical inventory
|
||||
// (DOKS clusters, droplets, block-storage volumes, load balancers) cross-referenced
|
||||
// against what every cluster's Kubernetes actually claims, with the cost of each and
|
||||
// an orphan analysis that is safe BY CONSTRUCTION.
|
||||
//
|
||||
// THE RULE THIS PACKAGE EXISTS TO ENFORCE. A volume being detached, or carrying a
|
||||
// `k8s:<cluster-uuid>` tag for some other cluster, does NOT make it garbage. Those two
|
||||
// signals together would have condemned 4.39 TiB of live data belonging to running
|
||||
// clusters. The ONLY sound liveness test is a cross-reference against the
|
||||
// `spec.csi.volumeHandle` of every PersistentVolume in EVERY cluster — and it is only
|
||||
// a valid test when every cluster answered. So:
|
||||
//
|
||||
// - a volume is deletable only when NO PV in ANY cluster names it, and
|
||||
// - if even one cluster failed to scan, NOTHING is deletable (Complete=false).
|
||||
//
|
||||
// Absence of evidence is not evidence of absence: an unreachable cluster is treated as
|
||||
// a cluster that might be holding the volume. The analysis fails CLOSED.
|
||||
//
|
||||
// "No pod mounts it" is a REVIEW signal, never a delete signal — an idle Bound PVC is
|
||||
// an idle database, not garbage. Idle volumes are surfaced as a queue for a human and
|
||||
// are never counted as reclaimable.
|
||||
//
|
||||
// Analyze is a pure function of (DO inventory, cluster scans) so every rule above is
|
||||
// unit-testable without a network or a cluster.
|
||||
package infra
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/admin/core"
|
||||
"github.com/hanzoai/cloud/clients/admin/digitalocean"
|
||||
"github.com/hanzoai/cloud/clients/admin/money"
|
||||
)
|
||||
|
||||
// volumeGiBCents is DO's block-storage rate: $0.10 per GiB per month. Droplet LOCAL
|
||||
// disk is NOT billed at this rate (or at all, separately) — it is included in the
|
||||
// droplet's own price. Conflating the two invents terabytes of phantom cost.
|
||||
const volumeGiBCents = money.Cents(10)
|
||||
|
||||
// outlierShareBP flags any single resource costing at least this share of total fleet
|
||||
// spend, in basis points (500bp = 5%). One explicable rule, no magic thresholds.
|
||||
const outlierShareBP = money.Cents(500)
|
||||
|
||||
// Volume states. The state machine is total and ordered: attachment beats reference,
|
||||
// reference beats absence. Only Unreferenced is ever deletable.
|
||||
const (
|
||||
// StateAttached — DO reports the volume attached to a droplet. In use, now.
|
||||
StateAttached = "attached"
|
||||
// StateBound — detached, but a PV in some cluster claims it and that PV is Bound
|
||||
// to a PVC. This is live data between mounts; deleting it destroys a database.
|
||||
StateBound = "bound"
|
||||
// StateReleased — a PV claims it but is no longer Bound (Released/Available/Failed).
|
||||
// A genuine cleanup candidate, but the PV still exists, so a human retires the PV.
|
||||
StateReleased = "released"
|
||||
// StateUnreferenced — no PV in ANY scanned cluster names it. The ONLY deletable state.
|
||||
StateUnreferenced = "unreferenced"
|
||||
)
|
||||
|
||||
// Finding severities.
|
||||
const (
|
||||
SevCritical = "critical"
|
||||
SevWarn = "warn"
|
||||
SevInfo = "info"
|
||||
)
|
||||
|
||||
// firstParty are our own registries: anything here is ours by construction.
|
||||
var firstParty = []string{
|
||||
"ghcr.io/hanzoai/", "ghcr.io/luxfi/", "ghcr.io/zooai/",
|
||||
"registry.hanzo.ai/", "registry.lux.network/", "registry.zoo.network/",
|
||||
"registry.digitalocean.com/hanzo/",
|
||||
}
|
||||
|
||||
// knownVendors is the REVIEWED third-party set — upstream images we deliberately run.
|
||||
// Kept deliberately short: an image outside both lists is reported for a human to
|
||||
// judge, which is the point. Growing this list is a review decision, not a reflex.
|
||||
var knownVendors = []string{
|
||||
"docker.io/library/", "library/", "registry.k8s.io/", "k8s.gcr.io/", "quay.io/",
|
||||
"grafana/", "prom/", "prometheus/", "bitnami/", "minio/", "moby/",
|
||||
"digitalocean/", "docker.digitalocean.com/", "acmglobaltech/", "hanzozt/",
|
||||
}
|
||||
|
||||
// Inventory is the DigitalOcean account read — the half of the analysis input that
|
||||
// needs no cluster.
|
||||
type Inventory struct {
|
||||
Clusters []digitalocean.Cluster
|
||||
Droplets []digitalocean.Droplet
|
||||
Volumes []digitalocean.Volume
|
||||
LoadBalancers []digitalocean.LoadBalancer
|
||||
}
|
||||
|
||||
// PVRef is one PersistentVolume's identity: which DO volume it claims, and whether
|
||||
// that claim is still live.
|
||||
type PVRef struct {
|
||||
Name string
|
||||
Phase string
|
||||
VolumeHandle string
|
||||
ClaimNS string
|
||||
ClaimName string
|
||||
}
|
||||
|
||||
// PVCRef is one PersistentVolumeClaim.
|
||||
type PVCRef struct {
|
||||
Namespace string
|
||||
Name string
|
||||
Phase string
|
||||
Volume string
|
||||
}
|
||||
|
||||
// PodRef is one pod, reduced to what the board needs: where it runs, whether it is
|
||||
// healthy, which PVCs it mounts, and what images it runs.
|
||||
type PodRef struct {
|
||||
Namespace string
|
||||
Name string
|
||||
Phase string
|
||||
Reason string
|
||||
Node string
|
||||
Claims []string
|
||||
Images []string
|
||||
}
|
||||
|
||||
// NodeState is one Kubernetes node's own view of itself.
|
||||
type NodeState struct {
|
||||
Name string
|
||||
Ready bool
|
||||
Schedulable bool
|
||||
}
|
||||
|
||||
// ClusterScan is ONE cluster's Kubernetes truth. Err non-nil means the cluster did
|
||||
// not answer — which forces the whole analysis incomplete.
|
||||
type ClusterScan struct {
|
||||
ClusterID string
|
||||
Err error
|
||||
Nodes []NodeState
|
||||
PVs []PVRef
|
||||
PVCs []PVCRef
|
||||
Pods []PodRef
|
||||
}
|
||||
|
||||
// Snapshot is the whole board in one value.
|
||||
type Snapshot struct {
|
||||
At string `json:"at"`
|
||||
Complete bool `json:"complete"`
|
||||
IncompleteReason string `json:"incompleteReason"`
|
||||
Sources []core.SourceStatus `json:"sources"`
|
||||
Totals Totals `json:"totals"`
|
||||
Cost Cost `json:"cost"`
|
||||
Clusters []Cluster `json:"clusters"`
|
||||
Nodes []Node `json:"nodes"`
|
||||
Volumes []Volume `json:"volumes"`
|
||||
LoadBalancers []LoadBalancer `json:"loadBalancers"`
|
||||
Findings []Finding `json:"findings"`
|
||||
}
|
||||
|
||||
// Totals are fleet counts. LocalDiskGiB is broken out precisely so it can be shown
|
||||
// as NOT separately billed.
|
||||
type Totals struct {
|
||||
Clusters int `json:"clusters"`
|
||||
Nodes int `json:"nodes"`
|
||||
Volumes int `json:"volumes"`
|
||||
LoadBalancers int `json:"loadBalancers"`
|
||||
VolumeGiB int `json:"volumeGiB"`
|
||||
AttachedVolumes int `json:"attachedVolumes"`
|
||||
AttachedGiB int `json:"attachedGiB"`
|
||||
DetachedVolumes int `json:"detachedVolumes"`
|
||||
DetachedGiB int `json:"detachedGiB"`
|
||||
UnreferencedVolumes int `json:"unreferencedVolumes"`
|
||||
UnreferencedGiB int `json:"unreferencedGiB"`
|
||||
IdlePVCs int `json:"idlePVCs"`
|
||||
LocalDiskGiB int `json:"localDiskGiB"`
|
||||
}
|
||||
|
||||
// Cost is monthly spend in cents. Reclaimable counts ONLY unreferenced volumes —
|
||||
// never idle ones, which are live data awaiting a human verdict.
|
||||
type Cost struct {
|
||||
DropletsMonthly money.Cents `json:"dropletsMonthly"`
|
||||
VolumesMonthly money.Cents `json:"volumesMonthly"`
|
||||
LoadBalancersMonthly money.Cents `json:"loadBalancersMonthly"`
|
||||
TotalMonthly money.Cents `json:"totalMonthly"`
|
||||
ReclaimableMonthly money.Cents `json:"reclaimableMonthly"`
|
||||
}
|
||||
|
||||
// Cluster is one DOKS cluster with its scanned Kubernetes rollup.
|
||||
type Cluster struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Region string `json:"region"`
|
||||
Version string `json:"version"`
|
||||
Status string `json:"status"`
|
||||
NodePools int `json:"nodePools"`
|
||||
Nodes int `json:"nodes"`
|
||||
Pods int `json:"pods"`
|
||||
PVs int `json:"pvs"`
|
||||
PVCs int `json:"pvcs"`
|
||||
IdlePVCs int `json:"idlePVCs"`
|
||||
Scanned bool `json:"scanned"`
|
||||
ScanError string `json:"scanError"`
|
||||
MonthlyCents money.Cents `json:"monthlyCents"`
|
||||
}
|
||||
|
||||
// Node is one droplet, joined to the Kubernetes node of the same name.
|
||||
type Node struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Cluster string `json:"cluster"`
|
||||
ClusterID string `json:"clusterId"`
|
||||
Region string `json:"region"`
|
||||
Status string `json:"status"`
|
||||
SizeSlug string `json:"sizeSlug"`
|
||||
VCPUs int `json:"vcpus"`
|
||||
MemoryMiB int `json:"memoryMiB"`
|
||||
LocalDiskGiB int `json:"localDiskGiB"`
|
||||
MonthlyCents money.Cents `json:"monthlyCents"`
|
||||
CreatedAt string `json:"createdAt"`
|
||||
PrivateIP string `json:"privateIp"`
|
||||
PublicIP string `json:"publicIp"`
|
||||
Tags []string `json:"tags"`
|
||||
Ready bool `json:"ready"`
|
||||
Schedulable bool `json:"schedulable"`
|
||||
Pods int `json:"pods"`
|
||||
Volumes int `json:"volumes"`
|
||||
}
|
||||
|
||||
// Volume is one block-storage volume with its PROVEN cluster ownership.
|
||||
type Volume struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Region string `json:"region"`
|
||||
SizeGiB int `json:"sizeGiB"`
|
||||
MonthlyCents money.Cents `json:"monthlyCents"`
|
||||
State string `json:"state"`
|
||||
DropletIDs []int `json:"dropletIds"`
|
||||
NodeName string `json:"nodeName"`
|
||||
// Cluster/ClusterID are the PROVEN owner — resolved through a PV that names this
|
||||
// volume, never through the tag.
|
||||
Cluster string `json:"cluster"`
|
||||
ClusterID string `json:"clusterId"`
|
||||
// TagCluster is the `k8s:<uuid>` tag. ADVISORY ONLY: it outlives the cluster that
|
||||
// set it. Shown so the operator can see tag-vs-truth disagree, never acted on.
|
||||
TagCluster string `json:"tagCluster"`
|
||||
PV string `json:"pv"`
|
||||
PVPhase string `json:"pvPhase"`
|
||||
PVCNamespace string `json:"pvcNamespace"`
|
||||
PVCName string `json:"pvcName"`
|
||||
MountedBy []string `json:"mountedBy"`
|
||||
Idle bool `json:"idle"`
|
||||
CreatedAt string `json:"createdAt"`
|
||||
Deletable bool `json:"deletable"`
|
||||
BlockedReason string `json:"blockedReason"`
|
||||
}
|
||||
|
||||
// LoadBalancer is one DO load balancer, attributed to a cluster via its members.
|
||||
type LoadBalancer struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Region string `json:"region"`
|
||||
Status string `json:"status"`
|
||||
IP string `json:"ip"`
|
||||
SizeUnit int `json:"sizeUnit"`
|
||||
MonthlyCents money.Cents `json:"monthlyCents"`
|
||||
Droplets int `json:"droplets"`
|
||||
Cluster string `json:"cluster"`
|
||||
}
|
||||
|
||||
// Finding is one audit result — the "is anything bad" surface.
|
||||
type Finding struct {
|
||||
ID string `json:"id"`
|
||||
Severity string `json:"severity"`
|
||||
Kind string `json:"kind"`
|
||||
Title string `json:"title"`
|
||||
Detail string `json:"detail"`
|
||||
Resource string `json:"resource"`
|
||||
Cluster string `json:"cluster"`
|
||||
MonthlyCents money.Cents `json:"monthlyCents"`
|
||||
}
|
||||
|
||||
// pvHit is a PV that claims a given DO volume, plus the cluster it lives in.
|
||||
type pvHit struct {
|
||||
cluster string
|
||||
clusterID string
|
||||
pv PVRef
|
||||
}
|
||||
|
||||
// Analyze folds the DO inventory and the per-cluster Kubernetes scans into the board.
|
||||
// PURE: no clock, no network, no cluster — `at` is passed in so the result is
|
||||
// byte-reproducible in tests.
|
||||
func Analyze(inv Inventory, scans []ClusterScan, sources []core.SourceStatus, at time.Time) Snapshot {
|
||||
snap := Snapshot{
|
||||
At: at.UTC().Format(time.RFC3339),
|
||||
Sources: sources,
|
||||
}
|
||||
if snap.Sources == nil {
|
||||
snap.Sources = []core.SourceStatus{}
|
||||
}
|
||||
|
||||
scanByID := make(map[string]ClusterScan, len(scans))
|
||||
for _, s := range scans {
|
||||
scanByID[s.ClusterID] = s
|
||||
}
|
||||
nameByID := make(map[string]string, len(inv.Clusters))
|
||||
for _, c := range inv.Clusters {
|
||||
nameByID[c.ID] = c.Name
|
||||
}
|
||||
|
||||
// ---- completeness gate -------------------------------------------------------
|
||||
// Every cluster must have answered. One silent gap and no volume may be condemned.
|
||||
var unreachable []string
|
||||
for _, c := range inv.Clusters {
|
||||
s, ok := scanByID[c.ID]
|
||||
if !ok || s.Err != nil {
|
||||
unreachable = append(unreachable, c.Name)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(inv.Clusters) == 0:
|
||||
snap.IncompleteReason = "DigitalOcean returned no clusters — the set of places a volume could be in use is unknown."
|
||||
case len(unreachable) > 0:
|
||||
snap.IncompleteReason = fmt.Sprintf(
|
||||
"%d of %d clusters did not answer (%s) — a volume they hold would look unreferenced, so nothing is classified as deletable.",
|
||||
len(unreachable), len(inv.Clusters), strings.Join(unreachable, ", "))
|
||||
default:
|
||||
snap.Complete = true
|
||||
}
|
||||
|
||||
// ---- cross-cluster PV index --------------------------------------------------
|
||||
// THE safety index: every volume handle claimed by any PV in any cluster.
|
||||
byHandle := make(map[string]pvHit)
|
||||
// mounted[clusterID/ns/pvc] -> pods currently mounting it.
|
||||
mounted := make(map[string][]string)
|
||||
for _, s := range scans {
|
||||
if s.Err != nil {
|
||||
continue
|
||||
}
|
||||
cname := nameByID[s.ClusterID]
|
||||
for _, pv := range s.PVs {
|
||||
if h := strings.TrimSpace(pv.VolumeHandle); h != "" {
|
||||
byHandle[h] = pvHit{cluster: cname, clusterID: s.ClusterID, pv: pv}
|
||||
}
|
||||
}
|
||||
for _, p := range s.Pods {
|
||||
for _, claim := range p.Claims {
|
||||
k := claimKey(s.ClusterID, p.Namespace, claim)
|
||||
mounted[k] = append(mounted[k], p.Namespace+"/"+p.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- nodes -------------------------------------------------------------------
|
||||
nodeByName := make(map[string]NodeState)
|
||||
podsPerNode := make(map[string]int)
|
||||
for _, s := range scans {
|
||||
if s.Err != nil {
|
||||
continue
|
||||
}
|
||||
for _, n := range s.Nodes {
|
||||
nodeByName[n.Name] = n
|
||||
}
|
||||
for _, p := range s.Pods {
|
||||
if p.Node != "" {
|
||||
podsPerNode[p.Node]++
|
||||
}
|
||||
}
|
||||
}
|
||||
volsPerDroplet := make(map[int]int)
|
||||
for _, v := range inv.Volumes {
|
||||
for _, id := range v.DropletIDs {
|
||||
volsPerDroplet[id]++
|
||||
}
|
||||
}
|
||||
clusterByDroplet := make(map[int]string, len(inv.Droplets))
|
||||
snap.Nodes = make([]Node, 0, len(inv.Droplets))
|
||||
for _, d := range inv.Droplets {
|
||||
cid := clusterIDFromTags(d.Tags)
|
||||
clusterByDroplet[d.ID] = cid
|
||||
ks := nodeByName[d.Name]
|
||||
snap.Nodes = append(snap.Nodes, Node{
|
||||
ID: d.ID, Name: d.Name, Cluster: nameByID[cid], ClusterID: cid,
|
||||
Region: d.Region, Status: d.Status, SizeSlug: d.SizeSlug,
|
||||
VCPUs: d.VCPUs, MemoryMiB: d.MemoryMiB, LocalDiskGiB: d.LocalDiskGiB,
|
||||
MonthlyCents: d.MonthlyCents, CreatedAt: d.CreatedAt,
|
||||
PrivateIP: d.PrivateIP, PublicIP: d.PublicIP, Tags: nonNilStrings(d.Tags),
|
||||
Ready: ks.Ready, Schedulable: ks.Schedulable,
|
||||
Pods: podsPerNode[d.Name], Volumes: volsPerDroplet[d.ID],
|
||||
})
|
||||
snap.Cost.DropletsMonthly += d.MonthlyCents
|
||||
snap.Totals.LocalDiskGiB += d.LocalDiskGiB
|
||||
}
|
||||
dropletName := make(map[int]string, len(inv.Droplets))
|
||||
for _, d := range inv.Droplets {
|
||||
dropletName[d.ID] = d.Name
|
||||
}
|
||||
|
||||
// ---- volumes: the state machine ----------------------------------------------
|
||||
snap.Volumes = make([]Volume, 0, len(inv.Volumes))
|
||||
for _, v := range inv.Volumes {
|
||||
hit, referenced := byHandle[v.ID]
|
||||
vol := Volume{
|
||||
ID: v.ID, Name: v.Name, Region: v.Region, SizeGiB: v.SizeGiB,
|
||||
MonthlyCents: money.Cents(v.SizeGiB) * volumeGiBCents,
|
||||
DropletIDs: nonNilInts(v.DropletIDs),
|
||||
TagCluster: nameByID[clusterIDFromTags(v.Tags)],
|
||||
CreatedAt: v.CreatedAt,
|
||||
MountedBy: []string{},
|
||||
}
|
||||
if len(v.DropletIDs) > 0 {
|
||||
vol.NodeName = dropletName[v.DropletIDs[0]]
|
||||
}
|
||||
if referenced {
|
||||
vol.Cluster, vol.ClusterID = hit.cluster, hit.clusterID
|
||||
vol.PV, vol.PVPhase = hit.pv.Name, hit.pv.Phase
|
||||
vol.PVCNamespace, vol.PVCName = hit.pv.ClaimNS, hit.pv.ClaimName
|
||||
if hit.pv.ClaimName != "" {
|
||||
vol.MountedBy = nonNilStrings(mounted[claimKey(hit.clusterID, hit.pv.ClaimNS, hit.pv.ClaimName)])
|
||||
}
|
||||
} else if len(v.DropletIDs) > 0 {
|
||||
// No PV names it, but it is physically mounted on a node — that node's
|
||||
// cluster owns it. Attachment is hard evidence, unlike the tag: it is the
|
||||
// live kernel state, so the cost rolls up to the right cluster.
|
||||
vol.ClusterID = clusterByDroplet[v.DropletIDs[0]]
|
||||
vol.Cluster = nameByID[vol.ClusterID]
|
||||
}
|
||||
|
||||
// Attachment beats reference; reference beats absence.
|
||||
switch {
|
||||
case len(v.DropletIDs) > 0:
|
||||
vol.State = StateAttached
|
||||
case referenced && strings.EqualFold(hit.pv.Phase, "Bound"):
|
||||
vol.State = StateBound
|
||||
case referenced:
|
||||
vol.State = StateReleased
|
||||
default:
|
||||
vol.State = StateUnreferenced
|
||||
}
|
||||
|
||||
// Idle is a REVIEW signal on live data, never a delete signal.
|
||||
vol.Idle = vol.State == StateBound && len(vol.MountedBy) == 0
|
||||
|
||||
vol.Deletable = snap.Complete && vol.State == StateUnreferenced
|
||||
vol.BlockedReason = blockedReason(vol, snap.Complete, snap.IncompleteReason)
|
||||
|
||||
snap.Cost.VolumesMonthly += vol.MonthlyCents
|
||||
snap.Totals.VolumeGiB += vol.SizeGiB
|
||||
switch vol.State {
|
||||
case StateAttached:
|
||||
snap.Totals.AttachedVolumes++
|
||||
snap.Totals.AttachedGiB += vol.SizeGiB
|
||||
default:
|
||||
snap.Totals.DetachedVolumes++
|
||||
snap.Totals.DetachedGiB += vol.SizeGiB
|
||||
}
|
||||
if vol.State == StateUnreferenced {
|
||||
snap.Totals.UnreferencedVolumes++
|
||||
snap.Totals.UnreferencedGiB += vol.SizeGiB
|
||||
// Reclaimable is exactly the unreferenced set — and only when the scan was
|
||||
// complete enough to have earned that verdict.
|
||||
if snap.Complete {
|
||||
snap.Cost.ReclaimableMonthly += vol.MonthlyCents
|
||||
}
|
||||
}
|
||||
if vol.Idle {
|
||||
snap.Totals.IdlePVCs++
|
||||
}
|
||||
snap.Volumes = append(snap.Volumes, vol)
|
||||
}
|
||||
|
||||
// ---- load balancers ----------------------------------------------------------
|
||||
snap.LoadBalancers = make([]LoadBalancer, 0, len(inv.LoadBalancers))
|
||||
for _, l := range inv.LoadBalancers {
|
||||
lb := LoadBalancer{
|
||||
ID: l.ID, Name: l.Name, Region: l.Region, Status: l.Status, IP: l.IP,
|
||||
SizeUnit: l.SizeUnit, MonthlyCents: l.MonthlyCents, Droplets: len(l.DropletIDs),
|
||||
}
|
||||
for _, id := range l.DropletIDs {
|
||||
if cid := clusterByDroplet[id]; cid != "" {
|
||||
lb.Cluster = nameByID[cid]
|
||||
break
|
||||
}
|
||||
}
|
||||
snap.Cost.LoadBalancersMonthly += lb.MonthlyCents
|
||||
snap.LoadBalancers = append(snap.LoadBalancers, lb)
|
||||
}
|
||||
|
||||
// ---- cluster rollup ----------------------------------------------------------
|
||||
idleByCluster := make(map[string]int)
|
||||
costByCluster := make(map[string]money.Cents)
|
||||
for _, v := range snap.Volumes {
|
||||
if v.ClusterID != "" {
|
||||
costByCluster[v.ClusterID] += v.MonthlyCents
|
||||
if v.Idle {
|
||||
idleByCluster[v.ClusterID]++
|
||||
}
|
||||
}
|
||||
}
|
||||
nodesByCluster := make(map[string]int)
|
||||
for _, n := range snap.Nodes {
|
||||
if n.ClusterID != "" {
|
||||
nodesByCluster[n.ClusterID]++
|
||||
costByCluster[n.ClusterID] += n.MonthlyCents
|
||||
}
|
||||
}
|
||||
snap.Clusters = make([]Cluster, 0, len(inv.Clusters))
|
||||
for _, c := range inv.Clusters {
|
||||
row := Cluster{
|
||||
ID: c.ID, Name: c.Name, Region: c.Region, Version: c.Version,
|
||||
Status: c.Status, NodePools: c.NodePools, Nodes: nodesByCluster[c.ID],
|
||||
IdlePVCs: idleByCluster[c.ID], MonthlyCents: costByCluster[c.ID],
|
||||
}
|
||||
if s, ok := scanByID[c.ID]; ok {
|
||||
if s.Err != nil {
|
||||
row.ScanError = s.Err.Error()
|
||||
} else {
|
||||
row.Scanned = true
|
||||
row.Pods, row.PVs, row.PVCs = len(s.Pods), len(s.PVs), len(s.PVCs)
|
||||
}
|
||||
} else {
|
||||
row.ScanError = "not scanned"
|
||||
}
|
||||
snap.Clusters = append(snap.Clusters, row)
|
||||
}
|
||||
|
||||
snap.Totals.Clusters = len(snap.Clusters)
|
||||
snap.Totals.Nodes = len(snap.Nodes)
|
||||
snap.Totals.Volumes = len(snap.Volumes)
|
||||
snap.Totals.LoadBalancers = len(snap.LoadBalancers)
|
||||
snap.Cost.TotalMonthly = snap.Cost.DropletsMonthly + snap.Cost.VolumesMonthly + snap.Cost.LoadBalancersMonthly
|
||||
|
||||
snap.Findings = findings(snap, scans, nameByID)
|
||||
return snap
|
||||
}
|
||||
|
||||
// blockedReason states, in the operator's language, exactly why a volume may not be
|
||||
// deleted. An empty string means it may.
|
||||
func blockedReason(v Volume, complete bool, incomplete string) string {
|
||||
if v.Deletable {
|
||||
return ""
|
||||
}
|
||||
switch {
|
||||
case !complete:
|
||||
return incomplete
|
||||
case v.State == StateAttached:
|
||||
if v.NodeName != "" {
|
||||
return "Attached to " + v.NodeName + " and in use."
|
||||
}
|
||||
return "Attached to a droplet and in use."
|
||||
case v.State == StateBound:
|
||||
return fmt.Sprintf("Live data: PV %s is Bound to %s/%s in %s.", v.PV, v.PVCNamespace, v.PVCName, v.Cluster)
|
||||
case v.State == StateReleased:
|
||||
return fmt.Sprintf("PV %s in %s still references it (%s) — retire the PV first.", v.PV, v.Cluster, v.PVPhase)
|
||||
}
|
||||
return "Not eligible for deletion."
|
||||
}
|
||||
|
||||
// findings is the audit pass: what a human should look at, worst first.
|
||||
func findings(s Snapshot, scans []ClusterScan, nameByID map[string]string) []Finding {
|
||||
out := []Finding{}
|
||||
|
||||
if !s.Complete {
|
||||
out = append(out, Finding{
|
||||
ID: "scan-incomplete", Severity: SevCritical, Kind: "scan-incomplete",
|
||||
Title: "Cluster scan incomplete — deletion disabled",
|
||||
Detail: s.IncompleteReason,
|
||||
})
|
||||
}
|
||||
|
||||
for _, v := range s.Volumes {
|
||||
switch {
|
||||
case v.State == StateUnreferenced && s.Complete:
|
||||
out = append(out, Finding{
|
||||
ID: "unref/" + v.ID, Severity: SevWarn, Kind: "unreferenced-volume",
|
||||
Title: fmt.Sprintf("Unreferenced volume %s (%d GiB)", v.Name, v.SizeGiB),
|
||||
Detail: "No PersistentVolume in any cluster references this volume. " +
|
||||
"Verified against every cluster, so it is safe to snapshot and delete.",
|
||||
Resource: v.ID, MonthlyCents: v.MonthlyCents,
|
||||
})
|
||||
case v.State == StateReleased:
|
||||
out = append(out, Finding{
|
||||
ID: "released/" + v.ID, Severity: SevWarn, Kind: "released-pv",
|
||||
Title: fmt.Sprintf("Released PV holding %s (%d GiB)", v.Name, v.SizeGiB),
|
||||
Detail: fmt.Sprintf("PV %s is %s. Retire the PV to release the volume.", v.PV, v.PVPhase),
|
||||
Resource: v.ID, Cluster: v.Cluster, MonthlyCents: v.MonthlyCents,
|
||||
})
|
||||
case v.Idle:
|
||||
out = append(out, Finding{
|
||||
ID: "idle/" + v.ID, Severity: SevInfo, Kind: "idle-pvc",
|
||||
Title: fmt.Sprintf("Idle volume %s (%d GiB) — no pod mounts it", v.Name, v.SizeGiB),
|
||||
Detail: fmt.Sprintf("PVC %s/%s is Bound but no running pod mounts it. "+
|
||||
"REVIEW ONLY: this is live data (typically a stopped database), not garbage.",
|
||||
v.PVCNamespace, v.PVCName),
|
||||
Resource: v.ID, Cluster: v.Cluster, MonthlyCents: v.MonthlyCents,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Unhealthy pods + unknown images, per cluster.
|
||||
type imgSeen struct {
|
||||
pods int
|
||||
cluster string
|
||||
}
|
||||
unknown := map[string]*imgSeen{}
|
||||
for _, sc := range scans {
|
||||
if sc.Err != nil {
|
||||
continue
|
||||
}
|
||||
cname := nameByID[sc.ClusterID]
|
||||
for _, p := range sc.Pods {
|
||||
if bad := podProblem(p); bad != "" {
|
||||
out = append(out, Finding{
|
||||
ID: "pod/" + sc.ClusterID + "/" + p.Namespace + "/" + p.Name, Severity: SevWarn,
|
||||
Kind: "pod-unhealthy", Title: fmt.Sprintf("Pod %s/%s is %s", p.Namespace, p.Name, bad),
|
||||
Detail: fmt.Sprintf("Phase %s%s on node %s.", p.Phase, reasonSuffix(p.Reason), p.Node),
|
||||
Resource: p.Namespace + "/" + p.Name, Cluster: cname,
|
||||
})
|
||||
}
|
||||
for _, img := range p.Images {
|
||||
if knownImage(img) {
|
||||
continue
|
||||
}
|
||||
repo := imageRepo(img)
|
||||
if e, ok := unknown[repo]; ok {
|
||||
e.pods++
|
||||
} else {
|
||||
unknown[repo] = &imgSeen{pods: 1, cluster: cname}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for repo, e := range unknown {
|
||||
out = append(out, Finding{
|
||||
ID: "image/" + repo, Severity: SevWarn, Kind: "unknown-image",
|
||||
Title: "Unrecognised container image: " + repo,
|
||||
Detail: fmt.Sprintf("Run by %d pod(s), from neither our registries nor the reviewed vendor set.", e.pods),
|
||||
Resource: repo, Cluster: e.cluster,
|
||||
})
|
||||
}
|
||||
|
||||
// Cost outliers: any single resource at or above outlierShareBP of total spend.
|
||||
if s.Cost.TotalMonthly > 0 {
|
||||
threshold := s.Cost.TotalMonthly * outlierShareBP / 10000
|
||||
for _, n := range s.Nodes {
|
||||
if n.MonthlyCents >= threshold {
|
||||
out = append(out, Finding{
|
||||
ID: "cost/node/" + n.Name, Severity: SevInfo, Kind: "cost-outlier",
|
||||
Title: fmt.Sprintf("Node %s is %s of fleet spend", n.Name, shareLabel(n.MonthlyCents, s.Cost.TotalMonthly)),
|
||||
Detail: fmt.Sprintf("%s, %d vCPU / %d MiB.", n.SizeSlug, n.VCPUs, n.MemoryMiB),
|
||||
Resource: n.Name, Cluster: n.Cluster, MonthlyCents: n.MonthlyCents,
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, v := range s.Volumes {
|
||||
if v.MonthlyCents >= threshold {
|
||||
out = append(out, Finding{
|
||||
ID: "cost/volume/" + v.ID, Severity: SevInfo, Kind: "cost-outlier",
|
||||
Title: fmt.Sprintf("Volume %s is %s of fleet spend", v.Name, shareLabel(v.MonthlyCents, s.Cost.TotalMonthly)),
|
||||
Detail: fmt.Sprintf("%d GiB, %s.", v.SizeGiB, v.State),
|
||||
Resource: v.ID, Cluster: v.Cluster, MonthlyCents: v.MonthlyCents,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rank := map[string]int{SevCritical: 0, SevWarn: 1, SevInfo: 2}
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
if rank[out[i].Severity] != rank[out[j].Severity] {
|
||||
return rank[out[i].Severity] < rank[out[j].Severity]
|
||||
}
|
||||
if out[i].MonthlyCents != out[j].MonthlyCents {
|
||||
return out[i].MonthlyCents > out[j].MonthlyCents
|
||||
}
|
||||
return out[i].ID < out[j].ID
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// podProblem names the failure a pod is in, or "" when it is fine.
|
||||
func podProblem(p PodRef) string {
|
||||
switch {
|
||||
case strings.EqualFold(p.Reason, "Evicted"):
|
||||
return "Evicted"
|
||||
case strings.EqualFold(p.Phase, "Failed"):
|
||||
return "Failed"
|
||||
case strings.Contains(p.Reason, "CrashLoopBackOff"):
|
||||
return "CrashLoopBackOff"
|
||||
case strings.Contains(p.Reason, "ImagePullBackOff"), strings.Contains(p.Reason, "ErrImagePull"):
|
||||
return "ImagePullBackOff"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func reasonSuffix(r string) string {
|
||||
if strings.TrimSpace(r) == "" {
|
||||
return ""
|
||||
}
|
||||
return " (" + r + ")"
|
||||
}
|
||||
|
||||
// knownImage reports whether an image comes from our registries or the reviewed
|
||||
// vendor set.
|
||||
func knownImage(img string) bool {
|
||||
l := strings.ToLower(strings.TrimSpace(img))
|
||||
l = strings.TrimPrefix(l, "docker.io/")
|
||||
for _, p := range firstParty {
|
||||
if strings.HasPrefix(l, strings.TrimPrefix(p, "docker.io/")) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, p := range knownVendors {
|
||||
if strings.HasPrefix(l, strings.TrimPrefix(p, "docker.io/")) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// A bare `name:tag` with no slash is an official Docker Hub library image.
|
||||
return !strings.Contains(strings.SplitN(l, ":", 2)[0], "/")
|
||||
}
|
||||
|
||||
// imageRepo strips the tag/digest so findings group by repository, not by build.
|
||||
func imageRepo(img string) string {
|
||||
s := strings.TrimSpace(img)
|
||||
if i := strings.Index(s, "@"); i > 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
if i := strings.LastIndex(s, ":"); i > strings.LastIndex(s, "/") {
|
||||
s = s[:i]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// shareLabel renders a cents-of-total share as a percentage.
|
||||
func shareLabel(part, total money.Cents) string {
|
||||
if total <= 0 {
|
||||
return "0%"
|
||||
}
|
||||
return fmt.Sprintf("%.1f%%", float64(part)*100/float64(total))
|
||||
}
|
||||
|
||||
// clusterIDFromTags extracts the DOKS cluster UUID from a `k8s:<uuid>` resource tag.
|
||||
// On droplets this is authoritative (DOKS owns the droplet); on VOLUMES it is
|
||||
// advisory only — see the Volume.TagCluster doc.
|
||||
func clusterIDFromTags(tags []string) string {
|
||||
for _, t := range tags {
|
||||
v := strings.TrimPrefix(t, "k8s:")
|
||||
if v == t || v == "" {
|
||||
continue
|
||||
}
|
||||
// Cluster tags are UUIDs; DOKS also stamps role tags like `k8s:worker`.
|
||||
if len(v) == 36 && strings.Count(v, "-") == 4 {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func claimKey(clusterID, ns, name string) string { return clusterID + "/" + ns + "/" + name }
|
||||
|
||||
func nonNilStrings(s []string) []string {
|
||||
if s == nil {
|
||||
return []string{}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func nonNilInts(s []int) []int {
|
||||
if s == nil {
|
||||
return []int{}
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,390 @@
|
||||
package infra
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/admin/digitalocean"
|
||||
)
|
||||
|
||||
// Two live clusters. The near-miss that motivated this package involved volumes tagged
|
||||
// for one cluster whose PVs actually live in another, so every fixture here has two.
|
||||
const (
|
||||
cidA = "aaaaaaaa-1111-2222-3333-444444444444"
|
||||
cidB = "bbbbbbbb-1111-2222-3333-444444444444"
|
||||
)
|
||||
|
||||
func baseInventory() Inventory {
|
||||
return Inventory{
|
||||
Clusters: []digitalocean.Cluster{
|
||||
{ID: cidA, Name: "hanzo-k8s", Region: "sfo3", Status: "running"},
|
||||
{ID: cidB, Name: "lux-k8s", Region: "sfo3", Status: "running"},
|
||||
},
|
||||
Droplets: []digitalocean.Droplet{{
|
||||
ID: 101, Name: "node-a1", Region: "sfo3", Status: "active",
|
||||
SizeSlug: "s-8vcpu-16gb-amd", VCPUs: 8, MemoryMiB: 16384,
|
||||
LocalDiskGiB: 320, MonthlyCents: 11200,
|
||||
Tags: []string{"k8s", "k8s:" + cidA, "k8s:worker"},
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func scansOK() []ClusterScan {
|
||||
return []ClusterScan{{ClusterID: cidA}, {ClusterID: cidB}}
|
||||
}
|
||||
|
||||
func volByID(t *testing.T, s Snapshot, id string) Volume {
|
||||
t.Helper()
|
||||
v, ok := findVolume(s, id)
|
||||
if !ok {
|
||||
t.Fatalf("volume %s missing from snapshot", id)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func analyze(inv Inventory, scans []ClusterScan) Snapshot {
|
||||
return Analyze(inv, scans, nil, time.Unix(0, 0).UTC())
|
||||
}
|
||||
|
||||
// TestCrossClusterPVProtectsMisTaggedVolume is THE regression test. A detached volume
|
||||
// tagged `k8s:<cluster A>` whose PV actually lives in cluster B must be classified from
|
||||
// the PV, not the tag. Trusting the tag here is what nearly destroyed 4.39 TiB.
|
||||
func TestCrossClusterPVProtectsMisTaggedVolume(t *testing.T) {
|
||||
inv := baseInventory()
|
||||
inv.Volumes = []digitalocean.Volume{{
|
||||
ID: "vol-mistagged", Name: "pvc-neon-pageserver", Region: "sfo3", SizeGiB: 50,
|
||||
DropletIDs: nil, // detached
|
||||
Tags: []string{"k8s:" + cidA}, // tag says cluster A …
|
||||
}}
|
||||
scans := scansOK()
|
||||
// … but the PV that owns it lives in cluster B, and is Bound to a live PVC.
|
||||
scans[1].PVs = []PVRef{{
|
||||
Name: "pv-neon", Phase: "Bound", VolumeHandle: "vol-mistagged",
|
||||
ClaimNS: "neon", ClaimName: "pageserver-data",
|
||||
}}
|
||||
|
||||
got := analyze(inv, scans)
|
||||
v := volByID(t, got, "vol-mistagged")
|
||||
|
||||
if v.State != StateBound {
|
||||
t.Fatalf("state = %q, want %q — a detached, mis-tagged volume with a Bound PV is LIVE DATA", v.State, StateBound)
|
||||
}
|
||||
if v.Deletable {
|
||||
t.Fatal("volume marked deletable: this is the 4.39 TiB data-loss bug")
|
||||
}
|
||||
if v.Cluster != "lux-k8s" {
|
||||
t.Errorf("proven cluster = %q, want lux-k8s (from the PV, not the tag)", v.Cluster)
|
||||
}
|
||||
if v.TagCluster != "hanzo-k8s" {
|
||||
t.Errorf("tagCluster = %q, want hanzo-k8s (advisory, surfaced so tag-vs-truth is visible)", v.TagCluster)
|
||||
}
|
||||
if got.Cost.ReclaimableMonthly != 0 {
|
||||
t.Errorf("reclaimable = %d, want 0", got.Cost.ReclaimableMonthly)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIncompleteScanBlocksEveryDeletion: one unreachable cluster and NOTHING is
|
||||
// deletable, even a volume no reachable cluster references. Absence of evidence is not
|
||||
// evidence of absence.
|
||||
func TestIncompleteScanBlocksEveryDeletion(t *testing.T) {
|
||||
inv := baseInventory()
|
||||
inv.Volumes = []digitalocean.Volume{{ID: "vol-orphan", Name: "stray", SizeGiB: 100}}
|
||||
scans := scansOK()
|
||||
scans[1].Err = errors.New("dial tcp: i/o timeout")
|
||||
|
||||
got := analyze(inv, scans)
|
||||
if got.Complete {
|
||||
t.Fatal("Complete = true with an unreachable cluster")
|
||||
}
|
||||
v := volByID(t, got, "vol-orphan")
|
||||
if v.State != StateUnreferenced {
|
||||
t.Errorf("state = %q, want %q (state is observable; the VERDICT is what is withheld)", v.State, StateUnreferenced)
|
||||
}
|
||||
if v.Deletable {
|
||||
t.Fatal("deletable with an incomplete scan — fail-closed violated")
|
||||
}
|
||||
if got.Cost.ReclaimableMonthly != 0 {
|
||||
t.Errorf("reclaimable = %d, want 0 when the scan is incomplete", got.Cost.ReclaimableMonthly)
|
||||
}
|
||||
if v.BlockedReason == "" || !strings.Contains(v.BlockedReason, "lux-k8s") {
|
||||
t.Errorf("blockedReason = %q, want it to name the unreachable cluster", v.BlockedReason)
|
||||
}
|
||||
if got.Findings[0].Kind != "scan-incomplete" || got.Findings[0].Severity != SevCritical {
|
||||
t.Errorf("first finding = %+v, want a critical scan-incomplete", got.Findings[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestNoClustersIsIncomplete: an empty cluster list means the set of places a volume
|
||||
// could be in use is unknown, which must not read as "referenced by nothing".
|
||||
func TestNoClustersIsIncomplete(t *testing.T) {
|
||||
inv := Inventory{Volumes: []digitalocean.Volume{{ID: "v1", Name: "x", SizeGiB: 10}}}
|
||||
got := analyze(inv, nil)
|
||||
if got.Complete {
|
||||
t.Fatal("Complete = true with zero clusters")
|
||||
}
|
||||
if volByID(t, got, "v1").Deletable {
|
||||
t.Fatal("deletable with zero clusters known")
|
||||
}
|
||||
}
|
||||
|
||||
// TestVolumeStateMachine covers the full ordering: attachment beats reference,
|
||||
// reference beats absence, and only the unreferenced volume is ever deletable.
|
||||
func TestVolumeStateMachine(t *testing.T) {
|
||||
inv := baseInventory()
|
||||
inv.Volumes = []digitalocean.Volume{
|
||||
{ID: "v-attached", Name: "attached", SizeGiB: 10, DropletIDs: []int{101}, Tags: []string{"k8s:" + cidA}},
|
||||
{ID: "v-bound", Name: "bound", SizeGiB: 20},
|
||||
{ID: "v-released", Name: "released", SizeGiB: 30},
|
||||
{ID: "v-unref", Name: "unref", SizeGiB: 40},
|
||||
// Attached AND referenced by a Bound PV: attachment wins.
|
||||
{ID: "v-both", Name: "both", SizeGiB: 50, DropletIDs: []int{101}},
|
||||
}
|
||||
scans := scansOK()
|
||||
scans[0].PVs = []PVRef{
|
||||
{Name: "pv-bound", Phase: "Bound", VolumeHandle: "v-bound", ClaimNS: "ns", ClaimName: "c1"},
|
||||
{Name: "pv-rel", Phase: "Released", VolumeHandle: "v-released", ClaimNS: "ns", ClaimName: "c2"},
|
||||
{Name: "pv-both", Phase: "Bound", VolumeHandle: "v-both", ClaimNS: "ns", ClaimName: "c3"},
|
||||
}
|
||||
scans[0].Pods = []PodRef{{Namespace: "ns", Name: "p1", Node: "node-a1", Claims: []string{"c1", "c3"}}}
|
||||
|
||||
got := analyze(inv, scans)
|
||||
if !got.Complete {
|
||||
t.Fatalf("Complete = false, want true: %s", got.IncompleteReason)
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
id, want string
|
||||
deletable bool
|
||||
}{
|
||||
{"v-attached", StateAttached, false},
|
||||
{"v-bound", StateBound, false},
|
||||
{"v-released", StateReleased, false},
|
||||
{"v-unref", StateUnreferenced, true},
|
||||
{"v-both", StateAttached, false},
|
||||
} {
|
||||
v := volByID(t, got, tc.id)
|
||||
if v.State != tc.want {
|
||||
t.Errorf("%s: state = %q, want %q", tc.id, v.State, tc.want)
|
||||
}
|
||||
if v.Deletable != tc.deletable {
|
||||
t.Errorf("%s: deletable = %v, want %v (reason %q)", tc.id, v.Deletable, tc.deletable, v.BlockedReason)
|
||||
}
|
||||
if !v.Deletable && v.BlockedReason == "" {
|
||||
t.Errorf("%s: not deletable but no reason given", tc.id)
|
||||
}
|
||||
if v.Deletable && v.BlockedReason != "" {
|
||||
t.Errorf("%s: deletable but carries reason %q", tc.id, v.BlockedReason)
|
||||
}
|
||||
}
|
||||
// Exactly one volume is reclaimable: 40 GiB × $0.10 = $4.00.
|
||||
if got.Cost.ReclaimableMonthly != 400 {
|
||||
t.Errorf("reclaimable = %d cents, want 400", got.Cost.ReclaimableMonthly)
|
||||
}
|
||||
// v-bound is mounted by p1; v-both is mounted by p1 too — neither is idle.
|
||||
if got.Totals.IdlePVCs != 0 {
|
||||
t.Errorf("idlePVCs = %d, want 0", got.Totals.IdlePVCs)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIdleIsReviewNotReclaimable: a Bound volume no pod mounts is flagged for review
|
||||
// but is never deletable and never counted as money we can get back.
|
||||
func TestIdleIsReviewNotReclaimable(t *testing.T) {
|
||||
inv := baseInventory()
|
||||
inv.Volumes = []digitalocean.Volume{{ID: "v-idle", Name: "registry-data", SizeGiB: 50}}
|
||||
scans := scansOK()
|
||||
scans[1].PVs = []PVRef{{Name: "pv-idle", Phase: "Bound", VolumeHandle: "v-idle", ClaimNS: "registry", ClaimName: "registry-data"}}
|
||||
// No pod mounts registry-data.
|
||||
scans[1].Pods = []PodRef{{Namespace: "other", Name: "unrelated", Claims: []string{"something-else"}}}
|
||||
|
||||
got := analyze(inv, scans)
|
||||
v := volByID(t, got, "v-idle")
|
||||
if !v.Idle {
|
||||
t.Fatal("idle = false, want true (Bound but unmounted)")
|
||||
}
|
||||
if v.Deletable {
|
||||
t.Fatal("an idle volume must never be deletable — it is a stopped database, not garbage")
|
||||
}
|
||||
if got.Cost.ReclaimableMonthly != 0 {
|
||||
t.Errorf("reclaimable = %d, want 0: idle capacity is not reclaimable", got.Cost.ReclaimableMonthly)
|
||||
}
|
||||
if got.Totals.IdlePVCs != 1 {
|
||||
t.Errorf("idlePVCs = %d, want 1", got.Totals.IdlePVCs)
|
||||
}
|
||||
var f *Finding
|
||||
for i := range got.Findings {
|
||||
if got.Findings[i].Kind == "idle-pvc" {
|
||||
f = &got.Findings[i]
|
||||
}
|
||||
}
|
||||
if f == nil {
|
||||
t.Fatal("no idle-pvc finding")
|
||||
}
|
||||
if f.Severity != SevInfo {
|
||||
t.Errorf("idle-pvc severity = %q, want info (a review queue, not an alarm)", f.Severity)
|
||||
}
|
||||
if !strings.Contains(f.Detail, "REVIEW ONLY") {
|
||||
t.Errorf("idle-pvc detail must say it is review-only, got %q", f.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLegacyFlexVolumePVProtects: a pre-CSI PV still shields its volume.
|
||||
func TestLegacyFlexVolumePVProtects(t *testing.T) {
|
||||
inv := baseInventory()
|
||||
inv.Volumes = []digitalocean.Volume{{ID: "v-flex", Name: "legacy", SizeGiB: 10}}
|
||||
scans := scansOK()
|
||||
scans[0].PVs = []PVRef{{Name: "pv-flex", Phase: "Bound", VolumeHandle: "v-flex", ClaimNS: "old", ClaimName: "data"}}
|
||||
if volByID(t, analyze(inv, scans), "v-flex").Deletable {
|
||||
t.Fatal("a flexVolume-referenced volume must not be deletable")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCostMathAndLocalDiskSeparation: block storage is billed per GiB; droplet local
|
||||
// disk is included in the droplet price and must never be added to storage cost.
|
||||
func TestCostMathAndLocalDiskSeparation(t *testing.T) {
|
||||
inv := baseInventory()
|
||||
inv.Volumes = []digitalocean.Volume{
|
||||
{ID: "a", Name: "a", SizeGiB: 200},
|
||||
{ID: "b", Name: "b", SizeGiB: 300},
|
||||
}
|
||||
inv.LoadBalancers = []digitalocean.LoadBalancer{
|
||||
{ID: "lb1", Name: "ingress", SizeUnit: 1, MonthlyCents: 1200, DropletIDs: []int{101}},
|
||||
}
|
||||
got := analyze(inv, scansOK())
|
||||
|
||||
if got.Cost.VolumesMonthly != 5000 {
|
||||
t.Errorf("volumes = %d cents, want 5000 (500 GiB × $0.10)", got.Cost.VolumesMonthly)
|
||||
}
|
||||
if got.Cost.DropletsMonthly != 11200 {
|
||||
t.Errorf("droplets = %d cents, want 11200", got.Cost.DropletsMonthly)
|
||||
}
|
||||
if got.Cost.LoadBalancersMonthly != 1200 {
|
||||
t.Errorf("load balancers = %d cents, want 1200", got.Cost.LoadBalancersMonthly)
|
||||
}
|
||||
if got.Cost.TotalMonthly != 5000+11200+1200 {
|
||||
t.Errorf("total = %d cents, want %d", got.Cost.TotalMonthly, 5000+11200+1200)
|
||||
}
|
||||
// The 320 GiB of local disk is reported, but is NOT in any cost line.
|
||||
if got.Totals.LocalDiskGiB != 320 {
|
||||
t.Errorf("localDiskGiB = %d, want 320", got.Totals.LocalDiskGiB)
|
||||
}
|
||||
if got.Totals.VolumeGiB != 500 {
|
||||
t.Errorf("volumeGiB = %d, want 500 — local disk must never be folded into block storage", got.Totals.VolumeGiB)
|
||||
}
|
||||
if got.LoadBalancers[0].Cluster != "hanzo-k8s" {
|
||||
t.Errorf("lb cluster = %q, want hanzo-k8s (attributed via its member droplet)", got.LoadBalancers[0].Cluster)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNodeJoinAndClusterRollup: droplets join their Kubernetes node by name and roll
|
||||
// up into the owning cluster.
|
||||
func TestNodeJoinAndClusterRollup(t *testing.T) {
|
||||
inv := baseInventory()
|
||||
inv.Volumes = []digitalocean.Volume{{ID: "v1", Name: "v1", SizeGiB: 10, DropletIDs: []int{101}}}
|
||||
scans := scansOK()
|
||||
scans[0].Nodes = []NodeState{{Name: "node-a1", Ready: true, Schedulable: false}}
|
||||
scans[0].Pods = []PodRef{
|
||||
{Namespace: "hanzo", Name: "p1", Node: "node-a1"},
|
||||
{Namespace: "hanzo", Name: "p2", Node: "node-a1"},
|
||||
}
|
||||
got := analyze(inv, scans)
|
||||
|
||||
n := got.Nodes[0]
|
||||
if n.Cluster != "hanzo-k8s" || n.ClusterID != cidA {
|
||||
t.Errorf("node cluster = %q/%q, want hanzo-k8s", n.Cluster, n.ClusterID)
|
||||
}
|
||||
if !n.Ready || n.Schedulable {
|
||||
t.Errorf("node ready=%v schedulable=%v, want ready + cordoned", n.Ready, n.Schedulable)
|
||||
}
|
||||
if n.Pods != 2 || n.Volumes != 1 {
|
||||
t.Errorf("node pods=%d volumes=%d, want 2/1", n.Pods, n.Volumes)
|
||||
}
|
||||
var ca Cluster
|
||||
for _, c := range got.Clusters {
|
||||
if c.ID == cidA {
|
||||
ca = c
|
||||
}
|
||||
}
|
||||
if ca.Nodes != 1 || ca.Pods != 2 || !ca.Scanned {
|
||||
t.Errorf("cluster A rollup = %+v, want 1 node / 2 pods / scanned", ca)
|
||||
}
|
||||
// Node $112.00 + its 10 GiB volume $1.00.
|
||||
if ca.MonthlyCents != 11200+100 {
|
||||
t.Errorf("cluster monthly = %d, want 11300", ca.MonthlyCents)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnknownImageDetection: our registries and the reviewed vendor set stay quiet;
|
||||
// anything else is reported once per repository.
|
||||
func TestUnknownImageDetection(t *testing.T) {
|
||||
inv := baseInventory()
|
||||
scans := scansOK()
|
||||
scans[0].Pods = []PodRef{{Namespace: "hanzo", Name: "p1", Images: []string{
|
||||
"ghcr.io/hanzoai/cloud:v1.801.218",
|
||||
"ghcr.io/luxfi/node:v1.2.3",
|
||||
"registry.k8s.io/pause:3.9",
|
||||
"grafana/grafana:11.0.0",
|
||||
"acmglobaltech/thing:1",
|
||||
"redis:7", // bare official library image
|
||||
"evil.example.com/miner:latest", // ← the only one that should be reported
|
||||
}}}
|
||||
got := analyze(inv, scans)
|
||||
|
||||
var unknown []string
|
||||
for _, f := range got.Findings {
|
||||
if f.Kind == "unknown-image" {
|
||||
unknown = append(unknown, f.Resource)
|
||||
}
|
||||
}
|
||||
if len(unknown) != 1 || unknown[0] != "evil.example.com/miner" {
|
||||
t.Fatalf("unknown images = %v, want exactly [evil.example.com/miner]", unknown)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnhealthyPodFindings covers the pod states the audit surfaces.
|
||||
func TestUnhealthyPodFindings(t *testing.T) {
|
||||
inv := baseInventory()
|
||||
scans := scansOK()
|
||||
scans[0].Pods = []PodRef{
|
||||
{Namespace: "a", Name: "ok", Phase: "Running"},
|
||||
{Namespace: "a", Name: "gone", Phase: "Failed", Reason: "Evicted"},
|
||||
{Namespace: "a", Name: "loop", Phase: "Pending", Reason: "CrashLoopBackOff"},
|
||||
{Namespace: "a", Name: "pull", Phase: "Pending", Reason: "ImagePullBackOff"},
|
||||
}
|
||||
got := analyze(inv, scans)
|
||||
seen := map[string]bool{}
|
||||
for _, f := range got.Findings {
|
||||
if f.Kind == "pod-unhealthy" {
|
||||
seen[f.Resource] = true
|
||||
}
|
||||
}
|
||||
if len(seen) != 3 || !seen["a/gone"] || !seen["a/loop"] || !seen["a/pull"] {
|
||||
t.Fatalf("unhealthy pods = %v, want gone/loop/pull and not ok", seen)
|
||||
}
|
||||
}
|
||||
|
||||
// TestClusterTagParsing: only a UUID-shaped k8s: tag is a cluster id; role tags are not.
|
||||
func TestClusterTagParsing(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
tags []string
|
||||
want string
|
||||
}{
|
||||
{[]string{"k8s", "k8s:" + cidA, "k8s:worker"}, cidA},
|
||||
{[]string{"k8s", "k8s:worker"}, ""},
|
||||
{[]string{"unrelated"}, ""},
|
||||
{nil, ""},
|
||||
} {
|
||||
if got := clusterIDFromTags(tc.tags); got != tc.want {
|
||||
t.Errorf("clusterIDFromTags(%v) = %q, want %q", tc.tags, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestJSONArraysNeverNull: the console renders these directly; a null array is a crash.
|
||||
func TestJSONArraysNeverNull(t *testing.T) {
|
||||
got := analyze(Inventory{}, nil)
|
||||
if got.Volumes == nil || got.Nodes == nil || got.Clusters == nil ||
|
||||
got.LoadBalancers == nil || got.Findings == nil || got.Sources == nil {
|
||||
t.Fatalf("empty snapshot has nil slices: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
package infra
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/zap-proto/zip"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/audit"
|
||||
"github.com/hanzoai/cloud/clients/admin/core"
|
||||
"github.com/hanzoai/cloud/clients/admin/digitalocean"
|
||||
)
|
||||
|
||||
// cacheTTL bounds how stale a READ may be. It exists because one board is a fan-out
|
||||
// over the DO API plus every cluster's full pod/PV listing — not because staleness is
|
||||
// acceptable when it matters: every MUTATION re-scans from scratch, ignoring this.
|
||||
const cacheTTL = 60 * time.Second
|
||||
|
||||
// board holds the one cached snapshot behind /v1/admin/infra.
|
||||
type board struct {
|
||||
mu sync.Mutex
|
||||
snap Snapshot
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// Routes registers the DigitalOcean infrastructure board. SuperAdmin only: this is
|
||||
// the whole account's physical inventory and the controls that destroy parts of it.
|
||||
//
|
||||
// NOTE ON THE NOUN: this is INFRASTRUCTURE — droplets, volumes, DOKS clusters, load
|
||||
// balancers. The pre-existing /v1/fleet surface is compute workers and jobs. Different
|
||||
// nouns, deliberately not merged.
|
||||
func Routes(app *zip.App, s *cloud.Service[core.State]) {
|
||||
b := &board{}
|
||||
g := app.Group("/v1/admin")
|
||||
g.Get("/infra", core.Guard(s, b.read))
|
||||
g.Post("/infra/volumes/:id/snapshot", core.Guard(s, b.snapshotVolume))
|
||||
g.Delete("/infra/volumes/:id", core.Guard(s, b.deleteVolume))
|
||||
g.Post("/infra/nodes/:id/cordon", core.Guard(s, b.cordonNode))
|
||||
}
|
||||
|
||||
// read serves the board, from cache unless ?refresh=1.
|
||||
func (b *board) read(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
snap, err := b.load(c.Context(), s.State.DO, c.Query("refresh") != "")
|
||||
if err != nil {
|
||||
return core.Fail(c, err.Error())
|
||||
}
|
||||
return core.OK(c, snap)
|
||||
}
|
||||
|
||||
// load returns the snapshot, recomputing when forced or stale. A forced load is the
|
||||
// authority every mutation checks itself against.
|
||||
func (b *board) load(ctx context.Context, do *digitalocean.Client, force bool) (Snapshot, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if !force && !b.at.IsZero() && time.Since(b.at) < cacheTTL {
|
||||
return b.snap, nil
|
||||
}
|
||||
snap, err := collect(ctx, do)
|
||||
if err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
b.snap, b.at = snap, time.Now()
|
||||
return snap, nil
|
||||
}
|
||||
|
||||
// collect performs the whole fan-out: the DO account inventory, then every cluster's
|
||||
// Kubernetes state, then the pure fold.
|
||||
//
|
||||
// Only an unusable DO account is a hard error. A partial DO read (say load balancers
|
||||
// fail) still produces a board, with the failure named in Sources — EXCEPT for the
|
||||
// two reads the safety verdict depends on. Clusters and Volumes are load-bearing: if
|
||||
// either is missing, the analysis cannot honestly classify anything, so it degrades
|
||||
// via the completeness gate rather than pretending.
|
||||
func collect(ctx context.Context, do *digitalocean.Client) (Snapshot, error) {
|
||||
if do == nil || !do.Ready() {
|
||||
return Snapshot{}, fmt.Errorf("DO_API_TOKEN not configured — DigitalOcean inventory unavailable")
|
||||
}
|
||||
at := time.Now().UTC()
|
||||
stamp := at.Format(time.RFC3339)
|
||||
|
||||
var (
|
||||
inv Inventory
|
||||
sources []core.SourceStatus
|
||||
mu sync.Mutex
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
run := func(name string, fn func() (int, error)) {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
n, err := fn()
|
||||
mu.Lock()
|
||||
sources = append(sources, core.SrcOf(name, err, n, stamp))
|
||||
mu.Unlock()
|
||||
}()
|
||||
}
|
||||
run("do.clusters", func() (int, error) {
|
||||
v, err := do.Clusters(ctx)
|
||||
inv.Clusters = v
|
||||
return len(v), err
|
||||
})
|
||||
run("do.droplets", func() (int, error) {
|
||||
v, err := do.Droplets(ctx)
|
||||
inv.Droplets = v
|
||||
return len(v), err
|
||||
})
|
||||
run("do.volumes", func() (int, error) {
|
||||
v, err := do.Volumes(ctx)
|
||||
inv.Volumes = v
|
||||
return len(v), err
|
||||
})
|
||||
run("do.loadBalancers", func() (int, error) {
|
||||
v, err := do.LoadBalancers(ctx)
|
||||
inv.LoadBalancers = v
|
||||
return len(v), err
|
||||
})
|
||||
wg.Wait()
|
||||
|
||||
scans := Scan(ctx, do, inv.Clusters)
|
||||
for i, sc := range scans {
|
||||
name := "k8s." + inv.Clusters[i].Name
|
||||
rows := len(sc.PVs) + len(sc.PVCs) + len(sc.Pods) + len(sc.Nodes)
|
||||
sources = append(sources, core.SrcOf(name, sc.Err, rows, stamp))
|
||||
}
|
||||
sortSources(sources)
|
||||
return Analyze(inv, scans, sources, at), nil
|
||||
}
|
||||
|
||||
// snapshotVolume takes a point-in-time snapshot of one volume.
|
||||
func (b *board) snapshotVolume(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
id := strings.TrimSpace(c.Param("id"))
|
||||
snap, err := b.load(c.Context(), s.State.DO, true)
|
||||
if err != nil {
|
||||
return core.Fail(c, err.Error())
|
||||
}
|
||||
v, ok := findVolume(snap, id)
|
||||
if !ok {
|
||||
return core.Fail(c, "volume not found")
|
||||
}
|
||||
var body struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
_ = c.Bind(&body)
|
||||
out, err := takeSnapshot(c.Context(), s.State.DO, v, body.Name)
|
||||
if err != nil {
|
||||
core.EmitAudit(s, c, "infra.volume.snapshot", "do_volume", id, v, nil,
|
||||
audit.Outcome{Result: "failure", Status: 200, Reason: err.Error()})
|
||||
return core.Fail(c, err.Error())
|
||||
}
|
||||
core.EmitAudit(s, c, "infra.volume.snapshot", "do_volume", id, v, out,
|
||||
audit.Outcome{Result: "success", Status: 200})
|
||||
return core.OK(c, out)
|
||||
}
|
||||
|
||||
// deleteVolume destroys a volume — but ONLY one the server itself has just proven to
|
||||
// be referenced by no PersistentVolume in any cluster.
|
||||
//
|
||||
// The client's opinion is never trusted: deletability is recomputed here from a FRESH
|
||||
// complete cross-cluster scan (force=true, never the cache), so a volume that became
|
||||
// live between the operator loading the page and pressing the button is refused. If
|
||||
// any cluster is unreachable the scan is incomplete and NOTHING is deletable.
|
||||
func (b *board) deleteVolume(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
id := strings.TrimSpace(c.Param("id"))
|
||||
snap, err := b.load(c.Context(), s.State.DO, true)
|
||||
if err != nil {
|
||||
return core.Fail(c, err.Error())
|
||||
}
|
||||
v, ok := findVolume(snap, id)
|
||||
if !ok {
|
||||
return core.Fail(c, "volume not found")
|
||||
}
|
||||
if !v.Deletable {
|
||||
core.EmitAudit(s, c, "infra.volume.delete", "do_volume", id, v, nil,
|
||||
audit.Outcome{Result: "denied", Status: 200, Reason: v.BlockedReason})
|
||||
return core.Fail(c, "refusing to delete: "+v.BlockedReason)
|
||||
}
|
||||
|
||||
out := map[string]any{"deleted": false, "name": v.Name, "sizeGiB": v.SizeGiB,
|
||||
"freedMonthlyCents": v.MonthlyCents}
|
||||
// Snapshot first unless explicitly waived — the delete is irreversible, the
|
||||
// snapshot is the undo.
|
||||
if c.Query("snapshot") != "false" {
|
||||
shot, serr := takeSnapshot(c.Context(), s.State.DO, v, "")
|
||||
if serr != nil {
|
||||
core.EmitAudit(s, c, "infra.volume.delete", "do_volume", id, v, nil,
|
||||
audit.Outcome{Result: "failure", Status: 200, Reason: "snapshot failed: " + serr.Error()})
|
||||
return core.Fail(c, "snapshot failed, volume NOT deleted: "+serr.Error())
|
||||
}
|
||||
out["snapshotId"] = shot.ID
|
||||
}
|
||||
if err := s.State.DO.DeleteVolume(c.Context(), id); err != nil {
|
||||
core.EmitAudit(s, c, "infra.volume.delete", "do_volume", id, v, nil,
|
||||
audit.Outcome{Result: "failure", Status: 200, Reason: err.Error()})
|
||||
return core.Fail(c, err.Error())
|
||||
}
|
||||
out["deleted"] = true
|
||||
b.invalidate()
|
||||
core.EmitAudit(s, c, "infra.volume.delete", "do_volume", id, v, out,
|
||||
audit.Outcome{Result: "success", Status: 200})
|
||||
return core.OK(c, out)
|
||||
}
|
||||
|
||||
// cordonNode cordons/uncordons a node, optionally draining it.
|
||||
func (b *board) cordonNode(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
id, err := strconv.Atoi(strings.TrimSpace(c.Param("id")))
|
||||
if err != nil {
|
||||
return core.Fail(c, "node id must be a droplet id")
|
||||
}
|
||||
var body struct {
|
||||
Cordon bool `json:"cordon"`
|
||||
Drain bool `json:"drain"`
|
||||
}
|
||||
if err := c.Bind(&body); err != nil {
|
||||
return core.Fail(c, "invalid body")
|
||||
}
|
||||
snap, err := b.load(c.Context(), s.State.DO, false)
|
||||
if err != nil {
|
||||
return core.Fail(c, err.Error())
|
||||
}
|
||||
var node *Node
|
||||
for i := range snap.Nodes {
|
||||
if snap.Nodes[i].ID == id {
|
||||
node = &snap.Nodes[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if node == nil {
|
||||
return core.Fail(c, "node not found")
|
||||
}
|
||||
if node.ClusterID == "" {
|
||||
return core.Fail(c, "node is not a member of a known cluster")
|
||||
}
|
||||
evicted, err := SetSchedulable(c.Context(), s.State.DO, node.ClusterID, node.Name, !body.Cordon, body.Drain)
|
||||
out := map[string]any{"name": node.Name, "schedulable": !body.Cordon, "evicted": evicted}
|
||||
if err != nil {
|
||||
core.EmitAudit(s, c, "infra.node.cordon", "do_droplet", node.Name, node, out,
|
||||
audit.Outcome{Result: "failure", Status: 200, Reason: err.Error()})
|
||||
return core.Fail(c, err.Error())
|
||||
}
|
||||
b.invalidate()
|
||||
core.EmitAudit(s, c, "infra.node.cordon", "do_droplet", node.Name, node, out,
|
||||
audit.Outcome{Result: "success", Status: 200})
|
||||
return core.OK(c, out)
|
||||
}
|
||||
|
||||
// takeSnapshot names and takes a volume snapshot. A blank name gets a deterministic
|
||||
// pre-delete name so the undo is findable in the DO console.
|
||||
func takeSnapshot(ctx context.Context, do *digitalocean.Client, v Volume, name string) (digitalocean.Snapshot, error) {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
name = fmt.Sprintf("%s-predelete-%d", v.Name, time.Now().Unix())
|
||||
}
|
||||
return do.SnapshotVolume(ctx, v.ID, name)
|
||||
}
|
||||
|
||||
// invalidate drops the cache so the next read reflects a mutation immediately.
|
||||
func (b *board) invalidate() {
|
||||
b.mu.Lock()
|
||||
b.at = time.Time{}
|
||||
b.mu.Unlock()
|
||||
}
|
||||
|
||||
func findVolume(s Snapshot, id string) (Volume, bool) {
|
||||
for _, v := range s.Volumes {
|
||||
if v.ID == id {
|
||||
return v, true
|
||||
}
|
||||
}
|
||||
return Volume{}, false
|
||||
}
|
||||
|
||||
// sortSources keeps the freshness list stable across reads (map/goroutine order is not).
|
||||
func sortSources(rows []core.SourceStatus) {
|
||||
for i := 1; i < len(rows); i++ {
|
||||
for j := i; j > 0 && rows[j].Name < rows[j-1].Name; j-- {
|
||||
rows[j], rows[j-1] = rows[j-1], rows[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
package infra
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/admin/digitalocean"
|
||||
)
|
||||
|
||||
// fakeDO stands in for the DigitalOcean API. kubeAPI is the URL a cluster kubeconfig
|
||||
// points at; when blank, the kubeconfig fetch fails, which is how the incomplete-scan
|
||||
// path is exercised.
|
||||
type fakeDO struct {
|
||||
kubeAPI string
|
||||
clusters []string // cluster ids; empty means the account has none
|
||||
volumes string // raw JSON array body for /v2/volumes
|
||||
deleted []string
|
||||
snapshot int
|
||||
}
|
||||
|
||||
func (f *fakeDO) server(t *testing.T) *digitalocean.Client {
|
||||
t.Helper()
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/v2/droplets", func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprint(w, `{"droplets":[{"id":101,"name":"node-a1","status":"active","size_slug":"s-1vcpu-1gb",
|
||||
"vcpus":1,"memory":1024,"disk":25,"size":{"price_monthly":6},"region":{"slug":"sfo3"},
|
||||
"networks":{"v4":[{"type":"private","ip_address":"10.0.0.1"}]},
|
||||
"tags":["k8s","k8s:`+clusterUUID+`"]}],"meta":{"total":1}}`)
|
||||
})
|
||||
mux.HandleFunc("/v2/volumes", func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprintf(w, `{"volumes":%s,"meta":{"total":2}}`, f.volumes)
|
||||
})
|
||||
mux.HandleFunc("/v2/load_balancers", func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprint(w, `{"load_balancers":[],"meta":{"total":0}}`)
|
||||
})
|
||||
mux.HandleFunc("/v2/kubernetes/clusters", func(w http.ResponseWriter, r *http.Request) {
|
||||
rows := []string{}
|
||||
for _, id := range f.clusters {
|
||||
rows = append(rows, fmt.Sprintf(
|
||||
`{"id":%q,"name":"test-k8s","region":"sfo3","version":"1.35","status":{"state":"running"},"node_pools":[{"name":"p"}]}`, id))
|
||||
}
|
||||
fmt.Fprintf(w, `{"kubernetes_clusters":[%s],"meta":{"total":%d}}`, strings.Join(rows, ","), len(rows))
|
||||
})
|
||||
mux.HandleFunc("/v2/kubernetes/clusters/"+clusterUUID+"/kubeconfig", func(w http.ResponseWriter, r *http.Request) {
|
||||
if f.kubeAPI == "" {
|
||||
http.Error(w, `{"message":"cluster unreachable"}`, http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(w, `apiVersion: v1
|
||||
kind: Config
|
||||
clusters: [{name: c, cluster: {server: %s, insecure-skip-tls-verify: true}}]
|
||||
users: [{name: u, user: {token: t}}]
|
||||
contexts: [{name: x, context: {cluster: c, user: u}}]
|
||||
current-context: x
|
||||
`, f.kubeAPI)
|
||||
})
|
||||
mux.HandleFunc("/v2/volumes/", func(w http.ResponseWriter, r *http.Request) {
|
||||
id := strings.TrimPrefix(r.URL.Path, "/v2/volumes/")
|
||||
switch {
|
||||
case strings.HasSuffix(id, "/snapshots"):
|
||||
f.snapshot++
|
||||
fmt.Fprint(w, `{"snapshot":{"id":"snap-1","name":"s","size_gigabytes":40}}`)
|
||||
case r.Method == http.MethodDelete:
|
||||
f.deleted = append(f.deleted, id)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
default:
|
||||
http.Error(w, `{"message":"nope"}`, http.StatusNotFound)
|
||||
}
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
t.Cleanup(srv.Close)
|
||||
return digitalocean.NewWithBase(srv.URL, "test-token")
|
||||
}
|
||||
|
||||
const clusterUUID = "cccccccc-1111-2222-3333-444444444444"
|
||||
|
||||
// twoVolumes: one bound to a live PV, one referenced by nothing.
|
||||
const twoVolumes = `[
|
||||
{"id":"vol-live","name":"live","size_gigabytes":20,"region":{"slug":"sfo3"},"droplet_ids":[],"tags":["k8s:` + clusterUUID + `"]},
|
||||
{"id":"vol-junk","name":"junk","size_gigabytes":40,"region":{"slug":"sfo3"},"droplet_ids":[],"tags":["k8s:` + clusterUUID + `"]}
|
||||
]`
|
||||
|
||||
// fakeAPIServer serves the four core/v1 collections scanOne reads.
|
||||
func fakeAPIServer(t *testing.T) string {
|
||||
t.Helper()
|
||||
t.Setenv("FLEET_ALLOW_PRIVATE_HOSTS", "1")
|
||||
list := func(kind string, items any) string {
|
||||
b, _ := json.Marshal(items)
|
||||
return fmt.Sprintf(`{"apiVersion":"v1","kind":%q,"metadata":{},"items":%s}`, kind, b)
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/api/v1/persistentvolumes", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, list("PersistentVolumeList", []map[string]any{{
|
||||
"metadata": map[string]any{"name": "pv-live"},
|
||||
"spec": map[string]any{
|
||||
"csi": map[string]any{"driver": "dobs.csi.digitalocean.com", "volumeHandle": "vol-live"},
|
||||
"claimRef": map[string]any{"namespace": "db", "name": "data"},
|
||||
},
|
||||
"status": map[string]any{"phase": "Bound"},
|
||||
}}))
|
||||
})
|
||||
mux.HandleFunc("/api/v1/persistentvolumeclaims", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, list("PersistentVolumeClaimList", []map[string]any{{
|
||||
"metadata": map[string]any{"namespace": "db", "name": "data"},
|
||||
"spec": map[string]any{"volumeName": "pv-live"},
|
||||
"status": map[string]any{"phase": "Bound"},
|
||||
}}))
|
||||
})
|
||||
mux.HandleFunc("/api/v1/pods", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, list("PodList", []map[string]any{{
|
||||
"metadata": map[string]any{"namespace": "db", "name": "pg-0"},
|
||||
"spec": map[string]any{
|
||||
"nodeName": "node-a1",
|
||||
"containers": []map[string]any{{"name": "c", "image": "ghcr.io/hanzoai/base:v1"}},
|
||||
"volumes": []map[string]any{{"name": "v", "persistentVolumeClaim": map[string]any{"claimName": "data"}}},
|
||||
},
|
||||
"status": map[string]any{"phase": "Running"},
|
||||
}}))
|
||||
})
|
||||
mux.HandleFunc("/api/v1/nodes", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, list("NodeList", []map[string]any{{
|
||||
"metadata": map[string]any{"name": "node-a1"},
|
||||
"spec": map[string]any{},
|
||||
"status": map[string]any{"conditions": []map[string]any{{"type": "Ready", "status": "True"}}},
|
||||
}}))
|
||||
})
|
||||
srv := httptest.NewTLSServer(mux)
|
||||
t.Cleanup(srv.Close)
|
||||
return srv.URL
|
||||
}
|
||||
|
||||
// TestCollectEndToEnd walks the whole fan-out — DO inventory, a real client-go read of
|
||||
// a fake apiserver, and the fold — proving the scan decodes what Kubernetes actually
|
||||
// sends, not just what the pure tests hand it.
|
||||
func TestCollectEndToEnd(t *testing.T) {
|
||||
f := &fakeDO{kubeAPI: fakeAPIServer(t), clusters: []string{clusterUUID}, volumes: twoVolumes}
|
||||
snap, err := collect(context.Background(), f.server(t))
|
||||
if err != nil {
|
||||
t.Fatalf("collect: %v", err)
|
||||
}
|
||||
if !snap.Complete {
|
||||
t.Fatalf("Complete = false: %s", snap.IncompleteReason)
|
||||
}
|
||||
live, _ := findVolume(snap, "vol-live")
|
||||
if live.State != StateBound || live.Deletable {
|
||||
t.Errorf("vol-live = %s deletable=%v, want bound + not deletable", live.State, live.Deletable)
|
||||
}
|
||||
if live.PV != "pv-live" || live.PVCName != "data" {
|
||||
t.Errorf("vol-live PV binding not decoded: %+v", live)
|
||||
}
|
||||
if len(live.MountedBy) != 1 || live.MountedBy[0] != "db/pg-0" {
|
||||
t.Errorf("vol-live mountedBy = %v, want [db/pg-0]", live.MountedBy)
|
||||
}
|
||||
junk, _ := findVolume(snap, "vol-junk")
|
||||
if junk.State != StateUnreferenced || !junk.Deletable {
|
||||
t.Errorf("vol-junk = %s deletable=%v, want unreferenced + deletable", junk.State, junk.Deletable)
|
||||
}
|
||||
if snap.Cost.ReclaimableMonthly != 400 {
|
||||
t.Errorf("reclaimable = %d, want 400 (40 GiB)", snap.Cost.ReclaimableMonthly)
|
||||
}
|
||||
if n := snap.Nodes[0]; !n.Ready || n.Pods != 1 || n.Cluster != "test-k8s" {
|
||||
t.Errorf("node join wrong: %+v", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnreachableClusterFreezesEverything: the cluster exists but will not answer, so
|
||||
// the volume no reachable PV references must STILL be undeletable.
|
||||
func TestUnreachableClusterFreezesEverything(t *testing.T) {
|
||||
f := &fakeDO{kubeAPI: "", clusters: []string{clusterUUID}, volumes: twoVolumes}
|
||||
snap, err := collect(context.Background(), f.server(t))
|
||||
if err != nil {
|
||||
t.Fatalf("collect: %v", err)
|
||||
}
|
||||
if snap.Complete {
|
||||
t.Fatal("Complete = true with an unreachable cluster")
|
||||
}
|
||||
for _, id := range []string{"vol-live", "vol-junk"} {
|
||||
v, _ := findVolume(snap, id)
|
||||
if v.Deletable {
|
||||
t.Fatalf("%s deletable despite an unreachable cluster", id)
|
||||
}
|
||||
}
|
||||
if snap.Cost.ReclaimableMonthly != 0 {
|
||||
t.Errorf("reclaimable = %d, want 0", snap.Cost.ReclaimableMonthly)
|
||||
}
|
||||
// The failure must be named in Sources, not swallowed.
|
||||
var found bool
|
||||
for _, s := range snap.Sources {
|
||||
if s.Name == "k8s.test-k8s" && !s.OK && s.Error != "" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("cluster failure absent from sources: %+v", snap.Sources)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeleteRefusesNonDeletable proves the server never trusts the caller: asking to
|
||||
// delete a live volume is refused with a reason, and NOTHING is deleted upstream.
|
||||
func TestDeleteRefusesNonDeletable(t *testing.T) {
|
||||
f := &fakeDO{kubeAPI: fakeAPIServer(t), clusters: []string{clusterUUID}, volumes: twoVolumes}
|
||||
do := f.server(t)
|
||||
b := &board{}
|
||||
snap, err := b.load(context.Background(), do, true)
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
live, _ := findVolume(snap, "vol-live")
|
||||
if live.Deletable {
|
||||
t.Fatal("fixture wrong: vol-live must not be deletable")
|
||||
}
|
||||
if live.BlockedReason == "" {
|
||||
t.Error("no blockedReason for a live volume")
|
||||
}
|
||||
if len(f.deleted) != 0 {
|
||||
t.Fatalf("volumes deleted during a read: %v", f.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSnapshotThenDelete proves the undo exists: deleting takes a snapshot first.
|
||||
func TestSnapshotThenDelete(t *testing.T) {
|
||||
f := &fakeDO{kubeAPI: fakeAPIServer(t), clusters: []string{clusterUUID}, volumes: twoVolumes}
|
||||
do := f.server(t)
|
||||
snap, err := collect(context.Background(), do)
|
||||
if err != nil {
|
||||
t.Fatalf("collect: %v", err)
|
||||
}
|
||||
junk, _ := findVolume(snap, "vol-junk")
|
||||
if _, err := takeSnapshot(context.Background(), do, junk, ""); err != nil {
|
||||
t.Fatalf("snapshot: %v", err)
|
||||
}
|
||||
if f.snapshot != 1 {
|
||||
t.Fatalf("snapshots taken = %d, want 1", f.snapshot)
|
||||
}
|
||||
if err := do.DeleteVolume(context.Background(), junk.ID); err != nil {
|
||||
t.Fatalf("delete: %v", err)
|
||||
}
|
||||
if len(f.deleted) != 1 || f.deleted[0] != "vol-junk" {
|
||||
t.Fatalf("deleted = %v, want [vol-junk]", f.deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNoTokenIsHonest: an unconfigured deployment says so instead of rendering an
|
||||
// empty fleet that looks like a clean account.
|
||||
func TestNoTokenIsHonest(t *testing.T) {
|
||||
_, err := collect(context.Background(), digitalocean.New(""))
|
||||
if err == nil || !strings.Contains(err.Error(), "DO_API_TOKEN") {
|
||||
t.Fatalf("err = %v, want an explicit not-configured error", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package infra
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/admin/digitalocean"
|
||||
)
|
||||
|
||||
// TestLiveCollect runs the real fan-out against the real DigitalOcean account and the
|
||||
// real clusters. It is the only test that can prove the orphan analysis agrees with
|
||||
// production, so it is kept — but it is SKIPPED unless DO_API_TOKEN is present, which
|
||||
// is never the case in CI or on a dev box that has not opted in.
|
||||
//
|
||||
// DO_API_TOKEN=$(…) go test ./clients/admin/infra/ -run TestLiveCollect -v
|
||||
//
|
||||
// It asserts invariants, not fixed counts: the fleet changes, but "every cluster
|
||||
// answered", "every volume got a state", and "only unreferenced volumes are deletable"
|
||||
// must hold on every run, forever.
|
||||
func TestLiveCollect(t *testing.T) {
|
||||
token := os.Getenv("DO_API_TOKEN")
|
||||
if token == "" {
|
||||
t.Skip("DO_API_TOKEN not set — live fleet test skipped")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
snap, err := collect(ctx, digitalocean.New(token))
|
||||
if err != nil {
|
||||
t.Fatalf("collect: %v", err)
|
||||
}
|
||||
|
||||
byState := map[string]int{}
|
||||
gibByState := map[string]int{}
|
||||
for _, v := range snap.Volumes {
|
||||
byState[v.State]++
|
||||
gibByState[v.State] += v.SizeGiB
|
||||
}
|
||||
t.Logf("clusters=%d nodes=%d volumes=%d loadBalancers=%d complete=%v",
|
||||
snap.Totals.Clusters, snap.Totals.Nodes, snap.Totals.Volumes,
|
||||
snap.Totals.LoadBalancers, snap.Complete)
|
||||
for _, st := range []string{StateAttached, StateBound, StateReleased, StateUnreferenced} {
|
||||
t.Logf(" %-14s %4d volumes %7.2f TiB", st, byState[st], float64(gibByState[st])/1024)
|
||||
}
|
||||
t.Logf("local disk (INCLUDED in droplet price, not separately billed): %.2f TiB",
|
||||
float64(snap.Totals.LocalDiskGiB)/1024)
|
||||
t.Logf("cost/mo: droplets $%.2f volumes $%.2f lbs $%.2f TOTAL $%.2f reclaimable $%.2f",
|
||||
float64(snap.Cost.DropletsMonthly)/100, float64(snap.Cost.VolumesMonthly)/100,
|
||||
float64(snap.Cost.LoadBalancersMonthly)/100, float64(snap.Cost.TotalMonthly)/100,
|
||||
float64(snap.Cost.ReclaimableMonthly)/100)
|
||||
for _, c := range snap.Clusters {
|
||||
t.Logf(" %-16s nodes=%-3d pods=%-4d pvs=%-4d pvcs=%-4d idle=%-3d scanned=%v %s",
|
||||
c.Name, c.Nodes, c.Pods, c.PVs, c.PVCs, c.IdlePVCs, c.Scanned, c.ScanError)
|
||||
}
|
||||
var deletable []Volume
|
||||
for _, v := range snap.Volumes {
|
||||
if v.Deletable {
|
||||
deletable = append(deletable, v)
|
||||
}
|
||||
}
|
||||
t.Logf("DELETABLE: %d volumes", len(deletable))
|
||||
for _, v := range deletable {
|
||||
t.Logf(" %s %-40s %4d GiB $%.2f/mo", v.ID, v.Name, v.SizeGiB, float64(v.MonthlyCents)/100)
|
||||
}
|
||||
|
||||
// ---- invariants --------------------------------------------------------------
|
||||
if !snap.Complete {
|
||||
t.Fatalf("scan incomplete, so no verdict is trustworthy: %s", snap.IncompleteReason)
|
||||
}
|
||||
if snap.Totals.Clusters == 0 || snap.Totals.Nodes == 0 || snap.Totals.Volumes == 0 {
|
||||
t.Fatal("empty inventory from a live account")
|
||||
}
|
||||
for _, v := range snap.Volumes {
|
||||
if v.State == "" {
|
||||
t.Fatalf("volume %s has no state", v.ID)
|
||||
}
|
||||
if v.Deletable != (v.State == StateUnreferenced) {
|
||||
t.Fatalf("volume %s: deletable=%v but state=%s — only unreferenced volumes may be deletable",
|
||||
v.ID, v.Deletable, v.State)
|
||||
}
|
||||
if v.Deletable && len(v.DropletIDs) > 0 {
|
||||
t.Fatalf("volume %s is deletable while attached to %v", v.ID, v.DropletIDs)
|
||||
}
|
||||
if v.Deletable && v.PV != "" {
|
||||
t.Fatalf("volume %s is deletable while PV %s references it", v.ID, v.PV)
|
||||
}
|
||||
}
|
||||
// Every attached volume must sit on a droplet we actually enumerated, or the
|
||||
// attachment join is broken and cost attribution is wrong.
|
||||
nodes := map[int]bool{}
|
||||
for _, n := range snap.Nodes {
|
||||
nodes[n.ID] = true
|
||||
}
|
||||
for _, v := range snap.Volumes {
|
||||
for _, id := range v.DropletIDs {
|
||||
if !nodes[id] {
|
||||
t.Errorf("volume %s attached to unknown droplet %d", v.ID, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
if int64(snap.Cost.ReclaimableMonthly) != sumCents(deletable) {
|
||||
t.Errorf("reclaimable %d != sum of deletable volumes %d", snap.Cost.ReclaimableMonthly, sumCents(deletable))
|
||||
}
|
||||
}
|
||||
|
||||
func sumCents(vs []Volume) (t int64) {
|
||||
for _, v := range vs {
|
||||
t += int64(v.MonthlyCents)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
package infra
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
policyv1 "k8s.io/api/policy/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/admin/core"
|
||||
"github.com/hanzoai/cloud/clients/admin/digitalocean"
|
||||
"github.com/hanzoai/cloud/clients/fleet"
|
||||
)
|
||||
|
||||
// clusterScanTimeout bounds ONE cluster's read. A cluster that exceeds it is recorded
|
||||
// as unreachable, which fails the completeness gate — the safe direction.
|
||||
const clusterScanTimeout = 45 * time.Second
|
||||
|
||||
// kube opens an authenticated client for one DOKS cluster. DO hands back a
|
||||
// token-based kubeconfig against the cluster's public https endpoint; it still goes
|
||||
// through fleet.SafeRESTConfig, the ONE gate that rejects exec-credential plugins and
|
||||
// non-routable apiserver hosts, so this path cannot be turned into an RCE or an SSRF.
|
||||
func kube(ctx context.Context, do *digitalocean.Client, clusterID string) (*kubernetes.Clientset, error) {
|
||||
raw, err := do.Kubeconfig(ctx, clusterID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kubeconfig: %w", err)
|
||||
}
|
||||
cfg, err := fleet.SafeRESTConfig(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.Timeout = clusterScanTimeout
|
||||
return kubernetes.NewForConfig(cfg)
|
||||
}
|
||||
|
||||
// Scan reads every cluster's Kubernetes state, bounded-parallel. It ALWAYS returns
|
||||
// one row per cluster: a cluster that failed comes back with Err set rather than
|
||||
// being omitted, because a missing row and a healthy row must never be confusable —
|
||||
// that confusion is exactly what would condemn live data.
|
||||
func Scan(ctx context.Context, do *digitalocean.Client, clusters []digitalocean.Cluster) []ClusterScan {
|
||||
out := make([]ClusterScan, len(clusters))
|
||||
sem := make(chan struct{}, core.MaxCustomerConcurrency)
|
||||
var wg sync.WaitGroup
|
||||
for i, c := range clusters {
|
||||
wg.Add(1)
|
||||
go func(i int, c digitalocean.Cluster) {
|
||||
defer wg.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
cctx, cancel := context.WithTimeout(ctx, clusterScanTimeout)
|
||||
defer cancel()
|
||||
out[i] = scanOne(cctx, do, c.ID)
|
||||
}(i, c)
|
||||
}
|
||||
wg.Wait()
|
||||
return out
|
||||
}
|
||||
|
||||
// scanOne reads one cluster. Any error short-circuits with Err set.
|
||||
func scanOne(ctx context.Context, do *digitalocean.Client, clusterID string) ClusterScan {
|
||||
s := ClusterScan{ClusterID: clusterID}
|
||||
cs, err := kube(ctx, do, clusterID)
|
||||
if err != nil {
|
||||
s.Err = err
|
||||
return s
|
||||
}
|
||||
|
||||
pvs, err := cs.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
s.Err = fmt.Errorf("list persistentvolumes: %w", err)
|
||||
return s
|
||||
}
|
||||
for _, pv := range pvs.Items {
|
||||
s.PVs = append(s.PVs, PVRef{
|
||||
Name: pv.Name,
|
||||
Phase: string(pv.Status.Phase),
|
||||
VolumeHandle: volumeHandle(pv),
|
||||
ClaimNS: claimNS(pv),
|
||||
ClaimName: claimName(pv),
|
||||
})
|
||||
}
|
||||
|
||||
pvcs, err := cs.CoreV1().PersistentVolumeClaims(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
s.Err = fmt.Errorf("list persistentvolumeclaims: %w", err)
|
||||
return s
|
||||
}
|
||||
for _, p := range pvcs.Items {
|
||||
s.PVCs = append(s.PVCs, PVCRef{
|
||||
Namespace: p.Namespace, Name: p.Name,
|
||||
Phase: string(p.Status.Phase), Volume: p.Spec.VolumeName,
|
||||
})
|
||||
}
|
||||
|
||||
pods, err := cs.CoreV1().Pods(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
s.Err = fmt.Errorf("list pods: %w", err)
|
||||
return s
|
||||
}
|
||||
for _, p := range pods.Items {
|
||||
s.Pods = append(s.Pods, podRefOf(p))
|
||||
}
|
||||
|
||||
nodes, err := cs.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
|
||||
if err != nil {
|
||||
s.Err = fmt.Errorf("list nodes: %w", err)
|
||||
return s
|
||||
}
|
||||
for _, n := range nodes.Items {
|
||||
s.Nodes = append(s.Nodes, NodeState{
|
||||
Name: n.Name,
|
||||
Ready: nodeReady(n),
|
||||
Schedulable: !n.Spec.Unschedulable,
|
||||
})
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// volumeHandle extracts the backing DO volume ID a PV claims. Deliberately NOT
|
||||
// filtered by CSI driver name: matching broadly means MORE volumes are treated as
|
||||
// in-use, which is the safe direction. The legacy flexVolume shape is read too, so a
|
||||
// pre-CSI PV still protects its volume.
|
||||
func volumeHandle(pv corev1.PersistentVolume) string {
|
||||
if pv.Spec.CSI != nil && strings.TrimSpace(pv.Spec.CSI.VolumeHandle) != "" {
|
||||
return pv.Spec.CSI.VolumeHandle
|
||||
}
|
||||
if pv.Spec.FlexVolume != nil {
|
||||
if v := strings.TrimSpace(pv.Spec.FlexVolume.Options["volumeID"]); v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func claimNS(pv corev1.PersistentVolume) string {
|
||||
if pv.Spec.ClaimRef == nil {
|
||||
return ""
|
||||
}
|
||||
return pv.Spec.ClaimRef.Namespace
|
||||
}
|
||||
|
||||
func claimName(pv corev1.PersistentVolume) string {
|
||||
if pv.Spec.ClaimRef == nil {
|
||||
return ""
|
||||
}
|
||||
return pv.Spec.ClaimRef.Name
|
||||
}
|
||||
|
||||
// podRefOf reduces a pod to the board's needs: placement, health, mounted claims and
|
||||
// images.
|
||||
func podRefOf(p corev1.Pod) PodRef {
|
||||
r := PodRef{
|
||||
Namespace: p.Namespace, Name: p.Name,
|
||||
Phase: string(p.Status.Phase), Reason: p.Status.Reason, Node: p.Spec.NodeName,
|
||||
}
|
||||
for _, v := range p.Spec.Volumes {
|
||||
if v.PersistentVolumeClaim != nil {
|
||||
r.Claims = append(r.Claims, v.PersistentVolumeClaim.ClaimName)
|
||||
}
|
||||
}
|
||||
for _, c := range p.Spec.InitContainers {
|
||||
r.Images = append(r.Images, c.Image)
|
||||
}
|
||||
for _, c := range p.Spec.Containers {
|
||||
r.Images = append(r.Images, c.Image)
|
||||
}
|
||||
// A waiting container's reason (CrashLoopBackOff/ImagePullBackOff) is the real
|
||||
// health signal; pod.status.reason stays empty for those.
|
||||
for _, cs := range p.Status.ContainerStatuses {
|
||||
if cs.State.Waiting != nil && cs.State.Waiting.Reason != "" && r.Reason == "" {
|
||||
r.Reason = cs.State.Waiting.Reason
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func nodeReady(n corev1.Node) bool {
|
||||
for _, c := range n.Status.Conditions {
|
||||
if c.Type == corev1.NodeReady {
|
||||
return c.Status == corev1.ConditionTrue
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SetSchedulable cordons or uncordons a node, optionally draining it. Returns the
|
||||
// number of pods evicted.
|
||||
//
|
||||
// Drain uses the Eviction API, not delete: eviction respects PodDisruptionBudgets, so
|
||||
// a drain that would break a quorum is REFUSED by the apiserver rather than silently
|
||||
// taking a service down. DaemonSet and mirror pods are skipped — they are rescheduled
|
||||
// onto the same node by definition and evicting them is a no-op loop.
|
||||
func SetSchedulable(ctx context.Context, do *digitalocean.Client, clusterID, node string, schedulable, drain bool) (int, error) {
|
||||
cs, err := kube(ctx, do, clusterID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
patch := fmt.Sprintf(`{"spec":{"unschedulable":%t}}`, !schedulable)
|
||||
if _, err := cs.CoreV1().Nodes().Patch(ctx, node, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil {
|
||||
return 0, fmt.Errorf("cordon %s: %w", node, err)
|
||||
}
|
||||
if schedulable || !drain {
|
||||
return 0, nil
|
||||
}
|
||||
pods, err := cs.CoreV1().Pods(metav1.NamespaceAll).List(ctx, metav1.ListOptions{
|
||||
FieldSelector: "spec.nodeName=" + node,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("list pods on %s: %w", node, err)
|
||||
}
|
||||
evicted := 0
|
||||
for _, p := range pods.Items {
|
||||
if skipEviction(p) {
|
||||
continue
|
||||
}
|
||||
ev := &policyv1.Eviction{ObjectMeta: metav1.ObjectMeta{Namespace: p.Namespace, Name: p.Name}}
|
||||
if err := cs.CoreV1().Pods(p.Namespace).EvictV1(ctx, ev); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
continue
|
||||
}
|
||||
// A PDB refusal is the system working. Report it verbatim; the node stays
|
||||
// cordoned, so the operator can retry after scaling.
|
||||
return evicted, fmt.Errorf("evict %s/%s: %w", p.Namespace, p.Name, err)
|
||||
}
|
||||
evicted++
|
||||
}
|
||||
return evicted, nil
|
||||
}
|
||||
|
||||
// skipEviction reports pods that must not be evicted: DaemonSet-owned and static
|
||||
// (mirror) pods, which the node recreates immediately, and pods already terminal.
|
||||
func skipEviction(p corev1.Pod) bool {
|
||||
if _, mirror := p.Annotations[corev1.MirrorPodAnnotationKey]; mirror {
|
||||
return true
|
||||
}
|
||||
for _, o := range p.OwnerReferences {
|
||||
if o.Kind == "DaemonSet" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return p.Status.Phase == corev1.PodSucceeded || p.Status.Phase == corev1.PodFailed
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package admin
|
||||
|
||||
// products.go — GET /v1/admin/products, the fleet workload registry the operator
|
||||
// Infrastructure board (admin.hanzo.ai) renders: every operator App CR across the platform
|
||||
// namespaces with its declared vs running image tag, operator-reconciled health/phase, and
|
||||
// the drift verdict.
|
||||
//
|
||||
// SOURCE — reuse, never fork. The inventory is the SAME observation the native PaaS control
|
||||
// plane already computes for /v1/paas/apps (clients/paas: observeFleet → observeCR →
|
||||
// drift.go, one k8s dynamic client, one drift model). paas publishes it as an in-process
|
||||
// seam (paas.CurrentFleet, fleet.go); admin RESOLVES that seam and projects each AppView
|
||||
// onto the productRow the SPA decodes. There is no second k8s client and no second drift
|
||||
// definition — the admin board and the PaaS board can never disagree about what the fleet is
|
||||
// or what "drift" means. When the PaaS plane is not co-resident, or its k8s client did not
|
||||
// resolve, the registry is honestly empty — never a fabricated row.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/clients/admin/core"
|
||||
"github.com/hanzoai/cloud/clients/paas"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// products answers GET /v1/admin/products — the workload registry, optionally narrowed by
|
||||
// ?tier=, ?kind= (operator role), or ?env=. SuperAdmin only (core.Guard on the route).
|
||||
func products(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
rows, _, err := fleetProducts(c.Context())
|
||||
if err != nil {
|
||||
return core.Fail(c, err.Error())
|
||||
}
|
||||
kind := strings.TrimSpace(c.Query("kind"))
|
||||
tier := strings.TrimSpace(c.Query("tier"))
|
||||
env := strings.TrimSpace(c.Query("env"))
|
||||
out := make([]productRow, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
if kind != "" && r.Kind != kind {
|
||||
continue
|
||||
}
|
||||
if tier != "" && r.Tier != tier {
|
||||
continue
|
||||
}
|
||||
if env != "" && r.Env != env {
|
||||
continue
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return core.OKList(c, out, len(out))
|
||||
}
|
||||
|
||||
// productRollup is the fleet count the overview KPIs fold: total observed workloads, how many
|
||||
// are healthy (green), and how many are drifting.
|
||||
type productRollup struct{ Total, Active, Drift int }
|
||||
|
||||
// fleetProducts observes the platform fleet through the paas seam and projects it onto the
|
||||
// productRow board shape, returning the rows plus the rollup the overview KPIs read. A nil
|
||||
// seam (PaaS not co-resident) or an unready k8s client yields an honest-empty registry with a
|
||||
// nil error, so both the board and the KPIs degrade to empty rather than failing; only a hard
|
||||
// observation error (e.g. an RBAC denial listing apps.hanzo.ai) surfaces as an error.
|
||||
func fleetProducts(ctx context.Context) ([]productRow, productRollup, error) {
|
||||
fleet := paas.CurrentFleet()
|
||||
if fleet == nil {
|
||||
return []productRow{}, productRollup{}, nil
|
||||
}
|
||||
if ok, _ := fleet.Ready(); !ok {
|
||||
return []productRow{}, productRollup{}, nil
|
||||
}
|
||||
views, err := fleet.Observe(ctx)
|
||||
if err != nil {
|
||||
return nil, productRollup{}, err
|
||||
}
|
||||
rows := make([]productRow, 0, len(views))
|
||||
var roll productRollup
|
||||
for _, v := range views {
|
||||
r := productFromView(v)
|
||||
rows = append(rows, r)
|
||||
roll.Total++
|
||||
if r.Health == "green" {
|
||||
roll.Active++
|
||||
}
|
||||
if r.Drift {
|
||||
roll.Drift++
|
||||
}
|
||||
}
|
||||
return rows, roll, nil
|
||||
}
|
||||
|
||||
// productFromView projects a paas fleet AppView onto a productRow: the declared/running tags
|
||||
// + operator-reconciled health/phase verbatim, the drift verdict rolled to a boolean +
|
||||
// severity, and the derived infra tier for the board's grouping.
|
||||
func productFromView(v paas.AppView) productRow {
|
||||
return productRow{
|
||||
Name: v.App,
|
||||
Kind: v.Role, // the operator's OWN declared class (sql|kv|generic|ingress) or ""
|
||||
Tier: tierOf(v),
|
||||
Org: v.Org,
|
||||
Cluster: v.Cluster,
|
||||
Env: v.Env,
|
||||
Namespace: v.Namespace,
|
||||
Repo: v.Repo,
|
||||
Phase: v.Phase,
|
||||
DeclaredTag: v.DeclaredTag,
|
||||
RunningTag: v.RunningTag,
|
||||
LatestTag: v.LatestTag,
|
||||
Health: healthLabel(v.Health),
|
||||
Drift: v.Drift.Severity != paas.SeverityOK,
|
||||
DriftSeverity: string(v.Drift.Severity),
|
||||
Updated: "", // the CR carries no per-row reconcile timestamp; observation is live
|
||||
}
|
||||
}
|
||||
|
||||
// tierOf classifies a fleet workload into an infra TIER for the operator board's grouping
|
||||
// (cloud roles · managed-DB ring · edge · external daemons · PaaS deployments · general apps).
|
||||
//
|
||||
// HONESTY: this is a cloud-side DERIVATION over REAL fields — the operator App CR's declared
|
||||
// role, its namespace, and the image-repo family (what the workload actually RUNS) — NOT an
|
||||
// operator-declared tier. The operator does not label a tier today (only spec.role, and only
|
||||
// for sql/kv/generic/ingress), so the board groups on this derivation. A declarative
|
||||
// `hanzo.ai/tier` label on the App CRs would make it authoritative — a universe/operator
|
||||
// follow-up; until then this stays the single, documented classifier (one place, no fork).
|
||||
func tierOf(v paas.AppView) string {
|
||||
// A workload in a tenant namespace is a customer / PaaS deployment, not platform infra.
|
||||
// (Today the paas observer scans only the platform namespaces, so this is future-proofing
|
||||
// for when the scan federates tenant/other clusters.)
|
||||
if isTenantNamespace(v.Namespace) {
|
||||
return "paas"
|
||||
}
|
||||
// The operator's OWN declared role is authoritative when present.
|
||||
switch v.Role {
|
||||
case "sql", "kv":
|
||||
return "data"
|
||||
case "ingress":
|
||||
return "edge"
|
||||
}
|
||||
// Otherwise classify by the image-repo family — a real, stable property of the workload.
|
||||
switch repoName(v.Registry) {
|
||||
case "cloud":
|
||||
return "cloud" // the node-specialized cloud binary roles (cloud / cloud-reader / canary)
|
||||
case "sql", "kv", "datastore", "vector", "search", "search-fts5", "s3", "registry", "superbase", "base":
|
||||
return "data" // the managed-DB / storage ring
|
||||
case "ingress", "dns", "static":
|
||||
return "edge" // the ingress / DNS / static edge
|
||||
case "arcbuild", "o11y", "visor", "livekit", "git", "mpc", "zt", "analytics", "analytics-collector":
|
||||
return "daemon" // external daemons (build / observability / realtime / vcs / mpc / zt)
|
||||
}
|
||||
return "app" // a general platform service (chat, iam, console, engine, …)
|
||||
}
|
||||
|
||||
// healthLabel maps the paas health vocabulary ("" ⇒ unknown) onto the ProductHealth the SPA
|
||||
// decodes (green|yellow|red|unknown) — an unknown health is honest, never a fabricated green.
|
||||
func healthLabel(h string) string {
|
||||
if strings.TrimSpace(h) == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// repoName returns the final path segment of an image repository
|
||||
// (ghcr.io/hanzoai/cloud → cloud; docker.io/getmeili/meilisearch → meilisearch), the family
|
||||
// key tierOf classifies on.
|
||||
func repoName(registry string) string {
|
||||
registry = strings.TrimSpace(registry)
|
||||
if i := strings.LastIndex(registry, "/"); i >= 0 {
|
||||
return registry[i+1:]
|
||||
}
|
||||
return registry
|
||||
}
|
||||
|
||||
// isTenantNamespace reports whether ns is OUTSIDE the platform tier (hanzo/-testnet/-devnet) —
|
||||
// i.e. a customer / PaaS-tenant namespace. Kept in lockstep with the paas scanOrder.
|
||||
func isTenantNamespace(ns string) bool {
|
||||
switch strings.TrimSpace(ns) {
|
||||
case "hanzo", "hanzo-testnet", "hanzo-devnet", "":
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -134,22 +134,20 @@ func Revenue(s *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
}
|
||||
|
||||
// revenueOf reads one org's money view (balance + spend + plan/MRR). Returns (row, ok):
|
||||
// ok is false when the spend OR balance read failed, so the caller can mark the fleet
|
||||
// total PARTIAL rather than presenting an undercount as complete.
|
||||
// ok is false when the balance/spend read failed, so the caller can mark the fleet total
|
||||
// PARTIAL rather than presenting an undercount as complete.
|
||||
//
|
||||
// Balance + spend come from the ONE per-org money read (core.OrgMoney): co-resident that
|
||||
// reads cloud's native finance wallet — the SAME balances the ai gate enforces and the
|
||||
// overview folds — so the revenue board no longer reads the money source as down when
|
||||
// commerce is in-process. MRR stays a subscriptions read (a separate endpoint); its
|
||||
// absence degrades the row to pay-as-you-go and never marks the money source down.
|
||||
func revenueOf(s *cloud.Service[core.State], ctx context.Context, o iam.Org) (RevenueCustomer, bool) {
|
||||
row := RevenueCustomer{Org: o.Name, Display: core.Display(o.DisplayName, o.Name), Plan: "pay-as-you-go"}
|
||||
ok := true
|
||||
|
||||
if sp, err := s.State.Commerce.Spend(ctx, o.Name); err == nil {
|
||||
row.SpendCents = int64(sp.Consumed)
|
||||
} else {
|
||||
ok = false
|
||||
}
|
||||
if credits, err := s.State.Commerce.Credits(ctx, o.Name); err == nil {
|
||||
row.BalanceCents = int64(credits)
|
||||
} else {
|
||||
ok = false
|
||||
}
|
||||
spend, balance, ok := core.OrgMoney(s, ctx, o.Name)
|
||||
row.SpendCents = spend
|
||||
row.BalanceCents = balance
|
||||
if pl, err := s.State.Commerce.Plan(ctx, o.Name); err == nil {
|
||||
row.MRRCents = int64(pl.MRR)
|
||||
row.Plan = pl.Name
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
package admin
|
||||
|
||||
// Brand-named tenant-scope NEGATIVE tests for admin.<brand> (the operator cockpit).
|
||||
//
|
||||
// These make the generic escalation-line invariant concrete for the real brands the
|
||||
// ONE shared cockpit binary serves: an ADMITTED Lux white-label tenant admin
|
||||
// (z@lux.network, org=lux — the identity IAM mints on admin.lux.cloud) is HARD-PINNED to
|
||||
// the Lux subtree and can NEVER reach another brand's data (Zoo, Hanzo) or a fleet
|
||||
// DigitalOcean god-view. A cross-brand read here would break white-label isolation — the
|
||||
// exact thing admin.lux.cloud must guarantee for every tenant.
|
||||
//
|
||||
// Lux is ENABLED as a white-label tenant in each test (s.State.WLTenants), so the caller
|
||||
// is ADMITTED past GuardScoped — the denials below are the SCOPE CEILING (data hard-
|
||||
// limited to the caller's own org), not an admission refusal. Admission-refusal for a
|
||||
// NON-enabled org is covered by TestScope_NonWhiteLabelOrgAdminDenied.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// luxAdminHdr is a validated Lux-tenant org admin: a pinned own-org (lux) + the
|
||||
// unforgeable X-User-IsOrgAdmin bit SanitizeIdentity mints for a validated isAdmin
|
||||
// principal, but NO global X-User-IsAdmin (SuperAdmin) flag — exactly what the boundary
|
||||
// presents for z@lux.network signing into the cockpit at admin.lux.cloud.
|
||||
var luxAdminHdr = map[string]string{
|
||||
"X-Org-Id": "lux", "X-User-Id": "lux/z", "X-User-Email": "z@lux.network", "X-User-IsOrgAdmin": "true",
|
||||
}
|
||||
|
||||
// TestScope_LuxAdminCannotReachZooOrgs proves the cross-brand escalation line: a Lux
|
||||
// tenant admin who explicitly asks for Zoo's org data (?org=zoo) is hard-pinned to their
|
||||
// OWN subtree — the response is EXACTLY [lux], never zoo. The client-supplied ?org= is
|
||||
// ignored for a non-super caller; the scope is the sanitized org.
|
||||
func TestScope_LuxAdminCannotReachZooOrgs(t *testing.T) {
|
||||
iam := newScopeIAM()
|
||||
defer iam.server.Close()
|
||||
commerce := newFakeCommerce()
|
||||
defer commerce.server.Close()
|
||||
do, s, _ := mountService(t, iam.server.URL, commerce.server.URL, "")
|
||||
s.State.WLTenants = map[string]bool{"lux": true} // Lux ENABLED → admitted, then scoped.
|
||||
|
||||
resp, body := do("GET", "/v1/admin/orgs?org=zoo", luxAdminHdr)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("admitted Lux WL-tenant admin must reach the scoped orgs panel, got %d (%s)", resp.StatusCode, body)
|
||||
}
|
||||
var env struct {
|
||||
Data []orgRow `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &env); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(env.Data) != 1 || env.Data[0].Org != "lux" {
|
||||
t.Fatalf("Lux admin must see ONLY lux, got %+v — cross-brand leak into Zoo (?org=zoo honored)!", env.Data)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScope_LuxAdminCannotReachHanzoUsers proves the users read is HARD-PINNED: a Lux
|
||||
// admin listing "hanzo" users drives the IAM query with owner=lux, never hanzo. The pin
|
||||
// is the sanitized org; the ?org= is never trusted for a non-super caller.
|
||||
func TestScope_LuxAdminCannotReachHanzoUsers(t *testing.T) {
|
||||
iam := newScopeIAM()
|
||||
defer iam.server.Close()
|
||||
commerce := newFakeCommerce()
|
||||
defer commerce.server.Close()
|
||||
do, s, _ := mountService(t, iam.server.URL, commerce.server.URL, "")
|
||||
s.State.WLTenants = map[string]bool{"lux": true}
|
||||
|
||||
if resp, body := do("GET", "/v1/admin/users?org=hanzo", luxAdminHdr); resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("admitted Lux WL-tenant admin must reach the scoped users panel, got %d (%s)", resp.StatusCode, body)
|
||||
}
|
||||
iam.mu.Lock()
|
||||
owner := iam.lastUsersOwner
|
||||
iam.mu.Unlock()
|
||||
if owner != "lux" {
|
||||
t.Fatalf("Lux admin users read NOT hard-pinned: IAM saw owner=%q, want lux — cross-brand escalation into Hanzo!", owner)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScope_LuxAdminCannotReachDO proves the fleet ceiling: the DigitalOcean god-views —
|
||||
// compute (fleet infra) and finance (fleet DO billing) — are SuperAdmin-only (core.Guard).
|
||||
// An admitted Lux WL tenant is 403 on BOTH: a white-label tenant reads its own subtree but
|
||||
// NEVER a fleet DO resource. The gate refuses BEFORE the handler, so no DO call is made
|
||||
// (which also keeps this test free of any upstream I/O). That a SuperAdmin DOES pass these
|
||||
// gates is proven by TestGate_AllowsSuperAdmin + TestScope_SuperSeesAllOrgs.
|
||||
func TestScope_LuxAdminCannotReachDO(t *testing.T) {
|
||||
do, s, _ := mountService(t, "http://127.0.0.1:0", "http://127.0.0.1:0", "")
|
||||
s.State.WLTenants = map[string]bool{"lux": true} // even an ENABLED WL tenant is denied the DO god-views.
|
||||
|
||||
for _, path := range []string{"/v1/admin/compute", "/v1/admin/finance"} {
|
||||
if resp, body := do("GET", path, luxAdminHdr); resp.StatusCode != http.StatusForbidden {
|
||||
t.Errorf("Lux WL-tenant admin GET %s: got %d, want 403 — a WL tenant must never reach a fleet DO god-view (body=%s)",
|
||||
path, resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestScope_LuxAdminSpendCapWriteHardPinned pins the highest-value cross-tenant vector — a
|
||||
// STATE-CHANGING write. A Lux admin who tries to set a spend cap on Zoo (POST
|
||||
// /v1/admin/spend-caps?org=zoo) must have the write hard-pinned to owner=lux downstream:
|
||||
// the ?org= is ignored for a non-super caller (targetOrg → sc.Orgs[0]). We record the
|
||||
// X-Org-Id commerce actually receives and assert it is lux, never zoo. The read-path pins
|
||||
// are covered above; this closes the write path.
|
||||
func TestScope_LuxAdminSpendCapWriteHardPinned(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var wroteOrg string
|
||||
commerce := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasSuffix(r.URL.Path, "/v1/billing/spend-alerts") && r.Method == http.MethodPost {
|
||||
mu.Lock()
|
||||
wroteOrg = r.Header.Get("X-Org-Id") // commerce.Forward pins the target org here
|
||||
mu.Unlock()
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
io.WriteString(w, `{"status":"ok"}`)
|
||||
}))
|
||||
defer commerce.Close()
|
||||
|
||||
do, s, _ := mountService(t, "http://127.0.0.1:0", commerce.URL, "")
|
||||
s.State.WLTenants = map[string]bool{"lux": true}
|
||||
|
||||
resp, body := do("POST", "/v1/admin/spend-caps?org=zoo", luxAdminHdr)
|
||||
if resp.StatusCode == http.StatusForbidden {
|
||||
t.Fatalf("admitted Lux WL admin must reach the scoped spend-caps WRITE, got 403 (%s)", body)
|
||||
}
|
||||
mu.Lock()
|
||||
org := wroteOrg
|
||||
mu.Unlock()
|
||||
if org != "lux" {
|
||||
t.Fatalf("Lux admin spend-cap WRITE targeted org=%q, want lux — cross-brand WRITE into Zoo (?org=zoo honored)!", org)
|
||||
}
|
||||
}
|
||||
@@ -70,6 +70,13 @@ var superHdr = map[string]string{"X-User-IsAdmin": "true", "X-Org-Id": "admin",
|
||||
var orgAdminHdr = map[string]string{"X-Org-Id": "maxpower", "X-User-Id": "maxpower/dave", "X-User-Email": "dave@maxpower.test", "X-User-IsOrgAdmin": "true"}
|
||||
var memberHdr = map[string]string{"X-Org-Id": "maxpower", "X-User-Id": "maxpower/eve", "X-User-Email": "eve@maxpower.test"}
|
||||
|
||||
// nonWLOrgAdminHdr is a VALIDATED org admin (the unforgeable X-User-IsOrgAdmin bit +
|
||||
// a pinned own-org) of an org that is NOT an enabled white-label tenant — "acme" is
|
||||
// absent from the harness WLTenants set. It is the caller the OLD gate wrongly admitted
|
||||
// (any org-admin) and the new gate must REFUSE: same 403 as an anonymous forge. It is
|
||||
// the whole point of the WL admission tier.
|
||||
var nonWLOrgAdminHdr = map[string]string{"X-Org-Id": "acme", "X-User-Id": "acme/carol", "X-User-Email": "carol@acme.test", "X-User-IsOrgAdmin": "true"}
|
||||
|
||||
func TestScope_SuperSeesAllOrgs(t *testing.T) {
|
||||
iam := newScopeIAM()
|
||||
defer iam.server.Close()
|
||||
@@ -178,6 +185,84 @@ func TestScope_PlatformRouteDeniesOrgAdminButScopedAdmits(t *testing.T) {
|
||||
if env.Data.Owner != "maxpower" {
|
||||
t.Fatalf("org admin me.Owner = %q, want maxpower", env.Data.Owner)
|
||||
}
|
||||
if !env.Data.IsWhiteLabel {
|
||||
t.Fatalf("admitted WL-tenant org admin me.IsWhiteLabel must be true")
|
||||
}
|
||||
if len(env.Data.ScopeOrgs) != 1 || env.Data.ScopeOrgs[0] != "maxpower" {
|
||||
t.Fatalf("WL-tenant me.ScopeOrgs = %v, want [maxpower] (own subtree only)", env.Data.ScopeOrgs)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScope_NonWhiteLabelOrgAdminDenied is the CORE new invariant: a validated org
|
||||
// admin whose org is NOT an enabled white-label tenant is REFUSED on EVERY org-scoped
|
||||
// panel — not scoped-down, REFUSED (403). The old GuardScoped admitted any org-admin;
|
||||
// the tightened gate requires WL enablement, so an ordinary customer's own-org admin can
|
||||
// never open the cockpit. Same 403 a non-admin member gets.
|
||||
func TestScope_NonWhiteLabelOrgAdminDenied(t *testing.T) {
|
||||
do := mount(t, "http://127.0.0.1:0", "http://127.0.0.1:0", "")
|
||||
for _, r := range scopedAdminRoutes {
|
||||
if resp, body := do(r.method, r.path, nonWLOrgAdminHdr); resp.StatusCode != http.StatusForbidden {
|
||||
t.Errorf("%s %s [org admin of NON-WL org]: got %d, want 403 — WL admission tier bypassed (body=%s)",
|
||||
r.method, r.path, resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestScope_WhiteLabelTenantDeniedGodViews proves the SUBTREE ceiling: an ADMITTED WL
|
||||
// tenant (maxpower is enabled) still gets 403 on EVERY platform god-view (core.Guard) —
|
||||
// finance/revenue/metrics/o11y/providers-credit/audit/customers/flags/…. A WL tenant can
|
||||
// read its own subtree panels but NEVER a fleet number. This is the no-fleet-leak line.
|
||||
func TestScope_WhiteLabelTenantDeniedGodViews(t *testing.T) {
|
||||
do := mount(t, "http://127.0.0.1:0", "http://127.0.0.1:0", "")
|
||||
for _, r := range platformAdminRoutes {
|
||||
if resp, body := do(r.method, r.path, orgAdminHdr); resp.StatusCode != http.StatusForbidden {
|
||||
t.Errorf("%s %s [enabled WL tenant]: got %d, want 403 — a WL tenant must never reach a fleet god-view (body=%s)",
|
||||
r.method, r.path, resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestScope_WhiteLabelDefaultFailClosed proves the platform default is fleet-only: with
|
||||
// NO org enabled (WLTenants cleared), even a validated org admin is 403 on the scoped
|
||||
// panels — the cockpit is SuperAdmin-only until an org is explicitly onboarded.
|
||||
func TestScope_WhiteLabelDefaultFailClosed(t *testing.T) {
|
||||
iam := newScopeIAM()
|
||||
defer iam.server.Close()
|
||||
commerce := newFakeCommerce()
|
||||
defer commerce.server.Close()
|
||||
do, s, _ := mountService(t, iam.server.URL, commerce.server.URL, "")
|
||||
s.State.WLTenants = nil // fleet-only default: no white-label tenant enabled
|
||||
for _, r := range scopedAdminRoutes {
|
||||
if resp, body := do(r.method, r.path, orgAdminHdr); resp.StatusCode != http.StatusForbidden {
|
||||
t.Errorf("%s %s [org admin, NO WL enabled]: got %d, want 403 — default must be fleet-only (body=%s)",
|
||||
r.method, r.path, resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
// A SuperAdmin is unaffected by the allowlist — still cross-tenant.
|
||||
if resp, _ := do("GET", "/v1/admin/me", superHdr); resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("SuperAdmin must be admitted regardless of WLTenants, got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// TestState_IsWhiteLabelTenant unit-pins the ONE admission read: fail-closed on a
|
||||
// nil/empty set, verbatim (trimmed) match otherwise — no fold that could collapse a
|
||||
// distinct owner into an enabled one.
|
||||
func TestState_IsWhiteLabelTenant(t *testing.T) {
|
||||
var zero core.State // nil WLTenants
|
||||
if zero.IsWhiteLabelTenant("maxpower") {
|
||||
t.Fatal("nil WLTenants must be fail-closed (deny)")
|
||||
}
|
||||
empty := core.State{WLTenants: map[string]bool{}}
|
||||
if empty.IsWhiteLabelTenant("maxpower") {
|
||||
t.Fatal("empty WLTenants must be fail-closed (deny)")
|
||||
}
|
||||
on := core.State{WLTenants: map[string]bool{"maxpower": true}}
|
||||
if !on.IsWhiteLabelTenant("maxpower") || !on.IsWhiteLabelTenant(" maxpower ") {
|
||||
t.Fatal("enabled org must be admitted (trimmed)")
|
||||
}
|
||||
if on.IsWhiteLabelTenant("acme") || on.IsWhiteLabelTenant("") || on.IsWhiteLabelTenant("MAXPOWER") {
|
||||
t.Fatal("absent/empty/case-different org must be denied (verbatim match, no fold)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestScope_MemberWithoutOrgAdminDenied closes the same-tenant over-visibility gap: a
|
||||
|
||||
+32
-10
@@ -19,6 +19,17 @@ type adminMe struct {
|
||||
Email string `json:"email"`
|
||||
DisplayName string `json:"displayName"`
|
||||
IsSuperAdmin bool `json:"isSuperAdmin"`
|
||||
// IsWhiteLabel marks the admitted NON-super tier: an admin of an enabled
|
||||
// white-label tenant org. Mutually exclusive with IsSuperAdmin (the gate lets
|
||||
// exactly one tier through). The operator SPA reads it to render the SUBTREE
|
||||
// cockpit — the fleet god-view nav (finance/revenue/metrics/o11y/providers) is
|
||||
// hidden — while a super sees the whole fleet.
|
||||
IsWhiteLabel bool `json:"isWhiteLabel"`
|
||||
// ScopeOrgs is the caller's visible tenant window: empty for a SuperAdmin (means
|
||||
// ALL orgs), or the WL tenant's own subtree (today the singleton {org}). The SPA
|
||||
// threads it through the faceting/drill-down layer so a WL tenant can never widen
|
||||
// a filter past their subtree.
|
||||
ScopeOrgs []string `json:"scopeOrgs,omitempty"`
|
||||
}
|
||||
|
||||
// overviewData is the fleet overview tiles (OverviewData / GET /v1/admin/overview).
|
||||
@@ -88,17 +99,28 @@ type usageData struct {
|
||||
ByProduct []usageByProduct `json:"byProduct"`
|
||||
}
|
||||
|
||||
// productRow is one product/workload row (ProductRow / GET /v1/admin/products).
|
||||
// productRow is one product/workload row (ProductRow / GET /v1/admin/products) — the
|
||||
// projection of a paas fleet AppView (the operator App CR + its Deployment + the drift
|
||||
// verdict) onto the operator Infrastructure board. Tier is the DERIVED infra grouping
|
||||
// (tierOf, a cloud-side classification over the real image repo / role / namespace — NOT
|
||||
// an operator-declared field); Kind is the operator's OWN spec.role when it declares one.
|
||||
type productRow struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Org string `json:"org"`
|
||||
Cluster string `json:"cluster"`
|
||||
DeclaredTag string `json:"declaredTag"`
|
||||
RunningTag string `json:"runningTag"`
|
||||
Health string `json:"health"`
|
||||
Drift bool `json:"drift"`
|
||||
Updated string `json:"updated"`
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"` // operator App CR spec.role (sql|kv|generic|ingress) or ""
|
||||
Tier string `json:"tier"` // derived: cloud|data|edge|daemon|paas|app (grouping)
|
||||
Org string `json:"org"` // image namespace (hanzoai|luxfi|docker.io/…)
|
||||
Cluster string `json:"cluster"` // hanzo-k8s
|
||||
Env string `json:"env"` // main|test|dev (lifecycle namespace)
|
||||
Namespace string `json:"namespace"` // k8s namespace
|
||||
Repo string `json:"repo"` // owner/repo image coordinate
|
||||
Phase string `json:"phase"` // operator status.phase (Running/Creating/…)
|
||||
DeclaredTag string `json:"declaredTag"` // spec.image.tag on the App CR (declared truth)
|
||||
RunningTag string `json:"runningTag"` // observed from the live Deployment
|
||||
LatestTag string `json:"latestTag"` // newest released tag (GH release reader — empty until wired)
|
||||
Health string `json:"health"` // green|yellow|red|unknown
|
||||
Drift bool `json:"drift"` // any drift flag present
|
||||
DriftSeverity string `json:"driftSeverity"` // ok|yellow|red (rolled-up)
|
||||
Updated string `json:"updated"`
|
||||
}
|
||||
|
||||
// IAM wire shapes (iamOrg/iamUser) now live in clients/admin/iam as iam.Org /
|
||||
|
||||
@@ -205,12 +205,12 @@ func bounce(c *zip.Ctx, waitlistURL string) error {
|
||||
return c.NoContent(http.StatusFound)
|
||||
}
|
||||
|
||||
// apiKeyPrefixes are the Hanzo API-key prefixes. This MIRRORS cloud
|
||||
// auth_identity.go isAPIKey (the ONE authority) — kept local so admission stays
|
||||
// self-contained (no cloud-internal import) while agreeing on the exact contract:
|
||||
// a token with one of these prefixes is a possession-gated API key, not a session
|
||||
// principal. If cloud adds a prefix there, add it here.
|
||||
var apiKeyPrefixes = []string{"hk-", "sk-", "pk-", "fw_", "hz_"}
|
||||
// apiKeyPrefixes are the Hanzo API-key families: a published key (pk-), a secret
|
||||
// key (sk-), and hk- (sk- under an older name, retired once IAM renames it). This MIRRORS cloud auth_identity.go APIKeyPrefixes (the ONE
|
||||
// authority) — kept local so admission stays self-contained (no cloud-internal
|
||||
// import) while agreeing on the exact contract: a token with one of these
|
||||
// prefixes is a possession-gated API key, not a session principal.
|
||||
var apiKeyPrefixes = []string{"pk-", "sk-", "hk-"}
|
||||
|
||||
// carriesAPIKey reports whether the request authenticates with a Hanzo API key —
|
||||
// in the Authorization header (Bearer or Basic-username) or the common api-key /
|
||||
|
||||
+61
-2
@@ -32,6 +32,7 @@ package ads
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -114,6 +115,7 @@ func routes(app *zip.App, s *cloud.Service[state]) {
|
||||
g.Get("/campaigns/:id", cloud.Handle(s, getCampaign))
|
||||
g.Put("/campaigns/:id", cloud.Handle(s, updateCampaign))
|
||||
g.Delete("/campaigns/:id", cloud.Handle(s, deleteCampaign))
|
||||
g.Post("/campaigns/:id/launch", cloud.Handle(s, launchCampaignHandler))
|
||||
}
|
||||
|
||||
// ---- shared helpers (mirror clients/crm) ----
|
||||
@@ -224,7 +226,7 @@ func createCampaign(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
camp := Campaign{
|
||||
ID: id, Org: org, Name: name, Platform: platform, Status: status,
|
||||
ID: id, Org: org, Name: name, Platform: platform, Account: clip(body.Account), Status: status,
|
||||
Objective: clip(body.Objective), Budget: nonNeg(body.Budget), Spend: nonNeg(body.Spend),
|
||||
CreatedAt: now, UpdatedAt: now,
|
||||
}
|
||||
@@ -282,7 +284,7 @@ func updateCampaign(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
return zip.ErrBadRequest("status must be one of draft, active, paused, completed")
|
||||
}
|
||||
camp := Campaign{
|
||||
ID: idParam(c), Org: org, Name: name, Platform: platform, Status: status,
|
||||
ID: idParam(c), Org: org, Name: name, Platform: platform, Account: clip(body.Account), Status: status,
|
||||
Objective: clip(body.Objective), Budget: nonNeg(body.Budget), Spend: nonNeg(body.Spend),
|
||||
UpdatedAt: time.Now().Unix(),
|
||||
}
|
||||
@@ -308,6 +310,63 @@ func deleteCampaign(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
return c.NoContent(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// ---- launch (consumes the connector plane) ----
|
||||
|
||||
// launchCampaignHandler runs a stored ad campaign on its provider using the ORG'S
|
||||
// connected ad-account token. It is the standalone proof that /v1/ads consumes the
|
||||
// connector plane: no token is held here — LaunchPaid (provider.go) resolves it
|
||||
// from KMS through integrations.TokenFor and FAILS CLOSED when the org has not
|
||||
// connected the platform (424), so a launch can never spend on a connection the
|
||||
// org did not make. On success the provider campaign id is recorded (MarkLaunched)
|
||||
// and the campaign goes active. An optional body {account} sets/overrides the
|
||||
// target ad account when the stored campaign has none.
|
||||
func launchCampaignHandler(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
org, ok := tenant(c)
|
||||
if !ok {
|
||||
return zip.ErrForbidden("X-Org-Id required")
|
||||
}
|
||||
camp, err := s.State.store.GetCampaign(c.Context(), org, idParam(c))
|
||||
if err != nil {
|
||||
return mapErr(err, "campaign not found")
|
||||
}
|
||||
var body struct {
|
||||
Account string `json:"account"`
|
||||
}
|
||||
_ = c.Bind(&body)
|
||||
account := clip(body.Account)
|
||||
if account == "" {
|
||||
account = camp.Account
|
||||
}
|
||||
ref, lerr := LaunchPaid(c.Context(), org, PaidPlan{
|
||||
Platform: camp.Platform, Account: account, Name: camp.Name,
|
||||
Objective: camp.Objective, BudgetCents: camp.Budget,
|
||||
})
|
||||
if lerr != nil {
|
||||
return mapProviderErr(lerr)
|
||||
}
|
||||
saved, err := s.State.store.MarkLaunched(c.Context(), org, camp.ID, ref.Account, ref.ExternalID, time.Now().Unix())
|
||||
if err != nil {
|
||||
return mapErr(err, "campaign not found")
|
||||
}
|
||||
return c.JSON(http.StatusOK, saved)
|
||||
}
|
||||
|
||||
// mapProviderErr renders a provider-execution error as the honest HTTP status: a
|
||||
// missing/rejected connection is 424 (connect the account first), an unwired
|
||||
// platform is 501, a transient edge failure is 502.
|
||||
func mapProviderErr(err error) error {
|
||||
switch {
|
||||
case errors.Is(err, errNotConnected):
|
||||
return zip.Errorf(http.StatusFailedDependency, "connect your ad account for this platform first")
|
||||
case errors.Is(err, errUnsupportedPlatform):
|
||||
return zip.Errorf(http.StatusNotImplemented, "%v", err)
|
||||
case errors.Is(err, errUpstream):
|
||||
return zip.Errorf(http.StatusBadGateway, "%v", err)
|
||||
default:
|
||||
return zip.Errorf(http.StatusBadRequest, "%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- summary ----
|
||||
|
||||
func summary(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
package ads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/campaign"
|
||||
)
|
||||
|
||||
// campaign_paid_test.go is the END-TO-END proof of the paid GTM channel: the SAME
|
||||
// adapter apps/wire_seams.go registers (campaign.Plan → ads.PaidPlan → LaunchPaid)
|
||||
// driven through the campaign.Channel interface, so the whole chain
|
||||
// campaign → ads → integrations.TokenFor → provider is exercised against an
|
||||
// httptest Meta stub. It lives in package ads because only this package can point
|
||||
// the connector-custody seam (tokenFor) and the Meta base at test doubles;
|
||||
// importing clients/campaign here is acyclic (campaign never imports ads).
|
||||
|
||||
// paidChannelForTest builds the exact paid-channel adapter the composition root
|
||||
// wires, so the test exercises the real seam shape, not a bespoke one.
|
||||
func paidChannelForTest() campaign.Channel {
|
||||
return campaign.NewChannel(campaign.KindPaid,
|
||||
func(ctx context.Context, org string, p campaign.Plan) (campaign.Ref, error) {
|
||||
r, err := LaunchPaid(ctx, org, PaidPlan{
|
||||
Platform: p.Platform, Account: p.Account, Name: p.Name,
|
||||
Objective: p.Objective, BudgetCents: p.BudgetCents, ScheduleAt: p.ScheduleAt,
|
||||
})
|
||||
return campaign.Ref{Platform: r.Platform, Account: r.Account, ExternalID: r.ExternalID, Status: r.Status, Detail: r.Detail}, err
|
||||
},
|
||||
func(ctx context.Context, org string, ref campaign.Ref) (int64, error) {
|
||||
return PaidSpend(ctx, org, PaidRef{Platform: ref.Platform, Account: ref.Account, ExternalID: ref.ExternalID})
|
||||
},
|
||||
func(ctx context.Context, org string, ref campaign.Ref) error {
|
||||
return PausePaid(ctx, org, PaidRef{Platform: ref.Platform, Account: ref.Account, ExternalID: ref.ExternalID})
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// TestCampaignPaidChannel_EndToEnd: a campaign's paid channel launches an ad
|
||||
// campaign on Meta using the ORG'S connected token, then reads spend — the full
|
||||
// capability→connector→provider chain.
|
||||
func TestCampaignPaidChannel_EndToEnd(t *testing.T) {
|
||||
stubToken(t, func(_ context.Context, org, provider, _ string) ([]byte, error) {
|
||||
if provider != "meta_ads" {
|
||||
t.Fatalf("paid meta channel must consume meta_ads, got %q", provider)
|
||||
}
|
||||
return []byte("tok-" + org), nil
|
||||
})
|
||||
var gotAuth string
|
||||
stubMeta(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/act_123/campaigns" {
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
_, _ = w.Write([]byte(`{"id":"120210000000009"}`))
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/120210000000009/insights" {
|
||||
_, _ = w.Write([]byte(`{"data":[{"spend":"7.50"}]}`))
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
})
|
||||
|
||||
paid := paidChannelForTest()
|
||||
if paid.Kind() != campaign.KindPaid {
|
||||
t.Fatalf("channel kind want paid, got %q", paid.Kind())
|
||||
}
|
||||
|
||||
ref, err := paid.Launch(context.Background(), "acme", campaign.Plan{
|
||||
CampaignID: "cmp_1", Platform: "meta", Account: "123", Name: "GTM Launch",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("paid Launch: %v", err)
|
||||
}
|
||||
if ref.ExternalID != "120210000000009" || ref.Status != "live" {
|
||||
t.Fatalf("ref: %+v", ref)
|
||||
}
|
||||
if gotAuth != "Bearer tok-acme" {
|
||||
t.Fatalf("Meta must be called with acme's connector token, got %q", gotAuth)
|
||||
}
|
||||
|
||||
// Spend read composes the same token door → provider insights.
|
||||
cents, err := paid.Spend(context.Background(), "acme", ref)
|
||||
if err != nil {
|
||||
t.Fatalf("paid Spend: %v", err)
|
||||
}
|
||||
if cents != 750 {
|
||||
t.Fatalf("spend want 750 cents, got %d", cents)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCampaignPaidChannel_ConnectorDisabledBlocksSend: a campaign whose org has
|
||||
// not connected the ad account cannot launch — the channel fails closed and the
|
||||
// provider is never called (no spend on an unmade connection).
|
||||
func TestCampaignPaidChannel_ConnectorDisabledBlocksSend(t *testing.T) {
|
||||
stubToken(t, func(_ context.Context, _, _, _ string) ([]byte, error) {
|
||||
return nil, errors.New("integrations: meta_ads not connected for org")
|
||||
})
|
||||
hit := false
|
||||
stubMeta(t, func(w http.ResponseWriter, _ *http.Request) {
|
||||
hit = true
|
||||
_, _ = w.Write([]byte(`{"id":"nope"}`))
|
||||
})
|
||||
|
||||
paid := paidChannelForTest()
|
||||
_, err := paid.Launch(context.Background(), "acme", campaign.Plan{
|
||||
CampaignID: "cmp_1", Platform: "meta", Account: "123", Name: "GTM",
|
||||
})
|
||||
if !errors.Is(err, errNotConnected) {
|
||||
t.Fatalf("want errNotConnected, got %v", err)
|
||||
}
|
||||
if hit {
|
||||
t.Fatalf("provider must NOT be called when the org's connector is disabled")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
package ads
|
||||
|
||||
// provider.go is the ad-network EXECUTION edge: the ONE place /v1/ads consumes the
|
||||
// connector plane. An ad campaign runs on a provider (Meta/Google/…) using the
|
||||
// ORG'S OWN connector token — resolved at call time from KMS through the
|
||||
// integrations.TokenFor custody seam, never held in this process, never in a
|
||||
// manifest. This closes the gap the ads store left open: a stored Campaign is now
|
||||
// LAUNCHABLE against the real provider, and it is what the /v1/campaign paid
|
||||
// channel fans out to (apps/wire_seams.go adapts LaunchPaid/PaidSpend/PausePaid
|
||||
// onto campaign.Channel).
|
||||
//
|
||||
// FAIL-CLOSED CUSTODY. Every operation resolves the org's token FIRST; any reason
|
||||
// it cannot be produced — the org never connected the ad account, the integrations
|
||||
// plane is unmounted, KMS is down — is errNotConnected, and NO provider call is
|
||||
// made. The token rides the Authorization header only (never the URL, never a log,
|
||||
// never argv), exactly the meta.go connector discipline.
|
||||
//
|
||||
// LEAST SURPRISE ON SPEND. A launch creates the campaign OBJECT on the provider;
|
||||
// delivery (and therefore spend) does not begin until the ad-set/ad legs are wired
|
||||
// (the documented ads follow-up), so a launch cannot silently burn budget. Spend is
|
||||
// READ back from the provider's insights (PaidSpend) — the connector's reported
|
||||
// number the /v1/campaign metrics plane joins with the analytics funnel.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/integrations"
|
||||
)
|
||||
|
||||
const (
|
||||
// accessTokenSecret is the KMS secret name every ad connector custodies its
|
||||
// long-lived token under (meta.go/google_marketing.go: "access_token").
|
||||
accessTokenSecret = "access_token"
|
||||
// launchTimeout bounds a single provider call so a slow ad edge never wedges a
|
||||
// launch or a metrics read.
|
||||
launchTimeout = 20 * time.Second
|
||||
)
|
||||
|
||||
// platformConnector maps an ads Platform to its integrations connector id. The
|
||||
// paid channel consumes the connector via TokenFor(org, <id>, "access_token").
|
||||
// A platform with no entry has no connector wired → errUnsupportedPlatform (the
|
||||
// token is never even sought), so the map is the ONE source of "which ad networks
|
||||
// this deployment can run".
|
||||
var platformConnector = map[string]string{
|
||||
"meta": "meta_ads",
|
||||
"google": "google_ads",
|
||||
"tiktok": "tiktok_ads",
|
||||
"reddit": "reddit_ads",
|
||||
"linkedin": "linkedin_ads",
|
||||
"microsoft": "microsoft_ads",
|
||||
}
|
||||
|
||||
var (
|
||||
// errNotConnected — the org has no usable connection for the platform (no
|
||||
// token, or the provider rejected it). Fail-closed: no spend, no fabrication.
|
||||
errNotConnected = errors.New("ads: ad account not connected for org")
|
||||
// errUnsupportedPlatform — a valid platform whose provider execution is not yet
|
||||
// wired (the connector may be connected; only the create/read impl is missing).
|
||||
errUnsupportedPlatform = errors.New("ads: paid execution not wired for platform")
|
||||
// errUpstream — the ad platform edge failed transiently (network / non-auth non-2xx).
|
||||
errUpstream = errors.New("ads: ad platform edge unavailable")
|
||||
)
|
||||
|
||||
// tokenFor is the connector-custody seam. It defaults to integrations.TokenFor
|
||||
// (the ONE KMS-backed token door) and is a package var ONLY so a test can exercise
|
||||
// the provider path — the same reason meta.go's endpoints are package vars. It is
|
||||
// never reassigned in production.
|
||||
var tokenFor = integrations.TokenFor
|
||||
|
||||
// metaAdsBase is the Meta Graph API base. A package var so a test points create /
|
||||
// insights / pause at an httptest server; never mutated in production.
|
||||
var metaAdsBase = "https://graph.facebook.com/v21.0"
|
||||
|
||||
// adHTTP is the ONE bounded client for provider calls.
|
||||
var adHTTP = &http.Client{Timeout: launchTimeout}
|
||||
|
||||
// PaidPlan is the standalone contract for launching one ad campaign — the campaign
|
||||
// paid channel adapts campaign.Plan onto it (apps/wire_seams.go). BudgetCents is
|
||||
// the org's own (connector-paid) budget; ScheduleAt an optional start time.
|
||||
type PaidPlan struct {
|
||||
Platform string
|
||||
Account string // provider ad-account ref (Meta: act_<id> or <id>)
|
||||
Name string
|
||||
Objective string // provider objective enum; defaulted per-provider when empty
|
||||
BudgetCents int64
|
||||
ScheduleAt int64
|
||||
}
|
||||
|
||||
// PaidRef is a launched ad campaign's durable handle: the provider-side id plus the
|
||||
// platform/account needed to read spend or pause it.
|
||||
type PaidRef struct {
|
||||
Platform string
|
||||
Account string
|
||||
ExternalID string
|
||||
Status string
|
||||
Detail string
|
||||
}
|
||||
|
||||
// adToken resolves (connectorID, token) for a platform, fail-closed. An unmapped
|
||||
// platform never reaches KMS. An empty/absent token is errNotConnected.
|
||||
func adToken(ctx context.Context, org, platform string) (string, string, error) {
|
||||
connectorID, ok := platformConnector[strings.ToLower(strings.TrimSpace(platform))]
|
||||
if !ok {
|
||||
return "", "", fmt.Errorf("%w: %s", errUnsupportedPlatform, platform)
|
||||
}
|
||||
tok, err := tokenFor(ctx, org, connectorID, accessTokenSecret)
|
||||
if err != nil || len(strings.TrimSpace(string(tok))) == 0 {
|
||||
return connectorID, "", errNotConnected
|
||||
}
|
||||
return connectorID, strings.TrimSpace(string(tok)), nil
|
||||
}
|
||||
|
||||
// LaunchPaid creates the ad campaign on its provider using the org's connector
|
||||
// token. Fail-closed: it resolves the token BEFORE any provider call. Meta is
|
||||
// executed for real; other connected platforms return errUnsupportedPlatform
|
||||
// (the connector is verified, only the provider impl is the remaining gap).
|
||||
func LaunchPaid(ctx context.Context, org string, p PaidPlan) (PaidRef, error) {
|
||||
platform := strings.ToLower(strings.TrimSpace(p.Platform))
|
||||
connectorID, token, err := adToken(ctx, org, platform)
|
||||
if err != nil {
|
||||
return PaidRef{}, err
|
||||
}
|
||||
switch platform {
|
||||
case "meta":
|
||||
return metaCreateCampaign(ctx, token, p)
|
||||
default:
|
||||
return PaidRef{}, fmt.Errorf("%w: %s (connector %s connected)", errUnsupportedPlatform, platform, connectorID)
|
||||
}
|
||||
}
|
||||
|
||||
// PaidSpend reads the provider-reported spend (minor units) for a launched ad
|
||||
// campaign. Fail-closed on the token; honest 0 when the platform is unsupported.
|
||||
func PaidSpend(ctx context.Context, org string, ref PaidRef) (int64, error) {
|
||||
platform := strings.ToLower(strings.TrimSpace(ref.Platform))
|
||||
_, token, err := adToken(ctx, org, platform)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
switch platform {
|
||||
case "meta":
|
||||
return metaCampaignSpend(ctx, token, ref.ExternalID)
|
||||
default:
|
||||
return 0, fmt.Errorf("%w: %s", errUnsupportedPlatform, platform)
|
||||
}
|
||||
}
|
||||
|
||||
// PausePaid pauses a launched ad campaign on its provider. Fail-closed on the token.
|
||||
func PausePaid(ctx context.Context, org string, ref PaidRef) error {
|
||||
platform := strings.ToLower(strings.TrimSpace(ref.Platform))
|
||||
_, token, err := adToken(ctx, org, platform)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch platform {
|
||||
case "meta":
|
||||
return metaPauseCampaign(ctx, token, ref.ExternalID)
|
||||
default:
|
||||
return fmt.Errorf("%w: %s", errUnsupportedPlatform, platform)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Meta (Facebook/Instagram) ad campaign execution ─────────────────────────
|
||||
|
||||
// metaErr is Meta's Graph API error envelope.
|
||||
type metaErr struct {
|
||||
Message string `json:"message"`
|
||||
Type string `json:"type"`
|
||||
Code int `json:"code"`
|
||||
}
|
||||
|
||||
// metaAcct normalizes a Meta ad-account ref to the act_<id> form the campaigns
|
||||
// edge expects. Empty stays empty (caller rejects).
|
||||
func metaAcct(account string) string {
|
||||
a := strings.TrimSpace(account)
|
||||
if a == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(a, "act_") {
|
||||
return a
|
||||
}
|
||||
return "act_" + a
|
||||
}
|
||||
|
||||
// metaCreateCampaign POSTs a campaign to /act_<id>/campaigns. Objective defaults to
|
||||
// OUTCOME_TRAFFIC; special_ad_categories is the required empty set; the object is
|
||||
// created ACTIVE but does not deliver (no ad sets) so no spend starts on launch.
|
||||
func metaCreateCampaign(ctx context.Context, token string, p PaidPlan) (PaidRef, error) {
|
||||
account := metaAcct(p.Account)
|
||||
if account == "" {
|
||||
return PaidRef{}, fmt.Errorf("meta: ad account (account) is required")
|
||||
}
|
||||
objective := strings.TrimSpace(p.Objective)
|
||||
if objective == "" {
|
||||
objective = "OUTCOME_TRAFFIC"
|
||||
}
|
||||
form := url.Values{
|
||||
"name": {strings.TrimSpace(p.Name)},
|
||||
"objective": {objective},
|
||||
"status": {"ACTIVE"},
|
||||
"special_ad_categories": {"[]"},
|
||||
}
|
||||
var out struct {
|
||||
ID string `json:"id"`
|
||||
Error *metaErr `json:"error"`
|
||||
}
|
||||
if err := metaPost(ctx, metaAdsBase+"/"+account+"/campaigns", token, form, &out); err != nil {
|
||||
return PaidRef{}, err
|
||||
}
|
||||
if out.Error != nil {
|
||||
return PaidRef{}, fmt.Errorf("meta create campaign: %s", out.Error.Message)
|
||||
}
|
||||
if strings.TrimSpace(out.ID) == "" {
|
||||
return PaidRef{}, fmt.Errorf("meta create campaign returned no id")
|
||||
}
|
||||
return PaidRef{Platform: "meta", Account: account, ExternalID: out.ID, Status: "live"}, nil
|
||||
}
|
||||
|
||||
// metaCampaignSpend reads spend from /{campaignID}/insights?fields=spend. Meta
|
||||
// reports spend in the account currency's major unit as a string; it is converted
|
||||
// to minor units (cents). No insights row (campaign hasn't spent) → 0.
|
||||
func metaCampaignSpend(ctx context.Context, token, campaignID string) (int64, error) {
|
||||
if strings.TrimSpace(campaignID) == "" {
|
||||
return 0, nil
|
||||
}
|
||||
var out struct {
|
||||
Data []struct {
|
||||
Spend string `json:"spend"`
|
||||
} `json:"data"`
|
||||
Error *metaErr `json:"error"`
|
||||
}
|
||||
if err := metaGet(ctx, metaAdsBase+"/"+campaignID+"/insights?fields=spend", token, &out); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if out.Error != nil {
|
||||
return 0, fmt.Errorf("meta insights: %s", out.Error.Message)
|
||||
}
|
||||
if len(out.Data) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
dollars, err := strconv.ParseFloat(strings.TrimSpace(out.Data[0].Spend), 64)
|
||||
if err != nil || dollars < 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return int64(dollars*100 + 0.5), nil
|
||||
}
|
||||
|
||||
// metaPauseCampaign POSTs status=PAUSED to /{campaignID}.
|
||||
func metaPauseCampaign(ctx context.Context, token, campaignID string) error {
|
||||
if strings.TrimSpace(campaignID) == "" {
|
||||
return fmt.Errorf("meta: campaign id required")
|
||||
}
|
||||
var out struct {
|
||||
Success bool `json:"success"`
|
||||
Error *metaErr `json:"error"`
|
||||
}
|
||||
if err := metaPost(ctx, metaAdsBase+"/"+campaignID, token, url.Values{"status": {"PAUSED"}}, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
if out.Error != nil {
|
||||
return fmt.Errorf("meta pause: %s", out.Error.Message)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ── bounded HTTP (token on the Authorization header only) ────────────────────
|
||||
|
||||
func metaPost(ctx context.Context, endpoint, token string, form url.Values, out any) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %v", errUpstream, err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
return adDo(req, out)
|
||||
}
|
||||
|
||||
func metaGet(ctx context.Context, endpoint, token string, out any) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %v", errUpstream, err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
return adDo(req, out)
|
||||
}
|
||||
|
||||
// adDo runs a bounded provider request and decodes the JSON body. A 401/403 is
|
||||
// fail-closed as errNotConnected (the org's token is invalid/revoked — the
|
||||
// connection is not usable); any other non-2xx or transport error is errUpstream.
|
||||
// The body is bounded so a hostile edge cannot amplify memory.
|
||||
func adDo(req *http.Request, out any) error {
|
||||
resp, err := adHTTP.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %v", errUpstream, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
|
||||
if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
|
||||
return errNotConnected
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("%w: %d", errUpstream, resp.StatusCode)
|
||||
}
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(body, out); err != nil {
|
||||
return fmt.Errorf("%w: decode: %v", errUpstream, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package ads
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// stubToken overrides the connector-custody seam so the provider path is
|
||||
// exercised without standing up KMS + integrations. Restored on cleanup.
|
||||
func stubToken(t *testing.T, fn func(ctx context.Context, org, provider, name string) ([]byte, error)) {
|
||||
t.Helper()
|
||||
prev := tokenFor
|
||||
tokenFor = fn
|
||||
t.Cleanup(func() { tokenFor = prev })
|
||||
}
|
||||
|
||||
// stubMeta points the Meta base at an httptest server (restored on cleanup) and
|
||||
// returns the server so the test can inspect what the provider received.
|
||||
func stubMeta(t *testing.T, h http.HandlerFunc) *httptest.Server {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(h)
|
||||
prev := metaAdsBase
|
||||
metaAdsBase = srv.URL
|
||||
t.Cleanup(func() { metaAdsBase = prev; srv.Close() })
|
||||
return srv
|
||||
}
|
||||
|
||||
// TestLaunchPaid_MetaCreatesViaConnectorToken is the proof that /v1/ads consumes
|
||||
// the connector plane: LaunchPaid resolves the ORG'S token via the TokenFor seam
|
||||
// and creates the campaign on Meta with that token on the Authorization header.
|
||||
func TestLaunchPaid_MetaCreatesViaConnectorToken(t *testing.T) {
|
||||
var (
|
||||
gotAuth, gotPath, gotBody string
|
||||
gotTokenOrg, gotProvider string
|
||||
)
|
||||
stubToken(t, func(_ context.Context, org, provider, name string) ([]byte, error) {
|
||||
gotTokenOrg, gotProvider = org, provider
|
||||
if name != accessTokenSecret {
|
||||
t.Fatalf("token secret name want %q, got %q", accessTokenSecret, name)
|
||||
}
|
||||
return []byte("T0K3N-acme"), nil
|
||||
})
|
||||
stubMeta(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
gotPath = r.URL.Path
|
||||
_ = r.ParseForm()
|
||||
gotBody = r.Form.Encode()
|
||||
_, _ = w.Write([]byte(`{"id":"120210000000001"}`))
|
||||
})
|
||||
|
||||
ref, err := LaunchPaid(context.Background(), "acme", PaidPlan{
|
||||
Platform: "meta", Account: "123", Name: "Spring Launch", Objective: "OUTCOME_TRAFFIC",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("LaunchPaid: %v", err)
|
||||
}
|
||||
if ref.ExternalID != "120210000000001" || ref.Account != "act_123" || ref.Status != "live" {
|
||||
t.Fatalf("ref: %+v", ref)
|
||||
}
|
||||
if gotTokenOrg != "acme" || gotProvider != "meta_ads" {
|
||||
t.Fatalf("TokenFor called with (%q,%q), want (acme, meta_ads)", gotTokenOrg, gotProvider)
|
||||
}
|
||||
if gotAuth != "Bearer T0K3N-acme" {
|
||||
t.Fatalf("Authorization header want the org token, got %q", gotAuth)
|
||||
}
|
||||
if gotPath != "/act_123/campaigns" {
|
||||
t.Fatalf("create path want /act_123/campaigns, got %q", gotPath)
|
||||
}
|
||||
for _, want := range []string{"name=Spring", "objective=OUTCOME_TRAFFIC", "status=ACTIVE", "special_ad_categories="} {
|
||||
if !strings.Contains(gotBody, want) {
|
||||
t.Fatalf("create body missing %q: %q", want, gotBody)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestLaunchPaid_ConnectorDisabledNoProviderCall: when the org has not connected
|
||||
// the ad account (TokenFor fails), LaunchPaid fails closed and NEVER calls the
|
||||
// provider — no spend, no fabrication.
|
||||
func TestLaunchPaid_ConnectorDisabledNoProviderCall(t *testing.T) {
|
||||
stubToken(t, func(_ context.Context, _, _, _ string) ([]byte, error) {
|
||||
return nil, errors.New("integrations: meta_ads not connected for org")
|
||||
})
|
||||
hit := false
|
||||
stubMeta(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
hit = true
|
||||
_, _ = w.Write([]byte(`{"id":"should-not-happen"}`))
|
||||
})
|
||||
|
||||
_, err := LaunchPaid(context.Background(), "acme", PaidPlan{Platform: "meta", Account: "123", Name: "X"})
|
||||
if !errors.Is(err, errNotConnected) {
|
||||
t.Fatalf("want errNotConnected, got %v", err)
|
||||
}
|
||||
if hit {
|
||||
t.Fatalf("provider must NOT be called when the connector is disabled")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLaunchPaid_TenantIsolationTokenPerOrg: each org's launch carries ITS OWN
|
||||
// org's token to the provider — org A can never spend on org B's connection.
|
||||
func TestLaunchPaid_TenantIsolationTokenPerOrg(t *testing.T) {
|
||||
stubToken(t, func(_ context.Context, org, _, _ string) ([]byte, error) {
|
||||
return []byte("tok-" + org), nil // each org has a distinct token
|
||||
})
|
||||
var seen []string
|
||||
stubMeta(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
seen = append(seen, r.Header.Get("Authorization"))
|
||||
_, _ = w.Write([]byte(`{"id":"1"}`))
|
||||
})
|
||||
|
||||
if _, err := LaunchPaid(context.Background(), "acme", PaidPlan{Platform: "meta", Account: "1", Name: "A"}); err != nil {
|
||||
t.Fatalf("acme launch: %v", err)
|
||||
}
|
||||
if _, err := LaunchPaid(context.Background(), "maxpower", PaidPlan{Platform: "meta", Account: "2", Name: "B"}); err != nil {
|
||||
t.Fatalf("maxpower launch: %v", err)
|
||||
}
|
||||
if len(seen) != 2 || seen[0] != "Bearer tok-acme" || seen[1] != "Bearer tok-maxpower" {
|
||||
t.Fatalf("each launch must carry its own org token, got %v", seen)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLaunchPaid_AuthFailureIsNotConnected: a token the provider rejects (401) is
|
||||
// fail-closed as errNotConnected — the connection is not usable.
|
||||
func TestLaunchPaid_AuthFailureIsNotConnected(t *testing.T) {
|
||||
stubToken(t, func(_ context.Context, _, _, _ string) ([]byte, error) { return []byte("stale"), nil })
|
||||
stubMeta(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"invalid token"}}`))
|
||||
})
|
||||
_, err := LaunchPaid(context.Background(), "acme", PaidPlan{Platform: "meta", Account: "1", Name: "X"})
|
||||
if !errors.Is(err, errNotConnected) {
|
||||
t.Fatalf("401 want errNotConnected, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLaunchPaid_UnsupportedPlatformAfterToken: a mapped-but-unwired platform
|
||||
// resolves the connector token (proving consumption) then honestly reports the
|
||||
// execution gap — never a fabricated launch.
|
||||
func TestLaunchPaid_UnsupportedPlatformAfterToken(t *testing.T) {
|
||||
tokenSought := false
|
||||
stubToken(t, func(_ context.Context, _, provider, _ string) ([]byte, error) {
|
||||
tokenSought = true
|
||||
if provider != "google_ads" {
|
||||
t.Fatalf("google platform must resolve google_ads, got %q", provider)
|
||||
}
|
||||
return []byte("g-token"), nil
|
||||
})
|
||||
_, err := LaunchPaid(context.Background(), "acme", PaidPlan{Platform: "google", Account: "1", Name: "X"})
|
||||
if !errors.Is(err, errUnsupportedPlatform) {
|
||||
t.Fatalf("want errUnsupportedPlatform, got %v", err)
|
||||
}
|
||||
if !tokenSought {
|
||||
t.Fatalf("a mapped platform must resolve its connector token before reporting the gap")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLaunchPaid_UnmappedPlatformNeverSeeksToken: a platform with no connector is
|
||||
// rejected BEFORE any KMS/token lookup.
|
||||
func TestLaunchPaid_UnmappedPlatformNeverSeeksToken(t *testing.T) {
|
||||
stubToken(t, func(_ context.Context, _, _, _ string) ([]byte, error) {
|
||||
t.Fatalf("token must not be sought for an unmapped platform")
|
||||
return nil, nil
|
||||
})
|
||||
_, err := LaunchPaid(context.Background(), "acme", PaidPlan{Platform: "snapchat", Account: "1", Name: "X"})
|
||||
if !errors.Is(err, errUnsupportedPlatform) {
|
||||
t.Fatalf("want errUnsupportedPlatform, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPaidSpend_MetaInsightsCents: spend is read from Meta insights and converted
|
||||
// to minor units (cents), through the org's connector token.
|
||||
func TestPaidSpend_MetaInsightsCents(t *testing.T) {
|
||||
stubToken(t, func(_ context.Context, _, _, _ string) ([]byte, error) { return []byte("tok"), nil })
|
||||
var gotPath string
|
||||
stubMeta(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
_, _ = w.Write([]byte(`{"data":[{"spend":"12.34"}]}`))
|
||||
})
|
||||
cents, err := PaidSpend(context.Background(), "acme", PaidRef{Platform: "meta", ExternalID: "120210000000001"})
|
||||
if err != nil {
|
||||
t.Fatalf("PaidSpend: %v", err)
|
||||
}
|
||||
if cents != 1234 {
|
||||
t.Fatalf("spend want 1234 cents, got %d", cents)
|
||||
}
|
||||
if gotPath != "/120210000000001/insights" {
|
||||
t.Fatalf("insights path want /120210000000001/insights, got %q", gotPath)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPausePaid_MetaSetsPaused: pause posts status=PAUSED to the provider through
|
||||
// the org's connector token.
|
||||
func TestPausePaid_MetaSetsPaused(t *testing.T) {
|
||||
stubToken(t, func(_ context.Context, _, _, _ string) ([]byte, error) { return []byte("tok"), nil })
|
||||
var gotStatus string
|
||||
stubMeta(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = r.ParseForm()
|
||||
gotStatus = r.Form.Get("status")
|
||||
_, _ = w.Write([]byte(`{"success":true}`))
|
||||
})
|
||||
if err := PausePaid(context.Background(), "acme", PaidRef{Platform: "meta", ExternalID: "120210000000001"}); err != nil {
|
||||
t.Fatalf("PausePaid: %v", err)
|
||||
}
|
||||
if gotStatus != "PAUSED" {
|
||||
t.Fatalf("pause status want PAUSED, got %q", gotStatus)
|
||||
}
|
||||
}
|
||||
+82
-17
@@ -54,7 +54,9 @@ func openStore(path string) (*Store, error) {
|
||||
|
||||
// migrate creates the campaigns table. Idempotent (IF NOT EXISTS). The table
|
||||
// leads its lookup indexes with `org` so tenant isolation is a physical
|
||||
// property, not just a WHERE clause.
|
||||
// property, not just a WHERE clause. account + external_id link a stored campaign
|
||||
// to its launched provider execution (provider.go); they are added idempotently
|
||||
// so a DB created before the connector-execution edge existed gains them cleanly.
|
||||
func (s *Store) migrate() error {
|
||||
const ddl = `
|
||||
CREATE TABLE IF NOT EXISTS ads_campaigns (
|
||||
@@ -62,6 +64,8 @@ CREATE TABLE IF NOT EXISTS ads_campaigns (
|
||||
org TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
platform TEXT NOT NULL DEFAULT 'meta',
|
||||
account TEXT NOT NULL DEFAULT '',
|
||||
external_id TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'draft',
|
||||
objective TEXT NOT NULL DEFAULT '',
|
||||
budget INTEGER NOT NULL DEFAULT 0,
|
||||
@@ -76,9 +80,48 @@ CREATE INDEX IF NOT EXISTS ix_ads_campaigns_org_platform ON ads_campaigns(org, p
|
||||
if _, err := s.db.Exec(ddl); err != nil {
|
||||
return fmt.Errorf("ads migrate: %w", err)
|
||||
}
|
||||
// Idempotent column adds for DBs created before account/external_id existed.
|
||||
// The column names are package constants (never user input) → safe to inline.
|
||||
for col, spec := range map[string]string{
|
||||
"account": "TEXT NOT NULL DEFAULT ''",
|
||||
"external_id": "TEXT NOT NULL DEFAULT ''",
|
||||
} {
|
||||
if err := s.addColumnIfMissing("ads_campaigns", col, spec); err != nil {
|
||||
return fmt.Errorf("ads migrate column %s: %w", col, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// addColumnIfMissing ALTERs table to add col (with spec) only when absent. table/
|
||||
// col/spec are package constants, so the interpolation is injection-safe. Makes
|
||||
// the schema migration forward-only and idempotent across process restarts.
|
||||
func (s *Store) addColumnIfMissing(table, col, spec string) error {
|
||||
rows, err := s.db.Query(`PRAGMA table_info(` + table + `)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
for rows.Next() {
|
||||
var (
|
||||
cid, notnull, pk int
|
||||
name, ctype string
|
||||
dflt sql.NullString
|
||||
)
|
||||
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil {
|
||||
return err
|
||||
}
|
||||
if name == col {
|
||||
return nil // already present
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.Exec(`ALTER TABLE ` + table + ` ADD COLUMN ` + col + ` ` + spec)
|
||||
return err
|
||||
}
|
||||
|
||||
// Close closes the underlying database. Idempotent-safe via sql.DB.
|
||||
func (s *Store) Close() error { return s.db.Close() }
|
||||
|
||||
@@ -89,31 +132,33 @@ func (s *Store) Close() error { return s.db.Close() }
|
||||
// (draft/active/paused/completed) — both validated at the write layer against
|
||||
// the fixed vocabularies in ads.go.
|
||||
type Campaign struct {
|
||||
ID string `json:"id"`
|
||||
Org string `json:"-"`
|
||||
Name string `json:"name"`
|
||||
Platform string `json:"platform"`
|
||||
Status string `json:"status"`
|
||||
Objective string `json:"objective"`
|
||||
Budget int64 `json:"budget"`
|
||||
Spend int64 `json:"spend"`
|
||||
CreatedAt int64 `json:"createdAt"`
|
||||
UpdatedAt int64 `json:"updatedAt"`
|
||||
ID string `json:"id"`
|
||||
Org string `json:"-"`
|
||||
Name string `json:"name"`
|
||||
Platform string `json:"platform"`
|
||||
Account string `json:"account,omitempty"` // provider ad-account ref (Meta act_<id>)
|
||||
ExternalID string `json:"externalId,omitempty"` // provider campaign id after a launch
|
||||
Status string `json:"status"`
|
||||
Objective string `json:"objective"`
|
||||
Budget int64 `json:"budget"`
|
||||
Spend int64 `json:"spend"`
|
||||
CreatedAt int64 `json:"createdAt"`
|
||||
UpdatedAt int64 `json:"updatedAt"`
|
||||
}
|
||||
|
||||
const campaignCols = `id,org,name,platform,status,objective,budget,spend,created_at,updated_at`
|
||||
const campaignCols = `id,org,name,platform,account,external_id,status,objective,budget,spend,created_at,updated_at`
|
||||
|
||||
func scanCampaign(sc interface{ Scan(...any) error }) (Campaign, error) {
|
||||
var c Campaign
|
||||
err := sc.Scan(&c.ID, &c.Org, &c.Name, &c.Platform, &c.Status, &c.Objective,
|
||||
err := sc.Scan(&c.ID, &c.Org, &c.Name, &c.Platform, &c.Account, &c.ExternalID, &c.Status, &c.Objective,
|
||||
&c.Budget, &c.Spend, &c.CreatedAt, &c.UpdatedAt)
|
||||
return c, err
|
||||
}
|
||||
|
||||
func (s *Store) CreateCampaign(ctx context.Context, c Campaign) (Campaign, error) {
|
||||
if _, err := s.db.ExecContext(ctx,
|
||||
`INSERT INTO ads_campaigns (`+campaignCols+`) VALUES (?,?,?,?,?,?,?,?,?,?)`,
|
||||
c.ID, c.Org, c.Name, c.Platform, c.Status, c.Objective, c.Budget, c.Spend,
|
||||
`INSERT INTO ads_campaigns (`+campaignCols+`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
c.ID, c.Org, c.Name, c.Platform, c.Account, c.ExternalID, c.Status, c.Objective, c.Budget, c.Spend,
|
||||
c.CreatedAt, c.UpdatedAt); err != nil {
|
||||
return Campaign{}, fmt.Errorf("insert campaign: %w", err)
|
||||
}
|
||||
@@ -161,10 +206,13 @@ func (s *Store) ListCampaigns(ctx context.Context, org, status string, limit int
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// UpdateCampaign edits the user-owned fields. external_id is deliberately NOT in
|
||||
// the SET list: it is launch-owned (MarkLaunched sets it), so editing a campaign
|
||||
// never clobbers the link to its live provider execution.
|
||||
func (s *Store) UpdateCampaign(ctx context.Context, c Campaign) (Campaign, error) {
|
||||
res, err := s.db.ExecContext(ctx,
|
||||
`UPDATE ads_campaigns SET name=?,platform=?,status=?,objective=?,budget=?,spend=?,updated_at=? WHERE org=? AND id=?`,
|
||||
c.Name, c.Platform, c.Status, c.Objective, c.Budget, c.Spend, c.UpdatedAt, c.Org, c.ID)
|
||||
`UPDATE ads_campaigns SET name=?,platform=?,account=?,status=?,objective=?,budget=?,spend=?,updated_at=? WHERE org=? AND id=?`,
|
||||
c.Name, c.Platform, c.Account, c.Status, c.Objective, c.Budget, c.Spend, c.UpdatedAt, c.Org, c.ID)
|
||||
if err != nil {
|
||||
return Campaign{}, fmt.Errorf("update campaign: %w", err)
|
||||
}
|
||||
@@ -174,6 +222,23 @@ func (s *Store) UpdateCampaign(ctx context.Context, c Campaign) (Campaign, error
|
||||
return s.GetCampaign(ctx, c.Org, c.ID)
|
||||
}
|
||||
|
||||
// MarkLaunched records the provider execution on a stored campaign: its account +
|
||||
// external id, and status=active. Org-scoped — a cross-tenant id affects zero
|
||||
// rows (errNotFound), never a foreign mutation. This is the ONLY writer of
|
||||
// external_id, so the launch link is never clobbered by a user edit.
|
||||
func (s *Store) MarkLaunched(ctx context.Context, org, id, account, externalID string, updatedAt int64) (Campaign, error) {
|
||||
res, err := s.db.ExecContext(ctx,
|
||||
`UPDATE ads_campaigns SET account=?,external_id=?,status='active',updated_at=? WHERE org=? AND id=?`,
|
||||
account, externalID, updatedAt, org, id)
|
||||
if err != nil {
|
||||
return Campaign{}, fmt.Errorf("mark launched: %w", err)
|
||||
}
|
||||
if n, _ := res.RowsAffected(); n == 0 {
|
||||
return Campaign{}, errNotFound
|
||||
}
|
||||
return s.GetCampaign(ctx, org, id)
|
||||
}
|
||||
|
||||
func (s *Store) DeleteCampaign(ctx context.Context, org, id string) (bool, error) {
|
||||
res, err := s.db.ExecContext(ctx, `DELETE FROM ads_campaigns WHERE org=? AND id=?`, org, id)
|
||||
if err != nil {
|
||||
|
||||
@@ -59,6 +59,7 @@ import (
|
||||
"github.com/hanzoai/cloud/audit"
|
||||
"github.com/hanzoai/cloud/clients/authors"
|
||||
"github.com/hanzoai/cloud/clients/commerceinproc"
|
||||
"github.com/hanzoai/cloud/clients/flags"
|
||||
"github.com/hanzoai/cloud/clients/principal"
|
||||
"github.com/hanzoai/cloud/clients/treasury"
|
||||
"github.com/zap-proto/zip"
|
||||
@@ -78,8 +79,9 @@ const (
|
||||
// profit-share is computed on: the affiliate earns its rate of Hanzo's MARGIN, not
|
||||
// of the customer's gross bill, so a payout can never exceed the margin Hanzo
|
||||
// actually earned — and the customer's charge is never touched. A clearly-named
|
||||
// POLICY default (mirrors metered_ai's price default): ops sets the real gross
|
||||
// margin per deployment via AFFILIATE_MARGIN_BPS, cross-checking the finance board.
|
||||
// POLICY default. The REAL value is set in admin.hanzo.ai (platform switch
|
||||
// affiliate_margin_bps) against the finance board's actual cost of revenue; this
|
||||
// literal is only the value before anyone has set one.
|
||||
// 4000 = 40%. 10000 (100%) degrades to a gross-revenue share; 0 accrues nothing.
|
||||
defaultMarginBps int64 = 4000
|
||||
// grantCurrency is the ledger currency for a credits payout.
|
||||
@@ -128,17 +130,41 @@ func levelRateBps(level int, a Affiliate) int64 {
|
||||
}
|
||||
}
|
||||
|
||||
// marginBpsKey is the admin-editable platform switch carrying Hanzo's gross-margin
|
||||
// fraction. It is what the affiliate share is computed ON, so it moves with the real
|
||||
// cost of revenue and must be changeable at any time, by us, without a deploy.
|
||||
const marginBpsKey = "affiliate_margin_bps"
|
||||
|
||||
func init() {
|
||||
flags.Register(flags.Def{
|
||||
Key: marginBpsKey, Category: "Gateway", Type: flags.TypeInt,
|
||||
Default: strconv.FormatInt(defaultMarginBps, 10),
|
||||
Label: "Affiliate share base — gross margin (bps)",
|
||||
Desc: "Hanzo's gross margin on customer spend, in basis points, that every affiliate " +
|
||||
"commission is a rate OF. 4000 = 40%. Set from the finance board's real cost of " +
|
||||
"revenue; raise or lower it any time and the next accrual uses the new value. " +
|
||||
"10000 degrades to a gross-revenue share; 0 accrues nothing.",
|
||||
})
|
||||
}
|
||||
|
||||
// affiliateMarginBps resolves the platform gross-margin fraction (basis points) from
|
||||
// AFFILIATE_MARGIN_BPS, clamped to [0,10000], else the policy default. An invalid or
|
||||
// out-of-range value falls through to the default so a typo can never silently zero
|
||||
// out (or over-inflate) the share base.
|
||||
func affiliateMarginBps() int64 {
|
||||
v := strings.TrimSpace(os.Getenv("AFFILIATE_MARGIN_BPS"))
|
||||
if v == "" {
|
||||
return defaultMarginBps
|
||||
}
|
||||
n, err := strconv.ParseInt(v, 10, 64)
|
||||
if err != nil || n < 0 || n > bpsDenom {
|
||||
// the admin-editable switch, clamped to [0,10000], else the policy default. An unset
|
||||
// or out-of-range value falls through to the default so a bad edit can never silently
|
||||
// zero out (or over-inflate) the share base.
|
||||
//
|
||||
// Read LIVE, per accrual — never captured at boot. It used to come from
|
||||
// AFFILIATE_MARGIN_BPS and be snapshotted into state at Mount, which meant the number
|
||||
// could not be changed at all without a redeploy: editing the env changed nothing
|
||||
// until the pod restarted, and there was no admin control. Margin tracks real costs
|
||||
// and moves, so a boot-time constant was the wrong shape for it.
|
||||
func affiliateMarginBps() int64 { return clampMarginBps(int64(flags.Int(marginBpsKey))) }
|
||||
|
||||
// clampMarginBps bounds a configured margin to [0,10000], falling back to the
|
||||
// policy default outside it. Pure, so the bound is testable without the engine.
|
||||
// 0 and 10000 are both LEGAL (accrue nothing / share gross revenue) — only
|
||||
// genuinely impossible values fall back.
|
||||
func clampMarginBps(n int64) int64 {
|
||||
if n < 0 || n > bpsDenom {
|
||||
return defaultMarginBps
|
||||
}
|
||||
return n
|
||||
@@ -173,7 +199,6 @@ type state struct {
|
||||
commerce commerce
|
||||
clicks *clicks // in-memory coalescing buffer for public link-click pings
|
||||
linkBase string // https://hanzo.ai (brand host) — the ?aff link prefix
|
||||
marginBps int64 // platform gross-margin fraction the share is computed on
|
||||
auditStore *audit.Recorder // best-effort payout/accrual audit; nil disables it
|
||||
}
|
||||
|
||||
@@ -204,12 +229,11 @@ func Mount(app *zip.App, deps cloud.Deps) error {
|
||||
commerce: newCommerceClient(commerceinproc.BaseURL(os.Getenv("CLOUD_COMMERCE_HTTP_URL")), os.Getenv("COMMERCE_SERVICE_TOKEN")),
|
||||
clicks: newClicks(),
|
||||
linkBase: linkBase(deps),
|
||||
marginBps: affiliateMarginBps(),
|
||||
auditStore: deps.Audit,
|
||||
}}
|
||||
mounted = s
|
||||
routes(app, s)
|
||||
s.Log.Info("affiliates mounted", "brand", s.Brand, "linkBase", s.State.linkBase, "marginBps", s.State.marginBps, "commerce", s.State.commerce.configured())
|
||||
s.Log.Info("affiliates mounted", "brand", s.Brand, "linkBase", s.State.linkBase, "marginBps", affiliateMarginBps(), "commerce", s.State.commerce.configured())
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -297,7 +321,7 @@ func myAffiliates(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
"requestedCode": a.RequestedCode,
|
||||
"link": affiliateLink(s, a.Code),
|
||||
"rateBps": a.RateBps,
|
||||
"marginBps": s.State.marginBps,
|
||||
"marginBps": affiliateMarginBps(),
|
||||
"handle": a.Handle,
|
||||
"referredCount": referred,
|
||||
"accruedCents": a.AccruedCents,
|
||||
@@ -375,7 +399,7 @@ func myAffiliatesMe(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
"code": a.Code,
|
||||
"link": affiliateLink(s, a.Code),
|
||||
"rateBps": a.RateBps,
|
||||
"marginBps": s.State.marginBps,
|
||||
"marginBps": affiliateMarginBps(),
|
||||
"handle": a.Handle,
|
||||
"levels": levels,
|
||||
"downlineTotal": len(downline),
|
||||
@@ -832,7 +856,7 @@ func accrueSource(s *cloud.Service[state], ctx context.Context, sourceOrg string
|
||||
// The share base is Hanzo's MARGIN on this spend, computed ONCE (level-independent).
|
||||
// Every level's share is a rate of this margin, so their sum ≤ margin (share never
|
||||
// touches the customer's bill). No margin → nothing to share (fail-closed).
|
||||
margin := marginOf(spend, s.State.marginBps)
|
||||
margin := marginOf(spend, affiliateMarginBps())
|
||||
if margin <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
@@ -899,7 +923,7 @@ func sweepAffiliate(s *cloud.Service[state], ctx context.Context, a Affiliate) (
|
||||
s.Log.Warn("affiliates: spend read failed", "affiliate", a.ID, "source", src, "err", serr)
|
||||
continue
|
||||
}
|
||||
margin := marginOf(spend, s.State.marginBps)
|
||||
margin := marginOf(spend, affiliateMarginBps())
|
||||
commission := margin * levelRateBps(level, a) / bpsDenom
|
||||
if commission <= 0 {
|
||||
continue
|
||||
|
||||
@@ -86,11 +86,10 @@ func mount(t *testing.T) (*zip.App, *cloud.Service[state], *fakeCommerce) {
|
||||
s := &cloud.Service[state]{
|
||||
Base: cloud.NewBase(cloud.Deps{Logger: luxlog.New("test"), Brand: "hanzo"}, "affiliates"),
|
||||
State: state{
|
||||
store: store,
|
||||
commerce: fc,
|
||||
clicks: newClicks(),
|
||||
linkBase: "https://hanzo.ai",
|
||||
marginBps: defaultMarginBps, // realistic 40% gross margin — the share base
|
||||
store: store,
|
||||
commerce: fc,
|
||||
clicks: newClicks(),
|
||||
linkBase: "https://hanzo.ai",
|
||||
},
|
||||
}
|
||||
app := zip.New(zip.Config{Logger: luxlog.New("test")})
|
||||
|
||||
@@ -137,7 +137,7 @@ func myEarnings(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
}
|
||||
return c.JSON(http.StatusOK, map[string]any{
|
||||
"isAffiliate": true,
|
||||
"marginBps": s.State.marginBps,
|
||||
"marginBps": affiliateMarginBps(),
|
||||
"accruedCents": a.AccruedCents,
|
||||
"pendingCents": a.PendingCents(),
|
||||
"paidCents": a.PaidCents,
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package affiliates
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/flags"
|
||||
)
|
||||
|
||||
// The margin is the base every affiliate commission is a rate OF, so it has to
|
||||
// track our real cost of revenue and move when that moves. It used to come from
|
||||
// AFFILIATE_MARGIN_BPS and be snapshotted into state at Mount — two reasons it
|
||||
// could not change: no admin control, and even editing the env did nothing until
|
||||
// the pod restarted.
|
||||
func TestAffiliateMarginBps_IsARegisteredAdminSwitch(t *testing.T) {
|
||||
var def *flags.Def
|
||||
for _, d := range flags.Defs() {
|
||||
if d.Key == marginBpsKey {
|
||||
dd := d
|
||||
def = &dd
|
||||
break
|
||||
}
|
||||
}
|
||||
if def == nil {
|
||||
t.Fatalf("%s is not registered — it would not appear in admin.hanzo.ai, and flags.Int would return 0 (= no commission accrues at all)", marginBpsKey)
|
||||
}
|
||||
if def.Type != flags.TypeInt {
|
||||
t.Errorf("Type = %v, want TypeInt", def.Type)
|
||||
}
|
||||
if def.Env != "" {
|
||||
t.Errorf("Env = %q, want empty — margin must not be configurable by environment variable", def.Env)
|
||||
}
|
||||
if def.ReadOnly {
|
||||
t.Error("ReadOnly — the whole point is that we can change it at any time")
|
||||
}
|
||||
if def.Default != strconv.FormatInt(defaultMarginBps, 10) {
|
||||
t.Errorf("Default = %q, want %d", def.Default, defaultMarginBps)
|
||||
}
|
||||
}
|
||||
|
||||
// Unmounted / unset resolves to the policy default rather than 0. This is the
|
||||
// dangerous direction: 0 is a LEGITIMATE margin ("accrues nothing"), so a
|
||||
// zero-on-missing would silently switch off every affiliate commission instead of
|
||||
// failing loudly. flags.resolve falls back to Def.Default, which is why this holds.
|
||||
func TestAffiliateMarginBps_UnsetIsTheDefaultNotZero(t *testing.T) {
|
||||
if got := affiliateMarginBps(); got != defaultMarginBps {
|
||||
t.Fatalf("affiliateMarginBps() = %d, want %d — a missing value must not zero the share base", got, defaultMarginBps)
|
||||
}
|
||||
}
|
||||
|
||||
// A bad edit cannot over-inflate or negate the base: out-of-range falls back to
|
||||
// the policy default. In range (including 0 and 100%) is honoured as written.
|
||||
func TestAffiliateMarginBps_ClampsOutOfRange(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
in int64
|
||||
want int64
|
||||
}{
|
||||
{"negative", -1, defaultMarginBps},
|
||||
{"above 100%", bpsDenom + 1, defaultMarginBps},
|
||||
{"exactly 100% is legal", bpsDenom, bpsDenom},
|
||||
{"zero is legal (accrues nothing)", 0, 0},
|
||||
{"mid range", 2500, 2500},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := clampMarginBps(tc.in); got != tc.want {
|
||||
t.Fatalf("clampMarginBps(%d) = %d, want %d", tc.in, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The share base must be read at accrual time, not captured at boot — otherwise
|
||||
// changing it in admin does nothing until the next deploy, which is the failure
|
||||
// this replaced.
|
||||
func TestMargin_IsNotSnapshotAtBoot(t *testing.T) {
|
||||
base := marginOf(10000, affiliateMarginBps())
|
||||
if base != 10000*defaultMarginBps/bpsDenom {
|
||||
t.Fatalf("margin base = %d, want %d", base, 10000*defaultMarginBps/bpsDenom)
|
||||
}
|
||||
// Same call again must re-read rather than serve a cached snapshot.
|
||||
if again := marginOf(10000, affiliateMarginBps()); again != base {
|
||||
t.Fatalf("margin base not stable across reads: %d then %d", base, again)
|
||||
}
|
||||
}
|
||||
@@ -220,9 +220,13 @@ func rfc3339(unix int64) string {
|
||||
return time.Unix(unix, 0).UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
// toView projects a stored agent onto the wire. Model goes out through
|
||||
// cloud.ZenModel: writes already normalize, so in steady state this changes
|
||||
// nothing — it is the backstop that keeps a row written before the normalization
|
||||
// existed (or restored from an old backup) from publishing an upstream name.
|
||||
func toView(a Agent, runs int) agentView {
|
||||
return agentView{
|
||||
ID: a.ID, Name: a.Name, Model: a.Model, Description: a.Description,
|
||||
ID: a.ID, Name: a.Name, Model: cloud.ZenModel(a.Model), Description: a.Description,
|
||||
Tools: nonNil(a.Tools), Status: a.Status,
|
||||
ExecutionMode: a.ExecutionMode, Schedule: a.Schedule,
|
||||
ComputeRef: a.ComputeRef, ServiceAccountID: a.ServiceAccountID,
|
||||
@@ -231,9 +235,12 @@ func toView(a Agent, runs int) agentView {
|
||||
}
|
||||
}
|
||||
|
||||
// toRunView projects one execution onto the wire. Run history is customer-visible
|
||||
// too, and a run recorded before the migration carries the model it actually ran
|
||||
// on — so it is guarded the same way the agent is.
|
||||
func toRunView(r Run) runView {
|
||||
return runView{
|
||||
ID: r.ID, Status: r.Status, Model: r.Model, Input: r.Input, Output: r.Output,
|
||||
ID: r.ID, Status: r.Status, Model: cloud.ZenModel(r.Model), Input: r.Input, Output: r.Output,
|
||||
Error: r.Error, DurationMs: r.DurationMs, CreatedAt: rfc3339(r.CreatedAt),
|
||||
}
|
||||
}
|
||||
@@ -273,7 +280,11 @@ func Mount(app *zip.App, deps cloud.Deps) error {
|
||||
State: state{
|
||||
store: store,
|
||||
ai: deps.AI,
|
||||
defaultModel: strings.TrimSpace(deps.AIDefaultModel),
|
||||
// cloud.ZenModel guards the CONFIG boundary: an operator who points
|
||||
// CLOUD_AI_DEFAULT_MODEL at an upstream name still gets the Hanzo name
|
||||
// stamped on every agent seeded or created without one. The caller
|
||||
// boundary is guarded separately, in create/update.
|
||||
defaultModel: cloud.ZenModel(deps.AIDefaultModel),
|
||||
failoverModel: strings.TrimSpace(deps.AIFallbackModel),
|
||||
bill: cloud.NewResourceMeter(deps, meterKind),
|
||||
bus: newBus(),
|
||||
@@ -362,6 +373,11 @@ func create(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
// model falls back to the deployment default (a valid catalog model the
|
||||
// operator configured) — trusted, not re-validated — so a bot launched
|
||||
// without a model still runs. If neither is present the model is required.
|
||||
//
|
||||
// Then cloud.ZenModel normalizes: an upstream family name never enters the
|
||||
// registry, so it can never be served back out of one. The registry stores
|
||||
// exactly what we would show, which keeps the read guard in toView a no-op
|
||||
// rather than a lie about what the agent runs on.
|
||||
model := strings.TrimSpace(body.Model)
|
||||
if model == "" {
|
||||
if model = s.State.defaultModel; model == "" {
|
||||
@@ -370,6 +386,7 @@ func create(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
} else if err := validateModel(s, c.Context(), model); err != nil {
|
||||
return err
|
||||
}
|
||||
model = cloud.ZenModel(model)
|
||||
if len(body.Instructions) > maxInstructions {
|
||||
return zip.ErrBadRequest("instructions too large")
|
||||
}
|
||||
@@ -499,7 +516,7 @@ func update(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
if err := validateModel(s, c.Context(), m); err != nil {
|
||||
return err
|
||||
}
|
||||
a.Model = m
|
||||
a.Model = cloud.ZenModel(m)
|
||||
}
|
||||
if body.Instructions != nil {
|
||||
if len(*body.Instructions) > maxInstructions {
|
||||
@@ -956,7 +973,7 @@ func activity(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
}
|
||||
evs := make([]activityView, 0, len(runs)+2*len(rows))
|
||||
for _, r := range runs {
|
||||
kind, msg := "invoked", "Invoked "+r.Model
|
||||
kind, msg := "invoked", "Invoked "+cloud.ZenModel(r.Model)
|
||||
if r.Status == "error" {
|
||||
kind, msg = "failed", trimMsg(r.Error)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
package agents
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
)
|
||||
|
||||
// brand_test.go is the guard that stops the leak recurring.
|
||||
//
|
||||
// Hanzo serves the enso and zen families under its own names. An upstream family
|
||||
// name reaching a customer publishes which base sits behind a Hanzo model, so it
|
||||
// is a defect wherever it appears — an API payload, a UI string, a model list, a
|
||||
// log a customer can read.
|
||||
//
|
||||
// The tests below drive the agents registry over real HTTP with the WORST-CASE
|
||||
// inputs (an operator who configured an upstream default, a caller who POSTs an
|
||||
// upstream model, a row already in the database) and assert that no upstream
|
||||
// family name comes back out of ANY of it.
|
||||
|
||||
// scanUpstream fails the test if any upstream family name appears anywhere in
|
||||
// body. It scans the raw bytes rather than a decoded field, so a name leaking
|
||||
// through a message, a nested object or a field nobody thought to check is
|
||||
// caught just the same.
|
||||
func scanUpstream(t *testing.T, what string, body []byte) {
|
||||
t.Helper()
|
||||
for _, family := range []string{"deepseek", "qwen", "glm-", "kimi", "minimax"} {
|
||||
if bytes.Contains(bytes.ToLower(body), []byte(family)) {
|
||||
t.Errorf("BRAND LEAK: %s response contains upstream family %q\n%s", what, family, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNoUpstreamNameOnTheWire is the regression guard. It walks every
|
||||
// customer-visible read of the agents registry and scans the raw response.
|
||||
func TestNoUpstreamNameOnTheWire(t *testing.T) {
|
||||
// The adversarial deployment: an operator who set CLOUD_AI_DEFAULT_MODEL to an
|
||||
// upstream name, and a gateway whose catalog serves upstream names — exactly
|
||||
// the configuration that produced the live leak.
|
||||
ai := &catalogAI{content: "pong", ids: []string{"enso", "enso-flash", "deepseek-v4-flash", "glm-5.2"}}
|
||||
app := mountAppModel(t, ai, "deepseek-v4-flash")
|
||||
|
||||
// 1. An agent created with NO model. The configured default is an upstream
|
||||
// name; normalization must still store and answer the Hanzo name.
|
||||
code, body := do(t, app, http.MethodPost, "/v1/agents", "acme",
|
||||
map[string]any{"name": "defaulted", "instructions": "be terse"})
|
||||
if code != http.StatusCreated {
|
||||
t.Fatalf("create defaulted: want 201, got %d (%s)", code, body)
|
||||
}
|
||||
scanUpstream(t, "POST /v1/agents (defaulted)", body)
|
||||
var created agentView
|
||||
if err := json.Unmarshal(body, &created); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if created.Model != cloud.DefaultModel {
|
||||
t.Fatalf("defaulted agent model = %q, want %q", created.Model, cloud.DefaultModel)
|
||||
}
|
||||
|
||||
// 2. A caller who explicitly POSTs an upstream model the gateway really does
|
||||
// serve. It is accepted (not a 400 — the name was ours to leak, not theirs
|
||||
// to be punished for) but normalized, so it never enters the registry.
|
||||
code, body = do(t, app, http.MethodPost, "/v1/agents", "acme",
|
||||
map[string]any{"name": "pinned", "model": "deepseek-v4-flash"})
|
||||
if code != http.StatusCreated {
|
||||
t.Fatalf("create pinned: want 201, got %d (%s)", code, body)
|
||||
}
|
||||
scanUpstream(t, "POST /v1/agents (explicit upstream model)", body)
|
||||
|
||||
// 3. PATCH to an upstream model is normalized the same way.
|
||||
code, body = do(t, app, http.MethodPatch, "/v1/agents/pinned", "acme",
|
||||
map[string]any{"model": "glm-5.2"})
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("patch: want 200, got %d (%s)", code, body)
|
||||
}
|
||||
scanUpstream(t, "PATCH /v1/agents/:ref", body)
|
||||
|
||||
// 4. A run, and the run history and activity feed that record it.
|
||||
if code, body = do(t, app, http.MethodPost, "/v1/agents/defaulted/run", "acme",
|
||||
map[string]any{"input": "hi"}); code != http.StatusOK {
|
||||
t.Fatalf("run: want 200, got %d (%s)", code, body)
|
||||
}
|
||||
scanUpstream(t, "POST /v1/agents/:ref/run", body)
|
||||
|
||||
// 5. Every remaining read surface.
|
||||
for _, path := range []string{
|
||||
"/v1/agents",
|
||||
"/v1/agents/defaulted",
|
||||
"/v1/agents/pinned",
|
||||
"/v1/agents/defaulted/runs",
|
||||
"/v1/agents/activity",
|
||||
"/v1/agents/metrics",
|
||||
} {
|
||||
code, body := do(t, app, http.MethodGet, path, "acme", nil)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("GET %s: want 200, got %d (%s)", path, code, body)
|
||||
}
|
||||
scanUpstream(t, "GET "+path, body)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMigrateModelRewritesStoredRows proves the data migration: rows written
|
||||
// before normalization existed (des/dev/vi/verify-run on the live deployment)
|
||||
// are rewritten in place on store open, the rewrite is idempotent, and the
|
||||
// pre-migration value is retained so the change can be reversed.
|
||||
func TestMigrateModelRewritesStoredRows(t *testing.T) {
|
||||
dir := t.TempDir() + "/agents.db"
|
||||
ctx := context.Background()
|
||||
|
||||
// A store holding exactly what the live registry held.
|
||||
st, err := openStore(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
for _, name := range []string{"des", "dev", "vi", "verify-run"} {
|
||||
if err := st.Create(ctx, Agent{
|
||||
ID: "agent_" + name, Org: "hanzo", Name: name,
|
||||
Model: "deepseek-v4-flash", CreatedAt: 1, UpdatedAt: 1,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
// An agent already on a Hanzo model must NOT be touched.
|
||||
if err := st.Create(ctx, Agent{
|
||||
ID: "agent_enso", Org: "hanzo", Name: "enso",
|
||||
Model: "enso-pro", CreatedAt: 1, UpdatedAt: 1,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed enso: %v", err)
|
||||
}
|
||||
// Bypass the write-side normalization to plant the rows exactly as they exist
|
||||
// live, then run the migration the way a real deploy does — by opening the store.
|
||||
if _, err := st.db.Exec(`UPDATE agents SET model='deepseek-v4-flash' WHERE name<>'enso'`); err != nil {
|
||||
t.Fatalf("plant: %v", err)
|
||||
}
|
||||
if err := st.migrateModel(); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
|
||||
assertModels := func(when string) {
|
||||
t.Helper()
|
||||
list, err := st.List(ctx, "hanzo")
|
||||
if err != nil {
|
||||
t.Fatalf("list %s: %v", when, err)
|
||||
}
|
||||
if len(list) != 5 {
|
||||
t.Fatalf("%s: want 5 agents, got %d", when, len(list))
|
||||
}
|
||||
for _, a := range list {
|
||||
want := cloud.DefaultModel
|
||||
if a.Name == "enso" {
|
||||
want = "enso-pro" // an already-Hanzo model is left alone
|
||||
}
|
||||
if a.Model != want {
|
||||
t.Errorf("%s: agent %q model = %q, want %q", when, a.Name, a.Model, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
assertModels("after migrate")
|
||||
|
||||
// Idempotent: a second pass moves nothing.
|
||||
if err := st.migrateModel(); err != nil {
|
||||
t.Fatalf("migrate twice: %v", err)
|
||||
}
|
||||
assertModels("after second migrate")
|
||||
|
||||
// Reversible: the pre-migration value was retained for exactly the four rows
|
||||
// that moved, and putting it back restores them.
|
||||
var snapshots int
|
||||
if err := st.db.QueryRow(`SELECT count(*) FROM model_snapshot`).Scan(&snapshots); err != nil {
|
||||
t.Fatalf("count snapshot: %v", err)
|
||||
}
|
||||
if snapshots != 4 {
|
||||
t.Fatalf("model_snapshot has %d rows, want 4 (only the rows that moved)", snapshots)
|
||||
}
|
||||
if _, err := st.db.Exec(`UPDATE agents SET model = (SELECT model FROM model_snapshot WHERE agent_id = agents.id)
|
||||
WHERE id IN (SELECT agent_id FROM model_snapshot)`); err != nil {
|
||||
t.Fatalf("undo: %v", err)
|
||||
}
|
||||
list, err := st.List(ctx, "hanzo")
|
||||
if err != nil {
|
||||
t.Fatalf("list after undo: %v", err)
|
||||
}
|
||||
for _, a := range list {
|
||||
want := "deepseek-v4-flash"
|
||||
if a.Name == "enso" {
|
||||
want = "enso-pro"
|
||||
}
|
||||
if a.Model != want {
|
||||
t.Errorf("undo: agent %q model = %q, want %q", a.Name, a.Model, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSeedPersonalitiesUsesHanzoModel proves the built-in crew (dev/des/vi) is
|
||||
// seeded on a Hanzo model even when the deployment default is an upstream name —
|
||||
// the seed path that put deepseek-v4-flash on three live agents.
|
||||
func TestSeedPersonalitiesUsesHanzoModel(t *testing.T) {
|
||||
ai := &catalogAI{content: "pong", ids: []string{"enso", "deepseek-v4-flash"}}
|
||||
app := mountAppModel(t, ai, "deepseek-v4-flash")
|
||||
|
||||
n, err := SeedPersonalities(context.Background(), "acme")
|
||||
if err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
if n != len(personalities) {
|
||||
t.Fatalf("seeded %d personas, want %d", n, len(personalities))
|
||||
}
|
||||
code, body := do(t, app, http.MethodGet, "/v1/agents", "acme", nil)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("list: want 200, got %d (%s)", code, body)
|
||||
}
|
||||
scanUpstream(t, "GET /v1/agents after SeedPersonalities", body)
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package agents
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
sqlitedrv "github.com/hanzoai/sqlite"
|
||||
)
|
||||
|
||||
// TestMain makes this package's store-backed tests runnable on either build, the
|
||||
// same way the root package and clients/research already do it.
|
||||
//
|
||||
// Every test here opens an agents.db, and cek REFUSES to open a store without a
|
||||
// master key on an encryption-capable build — which, since hanzoai/sqlite v0.3,
|
||||
// includes the pure-Go build. Without this the WHOLE package fails to run, which
|
||||
// is exactly how the brand guard could go missing unnoticed.
|
||||
//
|
||||
// Supply a throwaway dev key ONLY when the build can encrypt AND the environment
|
||||
// did not already provide one, so a real injected key is never overridden.
|
||||
func TestMain(m *testing.M) {
|
||||
if sqlitedrv.EncryptionAvailable() && os.Getenv("CLOUD_KMS_MASTER_KEY_REF") == "" {
|
||||
_ = os.Setenv("CLOUD_KMS_MASTER_KEY_REF", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") // 32 zero bytes, dev-only
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/types"
|
||||
)
|
||||
|
||||
@@ -50,8 +51,10 @@ func TestHTTPCreateModelValidation(t *testing.T) {
|
||||
t.Fatalf("catalog model want 201, got %d (%s)", code, body)
|
||||
}
|
||||
|
||||
// An OMITTED model falls back to the deployment default (a valid catalog
|
||||
// model), so the agent is created AND runnable — not a 400.
|
||||
// An OMITTED model falls back to the deployment default, so the agent is
|
||||
// created AND runnable — not a 400. This deployment's default is an UPSTREAM
|
||||
// name, so what actually lands is cloud.DefaultModel: the brand boundary holds
|
||||
// even against an operator who misconfigured CLOUD_AI_DEFAULT_MODEL.
|
||||
code, body = do(t, app, http.MethodPost, "/v1/agents", "acme",
|
||||
map[string]any{"name": "defaulted", "instructions": "be terse"})
|
||||
if code != http.StatusCreated {
|
||||
@@ -59,8 +62,8 @@ func TestHTTPCreateModelValidation(t *testing.T) {
|
||||
}
|
||||
var created agentView
|
||||
_ = json.Unmarshal(body, &created)
|
||||
if created.Model != defaultModel {
|
||||
t.Fatalf("omitted model must store the deployment default %q, got %q", defaultModel, created.Model)
|
||||
if created.Model != cloud.DefaultModel {
|
||||
t.Fatalf("omitted model must store the Hanzo default %q, got %q", cloud.DefaultModel, created.Model)
|
||||
}
|
||||
// The defaulted agent runs (its stored default model is a real catalog model).
|
||||
if code, body := do(t, app, http.MethodPost, "/v1/agents/defaulted/run", "acme",
|
||||
|
||||
@@ -190,6 +190,7 @@ func mountSessions(s *cloud.Service[state], app *zip.App) {
|
||||
g.Patch("/sessions/:id", cloud.Handle(s, patchSession))
|
||||
g.Get("/sessions/:id/tree", cloud.Handle(s, sessionTree))
|
||||
g.Post("/sessions/:id/events", cloud.Handle(s, appendSessionEvent))
|
||||
g.Get("/sessions/:id/control", cloud.Handle(s, drainControl))
|
||||
g.Post("/sessions/:id/pause", cloud.Handle(s, pauseSession))
|
||||
g.Post("/sessions/:id/resume", cloud.Handle(s, resumeSession))
|
||||
g.Post("/sessions/:id/stop", cloud.Handle(s, stopSession))
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
package agents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// sessions_control_drain.go is the CLI-facing half of the control channel. The
|
||||
// dashboard POSTs steering commands (pause/resume/stop/message) which `control`
|
||||
// records as durable KindControl events; a locally-started `hanzo code` session
|
||||
// is NOT task-backed, so those events are never forwarded to a tasks engine —
|
||||
// the running surface consumes them itself by polling here.
|
||||
//
|
||||
// GET /v1/agents/sessions/:id/control?after=<seq> returns the control commands
|
||||
// newer than the caller's cursor, oldest first, with a cursor to poll from next.
|
||||
// It is READ-ONLY and org-scoped (org is the ONLY tenant key): the same validated
|
||||
// principal + same-org ownership that guards the session guards this, so a poller
|
||||
// only ever drains its OWN session's commands and a foreign id is a clean 404.
|
||||
|
||||
// controlCommandView is one steering command as the CLI consumes it.
|
||||
type controlCommandView struct {
|
||||
Seq int64 `json:"seq"`
|
||||
Command string `json:"command"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Payload json.RawMessage `json:"payload,omitempty"`
|
||||
}
|
||||
|
||||
// drainControl returns KindControl commands with seq > ?after for the caller's
|
||||
// own session. Bounded (200/poll) and cursor-driven, so a steady poll is cheap
|
||||
// and never redelivers an applied command.
|
||||
func drainControl(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
org, ok := tenant(c)
|
||||
if !ok {
|
||||
return zip.ErrForbidden("X-Org-Id required")
|
||||
}
|
||||
id := idParam(c)
|
||||
if _, err := s.State.store.GetSession(c.Context(), org, id); err == errSessionNotFound {
|
||||
return zip.ErrNotFound("session not found")
|
||||
} else if err != nil {
|
||||
return zip.Errorf(http.StatusInternalServerError, "get: %v", err)
|
||||
}
|
||||
|
||||
after := int64(0)
|
||||
if q := strings.TrimSpace(c.Query("after")); q != "" {
|
||||
if n, perr := strconv.ParseInt(q, 10, 64); perr == nil && n >= 0 {
|
||||
after = n
|
||||
}
|
||||
}
|
||||
|
||||
evs, err := s.State.store.ListControlAfter(c.Context(), org, id, after, 200)
|
||||
if err != nil {
|
||||
return zip.Errorf(http.StatusInternalServerError, "drain control: %v", err)
|
||||
}
|
||||
|
||||
cursor := after
|
||||
cmds := make([]controlCommandView, 0, len(evs))
|
||||
for _, e := range evs {
|
||||
var cp controlPayload
|
||||
_ = json.Unmarshal([]byte(e.Payload), &cp)
|
||||
cmds = append(cmds, controlCommandView{
|
||||
Seq: e.Seq,
|
||||
Command: cp.Command,
|
||||
Message: cp.Message,
|
||||
Payload: cp.Payload,
|
||||
})
|
||||
if e.Seq > cursor {
|
||||
cursor = e.Seq
|
||||
}
|
||||
}
|
||||
return c.JSON(http.StatusOK, map[string]any{"commands": cmds, "cursor": cursor})
|
||||
}
|
||||
|
||||
// ListControlAfter returns a session's KindControl events with seq > since,
|
||||
// oldest first — the durable steering queue a running surface drains. Mirrors
|
||||
// ListEvents but filters to control so a chatty session's message/log/tool-call
|
||||
// events never dilute a poll.
|
||||
func (s *Store) ListControlAfter(ctx context.Context, org, sessionID string, since int64, limit int) ([]Event, error) {
|
||||
if limit <= 0 || limit > 1000 {
|
||||
limit = 200
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx,
|
||||
`SELECT id,session_id,org,seq,kind,actor,payload,created_at
|
||||
FROM agent_session_events WHERE org=? AND session_id=? AND kind=? AND seq>?
|
||||
ORDER BY seq ASC LIMIT ?`, org, sessionID, KindControl, since, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var out []Event
|
||||
for rows.Next() {
|
||||
var e Event
|
||||
if err := rows.Scan(&e.ID, &e.SessionID, &e.Org, &e.Seq, &e.Kind, &e.Actor,
|
||||
&e.Payload, &e.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package agents
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestListControlAfterFiltersCursorsAndScopes proves the CLI drain query: it
|
||||
// returns ONLY control events, oldest first, honours the ?after cursor so an
|
||||
// applied command is never redelivered, and is org-scoped so no co-tenant can
|
||||
// drain another org's steering queue.
|
||||
func TestListControlAfterFiltersCursorsAndScopes(t *testing.T) {
|
||||
s := testSessionStore(t)
|
||||
ctx := context.Background()
|
||||
if err := s.CreateSession(ctx, mkSession("acme", "root", "", "root")); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
// Interleave non-control noise with two control commands.
|
||||
_, _ = s.AppendEvent(ctx, Event{ID: genIDMust(t), SessionID: "root", Org: "acme", Kind: KindLog, CreatedAt: now})
|
||||
c1, err := s.AppendEvent(ctx, Event{ID: genIDMust(t), SessionID: "root", Org: "acme", Kind: KindControl, Payload: `{"command":"pause"}`, CreatedAt: now})
|
||||
if err != nil {
|
||||
t.Fatalf("append c1: %v", err)
|
||||
}
|
||||
_, _ = s.AppendEvent(ctx, Event{ID: genIDMust(t), SessionID: "root", Org: "acme", Kind: KindMessage, CreatedAt: now})
|
||||
c2, err := s.AppendEvent(ctx, Event{ID: genIDMust(t), SessionID: "root", Org: "acme", Kind: KindControl, Payload: `{"command":"stop"}`, CreatedAt: now})
|
||||
if err != nil {
|
||||
t.Fatalf("append c2: %v", err)
|
||||
}
|
||||
|
||||
// From the start: exactly the two control events, in seq order, no noise.
|
||||
got, err := s.ListControlAfter(ctx, "acme", "root", 0, 200)
|
||||
if err != nil {
|
||||
t.Fatalf("drain: %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("want 2 control events, got %d", len(got))
|
||||
}
|
||||
if got[0].Seq != c1.Seq || got[1].Seq != c2.Seq {
|
||||
t.Fatalf("want ordered [%d,%d], got [%d,%d]", c1.Seq, c2.Seq, got[0].Seq, got[1].Seq)
|
||||
}
|
||||
for _, e := range got {
|
||||
if e.Kind != KindControl {
|
||||
t.Fatalf("non-control kind leaked into drain: %s", e.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
// Cursor past the first command yields only the second (no redelivery).
|
||||
got2, err := s.ListControlAfter(ctx, "acme", "root", c1.Seq, 200)
|
||||
if err != nil {
|
||||
t.Fatalf("drain after cursor: %v", err)
|
||||
}
|
||||
if len(got2) != 1 || got2[0].Seq != c2.Seq {
|
||||
t.Fatalf("cursor drain want [%d], got %+v", c2.Seq, got2)
|
||||
}
|
||||
|
||||
// Tenant isolation: another org drains nothing from acme's session.
|
||||
evil, err := s.ListControlAfter(ctx, "evil", "root", 0, 200)
|
||||
if err != nil {
|
||||
t.Fatalf("evil drain: %v", err)
|
||||
}
|
||||
if len(evil) != 0 {
|
||||
t.Fatalf("cross-tenant control leak: %d events", len(evil))
|
||||
}
|
||||
}
|
||||
@@ -7,12 +7,14 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
// github.com/hanzoai/sqlite is the ONE Hanzo SQLite driver: it registers
|
||||
// the "sqlite" database/sql name under both build tags (cgo →
|
||||
// mattn+SQLCipher, encrypted at rest; !cgo → pure-Go modernc). Importing
|
||||
// modernc directly instead would double-register "sqlite" under CGO and
|
||||
// panic at init. Blank import registers the driver.
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/cek"
|
||||
_ "github.com/hanzoai/sqlite"
|
||||
)
|
||||
@@ -184,6 +186,72 @@ CREATE INDEX IF NOT EXISTS ix_runs_org_agent_created ON agent_runs(org, agent_na
|
||||
if err := s.migrateClaimKeys(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Rewrite any agent still holding an upstream model name to the Hanzo name.
|
||||
if err := s.migrateModel(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrateModel rewrites every agent whose stored model carries an upstream family
|
||||
// name to cloud.ZenModel of it — the data half of the brand boundary. Writes have
|
||||
// normalized since; this moves the rows that predate that.
|
||||
//
|
||||
// IDEMPOTENT: it selects the rows to move by the same predicate that decides where
|
||||
// they land, so after one pass nothing matches and a re-run is a no-op. Safe to run
|
||||
// on every store open, which is how it reaches every org's database without anyone
|
||||
// touching a pod.
|
||||
//
|
||||
// REVERSIBLE: the pre-migration model is written to model_snapshot first, keyed by
|
||||
// agent id with INSERT OR IGNORE, so the FIRST value ever seen is the one kept — a
|
||||
// second pass can never overwrite the true original. Undo is one statement:
|
||||
//
|
||||
// UPDATE agents SET model = (SELECT model FROM model_snapshot WHERE agent_id = agents.id)
|
||||
// WHERE id IN (SELECT agent_id FROM model_snapshot);
|
||||
//
|
||||
// agent_runs is deliberately NOT rewritten. A run is a record of what actually
|
||||
// happened and falsifying it would be worse than the leak; toRunView guards the
|
||||
// presentation instead.
|
||||
func (s *Store) migrateModel() error {
|
||||
if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS model_snapshot (
|
||||
agent_id TEXT PRIMARY KEY,
|
||||
model TEXT NOT NULL,
|
||||
at INTEGER NOT NULL
|
||||
)`); err != nil {
|
||||
return fmt.Errorf("migrate: model_snapshot: %w", err)
|
||||
}
|
||||
rows, err := s.db.Query(`SELECT DISTINCT model FROM agents`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("migrate: scan models: %w", err)
|
||||
}
|
||||
var stale []string
|
||||
for rows.Next() {
|
||||
var m string
|
||||
if err := rows.Scan(&m); err != nil {
|
||||
_ = rows.Close()
|
||||
return fmt.Errorf("migrate: scan model: %w", err)
|
||||
}
|
||||
if cloud.UpstreamModel(m) {
|
||||
stale = append(stale, m)
|
||||
}
|
||||
}
|
||||
_ = rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("migrate: models: %w", err)
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
for _, m := range stale {
|
||||
if _, err := s.db.Exec(
|
||||
`INSERT OR IGNORE INTO model_snapshot(agent_id, model, at) SELECT id, model, ? FROM agents WHERE model = ?`,
|
||||
now, m); err != nil {
|
||||
return fmt.Errorf("migrate: snapshot %q: %w", m, err)
|
||||
}
|
||||
if _, err := s.db.Exec(
|
||||
`UPDATE agents SET model = ?, updated_at = ? WHERE model = ?`,
|
||||
cloud.ZenModel(m), now, m); err != nil {
|
||||
return fmt.Errorf("migrate: rewrite %q: %w", m, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -139,9 +139,9 @@ func routes(app *zip.App, s *cloud.Service[state]) {
|
||||
|
||||
// Capture (WRITE) side — the ingest that fills hanzo.events. POST /v1/event
|
||||
// (event.go) is the ONE canonical front door serving EVERY auth context (IAM
|
||||
// bearer | pk_ publishable key | site-host-forced) and EVERY wire shape
|
||||
// (Event | [Event] | {batch}) into the ONE write core (ingestEvents). Every
|
||||
// other route below is a thin alias/shim delegating to it.
|
||||
// bearer | pk_ publishable key | site-host-forced | ANONYMOUS, public.go) and
|
||||
// EVERY wire shape (Event | [Event] | {batch}) into the ONE write core
|
||||
// (ingestEvents). Every other route below is a thin alias/shim delegating to it.
|
||||
app.Post("/v1/event", cloud.Handle(s, eventIngest))
|
||||
|
||||
// /v1/ingest — a THIN DEPRECATED ALIAS of /v1/event (delegates to the exact
|
||||
@@ -150,7 +150,6 @@ func routes(app *zip.App, s *cloud.Service[state]) {
|
||||
// /v1/errors is the type:'error' read lens (validated principal — reads never
|
||||
// accept the write-only key).
|
||||
app.Post("/v1/ingest", cloud.Handle(s, ingest))
|
||||
app.Post("/v1/ingest/keys", cloud.Handle(s, mintKey))
|
||||
app.Get("/v1/errors", cloud.Handle(s, errorsLens))
|
||||
|
||||
// DEPRECATED foreign-protocol ingest shims — external-SDK compat ONLY; no Hanzo
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// campaign.go is the in-process CAMPAIGN-METRICS seam over the ONE analytics
|
||||
// warehouse: the /v1/campaign plane (clients/campaign) reads a campaign's funnel
|
||||
// from HERE rather than opening a second store. A campaign's results ARE an
|
||||
// analytics query scoped to the campaign — the utm_campaign-tagged events in
|
||||
// hanzo.events — so there is one metrics plane, not a parallel one.
|
||||
//
|
||||
// TENANCY: identical to every other query this package builds. campaignWhere
|
||||
// binds the org (tenant_id) AND the campaign id (utm_campaign) AND the optional
|
||||
// variant (utm_content) POSITIONALLY — nothing user-derived is ever interpolated,
|
||||
// so a caller can only ever read its OWN org's campaign, and the utm_campaign
|
||||
// filter can never escape into SQL. The variant arg powers the creative-A/B
|
||||
// evidence read (utm_content) the experiment primitive composes.
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
aiobject "github.com/hanzoai/ai/object"
|
||||
)
|
||||
|
||||
// CampaignEvents is the per-campaign funnel read from hanzo.events, scoped to
|
||||
// (org, utm_campaign[, utm_content]). Impressions/clicks/conversions are counts
|
||||
// of the campaign's tagged events; Available is false (honest-empty) when the
|
||||
// events warehouse is not connected or the events table is not yet provisioned —
|
||||
// never fabricated. Spend is deliberately absent: it is the channel connector's
|
||||
// reported number, joined by the campaign plane, not an analytics value.
|
||||
type CampaignEvents struct {
|
||||
Available bool `json:"available"`
|
||||
Impressions int64 `json:"impressions"`
|
||||
Clicks int64 `json:"clicks"`
|
||||
Conversions int64 `json:"conversions"`
|
||||
Revenue float64 `json:"revenue"`
|
||||
Visitors int64 `json:"visitors"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
|
||||
// campaignWhere is the org + campaign (+ optional variant) predicate over
|
||||
// hanzo.events. org (tenant_id) and campaignID (utm_campaign) are ALWAYS bound;
|
||||
// variant (utm_content) is appended only when non-empty. Time bounds are bound as
|
||||
// datastore DateTime literals (the proven cloud_usage transport). Same isolation
|
||||
// boundary as eventsWhere — the org is a bound parameter, never interpolated.
|
||||
func campaignWhere(org, campaignID, variant string, start, end time.Time) (string, []any) {
|
||||
where := "timestamp >= ? AND timestamp < ? AND tenant_id = ? AND utm_campaign = ?"
|
||||
args := []any{tsLiteral(start), tsLiteral(end), org, campaignID}
|
||||
if variant != "" {
|
||||
where += " AND utm_content = ?"
|
||||
args = append(args, variant)
|
||||
}
|
||||
return where, args
|
||||
}
|
||||
|
||||
// CampaignMetrics reads the (org, campaignID) funnel from the ONE analytics
|
||||
// warehouse. variant=="" reads the whole campaign (all creatives); a non-empty
|
||||
// variant reads a single creative's slice (utm_content) — the evidence read for a
|
||||
// creative A/B. It degrades to honest-empty (Available=false, nil error) when the
|
||||
// datastore is not connected, so a campaign metrics view still renders its spend +
|
||||
// channels. A genuine query failure against a connected warehouse returns the
|
||||
// error (the caller logs it and shows honest-empty) — never a fabricated funnel.
|
||||
func CampaignMetrics(ctx context.Context, org, campaignID, variant string, start, end time.Time) (CampaignEvents, error) {
|
||||
out := CampaignEvents{Available: false, Source: eventsTable}
|
||||
if org == "" || campaignID == "" {
|
||||
return out, nil
|
||||
}
|
||||
if !aiobject.DatastoreEnabled() {
|
||||
return out, nil // honest-empty: no warehouse connected
|
||||
}
|
||||
where, args := campaignWhere(org, campaignID, variant, start, end)
|
||||
// Each countIf predicate is a server-chosen constant expression (never user
|
||||
// input); the only user-derived values — org, campaign, variant, time — stay
|
||||
// bound parameters via campaignWhere.
|
||||
sql := "SELECT " +
|
||||
"countIf(event = 'impression' OR event = 'ad_impression') AS impressions, " +
|
||||
"countIf(event = 'click' OR event = 'ad_click') AS clicks, " +
|
||||
"countIf(event = 'order_completed' OR event = 'signup' OR event = 'conversion') AS conversions, " +
|
||||
"toFloat64(sum(revenue)) AS revenue, " +
|
||||
"uniqExact(distinct_id) AS visitors " +
|
||||
"FROM " + eventsTable + " WHERE " + where
|
||||
rows, err := aiobject.DatastoreQuery(ctx, sql, args...)
|
||||
if err != nil {
|
||||
// Connected warehouse rejected/failed the query (or the events table is
|
||||
// absent): honest-empty for the caller, with the error surfaced for logs.
|
||||
return out, err
|
||||
}
|
||||
row := firstRow(rows)
|
||||
return CampaignEvents{
|
||||
Available: true,
|
||||
Impressions: aInt64(row["impressions"]),
|
||||
Clicks: aInt64(row["clicks"]),
|
||||
Conversions: aInt64(row["conversions"]),
|
||||
Revenue: aFloat64(row["revenue"]),
|
||||
Visitors: aInt64(row["visitors"]),
|
||||
Source: eventsTable,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestCampaignWhere_BindsOrgAndCampaignPositionally is the tenancy-invariant test
|
||||
// for the campaign-metrics seam: the org (tenant_id) and campaign (utm_campaign)
|
||||
// are ALWAYS bound parameters, never interpolated, so a caller can only read its
|
||||
// own org's campaign and a hostile campaign id can never escape into SQL.
|
||||
func TestCampaignWhere_BindsOrgAndCampaignPositionally(t *testing.T) {
|
||||
start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
end := time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC)
|
||||
where, args := campaignWhere("acme", "cmp_1", "", start, end)
|
||||
|
||||
if !strings.Contains(where, "tenant_id = ?") || !strings.Contains(where, "utm_campaign = ?") {
|
||||
t.Fatalf("org + campaign must be bound placeholders, got %q", where)
|
||||
}
|
||||
if strings.Contains(where, "utm_content") {
|
||||
t.Fatalf("no variant clause expected for whole-campaign read, got %q", where)
|
||||
}
|
||||
// args order: start, end, org, campaign.
|
||||
if len(args) != 4 || args[2] != "acme" || args[3] != "cmp_1" {
|
||||
t.Fatalf("args must bind [ts, ts, org, campaign], got %v", args)
|
||||
}
|
||||
// The hostile-slug proof: the org value is a bound arg, never text in the SQL.
|
||||
if strings.Contains(where, "acme") || strings.Contains(where, "cmp_1") {
|
||||
t.Fatalf("org/campaign must NOT be interpolated into SQL: %q", where)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCampaignWhere_VariantAppended: a non-empty variant adds a bound utm_content
|
||||
// clause (the creative-A/B evidence read) — still fully parameterized.
|
||||
func TestCampaignWhere_VariantAppended(t *testing.T) {
|
||||
start := time.Unix(0, 0).UTC()
|
||||
end := time.Unix(1000, 0).UTC()
|
||||
where, args := campaignWhere("acme", "cmp_1", "hero-b", start, end)
|
||||
if !strings.Contains(where, "utm_content = ?") {
|
||||
t.Fatalf("variant must add a bound utm_content clause, got %q", where)
|
||||
}
|
||||
if len(args) != 5 || args[4] != "hero-b" {
|
||||
t.Fatalf("variant must be the trailing bound arg, got %v", args)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCampaignMetrics_HonestEmptyWhenDatastoreDisabled: with no warehouse
|
||||
// connected (unit-test default), the seam returns honest-empty (Available=false)
|
||||
// and NO error — the campaign metrics view still renders spend + channels.
|
||||
func TestCampaignMetrics_HonestEmptyWhenDatastoreDisabled(t *testing.T) {
|
||||
ev, err := CampaignMetrics(context.Background(), "acme", "cmp_1", "", time.Now().Add(-time.Hour), time.Now())
|
||||
if err != nil {
|
||||
t.Fatalf("datastore-disabled must be honest-empty, not an error: %v", err)
|
||||
}
|
||||
if ev.Available {
|
||||
t.Fatalf("no warehouse connected ⇒ Available must be false, got %+v", ev)
|
||||
}
|
||||
if ev.Source != eventsTable {
|
||||
t.Fatalf("source should name the events table even when empty, got %q", ev.Source)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCampaignMetrics_EmptyIdentifiersFailClosed: an empty org or campaign never
|
||||
// queries — honest-empty, never a warehouse-wide read.
|
||||
func TestCampaignMetrics_EmptyIdentifiersFailClosed(t *testing.T) {
|
||||
for _, tc := range []struct{ org, camp string }{{"", "cmp_1"}, {"acme", ""}} {
|
||||
ev, err := CampaignMetrics(context.Background(), tc.org, tc.camp, "", time.Now().Add(-time.Hour), time.Now())
|
||||
if err != nil || ev.Available {
|
||||
t.Fatalf("empty (%q,%q) must be honest-empty, got avail=%v err=%v", tc.org, tc.camp, ev.Available, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -365,6 +365,44 @@ func buildEventsInsert(rows []eventRow) (string, []any) {
|
||||
|
||||
var emailRe = regexp.MustCompile(`(?i)[a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,}`)
|
||||
|
||||
// secretRe redacts credential shapes that leak in free-text — chiefly error
|
||||
// stacks/messages (which bypass the key-based denylist): bearer tokens, the key
|
||||
// families (pk-/sk-/hk-), and ?token=/api_key=/access_token=/password=/secret=
|
||||
// query params. Applied to every scrubbed string so a token in a URL property or
|
||||
// an exception frame is redacted before storage AND before the destinations
|
||||
// fan-out.
|
||||
//
|
||||
// A published key is not a secret — it ships in public bundles by design — but it
|
||||
// is redacted anyway: a key in an error frame is noise, and telling the two apart
|
||||
// here would be a second place that has to know the families.
|
||||
var secretRe = regexp.MustCompile(`(?i)(bearer\s+[a-z0-9._~+/\-]{8,}={0,2}|(?:pk|sk|hk)-[a-z0-9._\-]{8,}|[?&](?:access_token|refresh_token|id_token|api[_-]?key|token|password|secret|auth)=[^&\s"']+)`)
|
||||
|
||||
// scrubText redacts email- and credential-shaped substrings from a free-text
|
||||
// string. This is the ONE string scrubber; scrubValue and scrubException both
|
||||
// route through it so the redaction policy lives in one place.
|
||||
func scrubText(s string) string {
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
s = emailRe.ReplaceAllString(s, "[redacted]")
|
||||
s = secretRe.ReplaceAllString(s, "[redacted]")
|
||||
return s
|
||||
}
|
||||
|
||||
// scrubException returns a COPY of e with its free-text fields (Message, Stack)
|
||||
// redacted; never mutates the caller's struct. nil-safe. This is what makes a
|
||||
// type:'error' event safe to both store and fan out to third parties — the raw
|
||||
// stack/message can carry tokens, API URLs with query secrets, or PII.
|
||||
func scrubException(e *Exception) *Exception {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
c := *e
|
||||
c.Message = scrubText(c.Message)
|
||||
c.Stack = scrubText(c.Stack)
|
||||
return &c
|
||||
}
|
||||
|
||||
// denySubstr: any property key CONTAINING one of these (case-insensitive) is
|
||||
// dropped — the credential/secret family.
|
||||
var denySubstr = []string{
|
||||
@@ -413,7 +451,9 @@ func scrubMap(p map[string]any) map[string]any {
|
||||
func scrubValue(v any) any {
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
return emailRe.ReplaceAllString(t, "[redacted]")
|
||||
return scrubText(t)
|
||||
case *Exception:
|
||||
return scrubException(t)
|
||||
case map[string]any:
|
||||
return scrubMap(t)
|
||||
case []any:
|
||||
@@ -681,6 +721,9 @@ func ingestEvents(ctx context.Context, org, source string, evs []CaptureEvent) (
|
||||
if err := aiobject.DatastoreExec(ctx, stmt, args...); err != nil {
|
||||
return CaptureResult{}, warehouseErr("capture", err)
|
||||
}
|
||||
// Fan the accepted batch out to the downstream sink (destinations), detached and
|
||||
// fail-soft — never blocks or fails an ingest (forward.go). No-op when unset.
|
||||
fanOut(org, evs)
|
||||
return CaptureResult{Accepted: len(rows), Dropped: dropped}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
//go:build datastore_live
|
||||
|
||||
// Live end-to-end proof of the capture plane against a REAL datastore
|
||||
// (ClickHouse). It drives the ACTUAL POST /v1/analytics handler (bind → normalize
|
||||
// (Datastore). It drives the ACTUAL POST /v1/analytics handler (bind → normalize
|
||||
// → EnsureEventsTable → DatastoreExec) and then reads the rows back through the
|
||||
// EXACT SQL the /v1/analytics/overview + /top handlers run — so a green run proves
|
||||
// "emit → hanzo.events row lands → analytics read lens sees it".
|
||||
@@ -52,7 +52,7 @@ func TestLiveCaptureRoundTrip(t *testing.T) {
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
}
|
||||
if !aiobject.DatastoreEnabled() {
|
||||
t.Fatal("datastore did not connect (set DATASTORE_ADDR=127.0.0.1:9000 with a live ClickHouse)")
|
||||
t.Fatal("datastore did not connect (set DATASTORE_ADDR=127.0.0.1:9000 with a live Datastore)")
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -94,7 +94,7 @@ func TestLiveCaptureRoundTrip(t *testing.T) {
|
||||
t.Fatalf("accepted = %d, want 10", res.Accepted)
|
||||
}
|
||||
|
||||
// ClickHouse MergeTree inserts are visible immediately to a direct SELECT.
|
||||
// Datastore MergeTree inserts are visible immediately to a direct SELECT.
|
||||
// 1) Raw landing proof: per-event counts for THIS org.
|
||||
rows, err := aiobject.DatastoreQuery(ctx,
|
||||
"SELECT event, count() AS n FROM hanzo.events WHERE tenant_id = ? GROUP BY event ORDER BY event", org)
|
||||
|
||||
+55
-15
@@ -27,15 +27,22 @@
|
||||
// SERVER-SIDE and FAIL-CLOSED, in strict trust order:
|
||||
//
|
||||
// 1. a validated IAM bearer principal — its owner org;
|
||||
// 2. a write-only publishable key (pk_…) — HMAC-verified org, no IAM/DB hop (the
|
||||
// 2. a publishable key (pk-…) — IAM resolves it to its org; it can write but
|
||||
// SAME key publishable.go mints; folded in here so a pk_ caller uses /v1/event
|
||||
// directly);
|
||||
// 3. an out-of-band IAM access key (hk-/sk-…) — resolved through the ONE key seam
|
||||
// (cloud.OrgForKey).
|
||||
//
|
||||
// None of the above ⇒ 403. There is NO brand-host fallback on the canonical door
|
||||
// (that path stays only on the deprecated aliases), so /v1/event never writes an
|
||||
// event into a tenant IAM did not vouch for. The org is NEVER read from the body.
|
||||
// A caller that PRESENTED one of those credentials and did not resolve ⇒ 403 (a
|
||||
// misconfigured key is refused, not downgraded). A caller that presented NOTHING falls
|
||||
// through to the ANONYMOUS lane (public.go): logged-out marketing traffic is admitted
|
||||
// and attributed to the reserved public tenant, under a restricted kind/field allowlist
|
||||
// and its own size + rate bounds. It rejoins this pipeline at ingestDecoded, so decode,
|
||||
// write core, and receipt are shared — only admission differs.
|
||||
//
|
||||
// There is NO brand-host fallback on the canonical door (that path stays only on the
|
||||
// deprecated aliases), so /v1/event never writes an event into a REAL tenant IAM did
|
||||
// not vouch for. The org is NEVER read from the body — on either lane.
|
||||
//
|
||||
// The site-host carve (eventWithOrg) is the ONE exception to in-handler auth: on a
|
||||
// published site host the tenant is FORCED from the resolved Site BEFORE the handler
|
||||
@@ -89,15 +96,25 @@ func (e Event) toCapture() CaptureEvent {
|
||||
// 3. else a presented out-of-band IAM access key (hk-/sk-…) is resolved to its org
|
||||
// through the ONE key seam (resolveKeyOrg → cloud.OrgForKey).
|
||||
//
|
||||
// None matches ⇒ ("", false) → 403. There is NO brand-host fallback (that path
|
||||
// stays only on the deprecated aliases), so the canonical door is strictly authed —
|
||||
// IAM or a signed/resolvable key, never the request Host.
|
||||
// None matches ⇒ ("", false), which eventHandle answers by refusing a presented-but-
|
||||
// unresolvable credential and otherwise taking the anonymous lane. There is NO
|
||||
// brand-host fallback (that path stays only on the deprecated aliases), so a REAL
|
||||
// tenant here is only ever IAM or a signed/resolvable key, never the request Host.
|
||||
func eventTenant(c *zip.Ctx) (string, bool) {
|
||||
if org, ok := tenant(c); ok {
|
||||
return org, true
|
||||
}
|
||||
// ONE publishable key, and IAM issues it. A pk- on any ingest-shaped carrier
|
||||
// (Bearer, x-hanzo-ingest-key, ?ingest_key= for sendBeacon, which cannot set
|
||||
// headers) resolves through the SAME IAM seam as every other key. Cloud used
|
||||
// to mint and verify its own pk_ under an HMAC of CLOUD_INGEST_KEY_SECRET —
|
||||
// a second publishable-key family with its own prefix, secret and mint
|
||||
// endpoint, beside the one IAM already owned.
|
||||
//
|
||||
// Safe only because a pk- no longer authenticates: IdentityFromRequest
|
||||
// refuses it, so it attributes a write and never mints a reading principal.
|
||||
if key := ingestKey(c); key != "" {
|
||||
if org, ok := verifyPublishableKey(ingestSecret(), key); ok {
|
||||
if org, ok := resolveKeyOrg(c.Context(), key); ok {
|
||||
return org, true
|
||||
}
|
||||
}
|
||||
@@ -206,6 +223,16 @@ func ingestBody(c *zip.Ctx, org, source string) error {
|
||||
if err != nil {
|
||||
return zip.ErrBadRequest("malformed event payload")
|
||||
}
|
||||
return ingestDecoded(c, org, source, evs, 0)
|
||||
}
|
||||
|
||||
// ingestDecoded is the TAIL of the ingest pipeline, and the ONE place it lives: fold
|
||||
// type:'error' events (foldException) → the ONE write core (ingestEvents) → the honest
|
||||
// receipt. Every lane ends here, so "what happens to an admitted event" is written
|
||||
// once. org is the SERVER-resolved tenant; dropped is what admission already refused
|
||||
// upstream (0 on the vouched-for lane, so its behavior is unchanged), added to the
|
||||
// receipt so {accepted,dropped} always totals what the caller sent.
|
||||
func ingestDecoded(c *zip.Ctx, org, source string, evs []CaptureEvent, dropped int) error {
|
||||
for i := range evs {
|
||||
evs[i] = foldException(evs[i])
|
||||
}
|
||||
@@ -213,19 +240,32 @@ func ingestBody(c *zip.Ctx, org, source string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res.Dropped += dropped
|
||||
return c.JSON(http.StatusOK, res)
|
||||
}
|
||||
|
||||
// eventHandle is the canonical-door handler core: pluggable in-handler auth
|
||||
// (eventTenant, fail-closed) → the ONE ingest core. source tags the door so the
|
||||
// canonical /v1/event and the /v1/ingest deprecated alias share ONE implementation,
|
||||
// differing only in origin tag (and the alias's deprecation log).
|
||||
// eventHandle is the canonical-door handler core: resolve WHO is calling (eventTenant),
|
||||
// then run the pipeline. source tags the door so the canonical /v1/event and the
|
||||
// /v1/ingest deprecated alias share ONE implementation, differing only in origin tag
|
||||
// (and the alias's deprecation log).
|
||||
//
|
||||
// A resolved tenant goes straight to ingestBody with full capability, exactly as
|
||||
// before. A caller with NOTHING to resolve takes the anonymous lane (publicIngest,
|
||||
// public.go), which resolves the tenant to the reserved public bucket and applies an
|
||||
// admission policy — then rejoins THIS pipeline at ingestDecoded. Decode, write core,
|
||||
// and receipt are shared; only admission differs.
|
||||
func eventHandle(c *zip.Ctx, source string) error {
|
||||
org, ok := eventTenant(c)
|
||||
if !ok {
|
||||
if org, ok := eventTenant(c); ok {
|
||||
return ingestBody(c, org, source)
|
||||
}
|
||||
// A caller that PRESENTED an ingest credential which did not resolve is refused,
|
||||
// never downgraded: filing its events under the public tenant would hide a
|
||||
// misconfigured key in a partition its owner cannot read — a silent failure worse
|
||||
// than the 403. The anonymous lane is for a caller that presented nothing.
|
||||
if ingestKey(c) != "" || projectKey(c) != "" {
|
||||
return zip.ErrForbidden("valid bearer or a resolvable ingest key required")
|
||||
}
|
||||
return ingestBody(c, org, source)
|
||||
return publicIngest(c, source)
|
||||
}
|
||||
|
||||
// eventIngest answers POST /v1/event — the ONE canonical ingestion front door.
|
||||
|
||||
@@ -66,15 +66,16 @@ func TestMount_HostCarve_EventEmptyBatchOK(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestMount_HostCarve_EventDirectNoHostStillFailsClosed pins that the forced-org
|
||||
// carve is HOST-scoped: the SAME anonymous /v1/event body on a NON-site host runs the
|
||||
// normal canonical gate (eventTenant, no brand fallback) and is refused 403 — the
|
||||
// carve did not fire, so the strict door invariant is unweakened.
|
||||
func TestMount_HostCarve_EventDirectNoHostStillFailsClosed(t *testing.T) {
|
||||
// TestMount_HostCarve_EventDirectNoHostGetsNoOrg pins that the forced-org carve is
|
||||
// HOST-scoped: the SAME body on a NON-site host does not get a site org. The carve did
|
||||
// not fire, so the request runs the normal canonical gate — no principal and no key, so
|
||||
// it takes the ANONYMOUS lane, where the forged X-Org-Id and the custom event kind both
|
||||
// buy nothing: 200 with an all-dropped receipt, no row under `attacker`.
|
||||
func TestMount_HostCarve_EventDirectNoHostGetsNoOrg(t *testing.T) {
|
||||
app := carveApp(t, "hanzo")
|
||||
code := postHost(t, app, "evil.example.com", "/v1/event",
|
||||
`{"event":"signup_completed","distinctId":"d"}`, map[string]string{"X-Org-Id": "attacker"})
|
||||
if code != http.StatusForbidden {
|
||||
t.Fatalf("anonymous /v1/event on a non-site host want 403 (no carve, no brand fallback), got %d", code)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("anonymous /v1/event on a non-site host want 200 (anonymous lane, kind dropped), got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,10 +182,22 @@ func TestSourceStampedIntoProperties(t *testing.T) {
|
||||
// ADMITTED one reaches requireDatastore and returns 503 (no datastore in tests).
|
||||
// So "not 403" ⇒ the tenant gate admitted the request.
|
||||
|
||||
func TestEvent_NoPrincipalNoKeyForbidden(t *testing.T) {
|
||||
// TestEvent_NoPrincipalNoKeyIsAnonymous: a caller with NO principal and NO key is not
|
||||
// refused — it takes the anonymous lane (public.go), attributed to the reserved public
|
||||
// tenant. The canonical-Event wire carries no `type`, so canonicalType folds it to
|
||||
// "event", which is not on the anonymous allowlist: the request is answered 200 with an
|
||||
// honest all-dropped receipt. What IS refused is a presented credential that does not
|
||||
// resolve (TestEvent_UnresolvableKeyFailsClosedEvenOnBrandHost).
|
||||
func TestEvent_NoPrincipalNoKeyIsAnonymous(t *testing.T) {
|
||||
app := mountApp(t)
|
||||
if code, _ := doBody(t, app, http.MethodPost, "/v1/event", "", "", `{"event":"e","distinctId":"d"}`); code != http.StatusForbidden {
|
||||
t.Fatalf("no-principal no-key /v1/event want 403, got %d", code)
|
||||
code, body := doBody(t, app, http.MethodPost, "/v1/event", "", "", `{"event":"e","distinctId":"d"}`)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("no-principal no-key /v1/event want 200 (anonymous lane, kind dropped), got %d (%s)", code, body)
|
||||
}
|
||||
// A pageview on the same credential-less request IS stored — it reaches the
|
||||
// warehouse (503 here, no datastore in the harness).
|
||||
if code, body := doBody(t, app, http.MethodPost, "/v1/event", "", "", `{"batch":[{"type":"pageview"}]}`); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("anonymous pageview want 503 (admitted), got %d (%s)", code, body)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -222,19 +234,29 @@ func TestEvent_UnresolvableKeyFailsClosedEvenOnBrandHost(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestEvent_NoBrandHostFallback is THE distinguishing invariant: anonymous traffic
|
||||
// on a recognized brand host is ADMITTED by the deprecated /v1/analytics alias
|
||||
// (brand-public partition) but REFUSED by the canonical /v1/event — IAM is the
|
||||
// only tenant authority on the canonical door.
|
||||
// TestEvent_NoBrandHostFallback is THE distinguishing invariant, and it survives the
|
||||
// anonymous lane: the Host NEVER selects the tenant on the canonical door. The
|
||||
// deprecated /v1/analytics alias resolves anonymous traffic on a recognized brand host
|
||||
// to that BRAND's org (a real org, picked by a caller-settable header); the canonical
|
||||
// door ignores the Host entirely and attributes to the reserved public tenant. Both are
|
||||
// admitted — the difference is now WHICH tenant, which is the property that matters.
|
||||
func TestEvent_NoBrandHostFallback(t *testing.T) {
|
||||
app := mountApp(t)
|
||||
body := `{"event":"e","distinctId":"d"}`
|
||||
if code, _ := doHost(t, app, "/v1/event", "", "", "hanzo.ai", body); code != http.StatusForbidden {
|
||||
t.Fatalf("anonymous brand-host /v1/event must 403 (no brand fallback), got %d", code)
|
||||
// The same anonymous pageview on a brand host and on an unrelated host: the
|
||||
// canonical door admits both identically, so the Host bought nothing.
|
||||
pageview := `{"batch":[{"type":"pageview"}]}`
|
||||
for _, host := range []string{"hanzo.ai", "zoo.ngo", "evil.example.com"} {
|
||||
if code, body := doHost(t, app, "/v1/event", "", "", host, pageview); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("anonymous /v1/event on host %q want 503 (admitted to the public tenant), got %d (%s)", host, code, body)
|
||||
}
|
||||
}
|
||||
// Contrast: the deprecated alias still admits the same anonymous brand-host
|
||||
// traffic (503 = admitted, datastore down), proving the difference is by design.
|
||||
if code, _ := doHost(t, app, "/v1/analytics", "", "", "hanzo.ai", `{"batch":[{"type":"pageview"}]}`); code != http.StatusServiceUnavailable {
|
||||
// The tenant the canonical door uses is the reserved constant, never the brand.
|
||||
if org, _, _ := admitPublic([]CaptureEvent{{Type: "pageview"}}); org != publicTenant {
|
||||
t.Fatalf("canonical anonymous tenant = %q, want %q (never a brand org)", org, publicTenant)
|
||||
}
|
||||
// Contrast: the deprecated alias still resolves the same traffic to the BRAND org
|
||||
// (503 = admitted, datastore down), proving the two doors differ by design.
|
||||
if code, _ := doHost(t, app, "/v1/analytics", "", "", "hanzo.ai", pageview); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("deprecated alias still brand-admits (want 503), got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package analytics
|
||||
|
||||
import "testing"
|
||||
|
||||
// A stack/message carrying an email, a bearer token, an sk- key, and a
|
||||
// ?access_token= query secret must be redacted at rest AND in the folded event
|
||||
// that the destinations fan-out consumes (forward.go sees pre-warehouse-scrub).
|
||||
func TestFoldException_RedactsSecretsAndPII(t *testing.T) {
|
||||
e := CaptureEvent{
|
||||
Type: "error",
|
||||
Error: &Exception{
|
||||
Type: "Error",
|
||||
Message: "login failed for alice@example.com with sk-live-abcdef0123456789",
|
||||
Stack: "at fetch (https://api.x.com/v1?access_token=tok_abc123def456 )\n Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.payload.sig",
|
||||
},
|
||||
}
|
||||
got := foldException(e)
|
||||
ex, ok := got.Properties["$exception"].(*Exception)
|
||||
if !ok || ex == nil {
|
||||
t.Fatalf("$exception not an *Exception: %T", got.Properties["$exception"])
|
||||
}
|
||||
for _, s := range []string{ex.Message, ex.Stack} {
|
||||
if containsAny(s, "alice@example.com", "sk-live-abcdef0123456789", "tok_abc123def456", "eyJhbGciOiJIUzI1NiJ9") {
|
||||
t.Fatalf("secret/PII survived scrub: %q", s)
|
||||
}
|
||||
}
|
||||
// original struct must NOT be mutated (copy semantics)
|
||||
if e.Error.Message == ex.Message {
|
||||
t.Fatal("foldException mutated the caller's Exception")
|
||||
}
|
||||
// scrubValue must also handle *Exception directly (defense in depth)
|
||||
sv, _ := scrubValue(&Exception{Message: "x@y.com Bearer sk-abcdef0123456789"}).(*Exception)
|
||||
if sv == nil || containsAny(sv.Message, "x@y.com", "sk-abcdef0123456789") {
|
||||
t.Fatalf("scrubValue(*Exception) did not redact: %+v", sv)
|
||||
}
|
||||
}
|
||||
|
||||
func containsAny(hay string, needles ...string) bool {
|
||||
for _, n := range needles {
|
||||
if len(n) > 0 && indexOf(hay, n) >= 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
func indexOf(h, n string) int {
|
||||
for i := 0; i+len(n) <= len(h); i++ {
|
||||
if h[i:i+len(n)] == n {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// forward.go is the fan-out seam of the canonical event plane. After the ONE write
|
||||
// core (ingestEvents) commits a batch to hanzo.events, it hands a COPY of that batch
|
||||
// to an optional downstream sink — the destinations subsystem — which translates and
|
||||
// forwards each event to the org's connected ad/analytics platforms (GA4, Meta CAPI,
|
||||
// …). The seam is:
|
||||
//
|
||||
// - ONE-WAY. analytics never imports destinations; destinations calls SetSink from
|
||||
// its Mount. A nil sink means no fan-out (the default when destinations is off),
|
||||
// so this file changes nothing about ingest when the subsystem is absent.
|
||||
// - RAW. The sink receives the event BEFORE the warehouse privacy scrub, because a
|
||||
// server-side Conversions-API forwarder must hash the match keys (email/phone/
|
||||
// click ids) the warehouse deliberately drops. The org connected the destination
|
||||
// and owns that consent; the destination adapters SHA-256 every PII field before
|
||||
// it leaves the process.
|
||||
// - FAIL-SOFT. The sink runs detached (a panic-guarded goroutine) so a slow or
|
||||
// broken destination can never block, fail, or crash an ingest.
|
||||
package analytics
|
||||
|
||||
import "time"
|
||||
|
||||
// SinkEvent is one accepted event handed to the downstream fan-out. It carries the
|
||||
// resolved canonical name plus the commerce + identity fields a conversion needs;
|
||||
// Properties is the RAW (pre-scrub) property bag the translator lifts match keys and
|
||||
// custom data from. The tenant is the org argument to the sink, never a field here.
|
||||
type SinkEvent struct {
|
||||
MessageID string
|
||||
Name string
|
||||
DistinctID string
|
||||
AnonymousID string
|
||||
Time time.Time
|
||||
URL string
|
||||
Path string
|
||||
Referrer string
|
||||
Revenue float64
|
||||
Currency string
|
||||
ProductID string
|
||||
Quantity uint32
|
||||
Properties map[string]any
|
||||
}
|
||||
|
||||
// sink is the downstream fan-out hook, installed once by the destinations subsystem
|
||||
// at Mount (nil ⇒ no fan-out). A subsystem Mount runs before request traffic, so no
|
||||
// lock is needed on this package-global.
|
||||
var sink func(org string, evs []SinkEvent)
|
||||
|
||||
// SetSink installs (nil clears) the downstream fan-out hook.
|
||||
func SetSink(fn func(org string, evs []SinkEvent)) { sink = fn }
|
||||
|
||||
// fanOut hands the accepted batch to the sink, detached and fail-soft. org is the
|
||||
// SERVER-resolved tenant (already an owned copy from principal.Org). It builds
|
||||
// SinkEvents from the RAW events (skipping unroutable ones, mirroring the write
|
||||
// core's drop rule) and, if any remain and a sink is installed, dispatches them on a
|
||||
// panic-guarded goroutine so ingest is never blocked or failed by a destination.
|
||||
func fanOut(org string, evs []CaptureEvent) {
|
||||
fn := sink
|
||||
if fn == nil || len(evs) == 0 {
|
||||
return
|
||||
}
|
||||
// The public tenant never fans out. A destination is a connection an ORG made, and
|
||||
// this sink is handed the RAW pre-scrub event so a Conversions API can hash match
|
||||
// keys — so forwarding an unattested event would push it into an external platform
|
||||
// on an org's behalf. publicTenant holds no connection, so the lookup is already
|
||||
// empty; stating it here makes that a property of the SEAM rather than a property of
|
||||
// the destination table.
|
||||
if org == publicTenant {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
out := make([]SinkEvent, 0, len(evs))
|
||||
for _, e := range evs {
|
||||
name := resolveEventName(e)
|
||||
if name == "" {
|
||||
continue // unroutable — the write core dropped it too
|
||||
}
|
||||
out = append(out, SinkEvent{
|
||||
MessageID: firstNonEmptyStr(trim(e.MessageID), randID()),
|
||||
Name: name,
|
||||
DistinctID: trim(e.DistinctID),
|
||||
AnonymousID: trim(e.AnonymousID),
|
||||
Time: clampTS(e.Timestamp, now),
|
||||
URL: trim(e.URL),
|
||||
Path: trim(e.Path),
|
||||
Referrer: trim(e.Referrer),
|
||||
Revenue: e.Revenue,
|
||||
Currency: trim(e.Currency),
|
||||
ProductID: trim(e.ProductID),
|
||||
Quantity: e.Quantity,
|
||||
Properties: e.Properties,
|
||||
})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
defer func() { _ = recover() }()
|
||||
fn(org, out)
|
||||
}()
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestFanOutTranslatesAndFiresSink verifies the fan-out seam builds SinkEvents from
|
||||
// the RAW accepted batch (resolved name, commerce fields, un-scrubbed properties for
|
||||
// match keys), drops unroutable events, and dispatches them to the installed sink.
|
||||
func TestFanOutTranslatesAndFiresSink(t *testing.T) {
|
||||
got := make(chan []SinkEvent, 1)
|
||||
SetSink(func(org string, evs []SinkEvent) {
|
||||
if org == "acme" {
|
||||
got <- evs
|
||||
}
|
||||
})
|
||||
defer SetSink(nil)
|
||||
|
||||
fanOut("acme", []CaptureEvent{
|
||||
{Type: "event", Event: "order_completed", DistinctID: "u1", Revenue: 49, Currency: "USD",
|
||||
Properties: map[string]any{"email": "a@b.com"}},
|
||||
{Type: "pageview"}, // resolves to $pageview
|
||||
{Type: "event", Event: ""}, // unroutable → dropped (mirrors the write core)
|
||||
})
|
||||
|
||||
select {
|
||||
case evs := <-got:
|
||||
if len(evs) != 2 {
|
||||
t.Fatalf("want 2 routable events, got %d", len(evs))
|
||||
}
|
||||
if evs[0].Name != "order_completed" || evs[0].Revenue != 49 || evs[0].Currency != "USD" {
|
||||
t.Errorf("purchase event not carried: %+v", evs[0])
|
||||
}
|
||||
if evs[0].Properties["email"] != "a@b.com" {
|
||||
t.Errorf("raw (pre-scrub) properties must be carried for match keys: %+v", evs[0].Properties)
|
||||
}
|
||||
if evs[1].Name != "$pageview" {
|
||||
t.Errorf("pageview name = %q", evs[1].Name)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("sink was not called")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFanOutNilSinkIsNoOp verifies fan-out is inert (no panic, no goroutine) when no
|
||||
// sink is installed — the default when destinations is disabled.
|
||||
func TestFanOutNilSinkIsNoOp(t *testing.T) {
|
||||
SetSink(nil)
|
||||
fanOut("acme", []CaptureEvent{{Type: "event", Event: "order_completed"}})
|
||||
// Nothing to assert beyond "did not panic / block".
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package analytics
|
||||
|
||||
// outcomes.go — the MEASUREMENT seam the experiments primitive composes. An
|
||||
// experiment's per-variant metric is read from the ONE analytics events plane
|
||||
// (hanzo.events), never a second event store: outcomes are already captured by
|
||||
// distinct_id (capture.go), so an experiment only needs to fold them per subject and
|
||||
// join each subject to its flags variant. This is the scientific-growth loop's
|
||||
// MEASUREMENT half — analytics measures, the experiment tests, flags decides.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
aiobject "github.com/hanzoai/ai/object"
|
||||
)
|
||||
|
||||
// SubjectOutcome is one subject's (distinct_id's) participation in an experiment
|
||||
// window: whether it fired the Exposed (enrolled / saw the arm) event and whether it
|
||||
// fired the Converted (metric) event. It is the per-subject grain the experiments
|
||||
// primitive joins to a flags variant assignment to produce per-variant samples.
|
||||
type SubjectOutcome struct {
|
||||
Subject string
|
||||
Exposed bool
|
||||
Converted bool
|
||||
}
|
||||
|
||||
// Outcomes returns, for one org over [start,end), each subject's exposure +
|
||||
// conversion for an experiment's two event names, read from hanzo.events. It is the
|
||||
// measurement seam the experiments primitive composes: flags assignment joins to
|
||||
// these outcomes by distinct_id.
|
||||
//
|
||||
// TENANT ISOLATION is the eventsWhere invariant — org is bound POSITIONALLY, never
|
||||
// interpolated — and every event name is a BOUND parameter too, so neither a hostile
|
||||
// org slug nor a hostile event name can escape into SQL. exposureEvent may be ""
|
||||
// (then every returned subject is Exposed: the population is "appeared in-window");
|
||||
// metricEvent is required. Fails closed with a 503 when the warehouse is absent.
|
||||
func Outcomes(ctx context.Context, org, exposureEvent, metricEvent string, start, end time.Time) ([]SubjectOutcome, error) {
|
||||
if err := requireDatastore(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if metricEvent == "" {
|
||||
return nil, fmt.Errorf("analytics: outcomes needs a metric event")
|
||||
}
|
||||
if err := EnsureEventsTable(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sql, args := outcomesSQL(org, exposureEvent, metricEvent, start, end)
|
||||
rows, err := aiobject.DatastoreQuery(ctx, sql, args...)
|
||||
if err != nil {
|
||||
return nil, warehouseErr("outcomes", err)
|
||||
}
|
||||
out := make([]SubjectOutcome, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
subject := aString(r["subject"])
|
||||
if subject == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, SubjectOutcome{
|
||||
Subject: subject,
|
||||
Exposed: aInt64(r["exposed"]) > 0,
|
||||
Converted: aInt64(r["converted"]) > 0,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// outcomesSQL builds the per-subject exposure/conversion query over hanzo.events —
|
||||
// the pure, I/O-free core so the isolation invariant is testable without a warehouse.
|
||||
// TENANCY: org rides eventsWhere as a BOUND parameter (never interpolated) and every
|
||||
// event name is BOUND too, so nothing user-derived escapes into SQL. The SELECT
|
||||
// maxIf placeholders appear first in the string, then eventsWhere's [start,end,org],
|
||||
// then the event-set IN — the args slice follows that exact positional order.
|
||||
func outcomesSQL(org, exposureEvent, metricEvent string, start, end time.Time) (string, []any) {
|
||||
where, wargs := eventsWhere(org, start, end)
|
||||
if exposureEvent == "" {
|
||||
args := append([]any{metricEvent}, wargs...)
|
||||
return "SELECT distinct_id AS subject, 1 AS exposed, " +
|
||||
"maxIf(1, event = ?) AS converted FROM " + eventsTable +
|
||||
" WHERE " + where + " GROUP BY distinct_id", args
|
||||
}
|
||||
args := append([]any{exposureEvent, metricEvent}, wargs...)
|
||||
args = append(args, exposureEvent, metricEvent)
|
||||
return "SELECT distinct_id AS subject, " +
|
||||
"maxIf(1, event = ?) AS exposed, " +
|
||||
"maxIf(1, event = ?) AS converted FROM " + eventsTable +
|
||||
" WHERE " + where + " AND event IN (?, ?) GROUP BY distinct_id", args
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestOutcomesSQL_TenantIsolation is the isolation-invariant test for the experiments
|
||||
// measurement seam: the org is a BOUND argument (never interpolated into SQL) and
|
||||
// every event name is bound too, so a hostile org slug or event name can never escape
|
||||
// into the query — the same boundary every analytics builder holds.
|
||||
func TestOutcomesSQL_TenantIsolation(t *testing.T) {
|
||||
start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
end := time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC)
|
||||
org := "acme'; DROP TABLE hanzo.events;--"
|
||||
exposure := "$feature_flag_called"
|
||||
metric := "order_completed"
|
||||
|
||||
sql, args := outcomesSQL(org, exposure, metric, start, end)
|
||||
|
||||
// The hostile org must NOT appear in the SQL text — only as a bound arg.
|
||||
if strings.Contains(sql, org) {
|
||||
t.Fatalf("org slug interpolated into SQL (injection): %s", sql)
|
||||
}
|
||||
if strings.Contains(sql, "DROP TABLE") {
|
||||
t.Fatalf("SQL carries injected text: %s", sql)
|
||||
}
|
||||
// tenant_id is bound; the org is the trailing eventsWhere arg.
|
||||
if !strings.Contains(sql, "tenant_id = ?") {
|
||||
t.Fatalf("query must bind tenant_id positionally: %s", sql)
|
||||
}
|
||||
// args order: [exposure, metric, start, end, org, exposure, metric].
|
||||
if len(args) != 7 {
|
||||
t.Fatalf("want 7 bound args, got %d: %v", len(args), args)
|
||||
}
|
||||
if args[0] != exposure || args[1] != metric {
|
||||
t.Fatalf("SELECT maxIf binds must lead: %v", args[:2])
|
||||
}
|
||||
if args[4] != org {
|
||||
t.Fatalf("org must be the eventsWhere trailing bound arg, got %v", args[4])
|
||||
}
|
||||
if args[5] != exposure || args[6] != metric {
|
||||
t.Fatalf("event-set IN binds must trail: %v", args[5:])
|
||||
}
|
||||
}
|
||||
|
||||
// TestOutcomesSQL_NoExposureEvent covers the metric-only shape (exposure "" -> every
|
||||
// subject exposed): one bound event + the three eventsWhere binds, org still bound.
|
||||
func TestOutcomesSQL_NoExposureEvent(t *testing.T) {
|
||||
start := time.Now().Add(-24 * time.Hour)
|
||||
end := time.Now()
|
||||
sql, args := outcomesSQL("acme", "", "signup", start, end)
|
||||
if strings.Contains(sql, " IN (") {
|
||||
t.Fatalf("metric-only query must not build an event-set IN: %s", sql)
|
||||
}
|
||||
if !strings.Contains(sql, "1 AS exposed") {
|
||||
t.Fatalf("metric-only query marks every subject exposed: %s", sql)
|
||||
}
|
||||
if len(args) != 4 || args[0] != "signup" || args[3] != "acme" {
|
||||
t.Fatalf("args must be [metric, start, end, org], got %v", args)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// public.go — the ANONYMOUS lane of the ONE canonical event door.
|
||||
//
|
||||
// A logged-out visitor on a marketing surface carries no bearer and no key, so
|
||||
// eventTenant resolves nothing. This file is the lane such a request falls through
|
||||
// to, so a pageview or a browser error from a logged-out page lands in the warehouse
|
||||
// instead of being refused.
|
||||
//
|
||||
// Anonymous input is attested by nobody. It is therefore admitted under a policy the
|
||||
// vouched-for lane never applies, and the two lanes are SEPARATE FUNCTIONS rather
|
||||
// than one function with a mode flag: eventTenant → ingestBody is untouched by
|
||||
// anything in this file, and every restriction below is unreachable from it.
|
||||
//
|
||||
// The policy is in two pieces and no more: the request-scoped gates (publicIngest)
|
||||
// and the pure decision (admitPublic).
|
||||
//
|
||||
// - TENANT is publicTenant, a compile-time constant. admitPublic takes no request,
|
||||
// so no header, query, or body field can influence the tenant an anonymous row
|
||||
// carries: an anonymous write CANNOT land in a real org's partition, and there is
|
||||
// no input that makes it do so.
|
||||
// - KIND is an ALLOWLIST of two — pageview and error, what a marketing surface
|
||||
// emits. `identify` and `group` (which name a person and a group) and every
|
||||
// custom event are dropped, counted in the honest receipt, never stored.
|
||||
// - NAME is server-chosen FROM the kind ($pageview | $error), so the anonymous name
|
||||
// space is closed to two values: an anonymous caller can introduce neither a new
|
||||
// name into the read lenses nor unbounded cardinality into the table's ORDER BY
|
||||
// key.
|
||||
// - FIELDS are a PROJECTION, not a filter: admitPublic builds a fresh CaptureEvent
|
||||
// from the fields it names, so a field it does not name — personId, groupId,
|
||||
// revenue, productId, refCode, signupWeek, and the entire client property bag —
|
||||
// cannot reach the row. The only properties an anonymous row carries are the
|
||||
// server-folded $exception and the write core's $source.
|
||||
// - BYTES and COUNT are bounded first, and REFUSED rather than truncated.
|
||||
// - RATE is capped per client IP and, independently, per socket peer.
|
||||
// - DNT / Sec-GPC on the wire is honored: nothing is stored and the receipt says so.
|
||||
//
|
||||
// Everything admitted here flows through the SAME ONE write core (ingestEvents) into
|
||||
// the SAME hanzo.events table. One write path; this file only decides what a caller
|
||||
// nobody vouched for may put on it.
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// publicTenant is the reserved tenant EVERY anonymous event is attributed to. The
|
||||
// '$' prefix is load-bearing: an IAM org slug is lowercase ASCII alphanumerics and
|
||||
// '-' (the IAM slugifier emits nothing else), so this value lies outside the org
|
||||
// namespace and cannot collide with a real tenant. It is also the reason the anonymous
|
||||
// stream is legible: a row's tenant_id alone says whether IAM vouched for it.
|
||||
const publicTenant = "$public"
|
||||
|
||||
// maxPublicBytes / maxPublicBatch bound ONE anonymous request. @hanzo/event's default
|
||||
// batchSize is 20 and it also drains the queue on page-unload, so 50 leaves real
|
||||
// headroom; 64 KiB is the browser's own sendBeacon ceiling, which makes it the honest
|
||||
// cap for the transport that reaches here. Over either bound is REFUSED, never
|
||||
// truncated — a silent truncation would make the receipt a lie.
|
||||
const (
|
||||
maxPublicBytes = 64 << 10
|
||||
maxPublicBatch = 50
|
||||
)
|
||||
|
||||
// publicKinds is the ALLOWLIST of canonical kinds (canonicalType's closed set) an
|
||||
// anonymous caller may store. A kind absent here is dropped: `identify` and `group`
|
||||
// bind an event to a named person and a named group, and a bare `event` is the whole
|
||||
// custom product/billing/metering surface — none of which a caller nobody vouched for
|
||||
// may write. Adding a kind here is the ONLY way to widen the anonymous surface.
|
||||
var publicKinds = map[string]bool{"pageview": true, "error": true}
|
||||
|
||||
// publicRateWindow, publicRateLimit and publicPeerRateLimit cap anonymous ingest.
|
||||
// TWO independent buckets, because neither key alone suffices:
|
||||
//
|
||||
// - the CLIENT IP (leftmost X-Forwarded-For, via cloud.ClientIP) is the real
|
||||
// per-visitor key at the edge, but it is a header, so a caller reaching the pod
|
||||
// directly can rotate it and reset its own bucket at will;
|
||||
// - the SOCKET PEER cannot be rotated, but at the edge it is the ingress for ALL
|
||||
// public traffic, so it can only carry a total-volume ceiling, never a per-visitor
|
||||
// cap.
|
||||
//
|
||||
// Together they bound both a single source and the aggregate. Sized so real marketing
|
||||
// traffic never notices: a browser emits a handful of events per page load, so 300/min
|
||||
// per client is orders of magnitude of headroom, while 6000/min is the pod's anonymous
|
||||
// ingest ceiling. Per-pod and in-memory — a per-replica soft cap, which is what a
|
||||
// flood control needs to be, not a distributed quota.
|
||||
const (
|
||||
publicRateWindow = time.Minute
|
||||
publicRateLimit = 300
|
||||
publicPeerRateLimit = 6000
|
||||
)
|
||||
|
||||
// counter is a fixed-window request counter keyed on a caller string, with
|
||||
// opportunistic eviction so the map stays bounded at the edge's IP cardinality. This
|
||||
// is deliberately a counter and not the zip token-bucket primitive for the same reason
|
||||
// cloud's edge limiter is not: that primitive never evicts, which is fine for a
|
||||
// bounded per-org keyspace and unbounded growth when keyed on raw IPs.
|
||||
type counter struct {
|
||||
limit int
|
||||
window time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
seen map[string]*tally
|
||||
swept time.Time
|
||||
}
|
||||
|
||||
// tally is one key's window: how many requests it has spent, and when it resets.
|
||||
type tally struct {
|
||||
n int
|
||||
reset time.Time
|
||||
}
|
||||
|
||||
func newCounter(limit int, window time.Duration) *counter {
|
||||
return &counter{limit: limit, window: window, seen: map[string]*tally{}}
|
||||
}
|
||||
|
||||
// ok charges one request to key and reports whether it is within the window's limit.
|
||||
// A request over the limit is still charged, so a caller cannot ride a rejected
|
||||
// request for free.
|
||||
func (c *counter) ok(key string) bool {
|
||||
now := time.Now()
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if now.Sub(c.swept) >= c.window {
|
||||
c.swept = now
|
||||
for k, t := range c.seen {
|
||||
if now.After(t.reset) {
|
||||
delete(c.seen, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
t := c.seen[key]
|
||||
if t == nil || now.After(t.reset) {
|
||||
t = &tally{reset: now.Add(c.window)}
|
||||
c.seen[key] = t
|
||||
}
|
||||
t.n++
|
||||
return t.n <= c.limit
|
||||
}
|
||||
|
||||
// publicRate / publicPeerRate are the two anonymous-ingest buckets. Package-level
|
||||
// because the cap is a property of the pod, which outlives any request. Vars, not
|
||||
// consts, so a test can install a tighter pair.
|
||||
var (
|
||||
publicRate = newCounter(publicRateLimit, publicRateWindow)
|
||||
publicPeerRate = newCounter(publicPeerRateLimit, publicRateWindow)
|
||||
)
|
||||
|
||||
// publicRateOK charges one anonymous request against BOTH buckets and reports whether
|
||||
// it may proceed. An unkeyable caller shares one bucket rather than being exempt, so
|
||||
// an absent header never buys an uncapped lane. Both buckets are always charged (no
|
||||
// short-circuit) so a flood keeps counting against the ceiling even while its own
|
||||
// per-client bucket is already over.
|
||||
func publicRateOK(c *zip.Ctx) bool {
|
||||
client := cloud.ClientIP(c)
|
||||
if client == "" {
|
||||
client = "-"
|
||||
}
|
||||
within := publicRate.ok(client)
|
||||
return publicPeerRate.ok(peerIP(c)) && within
|
||||
}
|
||||
|
||||
// peerIP is the L4 socket peer — the one address a caller cannot set. Through the
|
||||
// ingress this is the proxy, so it keys the aggregate ceiling rather than a visitor.
|
||||
func peerIP(c *zip.Ctx) string {
|
||||
ip := c.Fiber().IP()
|
||||
if host, _, err := net.SplitHostPort(ip); err == nil {
|
||||
ip = host
|
||||
}
|
||||
if ip == "" {
|
||||
return "-"
|
||||
}
|
||||
return ip
|
||||
}
|
||||
|
||||
// optedOut reports whether the visitor signalled Do-Not-Track or Global Privacy
|
||||
// Control on the wire.
|
||||
//
|
||||
// This is a SECOND, independent line rather than a duplicate of the client's. On the
|
||||
// client, consent is the host app's `enabled` config (@hanzo/event turns the whole
|
||||
// client off when it is false, so an opted-out visitor normally emits no request at
|
||||
// all) — which means the app owns that decision and a request can still arrive
|
||||
// carrying the header: a surface that has not wired `enabled` to DNT/GPC, a browser
|
||||
// or extension that sets the header itself, or a privacy proxy that adds one. When it
|
||||
// does, the server stores nothing.
|
||||
func optedOut(c *zip.Ctx) bool {
|
||||
if strings.TrimSpace(c.Header("DNT")) == "1" {
|
||||
return true
|
||||
}
|
||||
return strings.TrimSpace(c.Header("Sec-GPC")) == "1"
|
||||
}
|
||||
|
||||
// admitPublic is the anonymous door's WHOLE attribution decision, and it is PURE over
|
||||
// the decoded batch: it returns the tenant the rows will carry, the events that may be
|
||||
// stored, and how many were dropped. It takes no *zip.Ctx — there is no header, query,
|
||||
// or body field it could read a tenant from, so the returned org is always
|
||||
// publicTenant. That is the isolation proof: not a check that can be bypassed, but an
|
||||
// argument list that cannot express the alternative.
|
||||
//
|
||||
// Each admitted event is REBUILT from the allowlisted fields rather than edited, so a
|
||||
// field this function does not name cannot reach the row. The stored name comes from
|
||||
// the kind (resolveEventName maps the empty name to $pageview / $error). Properties are
|
||||
// left nil: the shared tail (ingestDecoded) folds the typed error into
|
||||
// properties.$exception and the write core stamps $source, so an anonymous row's
|
||||
// properties hold exactly what the SERVER put there and nothing the caller sent.
|
||||
func admitPublic(evs []CaptureEvent) (string, []CaptureEvent, int) {
|
||||
out := make([]CaptureEvent, 0, len(evs))
|
||||
dropped := 0
|
||||
for _, e := range evs {
|
||||
kind := canonicalType(e.Type)
|
||||
if !publicKinds[kind] {
|
||||
dropped++
|
||||
continue
|
||||
}
|
||||
out = append(out, CaptureEvent{
|
||||
MessageID: e.MessageID,
|
||||
Type: kind,
|
||||
Timestamp: e.Timestamp,
|
||||
DistinctID: e.DistinctID,
|
||||
AnonymousID: e.AnonymousID,
|
||||
SessionID: e.SessionID,
|
||||
Product: e.Product,
|
||||
URL: e.URL,
|
||||
Path: e.Path,
|
||||
Referrer: e.Referrer,
|
||||
UTM: e.UTM,
|
||||
Library: e.Library,
|
||||
LibraryVer: e.LibraryVer,
|
||||
Error: e.Error,
|
||||
})
|
||||
}
|
||||
return publicTenant, out, dropped
|
||||
}
|
||||
|
||||
// publicIngest answers an ANONYMOUS POST on the canonical door: the request-scoped
|
||||
// gates (capture flag, rate, size, opt-out) then the pure decision (admitPublic) then
|
||||
// the ONE write core. source stays the front door's origin tag — the TENANT, not the
|
||||
// tag, is what records that a row arrived unattested.
|
||||
//
|
||||
// It is reached only from eventHandle, and only when the caller presented no
|
||||
// credential at all: a presented-but-unresolvable key is refused there rather than
|
||||
// downgraded to anonymous.
|
||||
func publicIngest(c *zip.Ctx, source string) error {
|
||||
// CLOUD_ANALYTICS_PUBLIC_CAPTURE is the ONE existing anonymous-capture switch
|
||||
// (it also gates the site-host carve). Off ⇒ the canonical door keeps its
|
||||
// strict, principal-only contract.
|
||||
if !publicCaptureEnabled() {
|
||||
return zip.ErrForbidden("valid bearer or a resolvable ingest key required")
|
||||
}
|
||||
if !publicRateOK(c) {
|
||||
return zip.Errorf(http.StatusTooManyRequests, "rate limit exceeded")
|
||||
}
|
||||
body := c.Body()
|
||||
if len(body) > maxPublicBytes {
|
||||
return zip.Errorf(http.StatusRequestEntityTooLarge, "event payload too large")
|
||||
}
|
||||
evs, err := decodeIngest(body)
|
||||
if err != nil {
|
||||
return zip.ErrBadRequest("malformed event payload")
|
||||
}
|
||||
if len(evs) > maxPublicBatch {
|
||||
return zip.ErrBadRequest("batch too large")
|
||||
}
|
||||
if optedOut(c) {
|
||||
return c.JSON(http.StatusOK, CaptureResult{Dropped: len(evs)})
|
||||
}
|
||||
// Rejoin the ONE pipeline: admission decided the tenant and the projection, and
|
||||
// ingestDecoded (event.go) does the rest exactly as it does for a bearer.
|
||||
org, admitted, dropped := admitPublic(evs)
|
||||
return ingestDecoded(c, org, source, admitted, dropped)
|
||||
}
|
||||
@@ -0,0 +1,566 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// ── harness ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// postAnon issues an ANONYMOUS POST — no X-User-Id, no X-Org-Id, no key of any kind —
|
||||
// with optional extra headers, returning the status and body. This is exactly the
|
||||
// shape a logged-out marketing page emits.
|
||||
func postAnon(t *testing.T, app *zip.App, path, body string, hdr map[string]string) (int, []byte) {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
for k, v := range hdr {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
resp, err := app.Fiber().Test(req)
|
||||
if err != nil {
|
||||
t.Fatalf("Test POST %s: %v", path, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return resp.StatusCode, b
|
||||
}
|
||||
|
||||
// receipt decodes the {accepted,dropped} contract.
|
||||
func receipt(t *testing.T, body []byte) CaptureResult {
|
||||
t.Helper()
|
||||
var r CaptureResult
|
||||
if err := json.Unmarshal(body, &r); err != nil {
|
||||
t.Fatalf("receipt not valid json: %v (%s)", err, body)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// tightenPublicRate installs fresh anonymous-ingest counters with the given limits and
|
||||
// restores the package pair afterwards, so a rate test is hermetic and cannot leak a
|
||||
// spent bucket into another test.
|
||||
func tightenPublicRate(t *testing.T, client, peer int) {
|
||||
t.Helper()
|
||||
origClient, origPeer := publicRate, publicPeerRate
|
||||
publicRate = newCounter(client, publicRateWindow)
|
||||
publicPeerRate = newCounter(peer, publicRateWindow)
|
||||
t.Cleanup(func() { publicRate, publicPeerRate = origClient, origPeer })
|
||||
}
|
||||
|
||||
// The marketing wire: @hanzo/event batches {batch:[…]} and a logged-out page sends it
|
||||
// with no credentials at all.
|
||||
const anonPageview = `{"batch":[{"type":"pageview","event":"$pageview","distinctId":"anon-1",` +
|
||||
`"sessionId":"s1","product":"site","url":"https://hanzo.ai/cloud","path":"/cloud",` +
|
||||
`"referrer":"https://google.com/","utm":{"source":"google","medium":"organic"},` +
|
||||
`"library":"@hanzo/event","libraryVersion":"0.3.3"}]}`
|
||||
|
||||
// ── admitPublic: the pure attribution decision ──────────────────────────────
|
||||
|
||||
// TestAdmitPublic_TenantIsAlwaysPublic is the ISOLATION PROOF. admitPublic takes no
|
||||
// request, so no header/query/body can reach it; here we additionally hand it every
|
||||
// org-naming field the wire has and confirm the tenant it returns is the reserved
|
||||
// constant every time.
|
||||
func TestAdmitPublic_TenantIsAlwaysPublic(t *testing.T) {
|
||||
hostile := [][]CaptureEvent{
|
||||
{{Type: "pageview"}},
|
||||
{{Type: "pageview", GroupID: "maxpower"}},
|
||||
{{Type: "pageview", PersonID: "victim-person"}},
|
||||
{{Type: "error", GroupID: "maxpower", PersonID: "victim", Error: &Exception{Message: "boom"}}},
|
||||
{{Type: "pageview", Properties: map[string]any{"org": "maxpower", "tenant_id": "maxpower"}}},
|
||||
{},
|
||||
nil,
|
||||
}
|
||||
for i, evs := range hostile {
|
||||
org, _, _ := admitPublic(evs)
|
||||
if org != publicTenant {
|
||||
t.Fatalf("case %d: admitPublic org = %q, want the reserved %q", i, org, publicTenant)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPublicTenantOutsideOrgNamespace pins WHY the sentinel is safe: an IAM org slug is
|
||||
// lowercase ASCII alphanumerics and '-' (that is all the IAM slugifier emits), so a
|
||||
// '$'-carrying tenant cannot collide with a real org.
|
||||
func TestPublicTenantOutsideOrgNamespace(t *testing.T) {
|
||||
if !strings.HasPrefix(publicTenant, "$") {
|
||||
t.Fatalf("publicTenant %q must carry the reserved '$' so no IAM slug can collide", publicTenant)
|
||||
}
|
||||
for _, r := range publicTenant[1:] {
|
||||
if !(r >= 'a' && r <= 'z') && !(r >= '0' && r <= '9') && r != '-' {
|
||||
t.Fatalf("publicTenant %q: unexpected byte %q", publicTenant, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAdmitPublic_ForeignOrgFieldsDropped: a foreign org can be NAMED in a body, and
|
||||
// the projection must not carry it forward. groupId (a group/org) and personId (a
|
||||
// person) are the two fields that bind an event to someone else's identity.
|
||||
func TestAdmitPublic_ForeignOrgFieldsDropped(t *testing.T) {
|
||||
_, out, _ := admitPublic([]CaptureEvent{{
|
||||
Type: "pageview",
|
||||
GroupID: "maxpower",
|
||||
PersonID: "victim-person",
|
||||
RefCode: "r1",
|
||||
Channel: "paid",
|
||||
SignupWeek: "2026-W30",
|
||||
ProductID: "prod_1",
|
||||
Quantity: 7,
|
||||
Revenue: 999.99,
|
||||
Currency: "USD",
|
||||
Properties: map[string]any{"org": "maxpower", "email": "v@example.com"},
|
||||
}})
|
||||
if len(out) != 1 {
|
||||
t.Fatalf("want 1 admitted event, got %d", len(out))
|
||||
}
|
||||
got := out[0]
|
||||
if got.GroupID != "" || got.PersonID != "" {
|
||||
t.Fatalf("identity fields survived the projection: groupId=%q personId=%q", got.GroupID, got.PersonID)
|
||||
}
|
||||
if got.Revenue != 0 || got.ProductID != "" || got.Quantity != 0 || got.Currency != "" {
|
||||
t.Fatalf("commerce fields survived: %+v", got)
|
||||
}
|
||||
if got.RefCode != "" || got.Channel != "" || got.SignupWeek != "" {
|
||||
t.Fatalf("attribution/cohort fields survived: %+v", got)
|
||||
}
|
||||
if len(got.Properties) != 0 {
|
||||
t.Fatalf("client property bag survived the projection: %v", got.Properties)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAdmitPublic_ForeignOrgNeverStamped drives the projection through the REAL
|
||||
// normalizer — the one function that stamps tenant_id — and proves the stored row
|
||||
// carries the public tenant, not the org the body named.
|
||||
func TestAdmitPublic_ForeignOrgNeverStamped(t *testing.T) {
|
||||
org, out, _ := admitPublic([]CaptureEvent{{Type: "pageview", GroupID: "maxpower"}})
|
||||
row, ok := normalizeEvent(org, time.Now(), out[0])
|
||||
if !ok {
|
||||
t.Fatal("want routable")
|
||||
}
|
||||
if row.tenant != publicTenant {
|
||||
t.Fatalf("row.tenant = %q, want %q — an anonymous write must never land in a real org", row.tenant, publicTenant)
|
||||
}
|
||||
if row.tenant == "maxpower" || row.groupID == "maxpower" {
|
||||
t.Fatalf("the body-named org reached the row: tenant=%q group=%q", row.tenant, row.groupID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAdmitPublic_KindAllowlist: pageview and error are admitted; identify, group, and
|
||||
// every custom event are dropped and counted. This is the allowlist, not a denylist —
|
||||
// an unknown type folds to "event" (canonicalType) and is therefore dropped too.
|
||||
func TestAdmitPublic_KindAllowlist(t *testing.T) {
|
||||
for _, kind := range []string{"pageview", "page", "error"} {
|
||||
_, out, dropped := admitPublic([]CaptureEvent{{Type: kind, Error: &Exception{Message: "m"}}})
|
||||
if len(out) != 1 || dropped != 0 {
|
||||
t.Fatalf("kind %q must be admitted, got out=%d dropped=%d", kind, len(out), dropped)
|
||||
}
|
||||
}
|
||||
for _, kind := range []string{"identify", "group", "event", "", "purchase", "metering", "BILLING"} {
|
||||
_, out, dropped := admitPublic([]CaptureEvent{{Type: kind, Event: "whatever"}})
|
||||
if len(out) != 0 || dropped != 1 {
|
||||
t.Fatalf("kind %q must be dropped, got out=%d dropped=%d", kind, len(out), dropped)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAdmitPublic_NameIsServerChosen: the anonymous name space is CLOSED to two values.
|
||||
// A caller cannot introduce a new event name into the read lenses, nor unbounded
|
||||
// cardinality into the table's ORDER BY key.
|
||||
func TestAdmitPublic_NameIsServerChosen(t *testing.T) {
|
||||
cases := []struct{ kind, sent, want string }{
|
||||
{"pageview", "attacker_chosen_name", "$pageview"},
|
||||
{"pageview", "", "$pageview"},
|
||||
{"error", strings.Repeat("x", 4096), "$error"},
|
||||
{"error", "", "$error"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
_, out, _ := admitPublic([]CaptureEvent{{Type: c.kind, Event: c.sent}})
|
||||
if len(out) != 1 {
|
||||
t.Fatalf("kind %q: want 1 admitted", c.kind)
|
||||
}
|
||||
if got := resolveEventName(out[0]); got != c.want {
|
||||
t.Fatalf("kind %q sent name %q ⇒ stored %q, want %q", c.kind, truncate(c.sent), got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func truncate(s string) string {
|
||||
if len(s) > 24 {
|
||||
return s[:24] + "…"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// TestAdmitPublic_OnlyServerProperties: an anonymous row's properties are exactly what
|
||||
// the SERVER put there — the $exception the shared tail folds and the $source the write
|
||||
// core stamps. No key the caller chose can be persisted. This walks the SAME composition
|
||||
// the handler does: admitPublic (projection) → foldException (ingestDecoded's tail) →
|
||||
// withSource (write core) → normalizeEvent (the row).
|
||||
func TestAdmitPublic_OnlyServerProperties(t *testing.T) {
|
||||
_, out, _ := admitPublic([]CaptureEvent{{
|
||||
Type: "error",
|
||||
Error: &Exception{Type: "TypeError", Message: "x is not a function"},
|
||||
Properties: map[string]any{"password": "hunter2", "custom": "junk", "$release": "v1"},
|
||||
}})
|
||||
if len(out) != 1 {
|
||||
t.Fatal("want 1 admitted error event")
|
||||
}
|
||||
// The projection itself carries NO properties — the client's bag is gone entirely.
|
||||
if len(out[0].Properties) != 0 {
|
||||
t.Fatalf("the client property bag survived the projection: %v", out[0].Properties)
|
||||
}
|
||||
// The shared tail folds the typed error, and nothing else appears.
|
||||
folded := foldException(out[0])
|
||||
if len(folded.Properties) != 1 {
|
||||
t.Fatalf("after the shared fold, properties must hold only $exception, got %v", folded.Properties)
|
||||
}
|
||||
if _, ok := folded.Properties["$exception"]; !ok {
|
||||
t.Fatalf("the typed error must be folded to $exception, got %v", folded.Properties)
|
||||
}
|
||||
// Through the real normalizer the stored JSON carries $exception + $source, nothing else.
|
||||
row, ok := normalizeEvent(publicTenant, time.Now(), CaptureEvent{
|
||||
Type: folded.Type, Properties: withSource(folded.Properties, sourceEvent),
|
||||
})
|
||||
if !ok {
|
||||
t.Fatal("want routable")
|
||||
}
|
||||
stored := decodeProps(t, row.properties)
|
||||
if len(stored) != 2 || stored["$source"] != "event" {
|
||||
t.Fatalf("stored properties = %v, want exactly {$exception,$source}", stored)
|
||||
}
|
||||
}
|
||||
|
||||
// ── the route: POST /v1/event with no credential at all ─────────────────────
|
||||
//
|
||||
// Observable proxy, as everywhere in this package: 403 ⇒ refused at the gate; 503 ⇒
|
||||
// ADMITTED and reached requireDatastore (no warehouse in the harness). A 200 means the
|
||||
// door answered without needing the warehouse — an all-dropped batch or an opt-out.
|
||||
|
||||
// TestPublic_AnonymousPageviewAccepted is the headline: the exact logged-out marketing
|
||||
// wire, no bearer and no key, is ADMITTED. Before this change it was 403 and every
|
||||
// marketing pageview was lost.
|
||||
func TestPublic_AnonymousPageviewAccepted(t *testing.T) {
|
||||
app := mountApp(t)
|
||||
code, body := postAnon(t, app, "/v1/event", anonPageview, nil)
|
||||
if code == http.StatusForbidden {
|
||||
t.Fatalf("anonymous pageview must be admitted on the canonical door, got 403 (%s)", body)
|
||||
}
|
||||
if code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("anonymous pageview want 503 (admitted, datastore down), got %d (%s)", code, body)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPublic_AnonymousErrorAccepted: the other half of what marketing needs — a browser
|
||||
// error from a logged-out page.
|
||||
func TestPublic_AnonymousErrorAccepted(t *testing.T) {
|
||||
app := mountApp(t)
|
||||
code, body := postAnon(t, app, "/v1/event",
|
||||
`{"batch":[{"type":"error","error":{"type":"TypeError","message":"boom"},"path":"/cloud"}]}`, nil)
|
||||
if code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("anonymous error want 503 (admitted, datastore down), got %d (%s)", code, body)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPublic_ForeignOrgClaimBuysNothing: an anonymous caller that names a foreign org
|
||||
// EVERY way the wire allows — the X-Org-Id header, a body org/tenant field, groupId —
|
||||
// is not refused (it is anonymous traffic) but gains nothing: the only kind it sent is
|
||||
// non-allowlisted, so the receipt is 200 accepted:0 dropped:1 and no row exists to
|
||||
// carry `maxpower`. The tenant it would have landed under is proven by
|
||||
// TestAdmitPublic_ForeignOrgNeverStamped.
|
||||
func TestPublic_ForeignOrgClaimBuysNothing(t *testing.T) {
|
||||
app := mountApp(t)
|
||||
code, body := postAnon(t, app, "/v1/event",
|
||||
`{"org":"maxpower","tenant_id":"maxpower","batch":[{"type":"event","event":"steal","groupId":"maxpower"}]}`,
|
||||
map[string]string{"X-Org-Id": "maxpower"})
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("forged-org anonymous batch want 200 (all dropped, no warehouse needed), got %d (%s)", code, body)
|
||||
}
|
||||
if r := receipt(t, body); r.Accepted != 0 || r.Dropped != 1 {
|
||||
t.Fatalf("receipt = %+v, want accepted:0 dropped:1", r)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPublic_NonAllowlistedKindRejected: a custom/product/billing event is refused
|
||||
// storage anonymously and reported in the honest receipt. A mixed batch keeps its
|
||||
// allowlisted events and drops the rest, which is why marketing telemetry lands while
|
||||
// the arbitrary surface stays shut.
|
||||
func TestPublic_NonAllowlistedKindRejected(t *testing.T) {
|
||||
app := mountApp(t)
|
||||
for _, body := range []string{
|
||||
`{"batch":[{"type":"event","event":"order_completed","revenue":99.5}]}`,
|
||||
`{"batch":[{"type":"identify","distinctId":"victim"}]}`,
|
||||
`{"batch":[{"type":"group","groupId":"maxpower"}]}`,
|
||||
} {
|
||||
code, got := postAnon(t, app, "/v1/event", body, nil)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("non-allowlisted kind %s want 200 all-dropped, got %d (%s)", body, code, got)
|
||||
}
|
||||
if r := receipt(t, got); r.Accepted != 0 || r.Dropped != 1 {
|
||||
t.Fatalf("non-allowlisted kind %s receipt = %+v, want accepted:0 dropped:1", body, r)
|
||||
}
|
||||
}
|
||||
// Mixed batch: the pageview survives (so the request reaches the warehouse → 503),
|
||||
// the custom event does not.
|
||||
code, got := postAnon(t, app, "/v1/event",
|
||||
`{"batch":[{"type":"pageview"},{"type":"event","event":"order_completed"}]}`, nil)
|
||||
if code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("mixed batch want 503 (the pageview is admitted), got %d (%s)", code, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPublic_OversizedRejected: both bounds REFUSE rather than truncate — a body past
|
||||
// maxPublicBytes is 413 and a batch past maxPublicBatch is 400. Neither is silently
|
||||
// trimmed, so the receipt can never overstate what was stored.
|
||||
func TestPublic_OversizedRejected(t *testing.T) {
|
||||
app := mountApp(t)
|
||||
|
||||
// Over the BYTE cap: one event whose URL alone exceeds maxPublicBytes.
|
||||
huge := `{"batch":[{"type":"pageview","url":"` + strings.Repeat("a", maxPublicBytes+1024) + `"}]}`
|
||||
if len(huge) <= maxPublicBytes {
|
||||
t.Fatalf("test body must exceed maxPublicBytes (%d), got %d", maxPublicBytes, len(huge))
|
||||
}
|
||||
code, body := postAnon(t, app, "/v1/event", huge, nil)
|
||||
if code != http.StatusRequestEntityTooLarge {
|
||||
t.Fatalf("oversized anonymous body want 413, got %d (%s)", code, body)
|
||||
}
|
||||
|
||||
// Over the COUNT cap, while staying comfortably under the byte cap: many tiny events.
|
||||
evs := make([]string, maxPublicBatch+1)
|
||||
for i := range evs {
|
||||
evs[i] = `{"type":"pageview"}`
|
||||
}
|
||||
many := `{"batch":[` + strings.Join(evs, ",") + `]}`
|
||||
if len(many) > maxPublicBytes {
|
||||
t.Fatalf("count-cap body must stay under the byte cap, got %d", len(many))
|
||||
}
|
||||
code, body = postAnon(t, app, "/v1/event", many, nil)
|
||||
if code != http.StatusBadRequest {
|
||||
t.Fatalf("over-long anonymous batch want 400, got %d (%s)", code, body)
|
||||
}
|
||||
|
||||
// At the caps the request is admitted, so the bounds are not off by one.
|
||||
ok := make([]string, maxPublicBatch)
|
||||
for i := range ok {
|
||||
ok[i] = `{"type":"pageview"}`
|
||||
}
|
||||
code, body = postAnon(t, app, "/v1/event", `{"batch":[`+strings.Join(ok, ",")+`]}`, nil)
|
||||
if code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("batch AT the cap want 503 (admitted), got %d (%s)", code, body)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPublic_RateLimited: anonymous ingest is capped per client IP, and the cap is
|
||||
// charged even when the caller supplies no forwarding header (an unkeyable caller
|
||||
// shares one bucket rather than being exempt).
|
||||
func TestPublic_RateLimited(t *testing.T) {
|
||||
app := mountApp(t)
|
||||
tightenPublicRate(t, 1, 1000)
|
||||
|
||||
if code, body := postAnon(t, app, "/v1/event", anonPageview,
|
||||
map[string]string{"X-Forwarded-For": "203.0.113.7"}); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("first anonymous request want 503 (admitted), got %d (%s)", code, body)
|
||||
}
|
||||
if code, _ := postAnon(t, app, "/v1/event", anonPageview,
|
||||
map[string]string{"X-Forwarded-For": "203.0.113.7"}); code != http.StatusTooManyRequests {
|
||||
t.Fatalf("second anonymous request from the same IP want 429, got %d", code)
|
||||
}
|
||||
// A different client IP has its own bucket.
|
||||
if code, _ := postAnon(t, app, "/v1/event", anonPageview,
|
||||
map[string]string{"X-Forwarded-For": "198.51.100.4"}); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("a different client IP must have its own bucket, got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPublic_PeerCeiling: the socket-peer bucket is charged independently, so a flood
|
||||
// that ROTATES X-Forwarded-For (the header is caller-settable) still meets a ceiling.
|
||||
func TestPublic_PeerCeiling(t *testing.T) {
|
||||
app := mountApp(t)
|
||||
tightenPublicRate(t, 1000, 1)
|
||||
|
||||
if code, _ := postAnon(t, app, "/v1/event", anonPageview,
|
||||
map[string]string{"X-Forwarded-For": "203.0.113.1"}); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("first request want 503 (admitted), got %d", code)
|
||||
}
|
||||
if code, _ := postAnon(t, app, "/v1/event", anonPageview,
|
||||
map[string]string{"X-Forwarded-For": "203.0.113.2"}); code != http.StatusTooManyRequests {
|
||||
t.Fatalf("a rotated X-Forwarded-For must still meet the peer ceiling, got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPublic_OptOutHonored: DNT / Sec-GPC on the wire means nothing is stored. The
|
||||
// client's own consent gate is the host app's `enabled` flag, so an opted-out visitor
|
||||
// usually sends nothing at all; this is the independent server-side line for when the
|
||||
// header arrives anyway.
|
||||
func TestPublic_OptOutHonored(t *testing.T) {
|
||||
app := mountApp(t)
|
||||
for _, h := range []map[string]string{{"DNT": "1"}, {"Sec-GPC": "1"}} {
|
||||
code, body := postAnon(t, app, "/v1/event", anonPageview, h)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("opt-out %v want 200 (stored nothing, no warehouse touched), got %d (%s)", h, code, body)
|
||||
}
|
||||
if r := receipt(t, body); r.Accepted != 0 || r.Dropped != 1 {
|
||||
t.Fatalf("opt-out %v receipt = %+v, want accepted:0 dropped:1", h, r)
|
||||
}
|
||||
}
|
||||
// DNT:0 is NOT an opt-out — only the exact "1" signal is.
|
||||
if code, _ := postAnon(t, app, "/v1/event", anonPageview, map[string]string{"DNT": "0"}); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("DNT:0 must not suppress capture, got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPublic_CaptureFlagOff: CLOUD_ANALYTICS_PUBLIC_CAPTURE is the ONE existing
|
||||
// anonymous-capture switch, and turning it off restores the strict principal-only door.
|
||||
func TestPublic_CaptureFlagOff(t *testing.T) {
|
||||
t.Setenv(publicCaptureEnv, "false")
|
||||
app := mountApp(t)
|
||||
if code, body := postAnon(t, app, "/v1/event", anonPageview, nil); code != http.StatusForbidden {
|
||||
t.Fatalf("public capture off ⇒ anonymous /v1/event want 403, got %d (%s)", code, body)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPublic_PresentedKeyStillFailsClosed: the anonymous lane is for a caller that
|
||||
// presented NOTHING. A presented-but-unresolvable ingest key is still refused, never
|
||||
// downgraded into the public bucket — a misconfigured key must not silently file its
|
||||
// events where its owner cannot read them.
|
||||
func TestPublic_PresentedKeyStillFailsClosed(t *testing.T) {
|
||||
app := mountApp(t)
|
||||
stubResolver(t, func(string) (string, bool) { return "", false })
|
||||
if code := postKeyed(t, app, "/v1/event", "hanzo.ai",
|
||||
`{"api_key":"hk-bad","batch":[{"type":"pageview"}]}`, nil); code != http.StatusForbidden {
|
||||
t.Fatalf("presented-but-unresolvable api_key want 403 (fail closed, not anonymous), got %d", code)
|
||||
}
|
||||
// Same for a publishable key IAM cannot resolve, on the sendBeacon-friendly carriers.
|
||||
if code := postKeyed(t, app, "/v1/event", "hanzo.ai", anonPageview,
|
||||
map[string]string{"x-hanzo-ingest-key": "pk-nosuch"}); code != http.StatusForbidden {
|
||||
t.Fatalf("presented-but-unresolvable pk- want 403 (fail closed, not anonymous), got %d", code)
|
||||
}
|
||||
if code := postKeyed(t, app, "/v1/event?ingest_key=pk-nosuch", "hanzo.ai", anonPageview,
|
||||
nil); code != http.StatusForbidden {
|
||||
t.Fatalf("presented-but-unresolvable ?ingest_key= want 403 (fail closed, not anonymous), got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// ── the authenticated lane is UNCHANGED ─────────────────────────────────────
|
||||
|
||||
// TestAuthenticated_KeepsFullCapability is the additivity proof. Every restriction the
|
||||
// anonymous lane imposes — the kind allowlist, the batch cap, the field projection —
|
||||
// must NOT reach a validated principal. Each case below is refused or trimmed
|
||||
// anonymously and admitted whole with a bearer.
|
||||
func TestAuthenticated_KeepsFullCapability(t *testing.T) {
|
||||
app := mountApp(t)
|
||||
|
||||
// A custom product event with commerce fields and a property bag: dropped
|
||||
// anonymously, admitted (503, datastore down) for a real principal.
|
||||
commerce := `{"batch":[{"type":"event","event":"order_completed","revenue":99.5,` +
|
||||
`"productId":"prod_1","quantity":2,"currency":"USD","groupId":"acme-team",` +
|
||||
`"personId":"p1","properties":{"plan":"pro"}}]}`
|
||||
if code, body := postAnon(t, app, "/v1/event", commerce, nil); code != http.StatusOK {
|
||||
t.Fatalf("precondition: the commerce event must be dropped anonymously, got %d (%s)", code, body)
|
||||
}
|
||||
if code, body := doBody(t, app, http.MethodPost, "/v1/event", "user-dave", "acme", commerce); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("bearer commerce event want 503 (admitted, full capability), got %d (%s)", code, body)
|
||||
}
|
||||
|
||||
// identify/group name a person and a group: dropped anonymously, admitted with a bearer.
|
||||
for _, body := range []string{
|
||||
`{"batch":[{"type":"identify","distinctId":"u1","personId":"p1"}]}`,
|
||||
`{"batch":[{"type":"group","groupId":"acme-team"}]}`,
|
||||
} {
|
||||
if code, got := postAnon(t, app, "/v1/event", body, nil); code != http.StatusOK {
|
||||
t.Fatalf("precondition: %s must be dropped anonymously, got %d (%s)", body, code, got)
|
||||
}
|
||||
if code, got := doBody(t, app, http.MethodPost, "/v1/event", "user-dave", "acme", body); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("bearer %s want 503 (admitted), got %d (%s)", body, code, got)
|
||||
}
|
||||
}
|
||||
|
||||
// The public batch cap is NOT the authenticated cap: maxPublicBatch+1 events are
|
||||
// 400 anonymously and admitted with a bearer (the authenticated bound is maxBatch).
|
||||
evs := make([]string, maxPublicBatch+1)
|
||||
for i := range evs {
|
||||
evs[i] = `{"type":"pageview"}`
|
||||
}
|
||||
over := `{"batch":[` + strings.Join(evs, ",") + `]}`
|
||||
if code, _ := postAnon(t, app, "/v1/event", over, nil); code != http.StatusBadRequest {
|
||||
t.Fatalf("precondition: over-cap batch must be 400 anonymously, got %d", code)
|
||||
}
|
||||
if code, got := doBody(t, app, http.MethodPost, "/v1/event", "user-dave", "acme", over); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("bearer over-public-cap batch want 503 (public cap does not apply), got %d (%s)", code, got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAuthenticated_NotRateLimitedByPublicCounter: the anonymous flood cap must never
|
||||
// throttle a validated principal. With the anonymous buckets exhausted, a bearer still
|
||||
// gets through.
|
||||
func TestAuthenticated_NotRateLimitedByPublicCounter(t *testing.T) {
|
||||
app := mountApp(t)
|
||||
tightenPublicRate(t, 0, 0)
|
||||
if code, _ := postAnon(t, app, "/v1/event", anonPageview, nil); code != http.StatusTooManyRequests {
|
||||
t.Fatalf("precondition: a zero-limit anonymous bucket must 429, got %d", code)
|
||||
}
|
||||
if code, body := doBody(t, app, http.MethodPost, "/v1/event", "user-dave", "acme", anonPageview); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("bearer must be unaffected by the anonymous rate cap, got %d (%s)", code, body)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAuthenticated_OptOutNotHonoredForPrincipal: the DNT/GPC gate belongs to the
|
||||
// anonymous lane. An authenticated product client sending the header is unchanged —
|
||||
// first-party product telemetry is governed by the org's own consent, not by this door.
|
||||
func TestAuthenticated_OptOutNotHonoredForPrincipal(t *testing.T) {
|
||||
app := mountApp(t)
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/event", strings.NewReader(anonPageview))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-User-Id", "user-dave")
|
||||
req.Header.Set("X-Org-Id", "acme")
|
||||
req.Header.Set("DNT", "1")
|
||||
resp, err := app.Fiber().Test(req)
|
||||
if err != nil {
|
||||
t.Fatalf("Test: %v", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
if resp.StatusCode != http.StatusServiceUnavailable {
|
||||
t.Fatalf("bearer + DNT want 503 (authenticated lane untouched), got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// ── the fan-out seam refuses the public tenant ──────────────────────────────
|
||||
|
||||
// TestFanOut_PublicTenantNeverReachesDestinations: destinations receive the RAW
|
||||
// pre-scrub event and forward it to an org's connected ad platforms, so an unattested
|
||||
// event must never reach one.
|
||||
func TestFanOut_PublicTenantNeverReachesDestinations(t *testing.T) {
|
||||
var gotOrgs []string
|
||||
SetSink(func(org string, _ []SinkEvent) { gotOrgs = append(gotOrgs, org) })
|
||||
t.Cleanup(func() { SetSink(nil) })
|
||||
|
||||
fanOut(publicTenant, []CaptureEvent{{Type: "pageview", Event: "$pageview"}})
|
||||
// The sink dispatches on a goroutine; give a real fan-out time to land.
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if len(gotOrgs) != 0 {
|
||||
t.Fatalf("the public tenant must never fan out to destinations, got orgs %v", gotOrgs)
|
||||
}
|
||||
|
||||
// A real org still fans out — the guard is scoped to the sentinel, not a regression.
|
||||
fanOut("acme", []CaptureEvent{{Type: "pageview", Event: "$pageview"}})
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for len(gotOrgs) == 0 && time.Now().Before(deadline) {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if len(gotOrgs) != 1 || gotOrgs[0] != "acme" {
|
||||
t.Fatalf("a real org must still fan out, got %v", gotOrgs)
|
||||
}
|
||||
}
|
||||
@@ -12,46 +12,37 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// publishable.go — the FASTEST capture path: a write-only PUBLISHABLE KEY (pk_…)
|
||||
// that authenticates a direct-to-datastore ingest with ZERO network hop.
|
||||
// publishable.go — the public capture path: a PUBLISHABLE KEY (pk-…) attributes
|
||||
// a browser beacon to its tenant and writes straight to the datastore.
|
||||
//
|
||||
// POST /v1/ingest body: {batch:[WireEvent]} auth: pk_… -> {accepted,dropped}
|
||||
// POST /v1/ingest/keys mint a pk_ for the caller's org (validated principal)
|
||||
// GET /v1/errors recent type:'error' events for the org (read lens)
|
||||
// POST /v1/ingest body: {batch:[WireEvent]} auth: pk-… -> {accepted,dropped}
|
||||
// GET /v1/errors recent type:'error' events for the org (read lens)
|
||||
//
|
||||
// WHY a distinct key from the IAM hk-/sk-/pk- family: those resolve through IAM
|
||||
// (get-user?accessKey — a network round-trip) and mint a FULL principal that can
|
||||
// READ. A publishable key is meant to ship in a browser bundle, so it must be
|
||||
// write-only and cheap to verify. This key is:
|
||||
// ONE publishable key, and IAM issues it. pk- is publishable, sk- is secret, and
|
||||
// there is no third thing.
|
||||
//
|
||||
// - INGEST-ONLY BY CONSTRUCTION. The `pk_` (underscore) prefix is deliberately
|
||||
// NOT in isAPIKey's set (hk-/sk-/pk-/fw_/hz_, all dash/`fw_`/`hz_`), so the
|
||||
// identity boundary (SanitizeIdentity) and OrgForKey both REFUSE it — it can
|
||||
// never become a bearer principal, so it can never read. Its only door is the
|
||||
// ingest verifier below. Write-only is a property of WHICH resolver accepts
|
||||
// the value, not a flag on a row.
|
||||
// - ORG-SCOPED, SIGNED, NON-FORGEABLE. The org is carried in the key but sealed
|
||||
// under HMAC-SHA256(secret, org): a client cannot flip the org without the
|
||||
// secret. The server stamps tenant_id from the VERIFIED org, never from the
|
||||
// request body — the same tenant invariant the rest of the plane enforces.
|
||||
// - LOWEST LATENCY. Verification is one HMAC compute — no IAM call, no keys
|
||||
// table, no DB read. This is the no-Kafka, no-bridge, direct-to-ClickHouse
|
||||
// path; it funnels through the SAME write core (ingestEvents) into the SAME
|
||||
// hanzo.events table as every other adapter. One write path, many front doors.
|
||||
// This file used to mint and verify its OWN pk_ (underscore) under an
|
||||
// HMAC of CLOUD_INGEST_KEY_SECRET, with its own mint endpoint at
|
||||
// /v1/ingest/keys — a second publishable-key family sitting beside the one IAM
|
||||
// already owned. The underscore was load-bearing back then: pk_ was deliberately
|
||||
// kept OUT of isAPIKey's set, because anything isAPIKey resolved into "the same
|
||||
// principal a JWT yields", and a key meant for a browser bundle must not read.
|
||||
//
|
||||
// SECRET: the HMAC secret is CLOUD_INGEST_KEY_SECRET (KMS-injected by the
|
||||
// operator). Absent ⇒ mint and verify BOTH fail closed (503 / 403) — a deployment
|
||||
// without the secret never mints a forgeable key nor admits an unverifiable one.
|
||||
// That is fixed at the boundary instead of routed around: IdentityFromRequest now
|
||||
// refuses a pk- outright (cloud.IsPublishableKey), so publishable means
|
||||
// publishable no matter which door it arrives at. A pk- stays inside
|
||||
// APIKeyPrefixes on purpose — OrgForKey must resolve it to learn which tenant a
|
||||
// beacon belongs to. Resolvable, not authenticating.
|
||||
//
|
||||
// The tenant is whatever IAM resolves the key to, never a body or header claim,
|
||||
// so the tenant invariant the rest of the plane enforces holds here too. Every
|
||||
// door funnels through the SAME write core (ingestEvents) into the SAME
|
||||
// hanzo.events table: one write path, many front doors.
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -61,13 +52,11 @@ import (
|
||||
)
|
||||
|
||||
// ingestKeySecretEnv names the KMS-injected HMAC secret that seals a publishable
|
||||
// key's org. Absent ⇒ the publishable-key path is disabled (fails closed).
|
||||
const ingestKeySecretEnv = "CLOUD_INGEST_KEY_SECRET"
|
||||
|
||||
// publishablePrefix marks a write-only ingest key. Underscore (not the dash of
|
||||
// the isAPIKey family) is load-bearing: it keeps pk_ OUT of the bearer/principal
|
||||
// the isAPIKey family) is load-bearing: it keeps pk- OUT of the bearer/principal
|
||||
// path, so a publishable key is structurally read-incapable.
|
||||
const publishablePrefix = "pk_"
|
||||
const publishablePrefix = cloud.PublishablePrefix
|
||||
|
||||
// sourceIngest tags rows that arrived via the publishable-key direct ingest, so
|
||||
// the ONE hanzo.events table stays honest about origin (queryable as
|
||||
@@ -76,70 +65,9 @@ const publishablePrefix = "pk_"
|
||||
const sourceIngest = "ingest"
|
||||
|
||||
// sigBytes is the HMAC truncation length (128 bits) — ample against forgery while
|
||||
// keeping the key short enough to embed in a bundle.
|
||||
const sigBytes = 16
|
||||
|
||||
// ── key codec (pure) ─────────────────────────────────────────────────────────
|
||||
|
||||
// ingestSecret returns the configured HMAC secret, or "" when unset (path off).
|
||||
func ingestSecret() string { return strings.TrimSpace(os.Getenv(ingestKeySecretEnv)) }
|
||||
|
||||
// keySig computes the org signature under the secret: HMAC-SHA256(secret, org),
|
||||
// truncated to sigBytes. The org is the only signed input — the tenant a key can
|
||||
// ever write into is fixed at mint time and cannot be shifted without the secret.
|
||||
func keySig(secret, org string) []byte {
|
||||
m := hmac.New(sha256.New, []byte(secret))
|
||||
m.Write([]byte(org))
|
||||
return m.Sum(nil)[:sigBytes]
|
||||
}
|
||||
|
||||
// mintPublishableKey mints "pk_<b64url(org)>.<b64url(sig)>" for org under secret.
|
||||
// '.' is the delimiter because it is OUTSIDE the base64url alphabet (which uses
|
||||
// '-' and '_'), so the two segments split unambiguously. Returns ("",false) when
|
||||
// the secret is unconfigured (fail closed) or org is empty.
|
||||
func mintPublishableKey(secret, org string) (string, bool) {
|
||||
org = strings.TrimSpace(org)
|
||||
if secret == "" || org == "" {
|
||||
return "", false
|
||||
}
|
||||
b64 := base64.RawURLEncoding
|
||||
return publishablePrefix + b64.EncodeToString([]byte(org)) + "." + b64.EncodeToString(keySig(secret, org)), true
|
||||
}
|
||||
|
||||
// verifyPublishableKey resolves a presented key to its org, or ("",false) if the
|
||||
// key is not a well-formed, correctly-signed publishable key under the configured
|
||||
// secret. FAILS CLOSED: unconfigured secret, wrong prefix, malformed segments, or
|
||||
// a signature mismatch all return not-ok. Constant-time signature compare. Pure:
|
||||
// no I/O, so tests drive it directly.
|
||||
func verifyPublishableKey(secret, key string) (string, bool) {
|
||||
key = strings.TrimSpace(key)
|
||||
if secret == "" || !strings.HasPrefix(key, publishablePrefix) {
|
||||
return "", false
|
||||
}
|
||||
body := key[len(publishablePrefix):]
|
||||
dot := strings.IndexByte(body, '.')
|
||||
if dot <= 0 || dot == len(body)-1 {
|
||||
return "", false
|
||||
}
|
||||
b64 := base64.RawURLEncoding
|
||||
orgBytes, err := b64.DecodeString(body[:dot])
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
sig, err := b64.DecodeString(body[dot+1:])
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
org := string(orgBytes)
|
||||
if org == "" || len(org) > maxIngestOrgLen {
|
||||
return "", false
|
||||
}
|
||||
if subtle.ConstantTimeCompare(sig, keySig(secret, org)) != 1 {
|
||||
return "", false
|
||||
}
|
||||
return org, true
|
||||
}
|
||||
|
||||
// maxIngestOrgLen bounds a decoded org (it becomes a warehouse partition key),
|
||||
// mirroring the cap OrgForKey applies to an IAM-resolved owner.
|
||||
const maxIngestOrgLen = 128
|
||||
@@ -149,7 +77,7 @@ const maxIngestOrgLen = 128
|
||||
// ingestKey pulls the presented publishable key, in priority order: the
|
||||
// Authorization: Bearer header (the common browser-fetch shape), the
|
||||
// x-hanzo-ingest-key header, then the ?ingest_key= query (navigator.sendBeacon
|
||||
// cannot set headers). Only a pk_-prefixed value is returned — an unrelated
|
||||
// cannot set headers). Only a pk--prefixed value is returned — an unrelated
|
||||
// bearer (a real JWT/IAM key) is ignored here so this door never shadows the
|
||||
// identity path. "" when none is present.
|
||||
func ingestKey(c *zip.Ctx) string {
|
||||
@@ -198,7 +126,11 @@ func foldException(e CaptureEvent) CaptureEvent {
|
||||
for k, v := range e.Properties {
|
||||
props[k] = v
|
||||
}
|
||||
props["$exception"] = e.Error
|
||||
// Redact the exception's free-text (message/stack) at the fold point so the
|
||||
// stored row AND the raw destinations fan-out (forward.go, which sees events
|
||||
// BEFORE the warehouse scrub) both carry a clean $exception — never a token,
|
||||
// query secret, or PII lifted from a stack frame.
|
||||
props["$exception"] = scrubException(e.Error)
|
||||
e.Properties = props
|
||||
e.Error = nil
|
||||
return e
|
||||
@@ -207,41 +139,21 @@ func foldException(e CaptureEvent) CaptureEvent {
|
||||
// ── handlers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// ingest answers POST /v1/ingest — a THIN DEPRECATED ALIAS of the canonical door.
|
||||
// Since /v1/event now natively accepts the publishable key (pk_…, via eventTenant)
|
||||
// Since /v1/event now natively accepts the publishable key (pk-…, via eventTenant)
|
||||
// AND the {batch:[…]} wire (via decodeIngest), /v1/ingest is redundant: it delegates
|
||||
// to the EXACT canonical handler logic (eventHandle) — the SAME pluggable auth,
|
||||
// tolerant decode, error-fold, and ONE write core — differing only in a one-shot
|
||||
// deprecation log and the $source=ingest origin tag for the migration signal.
|
||||
// Existing pk_ callers keep working unchanged; there is ONE implementation.
|
||||
// Existing pk- callers keep working unchanged; there is ONE implementation.
|
||||
func ingest(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
deprecated(s, c, "/v1/event")
|
||||
return eventHandle(c, sourceIngest)
|
||||
}
|
||||
|
||||
// mintKey answers POST /v1/ingest/keys — an org owner (VALIDATED principal) mints
|
||||
// a publishable key for its OWN org. The key is org-scoped to the caller's tenant
|
||||
// (never a body-supplied org), so a caller can only ever mint a key that writes
|
||||
// into its own partition. Fails closed (503) when the secret is unconfigured.
|
||||
func mintKey(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
org, ok := tenant(c)
|
||||
if !ok {
|
||||
return zip.ErrForbidden("valid bearer required")
|
||||
}
|
||||
secret := ingestSecret()
|
||||
if secret == "" {
|
||||
return zip.Errorf(http.StatusServiceUnavailable, "publishable keys unavailable: ingest key secret not configured")
|
||||
}
|
||||
key, ok := mintPublishableKey(secret, org)
|
||||
if !ok {
|
||||
return zip.Errorf(http.StatusServiceUnavailable, "could not mint publishable key")
|
||||
}
|
||||
return c.JSON(http.StatusOK, map[string]any{"key": key, "org": org, "scope": "ingest"})
|
||||
}
|
||||
|
||||
// errorsLens answers GET /v1/errors — the error-tracking read view: recent
|
||||
// type:'error' events for the org, newest first. Tenant-scoped server-side and
|
||||
// gated on a VALIDATED principal (tenant()), NOT the publishable key — reads
|
||||
// require real auth, reinforcing that pk_ is write-only. The captured exception
|
||||
// require real auth, reinforcing that pk- is write-only. The captured exception
|
||||
// is surfaced straight from properties.$exception. limit defaults 50, caps 200.
|
||||
func errorsLens(s *cloud.Service[state], c *zip.Ctx) error {
|
||||
org, ok := tenant(c)
|
||||
|
||||
@@ -7,102 +7,29 @@ package analytics
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/hanzoai/cloud"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const testSecret = "test-ingest-secret-0123456789"
|
||||
|
||||
// mint→verify is a round trip: a key minted for an org verifies back to exactly
|
||||
// that org under the same secret.
|
||||
func TestPublishableKeyRoundTrip(t *testing.T) {
|
||||
for _, org := range []string{"acme", "hanzo", "maxpower", "org-with-dashes", "MixedCase"} {
|
||||
key, ok := mintPublishableKey(testSecret, org)
|
||||
if !ok {
|
||||
t.Fatalf("mint failed for %q", org)
|
||||
}
|
||||
got, ok := verifyPublishableKey(testSecret, key)
|
||||
if !ok {
|
||||
t.Fatalf("verify failed for freshly minted key %q", key)
|
||||
}
|
||||
if got != org {
|
||||
t.Fatalf("round trip org = %q, want %q", got, org)
|
||||
}
|
||||
// The ONE publishable spelling is IAM's pk-, and cloud only validates it. Cloud
|
||||
// used to mint and verify its OWN pk_ under an HMAC of CLOUD_INGEST_KEY_SECRET —
|
||||
// a second publishable-key family with its own prefix, secret and mint endpoint,
|
||||
// beside the one IAM already owned.
|
||||
//
|
||||
// The prefix is asserted against the ONE authority rather than a literal, and the
|
||||
// safety property it depends on is asserted with it: a pk- must resolve (so the
|
||||
// ingest door can attribute a beacon) and must NOT authenticate (so a key shipped
|
||||
// in a browser bundle is not a reading credential).
|
||||
func TestPublishablePrefixIsTheIAMFamily(t *testing.T) {
|
||||
if publishablePrefix != cloud.PublishablePrefix {
|
||||
t.Fatalf("publishable prefix = %q, want %q", publishablePrefix, cloud.PublishablePrefix)
|
||||
}
|
||||
}
|
||||
|
||||
// The key is write-only by construction: pk_ is NOT accepted by the isAPIKey
|
||||
// family, so it can never be minted into a bearer principal. (Guards the prefix
|
||||
// choice — an accidental switch to a dash prefix would silently make the key
|
||||
// readable.) We assert the prefix here; isAPIKey lives in the parent package.
|
||||
func TestPublishablePrefixIsUnderscore(t *testing.T) {
|
||||
key, _ := mintPublishableKey(testSecret, "acme")
|
||||
if key[:3] != "pk_" {
|
||||
t.Fatalf("publishable key must start with pk_ (write-only lane), got %q", key[:3])
|
||||
if !cloud.IsPublishableKey(publishablePrefix + "abc") {
|
||||
t.Fatal("a pk- must be recognised as publishable")
|
||||
}
|
||||
}
|
||||
|
||||
// verify FAILS CLOSED on every malformed / forged / unconfigured case.
|
||||
func TestVerifyFailsClosed(t *testing.T) {
|
||||
good, _ := mintPublishableKey(testSecret, "acme")
|
||||
cases := []struct {
|
||||
name, secret, key string
|
||||
}{
|
||||
{"no secret", "", good},
|
||||
{"wrong prefix", testSecret, "sk-abcdef"},
|
||||
{"empty", testSecret, ""},
|
||||
{"no delimiter", testSecret, "pk_YWNtZQ"},
|
||||
{"trailing delimiter", testSecret, "pk_YWNtZQ."},
|
||||
{"leading delimiter", testSecret, "pk_.YWNtZQ"},
|
||||
{"bad base64 org", testSecret, "pk_!!!.YWNtZQ"},
|
||||
{"bad base64 sig", testSecret, "pk_YWNtZQ.!!!"},
|
||||
{"wrong secret", "other-secret", good},
|
||||
{"tampered org keeps old sig", testSecret, forgeOrg(good, "evil")},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if org, ok := verifyPublishableKey(tc.secret, tc.key); ok {
|
||||
t.Errorf("%s: verify admitted a bad key → org %q (want fail closed)", tc.name, org)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// forgeOrg swaps the org segment of a real key while keeping its signature — the
|
||||
// canonical forgery attempt the HMAC must reject.
|
||||
func forgeOrg(key, newOrg string) string {
|
||||
// pk_<b64org>.<b64sig> — replace the b64org segment.
|
||||
dot := -1
|
||||
for i := 3; i < len(key); i++ {
|
||||
if key[i] == '.' {
|
||||
dot = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if dot < 0 {
|
||||
return key
|
||||
}
|
||||
enc := base64Raw(newOrg)
|
||||
return "pk_" + enc + key[dot:]
|
||||
}
|
||||
|
||||
func base64Raw(s string) string {
|
||||
const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
|
||||
src := []byte(s)
|
||||
var out []byte
|
||||
for i := 0; i < len(src); i += 3 {
|
||||
var b [3]byte
|
||||
n := copy(b[:], src[i:])
|
||||
out = append(out, alphabet[b[0]>>2])
|
||||
out = append(out, alphabet[(b[0]&0x03)<<4|b[1]>>4])
|
||||
if n > 1 {
|
||||
out = append(out, alphabet[(b[1]&0x0f)<<2|b[2]>>6])
|
||||
}
|
||||
if n > 2 {
|
||||
out = append(out, alphabet[b[2]&0x3f])
|
||||
}
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// foldException lifts a type:'error' event's exception into properties.$exception
|
||||
// and defaults the type, so the write core stores it as event_type='error'.
|
||||
func TestFoldException(t *testing.T) {
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"testing"
|
||||
@@ -27,7 +28,7 @@ import (
|
||||
// server-resolved org. This is exactly the row buildEventsInsert binds.
|
||||
// - AUTH dimension (HTTP layer): each auth context is ADMITTED (503, never 403),
|
||||
// proving the canonical door resolved a tenant for it. Each pure resolver
|
||||
// (verifyPublishableKey→org, resolveKeyOrg→org, the host-forced Site.Org) is
|
||||
// (resolveKeyOrg→org, the host-forced Site.Org) is
|
||||
// unit-proven elsewhere (publishable_test, capture_keyorg_test, hostcarve_test),
|
||||
// so admission + those proofs compose into "lands in the SAME tenant".
|
||||
|
||||
@@ -160,77 +161,95 @@ func assertRowArgsEqualExceptID(t *testing.T, name string, want, got []any) {
|
||||
}
|
||||
}
|
||||
|
||||
// ── pluggable auth on the ONE door: pk_ folded into /v1/event ─────────────────
|
||||
// ── pluggable auth on the ONE door: IAM's pk- folded into /v1/event ──────────
|
||||
|
||||
// TestEvent_PkKeyAdmitted proves the write-only publishable key (pk_…) is a
|
||||
// first-class auth mode ON the canonical door: a pk_ bearer for org acme is ADMITTED
|
||||
// (503, datastore down), so a pk_ caller uses /v1/event directly — no separate
|
||||
// /v1/ingest door required.
|
||||
func TestEvent_PkKeyAdmitted(t *testing.T) {
|
||||
t.Setenv(ingestKeySecretEnv, testSecret)
|
||||
app := mountApp(t)
|
||||
key, ok := mintPublishableKey(testSecret, "acme")
|
||||
if !ok {
|
||||
t.Fatal("mint pk_ failed")
|
||||
// stubKeyOrg points the ONE IAM key seam at a table for the test's duration.
|
||||
// resolveKeyOrg is a var precisely so the seam can be swapped without a network;
|
||||
// these exercise the DOOR, not IAM's resolution.
|
||||
func stubKeyOrg(t *testing.T, table map[string]string) {
|
||||
t.Helper()
|
||||
prev := resolveKeyOrg
|
||||
resolveKeyOrg = func(_ context.Context, key string) (string, bool) {
|
||||
org, ok := table[key]
|
||||
return org, ok
|
||||
}
|
||||
t.Cleanup(func() { resolveKeyOrg = prev })
|
||||
}
|
||||
|
||||
// A pk- is a first-class auth mode ON the canonical door: admitted (503,
|
||||
// datastore down), so a pk- caller uses /v1/event directly.
|
||||
func TestEvent_PkKeyAdmitted(t *testing.T) {
|
||||
stubKeyOrg(t, map[string]string{"pk-acme": "acme"})
|
||||
app := mountApp(t)
|
||||
code := postKeyed(t, app, "/v1/event", "", `{"batch":[{"type":"event","event":"signup_completed"}]}`,
|
||||
map[string]string{"Authorization": "Bearer " + key})
|
||||
map[string]string{"Authorization": "Bearer pk-acme"})
|
||||
if code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("pk_ on /v1/event want 503 (admitted, datastore down), got %d", code)
|
||||
t.Fatalf("pk- on /v1/event want 503 (admitted, datastore down), got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEvent_PkKeyForgedOrgIgnored: the tenant a pk_ writes into is the SIGNED org,
|
||||
// never the body/header claim — a pk_ for acme with a forged X-Org-Id + body org is
|
||||
// still admitted (as acme), proving the key's org wins.
|
||||
// The tenant a pk- writes into is the org IAM resolves it to, never a body or
|
||||
// header claim — the key's org wins over a forged one.
|
||||
func TestEvent_PkKeyForgedOrgIgnored(t *testing.T) {
|
||||
t.Setenv(ingestKeySecretEnv, testSecret)
|
||||
stubKeyOrg(t, map[string]string{"pk-acme": "acme"})
|
||||
app := mountApp(t)
|
||||
key, _ := mintPublishableKey(testSecret, "acme")
|
||||
code := postKeyed(t, app, "/v1/event", "hanzo.ai",
|
||||
`{"batch":[{"type":"pageview"}],"org":"attacker"}`,
|
||||
map[string]string{"Authorization": "Bearer " + key, "X-Org-Id": "attacker"})
|
||||
map[string]string{"Authorization": "Bearer pk-acme", "X-Org-Id": "attacker"})
|
||||
if code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("pk_ door with forged org want 503 (ingested as key org), got %d", code)
|
||||
t.Fatalf("pk- door with forged org want 503 (ingested as key org), got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEvent_BadPkKeyFailsClosed: a malformed/forged pk_ that does not verify, with no
|
||||
// other auth, is refused 403 — the canonical door fails closed (no brand-host escape).
|
||||
// A pk- IAM does not resolve, with no other auth, is refused 403 — fail closed.
|
||||
func TestEvent_BadPkKeyFailsClosed(t *testing.T) {
|
||||
t.Setenv(ingestKeySecretEnv, testSecret)
|
||||
stubKeyOrg(t, map[string]string{})
|
||||
app := mountApp(t)
|
||||
code := postKeyed(t, app, "/v1/event", "hanzo.ai", `{"batch":[{"type":"pageview"}]}`,
|
||||
map[string]string{"Authorization": "Bearer pk_deadbeef.deadbeef"})
|
||||
map[string]string{"Authorization": "Bearer pk-deadbeef"})
|
||||
if code != http.StatusForbidden {
|
||||
t.Fatalf("unverifiable pk_ on /v1/event want 403 (fail closed), got %d", code)
|
||||
t.Fatalf("unresolvable pk- on /v1/event want 403 (fail closed), got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// ── /v1/ingest is now a THIN ALIAS of the ONE handler ─────────────────────────
|
||||
|
||||
// TestIngestAlias_DelegatesToEventHandler proves /v1/ingest is the SAME
|
||||
// implementation as /v1/event: a pk_ caller is admitted (unchanged), AND — because it
|
||||
// delegates to eventHandle — it now ALSO admits an IAM bearer, while no-auth still
|
||||
// fails closed. One implementation, reached through two routes.
|
||||
// implementation as /v1/event: a pk- caller is admitted (unchanged), AND — because it
|
||||
// delegates to eventHandle — it ALSO admits an IAM bearer and takes the SAME anonymous
|
||||
// lane, under the same public tenant and the same allowlist. One implementation reached
|
||||
// through two routes: the alias cannot drift from the canonical door, which is the whole
|
||||
// point of it being an alias.
|
||||
func TestIngestAlias_DelegatesToEventHandler(t *testing.T) {
|
||||
t.Setenv(ingestKeySecretEnv, testSecret)
|
||||
stubKeyOrg(t, map[string]string{"pk-acme": "acme"})
|
||||
app := mountApp(t)
|
||||
key, _ := mintPublishableKey(testSecret, "acme")
|
||||
|
||||
// pk_ — the historical /v1/ingest auth — still works.
|
||||
// pk- — the historical /v1/ingest auth — still works.
|
||||
if code := postKeyed(t, app, "/v1/ingest", "", `{"batch":[{"type":"pageview"}]}`,
|
||||
map[string]string{"Authorization": "Bearer " + key}); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("pk_ on /v1/ingest want 503 (admitted), got %d", code)
|
||||
map[string]string{"Authorization": "Bearer pk-acme"}); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("pk- on /v1/ingest want 503 (admitted), got %d", code)
|
||||
}
|
||||
// IAM bearer — admitted too, because the alias IS eventHandle now.
|
||||
if code, _ := doBody(t, app, http.MethodPost, "/v1/ingest", "user-dave", "acme",
|
||||
`[{"event":"signup_completed","distinctId":"u1"}]`); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("bearer on /v1/ingest want 503 (admitted via eventHandle), got %d", code)
|
||||
}
|
||||
// No auth — fail closed, exactly like the canonical door.
|
||||
if code, _ := doBody(t, app, http.MethodPost, "/v1/ingest", "", "",
|
||||
`{"batch":[{"type":"pageview"}]}`); code != http.StatusForbidden {
|
||||
t.Fatalf("no-auth /v1/ingest want 403 (fail closed), got %d", code)
|
||||
// No auth at all — the anonymous lane, exactly like the canonical door: a pageview
|
||||
// is admitted (503, datastore down) under the public tenant.
|
||||
if code, body := doBody(t, app, http.MethodPost, "/v1/ingest", "", "",
|
||||
`{"batch":[{"type":"pageview"}]}`); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("no-auth /v1/ingest want 503 (anonymous lane, admitted), got %d (%s)", code, body)
|
||||
}
|
||||
// A non-allowlisted kind is dropped anonymously here too, not stored.
|
||||
if code, body := doBody(t, app, http.MethodPost, "/v1/ingest", "", "",
|
||||
`{"batch":[{"type":"event","event":"order_completed"}]}`); code != http.StatusOK {
|
||||
t.Fatalf("no-auth /v1/ingest custom kind want 200 all-dropped, got %d (%s)", code, body)
|
||||
}
|
||||
// A presented pk- that does NOT resolve still fails closed — never downgraded into
|
||||
// the anonymous lane, so a misconfigured key surfaces instead of silently filing
|
||||
// its events where its owner cannot read them.
|
||||
if code := postKeyed(t, app, "/v1/ingest", "", `{"batch":[{"type":"pageview"}]}`,
|
||||
map[string]string{"Authorization": "Bearer pk-nosuch"}); code != http.StatusForbidden {
|
||||
t.Fatalf("unresolvable pk- on /v1/ingest want 403 (fail closed), got %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
// Package ask is the UNIFIED GROUNDED ADVISOR: POST /v1/ask. A founder asks a plain-language
|
||||
// question ("what's my MRR?", "how long is my runway?") and gets an answer whose every figure is
|
||||
// a REAL value read from a domain endpoint in-process — never a number the model invented.
|
||||
//
|
||||
// ONE AND ONE WAY. /v1/ask is DISTINCT from /v1/chat/completions (the ai subsystem's RAW model
|
||||
// completions) and from /v1/agent (the tool-calling orchestrator). Raw model → /v1/chat/completions;
|
||||
// grounded advisor → /v1/ask. The advisor routes a question to the domain(s) that can ground it,
|
||||
// reads the REAL figures from each domain's own endpoint in-process, hands the model the EXACT
|
||||
// figures, and returns the grounded answer + the figures + the domain reads that backed them.
|
||||
//
|
||||
// THE FLOW.
|
||||
//
|
||||
// question → registry.Match (which domain grounds this?) → Contributor.Gather (replay the
|
||||
// domain's grounded READ in-process, under the caller's OWN creds) → the REAL facts
|
||||
// → narrate the facts with the model (prose only, never a number) → {answer, figures,
|
||||
// followups, sources, domain}
|
||||
//
|
||||
// GROUNDING CONTRACT (non-negotiable). Every figure in the answer is a real domain read; the
|
||||
// model only NARRATES the figures it is handed and can never override one — the figures array is
|
||||
// the Contributor's, computed BEFORE any model call and returned unaltered. If no domain can
|
||||
// ground the question, the advisor says so honestly rather than guessing. Per-tenant isolation is
|
||||
// inherited from the in-process replay carrying the caller's creds (agent.go's pattern): a
|
||||
// question can only ever surface the caller's own org's data.
|
||||
package ask
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/clients/principal"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// state is the advisor's own data: the contributor registry (the plug-in domains) and the
|
||||
// NARRATION-ONLY model seam. ai + model rephrase the grounded facts more naturally; they NEVER
|
||||
// source a figure. nil ai ⇒ the advisor returns its deterministic templated answer over the same
|
||||
// real figures — the numbers are identical whether the model plane is up or down.
|
||||
type state struct {
|
||||
registry *Registry
|
||||
ai cloud.AIClient
|
||||
model string
|
||||
}
|
||||
|
||||
// maxQuestion bounds the request body.
|
||||
const maxQuestion = 2000
|
||||
|
||||
// AskRequest is the POST /v1/ask body. The advisor path uses question (figures
|
||||
// grounding). The WEB grounding domain is selected by mode (search|news|research|
|
||||
// deep) and parameterized by the remaining fields; q is the answer-engine alias for
|
||||
// question. All web fields are optional and inert unless mode names a web domain.
|
||||
type AskRequest struct {
|
||||
Question string `json:"question"`
|
||||
Q string `json:"q"` // answer-engine alias for question
|
||||
|
||||
// Web grounding domain (mode-selected). Empty mode ⇒ the figure advisor, unchanged.
|
||||
Mode string `json:"mode"` // search|news|research|deep
|
||||
Sources []string `json:"sources"` // @hints appended to the web query: web,news,academic,github,reddit,x
|
||||
Model string `json:"model"` // override the narration/synthesis model
|
||||
Stream *bool `json:"stream"` // force SSE (else Accept: text/event-stream / ?stream=1)
|
||||
Language string `json:"language"` // web-search language (BCP-47-ish)
|
||||
MaxSources int `json:"maxSources"`
|
||||
MaxQueries int `json:"maxQueries"`
|
||||
FollowUps *bool `json:"followUps"` // default true
|
||||
System string `json:"system"` // override the synthesis system prompt
|
||||
}
|
||||
|
||||
// query is the caller's question, accepting either the advisor field (question) or
|
||||
// the answer-engine alias (q). Trimmed by the handler.
|
||||
func (r AskRequest) query() string {
|
||||
if q := strings.TrimSpace(r.Q); q != "" {
|
||||
return q
|
||||
}
|
||||
return r.Question
|
||||
}
|
||||
|
||||
// AskResponse is the /v1/ask contract: a natural-language answer grounded in Figures, the
|
||||
// followups worth asking next, the domain reads (Sources) the figures came from, and the Domain
|
||||
// that grounded the question ("" when none could). Every Figure is a real value; the Answer
|
||||
// narrates them.
|
||||
type AskResponse struct {
|
||||
Answer string `json:"answer"`
|
||||
Figures []Fact `json:"figures"`
|
||||
Followups []string `json:"followups"`
|
||||
Sources []string `json:"sources"`
|
||||
Domain string `json:"domain"`
|
||||
}
|
||||
|
||||
// Mount wires POST /v1/ask into cloud, building the contributor registry (books today) over the
|
||||
// SAME app so a contributor's Gather replays a domain's grounded read in-process. The narration
|
||||
// model comes from deps.AI. Mount is a distinct route, so it wins Fiber's first-match over the
|
||||
// ai /v1/* catch-all.
|
||||
func Mount(app *zip.App, deps cloud.Deps) error {
|
||||
b := cloud.NewBase(deps, "ask")
|
||||
svc := &cloud.Service[*state]{Base: b, State: &state{
|
||||
registry: NewRegistry(newBooksContributor(app)),
|
||||
ai: deps.AI,
|
||||
model: strings.TrimSpace(deps.AIDefaultModel),
|
||||
}}
|
||||
app.Post("/v1/ask", cloud.Handle(svc, askHandler))
|
||||
b.Log.Info("ask mounted", "prefix", "/v1/ask", "domains", "books,web", "web_modes", "search,news,research,deep")
|
||||
return nil
|
||||
}
|
||||
|
||||
// askHandler answers POST /v1/ask for the caller's OWN org. It gates the caller (a validated
|
||||
// principal is required — the SAME gate every data plane uses), classifies the question to a
|
||||
// grounded domain, gathers the REAL figures in-process under the caller's creds, and narrates
|
||||
// them. The figures and sources are the domain's, resolved BEFORE any model call and never
|
||||
// altered by it.
|
||||
func askHandler(s *cloud.Service[*state], c *zip.Ctx) error {
|
||||
if _, ok := principal.Org(c); !ok {
|
||||
return zip.ErrUnauthorized("sign in to ask")
|
||||
}
|
||||
var in AskRequest
|
||||
if err := c.Bind(&in); err != nil {
|
||||
return err
|
||||
}
|
||||
q := strings.TrimSpace(in.query())
|
||||
if q == "" {
|
||||
return zip.Errorf(http.StatusBadRequest, "question is required")
|
||||
}
|
||||
if len(q) > maxQuestion {
|
||||
q = q[:maxQuestion]
|
||||
}
|
||||
|
||||
// WEB grounding domain — selected explicitly by mode (search|news|research|deep). This is the
|
||||
// agentic search/deep-research path: it grounds on live web sources (not ledger figures),
|
||||
// streams the answer-engine envelope, and meters the caller. It is ADDITIVE — when no web mode
|
||||
// is set the advisor's figure path below runs exactly as before.
|
||||
if isWebMode(in.Mode) {
|
||||
return serveWeb(s, c, in, q)
|
||||
}
|
||||
|
||||
// Classify → the domain that can ground this question. No match ⇒ the honest fallback: the
|
||||
// advisor names what it CAN answer rather than fabricating a figure.
|
||||
domain := s.State.registry.Match(q)
|
||||
if domain == nil {
|
||||
return askJSON(c, honestFallback())
|
||||
}
|
||||
|
||||
// Gather the REAL figures in-process, under the caller's OWN credentials (so the read is
|
||||
// scoped to the caller's org and no other). A gather error degrades to the honest fallback —
|
||||
// never a guessed number.
|
||||
facts, sources, err := domain.Gather(c.Context(), credential(c))
|
||||
if err != nil {
|
||||
s.Log.Warn("ask gather failed", "domain", domain.Name(), "err", err)
|
||||
return askJSON(c, honestFallback())
|
||||
}
|
||||
|
||||
resp := AskResponse{
|
||||
Figures: facts,
|
||||
Followups: followups(domain.Name()),
|
||||
Sources: sources,
|
||||
Domain: domain.Name(),
|
||||
}
|
||||
// The answer over the grounded facts: the model narrates when wired, else a deterministic
|
||||
// template. Either way the prose restates figures the model was HANDED — it never sources one,
|
||||
// and the Figures array above is authoritative regardless of what the prose says.
|
||||
resp.Answer = narrate(s, c, q, facts)
|
||||
return askJSON(c, resp)
|
||||
}
|
||||
|
||||
// narrate produces the answer sentence over the EXACT grounded facts. It runs ONE completion that
|
||||
// rephrases the figures naturally, billed to the caller's HOME org and scoped to the caller's own
|
||||
// org. The prompt hands the model every figure and forbids changing a number — the model writes
|
||||
// prose only. It degrades to the deterministic template when no model is wired or the call fails,
|
||||
// so the answer always states the real figures. The Figures array is NOT touched here: a model
|
||||
// that hallucinates a number in its prose cannot override the grounded figure the caller receives.
|
||||
func narrate(s *cloud.Service[*state], c *zip.Ctx, question string, facts []Fact) string {
|
||||
tmpl := templateAnswer(facts)
|
||||
if s.State.ai == nil {
|
||||
return tmpl
|
||||
}
|
||||
org, _ := principal.Org(c) // already gated non-empty at the handler entry
|
||||
res, err := s.State.ai.ChatCompletion(c.Context(), &cloud.ChatRequest{
|
||||
Model: s.State.model,
|
||||
Prompt: narratePrompt(question, facts, tmpl),
|
||||
Org: org,
|
||||
BillingOrg: principal.Ledger(c),
|
||||
})
|
||||
if err != nil || res == nil || strings.TrimSpace(res.Content) == "" {
|
||||
return tmpl
|
||||
}
|
||||
return strings.TrimSpace(res.Content)
|
||||
}
|
||||
|
||||
// narratePrompt is the grounded narration prompt: it hands the model EVERY figure verbatim and
|
||||
// forbids inventing, rounding, or altering a number. The figures are listed so a test can assert
|
||||
// the exact grounded value is what the model was fed.
|
||||
func narratePrompt(question string, facts []Fact, draft string) string {
|
||||
var fb strings.Builder
|
||||
for _, f := range facts {
|
||||
fb.WriteString("- " + f.Label + ": " + f.Value)
|
||||
if f.Period != "" {
|
||||
fb.WriteString(" (" + f.Period + ")")
|
||||
}
|
||||
fb.WriteString("\n")
|
||||
}
|
||||
return "You are a precise business advisor. Answer the founder's question in ONE or TWO natural sentences.\n" +
|
||||
"You MUST use these figures EXACTLY as given — never invent, round, or alter a number:\n" +
|
||||
fb.String() +
|
||||
"\nQuestion: " + question +
|
||||
"\nGrounded draft (rephrase naturally, keep every figure identical): " + draft +
|
||||
"\nReturn only the answer."
|
||||
}
|
||||
|
||||
// templateAnswer is the deterministic sentence over the grounded facts — the answer when no model
|
||||
// is wired, and the draft the model rephrases. It states the real figures directly, so the advisor
|
||||
// is correct with or without the model plane.
|
||||
func templateAnswer(facts []Fact) string {
|
||||
if len(facts) == 0 {
|
||||
return "There are no figures to report for this period yet."
|
||||
}
|
||||
parts := make([]string, 0, len(facts))
|
||||
for _, f := range facts {
|
||||
parts = append(parts, f.Label+" "+f.Value)
|
||||
}
|
||||
period := facts[0].Period
|
||||
lead := "Here are your latest figures"
|
||||
if period != "" {
|
||||
lead += " for " + period
|
||||
}
|
||||
return lead + ": " + strings.Join(parts, ", ") + "."
|
||||
}
|
||||
|
||||
// honestFallback is the answer when NO domain can ground the question. It names what the advisor
|
||||
// CAN answer and offers grounded questions to ask instead — and carries ZERO figures, because a
|
||||
// figure the advisor cannot ground is a figure it must not state.
|
||||
func honestFallback() AskResponse {
|
||||
return AskResponse{
|
||||
Answer: "I can answer questions about your finances today — MRR, revenue, burn, runway, margin, cash, and P&L. Infra and usage advisors are coming.",
|
||||
Figures: []Fact{},
|
||||
Followups: []string{"What's my MRR?", "How long is my runway?", "What is my gross margin?"},
|
||||
Sources: []string{},
|
||||
Domain: "",
|
||||
}
|
||||
}
|
||||
|
||||
// followups returns sharp next questions for a domain — deterministic, so the advisor always
|
||||
// offers a path forward. Extended per domain as new contributors join.
|
||||
func followups(domain string) []string {
|
||||
switch domain {
|
||||
case "books":
|
||||
return []string{"How long is my runway?", "What is my gross margin?", "How much of revenue is recurring?"}
|
||||
default:
|
||||
return []string{"What's my MRR?", "How long is my runway?"}
|
||||
}
|
||||
}
|
||||
|
||||
// credential extracts the caller's replayable credential + already-validated identity headers, so
|
||||
// a contributor's in-process replay runs as the CALLER — scoped to the caller's own org. It
|
||||
// forwards both the bearer/session creds (which the identity middleware re-mints identity from on
|
||||
// the replayed request) AND the minted identity headers (X-Org-Id / X-User-Id / …), which are
|
||||
// already validated at this handler's own principal gate: in production either path yields the
|
||||
// caller's own validated identity, and in a middleware-free test the identity headers are what
|
||||
// scope the read. It is the SAME replayable set agent.go carries, plus the identity headers a
|
||||
// grounded READ resolves its org from.
|
||||
func credential(c *zip.Ctx) map[string]string {
|
||||
cred := map[string]string{}
|
||||
for _, h := range []string{
|
||||
"Authorization", "X-Authorization", "Cookie", "Accept-Language", "X-Forwarded-For",
|
||||
"X-Org-Id", "X-User-Id", "X-User-Owner", "X-User-IsOrgAdmin", "X-Project-Id",
|
||||
"X-Billing-Account-Id", "X-Hanzo-Test",
|
||||
} {
|
||||
if v := c.Header(h); v != "" {
|
||||
cred[h] = v
|
||||
}
|
||||
}
|
||||
return cred
|
||||
}
|
||||
|
||||
// askJSON writes an advisor payload with no-store (per-org figures must never be cached).
|
||||
func askJSON(c *zip.Ctx, v any) error {
|
||||
c.SetHeader("Cache-Control", "no-store")
|
||||
return c.JSON(http.StatusOK, v)
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
package ask
|
||||
|
||||
// ask_test.go — proofs for the unified grounded advisor. The whole point is GROUNDING: every
|
||||
// figure the advisor states is a REAL value read from a domain endpoint in-process; the model
|
||||
// only narrates the figures it is handed and can NEVER override one. These tests stand up a fake
|
||||
// in-process app whose /v1/books/metrics returns known figures scoped to the org the replay
|
||||
// carries, plus a recording fakeAI, and assert:
|
||||
//
|
||||
// 1. a financial question returns the REAL figure the books read produced, cited in sources;
|
||||
// 2. the model is fed the EXACT figure (the prompt contains it) and a hallucinated number in the
|
||||
// model's reply does NOT override the grounded figure the caller receives;
|
||||
// 3. a non-groundable question returns the honest fallback with ZERO fabricated figures;
|
||||
// 4. org isolation — the replay carries the CALLER's org, so a books read can only ever surface
|
||||
// the caller's own org's data, never another's.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/clients/principal"
|
||||
"github.com/hanzoai/cloud/types"
|
||||
luxlog "github.com/luxfi/log"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// recordingAI is the narration double: it records the LAST prompt it was handed (so a test can
|
||||
// assert the model was fed the exact grounded figure) and replies with a fixed string that may
|
||||
// contain a HALLUCINATED number — the grounded figures array must survive it unchanged.
|
||||
type recordingAI struct {
|
||||
reply string
|
||||
lastPrompt string
|
||||
}
|
||||
|
||||
func (r *recordingAI) ChatCompletion(_ context.Context, req *types.ChatRequest) (*types.ChatResponse, error) {
|
||||
r.lastPrompt = req.Prompt
|
||||
return &types.ChatResponse{Content: r.reply}, nil
|
||||
}
|
||||
|
||||
func (r *recordingAI) Embed(_ context.Context, _ *types.EmbedRequest) ([][]float32, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// fakeBooks mounts a stand-in GET /v1/books/metrics that returns figures scoped to the org it
|
||||
// SEES on the request — the same principal.Org gate the real books read uses. A figure is tagged
|
||||
// with the org, so a test can prove the advisor only ever surfaces the caller's own org's data.
|
||||
// This is the grounded read the books contributor replays in-process.
|
||||
func fakeBooks(app *zip.App, mrrByOrg map[string]string) {
|
||||
app.Get("/v1/books/metrics", func(c *zip.Ctx) error {
|
||||
org, ok := principal.Org(c)
|
||||
if !ok {
|
||||
return zip.ErrUnauthorized("sign in")
|
||||
}
|
||||
mrr, seen := mrrByOrg[org]
|
||||
if !seen {
|
||||
mrr = "$0"
|
||||
}
|
||||
return c.JSON(http.StatusOK, map[string]any{
|
||||
"figures": []Fact{
|
||||
{Label: "MRR", Value: mrr, Period: "2026-07"},
|
||||
{Label: "org-echo", Value: org, Period: "2026-07"},
|
||||
},
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// newAskApp stands up an app with the fake books read + the ask advisor over a recording AI.
|
||||
func newAskApp(t *testing.T, ai types.AIClient, mrrByOrg map[string]string) *zip.App {
|
||||
t.Helper()
|
||||
app := zip.New(zip.Config{Logger: luxlog.New("test"), DisableStartupMessage: true})
|
||||
fakeBooks(app, mrrByOrg)
|
||||
if err := Mount(app, cloud.Deps{Logger: luxlog.New("test"), DataDir: t.TempDir(), AI: ai}); err != nil {
|
||||
t.Fatalf("Mount: %v", err)
|
||||
}
|
||||
return app
|
||||
}
|
||||
|
||||
// ask POSTs a question as a VALIDATED principal for org (X-User-Id set, exactly as the gateway
|
||||
// mints it — the test app has no sanitizer). Empty org exercises the anonymous 403 path.
|
||||
func ask(t *testing.T, app *zip.App, org, question string) (int, AskResponse) {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(AskRequest{Question: question})
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/ask", strings.NewReader(string(body)))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if org != "" {
|
||||
req.Header.Set("X-Org-Id", org)
|
||||
req.Header.Set("X-User-Id", "u-"+org)
|
||||
}
|
||||
resp, err := app.Fiber().Test(req)
|
||||
if err != nil {
|
||||
t.Fatalf("ask %q: %v", question, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
var out AskResponse
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
if err := json.Unmarshal(b, &out); err != nil {
|
||||
t.Fatalf("decode: %v (%s)", err, b)
|
||||
}
|
||||
}
|
||||
return resp.StatusCode, out
|
||||
}
|
||||
|
||||
func figure(r AskResponse, label string) (string, bool) {
|
||||
for _, f := range r.Figures {
|
||||
if f.Label == label {
|
||||
return f.Value, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func hasSource(r AskResponse, src string) bool {
|
||||
for _, s := range r.Sources {
|
||||
if s == src {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// TestGroundedFinancialAnswer: a financial question returns the REAL figure the books read
|
||||
// produced, tagged to the books domain and cited in sources.
|
||||
func TestGroundedFinancialAnswer(t *testing.T) {
|
||||
ai := &recordingAI{reply: "Your MRR is $4,200 for July."}
|
||||
app := newAskApp(t, ai, map[string]string{"acme": "$4,200"})
|
||||
|
||||
code, r := ask(t, app, "acme", "what is my MRR right now?")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("want 200, got %d", code)
|
||||
}
|
||||
if r.Domain != "books" {
|
||||
t.Fatalf("financial question must route to the books domain, got %q", r.Domain)
|
||||
}
|
||||
if v, ok := figure(r, "MRR"); !ok || v != "$4,200" {
|
||||
t.Fatalf("MRR figure must be the real $4,200 from the books read, got %q (ok=%v)", v, ok)
|
||||
}
|
||||
if !hasSource(r, "books/metrics") {
|
||||
t.Fatalf("answer must cite the books/metrics read, got %v", r.Sources)
|
||||
}
|
||||
}
|
||||
|
||||
// TestModelFedExactFigureAndCannotOverride is THE grounding proof: the model is handed the EXACT
|
||||
// figure in its prompt, and even when it replies with a HALLUCINATED number the grounded figure
|
||||
// the caller receives is unchanged. The prose may carry the model's words; the figures array is
|
||||
// the ledger's, never the model's.
|
||||
func TestModelFedExactFigureAndCannotOverride(t *testing.T) {
|
||||
// The model hallucinates $9,999 in its narration — a number that is NOT the real figure.
|
||||
ai := &recordingAI{reply: "Your MRR is a whopping $9,999 this month!"}
|
||||
app := newAskApp(t, ai, map[string]string{"acme": "$4,200"})
|
||||
|
||||
_, r := ask(t, app, "acme", "how's my recurring revenue?")
|
||||
|
||||
// (a) the model was fed the EXACT grounded figure.
|
||||
if !strings.Contains(ai.lastPrompt, "$4,200") {
|
||||
t.Fatalf("narration prompt must contain the real figure $4,200, got:\n%s", ai.lastPrompt)
|
||||
}
|
||||
// (b) the grounded figure the caller receives is the REAL one — the hallucination did not
|
||||
// override it. figures[] comes from the domain read, never from the model's reply.
|
||||
if v, _ := figure(r, "MRR"); v != "$4,200" {
|
||||
t.Fatalf("grounded MRR figure must stay $4,200 despite the model's $9,999, got %q", v)
|
||||
}
|
||||
if v, _ := figure(r, "MRR"); strings.Contains(v, "9,999") {
|
||||
t.Fatalf("the hallucinated $9,999 must NEVER become a grounded figure, got %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNoModelStillGrounded: with no AI wired the advisor still answers with the REAL figures — the
|
||||
// deterministic template states them, so the numbers are identical whether the model is up or down.
|
||||
func TestNoModelStillGrounded(t *testing.T) {
|
||||
app := newAskApp(t, nil, map[string]string{"acme": "$4,200"})
|
||||
_, r := ask(t, app, "acme", "what's my mrr?")
|
||||
if v, _ := figure(r, "MRR"); v != "$4,200" {
|
||||
t.Fatalf("figure must be the real $4,200 with no model, got %q", v)
|
||||
}
|
||||
if !strings.Contains(r.Answer, "$4,200") {
|
||||
t.Fatalf("templated answer must state the real figure, got %q", r.Answer)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHonestFallbackNoFabrication: a question no domain can ground returns the honest fallback —
|
||||
// it names what the advisor CAN answer and carries ZERO figures. It must NEVER invent a number.
|
||||
func TestHonestFallbackNoFabrication(t *testing.T) {
|
||||
ai := &recordingAI{reply: "42 widgets shipped."} // the model would happily make something up
|
||||
app := newAskApp(t, ai, map[string]string{"acme": "$4,200"})
|
||||
|
||||
code, r := ask(t, app, "acme", "how many widgets did we ship to Mars?")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("want 200, got %d", code)
|
||||
}
|
||||
if r.Domain != "" {
|
||||
t.Fatalf("an ungroundable question must have no domain, got %q", r.Domain)
|
||||
}
|
||||
if len(r.Figures) != 0 {
|
||||
t.Fatalf("the fallback must carry ZERO figures, got %+v", r.Figures)
|
||||
}
|
||||
if len(r.Sources) != 0 {
|
||||
t.Fatalf("the fallback must cite no sources, got %v", r.Sources)
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(r.Answer), "finance") {
|
||||
t.Fatalf("the fallback must name what it CAN answer, got %q", r.Answer)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOrgIsolation proves the in-process replay carries the CALLER's org: acme sees acme's figure,
|
||||
// beta sees beta's, and neither can ever surface the other's data — tenant isolation is inherited
|
||||
// from the caller's own creds on the replay, never re-implemented.
|
||||
func TestOrgIsolation(t *testing.T) {
|
||||
app := newAskApp(t, nil, map[string]string{"acme": "$4,200", "beta": "$77,000"})
|
||||
|
||||
_, a := ask(t, app, "acme", "what's my mrr?")
|
||||
if v, _ := figure(a, "MRR"); v != "$4,200" {
|
||||
t.Fatalf("acme must see its own $4,200, got %q", v)
|
||||
}
|
||||
if echo, _ := figure(a, "org-echo"); echo != "acme" {
|
||||
t.Fatalf("the books read must have been scoped to acme, got org-echo=%q", echo)
|
||||
}
|
||||
|
||||
_, b := ask(t, app, "beta", "what's my mrr?")
|
||||
if v, _ := figure(b, "MRR"); v != "$77,000" {
|
||||
t.Fatalf("beta must see its own $77,000, got %q", v)
|
||||
}
|
||||
if v, _ := figure(b, "MRR"); v == "$4,200" {
|
||||
t.Fatalf("beta must NEVER surface acme's $4,200")
|
||||
}
|
||||
if echo, _ := figure(b, "org-echo"); echo != "beta" {
|
||||
t.Fatalf("the books read must have been scoped to beta, got org-echo=%q", echo)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnonymousRefused: /v1/ask is a data plane — a request with no validated principal is 401,
|
||||
// so an off-gateway forge can neither probe nor read a ledger through the advisor.
|
||||
func TestAnonymousRefused(t *testing.T) {
|
||||
app := newAskApp(t, nil, map[string]string{"acme": "$4,200"})
|
||||
// Forged X-Org-Id with NO X-User-Id (no validated principal) — the anonymous forge.
|
||||
body, _ := json.Marshal(AskRequest{Question: "what's my mrr?"})
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/ask", strings.NewReader(string(body)))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-Org-Id", "acme")
|
||||
resp, err := app.Fiber().Test(req)
|
||||
if err != nil {
|
||||
t.Fatalf("test: %v", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusUnauthorized {
|
||||
t.Fatalf("forged org with no principal must be 401, got %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// TestClassifierMatchesFinancialVocab locks the books classifier: the founder-vocabulary that
|
||||
// grounds against the ledger routes to books, and off-topic questions do not.
|
||||
func TestClassifierMatchesFinancialVocab(t *testing.T) {
|
||||
reg := NewRegistry(newBooksContributor(nil))
|
||||
for _, q := range []string{"what's my MRR?", "how long is my runway", "are we profitable?", "how much cash do we have", "what's my gross margin", "show me the P&L", "how much did we make"} {
|
||||
if reg.Match(q) == nil {
|
||||
t.Fatalf("financial question %q must match the books contributor", q)
|
||||
}
|
||||
}
|
||||
for _, q := range []string{"what's the weather", "how many users signed up", "deploy the app"} {
|
||||
if c := reg.Match(q); c != nil {
|
||||
t.Fatalf("off-topic question %q must NOT match any domain, matched %q", q, c.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package ask
|
||||
|
||||
// books.go — the FIRST contributor: BOOKS (financial). It grounds MRR/ARR/revenue/burn/runway/
|
||||
// margin/cash/deferred-revenue/P&L/balance questions by replaying the books domain's OWN
|
||||
// grounded read — GET /v1/books/metrics — in-process under the caller's own credentials, then
|
||||
// surfacing the REAL figures it returns. It NEVER recomputes: books stays the single source of
|
||||
// the numbers AND their formatting (the endpoint returns formatUSD-formatted figures). Per-tenant
|
||||
// isolation is inherited from the replay carrying the caller's own X-Org-Id — a books read can
|
||||
// only ever return the caller's own org's ledger.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
|
||||
fiber "github.com/zap-proto/fiber/v3"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// booksMetricsPath is the books domain's grounded read the contributor replays. It is the ONE
|
||||
// endpoint that returns the deterministic metric snapshot as formatted figures.
|
||||
const booksMetricsPath = "/v1/books/metrics"
|
||||
|
||||
// maxMetricsResponse bounds the in-process read so a broken upstream cannot balloon memory.
|
||||
const maxMetricsResponse = 1 << 20
|
||||
|
||||
// booksContributor grounds financial questions against the books domain. It holds the app it
|
||||
// replays against (the SAME binary; the read flows the normal middleware chain), exactly as
|
||||
// agent.go's aiCompleter holds the app to replay /v1/chat/completions.
|
||||
type booksContributor struct{ app *zip.App }
|
||||
|
||||
func newBooksContributor(app *zip.App) *booksContributor { return &booksContributor{app: app} }
|
||||
|
||||
func (booksContributor) Name() string { return "books" }
|
||||
|
||||
// CanAnswer is the books classifier: does the question ask about money the ledger knows? Keyword
|
||||
// match over the founder-vocabulary the metrics engine grounds. First-match in the registry, so
|
||||
// a books question routes here deterministically. An LLM classifier can replace this body later
|
||||
// without touching the router or the seam.
|
||||
func (booksContributor) CanAnswer(question string) bool {
|
||||
l := strings.ToLower(question)
|
||||
for _, kw := range booksKeywords {
|
||||
if strings.Contains(l, kw) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// booksKeywords is the financial vocabulary that routes a question to the books ledger. It
|
||||
// mirrors the books intent router's keywords so /v1/ask grounds exactly what /v1/books/ask does.
|
||||
var booksKeywords = []string{
|
||||
"mrr", "arr", "recurring", "subscription", "annualized",
|
||||
"revenue", "sales", "top line", "income", "how much did we make", "how much money",
|
||||
"burn", "spend", "spending", "expense", "expenses", "opex", "costs", "cost of",
|
||||
"runway", "how long", "cash last", "out of money", "out of cash",
|
||||
"margin", "profitab", "profit", "net income", "bottom line", "earnings", "break even", "break-even",
|
||||
"cash", "bank", "in the bank", "balance", "cogs",
|
||||
"deferred", "wallet", "prepaid", "liabilit", "owe",
|
||||
"p&l", "pnl", "p and l", "financ",
|
||||
}
|
||||
|
||||
// Gather replays GET /v1/books/metrics in-process under the caller's own credentials and returns
|
||||
// the REAL figures it read, with the domain read that backed them. The figures are books' own —
|
||||
// this contributor formats nothing and invents nothing; an empty ledger yields honest zero
|
||||
// figures. The caller's creds carry the caller's org, so the read is scoped to that org and no
|
||||
// other — tenant isolation is inherited, never re-implemented here.
|
||||
func (b booksContributor) Gather(ctx context.Context, cred map[string]string) ([]Fact, []string, error) {
|
||||
req := httptest.NewRequest(http.MethodGet, booksMetricsPath, nil).WithContext(ctx)
|
||||
for k, v := range cred {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
resp, err := b.app.Fiber().Test(req, fiber.TestConfig{Timeout: 0})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("books metrics replay: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return nil, nil, fmt.Errorf("books metrics status %d", resp.StatusCode)
|
||||
}
|
||||
var out struct {
|
||||
Figures []Fact `json:"figures"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, maxMetricsResponse)).Decode(&out); err != nil {
|
||||
return nil, nil, fmt.Errorf("books metrics decode: %w", err)
|
||||
}
|
||||
return out.Figures, []string{"books/metrics"}, nil
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package ask
|
||||
|
||||
// registry.go — the CONTRIBUTOR seam. /v1/ask is a UNIFIED grounded advisor: a plain-language
|
||||
// question is routed to the domain(s) that can ground it, each domain hands back REAL figures
|
||||
// read in-process, and the model only narrates the figures it is handed. A Contributor is one
|
||||
// such domain (books today; o11y/metrics/billing tomorrow). New domains plug in here WITHOUT
|
||||
// touching the router: they implement Contributor and are appended to the registry at Mount.
|
||||
//
|
||||
// THE GROUNDING CONTRACT. A Contributor never invents a number. Gather replays a domain's own
|
||||
// grounded READ endpoint in-process under the CALLER'S own credentials (agent.go's replay
|
||||
// pattern), so the figures are the domain's ground truth and per-tenant isolation is inherited
|
||||
// — a question can only ever surface the caller's own org's data. If no Contributor CanAnswer,
|
||||
// the advisor says so honestly rather than guessing.
|
||||
|
||||
import "context"
|
||||
|
||||
// Fact is one grounded figure a Contributor read from its domain: a label, its formatted value,
|
||||
// and the period it covers. It is the exact {label,value,period?} shape the /v1/ask answer
|
||||
// surfaces — the model narrates these, it never produces one.
|
||||
type Fact struct {
|
||||
Label string `json:"label"`
|
||||
Value string `json:"value"`
|
||||
Period string `json:"period,omitempty"`
|
||||
}
|
||||
|
||||
// Contributor is one grounded domain behind the advisor. It is the ONE plug-in seam: a domain
|
||||
// declares WHICH questions it can ground (CanAnswer) and HOW it reads the real figures (Gather),
|
||||
// and the router composes it with no domain-specific branch of its own.
|
||||
//
|
||||
// - Name reports the domain id surfaced as the answer's `domain` and used in traces.
|
||||
// - CanAnswer is the lightweight classifier: does this domain ground THIS question? Keyword/
|
||||
// intent today; an LLM classifier can replace the body without changing the seam.
|
||||
// - Gather reads the REAL figures IN-PROCESS under the caller's own creds (the replayable
|
||||
// header set), returning the facts and the domain reads (sources) that backed them. It
|
||||
// never writes and never fabricates: an empty/zero domain yields honest zero figures.
|
||||
type Contributor interface {
|
||||
Name() string
|
||||
CanAnswer(question string) bool
|
||||
Gather(ctx context.Context, cred map[string]string) (facts []Fact, sources []string, err error)
|
||||
}
|
||||
|
||||
// Registry is the ordered set of contributors the router consults. It is populated once at
|
||||
// Mount and read at request time, so adding a domain is a one-line append at the composition
|
||||
// point — never a router edit.
|
||||
type Registry struct {
|
||||
contributors []Contributor
|
||||
}
|
||||
|
||||
// NewRegistry builds the registry from the contributors wired at Mount, in priority order
|
||||
// (first match wins the classification).
|
||||
func NewRegistry(cs ...Contributor) *Registry {
|
||||
return &Registry{contributors: append([]Contributor(nil), cs...)}
|
||||
}
|
||||
|
||||
// Register appends a contributor — the plug-in point a new domain calls to join the advisor
|
||||
// without the router knowing it exists.
|
||||
func (r *Registry) Register(c Contributor) { r.contributors = append(r.contributors, c) }
|
||||
|
||||
// Match returns the FIRST contributor that can ground the question, or nil when none can. It is
|
||||
// the classification hook: deterministic first-match today (books is the sole domain), and the
|
||||
// ONE place an LLM-based classifier or a multi-domain fan-out slots in later — the router only
|
||||
// ever asks the registry "who can answer this?", never how the decision is made.
|
||||
func (r *Registry) Match(question string) Contributor {
|
||||
for _, c := range r.contributors {
|
||||
if c.CanAnswer(question) {
|
||||
return c
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,363 @@
|
||||
package ask
|
||||
|
||||
// relevance.go — the WEB grounding domain's mode registry (one parameterized code
|
||||
// path for search/news/research/deep), its per-answer price policy, web-query
|
||||
// construction, and the SERVER-SIDE source ranking. This is the "web/news/academic/
|
||||
// research" domain of the advisor: unlike a figure domain (books), it grounds on
|
||||
// live web sources rather than ledger figures. The ranking is the relevance fix:
|
||||
// keyless meta-search returns broadly-matched pages, so the loop dedupes by
|
||||
// URL+host and re-orders by query-term overlap before grounding — off-topic hits
|
||||
// (which scored zero, e.g. an unrelated page for "Rich Hickey") sink.
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud/clients/websearch"
|
||||
)
|
||||
|
||||
// webMode is one web-grounding profile. plan toggles multi-query expansion;
|
||||
// maxQueries and maxSources bound the loop; newsBias recency-biases the web query;
|
||||
// system is the synthesis prompt; feeCents is the default per-answer price; models
|
||||
// is the synthesis model fallback chain (primary first) — research/deep lead with a
|
||||
// strong model for a quality report, search/news with a fast one.
|
||||
type webMode struct {
|
||||
name string
|
||||
plan bool
|
||||
maxQueries int
|
||||
maxSources int
|
||||
newsBias bool
|
||||
system string
|
||||
feeCents int64
|
||||
models []string
|
||||
}
|
||||
|
||||
// webModes is the registry. search/news are fast single-pass; research/deep plan
|
||||
// sub-queries over a wider source set and synthesize a report. Deeper modes cost
|
||||
// more (they do more work) — a legitimate product dimension, priced as policy.
|
||||
//
|
||||
// models is the per-mode synthesis chain: research/deep default to zen5 (a capable
|
||||
// Hanzo model that streams FREE on the binary's M2M identity — a strong, always-
|
||||
// reachable report writer), search/news to zen5-flash (the fast tier). zen5-flash
|
||||
// backs research and zen5 backs search, so either mode still answers if its primary
|
||||
// is down; the cloud-wide default is appended as a final backstop in synthModels.
|
||||
var webModes = map[string]webMode{
|
||||
"search": {name: "search", plan: false, maxQueries: 1, maxSources: 6, system: answerSystem, feeCents: 2, models: []string{"zen5-flash", "zen5"}},
|
||||
"news": {name: "news", plan: false, maxQueries: 1, maxSources: 6, newsBias: true, system: answerSystem, feeCents: 2, models: []string{"zen5-flash", "zen5"}},
|
||||
"research": {name: "research", plan: true, maxQueries: 4, maxSources: 12, system: researchSystem, feeCents: 5, models: []string{"zen5", "zen5-flash"}},
|
||||
"deep": {name: "deep", plan: true, maxQueries: 6, maxSources: 16, system: researchSystem, feeCents: 10, models: []string{"zen5", "zen5-flash"}},
|
||||
}
|
||||
|
||||
// isWebMode reports whether a request mode selects the web grounding domain. An
|
||||
// empty/unknown mode is NOT a web request — the advisor's figure path handles it,
|
||||
// so the existing /v1/ask behavior is untouched when no web mode is set.
|
||||
func isWebMode(name string) bool {
|
||||
_, ok := webModes[strings.ToLower(strings.TrimSpace(name))]
|
||||
return ok
|
||||
}
|
||||
|
||||
// resolveWebMode maps a request mode to a registry entry, defaulting to search.
|
||||
// Only called after isWebMode has confirmed a web request.
|
||||
func resolveWebMode(name string) webMode {
|
||||
if m, ok := webModes[strings.ToLower(strings.TrimSpace(name))]; ok {
|
||||
return m
|
||||
}
|
||||
return webModes["search"]
|
||||
}
|
||||
|
||||
const (
|
||||
answerSystem = "You are Hanzo, an AI answer engine. Answer the question directly and accurately, grounded in the numbered web sources provided. " +
|
||||
"Lead with the answer; be concise, factual, and well structured (short paragraphs, bullets where they help). " +
|
||||
"Cite inline as Markdown links [source title](url) immediately after the claim each source supports, and cite generously. " +
|
||||
"Do NOT add a References or Sources section, footnote markers, or bare URLs — citations are inline links only. " +
|
||||
"If the sources conflict or are insufficient, say so plainly and answer from general knowledge while noting the uncertainty. Never fabricate facts or URLs."
|
||||
|
||||
researchSystem = "You are Hanzo Deep Research. Synthesize a thorough, well-organized report answering the question from the numbered web sources. " +
|
||||
"Use clear section headings, compare sources, and surface the strongest evidence. " +
|
||||
"Cite inline as Markdown links [title](url) after each supported claim. " +
|
||||
"Do NOT add a trailing References section or bare URLs. Note gaps or disagreements between sources. Never fabricate facts or URLs."
|
||||
)
|
||||
|
||||
// feeCents resolves the per-answer price in cents for a web mode, most specific
|
||||
// first: CLOUD_ASK_FEE_CENTS_<MODE> → CLOUD_ASK_FEE_CENTS → the mode default. A
|
||||
// value of 0 makes the mode free (and un-gated); a negative/invalid env value is
|
||||
// ignored so a typo can never make a paid mode free. Mirrors ResourceFeeCents.
|
||||
func feeCents(mode string, def int64) int64 {
|
||||
if v, ok := envCents("CLOUD_ASK_FEE_CENTS_" + strings.ToUpper(mode)); ok {
|
||||
return v
|
||||
}
|
||||
if v, ok := envCents("CLOUD_ASK_FEE_CENTS"); ok {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func envCents(key string) (int64, bool) {
|
||||
s := strings.TrimSpace(os.Getenv(key))
|
||||
if s == "" {
|
||||
return 0, false
|
||||
}
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil || n < 0 {
|
||||
return 0, false
|
||||
}
|
||||
return n, true
|
||||
}
|
||||
|
||||
// defaultSynthModel is the resilient synthesis anchor: a capable Hanzo model that
|
||||
// streams FREE on the binary's M2M identity, so it stays reachable even when a paid
|
||||
// or third-party model is throttled, out of balance, or its provider is down. It is
|
||||
// the last-resort backstop if a mode/env/config chain ever resolves to nothing.
|
||||
const defaultSynthModel = "zen5"
|
||||
|
||||
// synthModels resolves the SYNTHESIS model fallback chain for a web request, most
|
||||
// specific first. The loop tries these in order until one returns a real answer, so
|
||||
// a single model's outage advances to the next capable model instead of emitting a
|
||||
// degraded "model unavailable" note:
|
||||
//
|
||||
// 1. caller's explicit model (in.Model) — honored outright, their one choice
|
||||
// 2. per-mode env override CLOUD_ASK_MODEL_<MODE>, else global CLOUD_ASK_MODEL
|
||||
// 3. the mode's capable defaults (research/deep → a strong model, search/news → a
|
||||
// fast one) — every entry a catalog model that streams free on the M2M identity
|
||||
// 4. the cloud-wide default (deps.AIDefaultModel) as a final backstop
|
||||
//
|
||||
// Blanks are dropped and duplicates collapsed (order preserved). An empty result —
|
||||
// impossible in practice, the registry always seeds a mode default — falls back to
|
||||
// defaultSynthModel so synthesis always has at least one model to try.
|
||||
func synthModels(reqModel string, m webMode, def string) []string {
|
||||
if r := strings.TrimSpace(reqModel); r != "" {
|
||||
return []string{r}
|
||||
}
|
||||
chain := make([]string, 0, len(m.models)+2)
|
||||
if env := envModel("CLOUD_ASK_MODEL_"+strings.ToUpper(m.name), "CLOUD_ASK_MODEL"); env != "" {
|
||||
chain = append(chain, env)
|
||||
}
|
||||
chain = append(chain, m.models...)
|
||||
chain = append(chain, def)
|
||||
|
||||
seen := make(map[string]bool, len(chain))
|
||||
out := make([]string, 0, len(chain))
|
||||
for _, s := range chain {
|
||||
if s = strings.TrimSpace(s); s == "" || seen[s] {
|
||||
continue
|
||||
}
|
||||
seen[s] = true
|
||||
out = append(out, s)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return []string{defaultSynthModel}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// envModel returns the first non-empty, trimmed environment value among keys.
|
||||
func envModel(keys ...string) string {
|
||||
for _, k := range keys {
|
||||
if v := strings.TrimSpace(os.Getenv(k)); v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// pickSystem lets a caller override the synthesis prompt; else the mode's default.
|
||||
func pickSystem(req, def string) string {
|
||||
if s := strings.TrimSpace(req); s != "" {
|
||||
return s
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// clampPositive returns def when req<=0, else req bounded ABOVE by def — a caller
|
||||
// may ask for fewer queries/sources but never more than the mode's ceiling, so the
|
||||
// loop's cost stays bounded regardless of input.
|
||||
func clampPositive(req, def int) int {
|
||||
if req <= 0 || req > def {
|
||||
return def
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
// knownSourceHints are the @source tokens appended to the web query (SDK parity).
|
||||
// "web" is the default (no hint). Unknown tokens are dropped so the query is not
|
||||
// polluted by arbitrary caller input.
|
||||
var knownSourceHints = map[string]bool{
|
||||
"news": true, "academic": true, "github": true, "reddit": true, "x": true,
|
||||
}
|
||||
|
||||
// buildWebQuery composes the web-search string: the question, a recency bias for
|
||||
// news mode, and any recognized @source hints appended as tokens.
|
||||
func buildWebQuery(q string, m webMode, sources []string) string {
|
||||
wq := q
|
||||
if m.newsBias {
|
||||
wq = q + " latest news " + strconv.Itoa(time.Now().Year())
|
||||
}
|
||||
var hints []string
|
||||
for _, s := range sources {
|
||||
t := strings.ToLower(strings.TrimSpace(s))
|
||||
if knownSourceHints[t] {
|
||||
hints = append(hints, t)
|
||||
}
|
||||
}
|
||||
if len(hints) > 0 {
|
||||
wq = wq + " " + strings.Join(hints, " ")
|
||||
}
|
||||
return wq
|
||||
}
|
||||
|
||||
// webSource is one web source backing a web-grounded answer — the @hanzo/ai
|
||||
// SearchSource shape. Distinct from Fact (a books-style grounded figure): the web
|
||||
// domain grounds on live sources, not ledger numbers.
|
||||
type webSource struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Snippet string `json:"snippet"`
|
||||
Engine string `json:"engine,omitempty"`
|
||||
Favicon string `json:"favicon"`
|
||||
}
|
||||
|
||||
// rankSources dedupes results (one per URL and one per host, preserving discovery
|
||||
// order on ties) and orders them by relevance to the query, then caps at limit.
|
||||
// Relevance = query-term overlap weighted toward the title plus a whole-phrase
|
||||
// bonus, so a page that actually mentions the subject outranks a broad match.
|
||||
func rankSources(query string, results []websearch.Result, limit int) []webSource {
|
||||
terms := queryTerms(query)
|
||||
phrase := strings.ToLower(strings.TrimSpace(query))
|
||||
|
||||
type scored struct {
|
||||
src webSource
|
||||
score int
|
||||
idx int
|
||||
}
|
||||
seenURL := make(map[string]bool)
|
||||
seenHost := make(map[string]bool)
|
||||
list := make([]scored, 0, len(results))
|
||||
|
||||
for i, r := range results {
|
||||
if r.URL == "" || seenURL[r.URL] {
|
||||
continue
|
||||
}
|
||||
host := hostOf(r.URL)
|
||||
if host == "" || seenHost[host] {
|
||||
continue
|
||||
}
|
||||
seenURL[r.URL] = true
|
||||
seenHost[host] = true
|
||||
list = append(list, scored{
|
||||
src: webSource{
|
||||
URL: r.URL,
|
||||
Title: orHost(r.Title, host),
|
||||
Snippet: clip(r.Content, 600),
|
||||
Engine: r.Engine,
|
||||
Favicon: favicon(host),
|
||||
},
|
||||
score: relevanceScore(terms, phrase, r.Title, r.Content),
|
||||
idx: i,
|
||||
})
|
||||
}
|
||||
|
||||
sort.SliceStable(list, func(a, b int) bool {
|
||||
if list[a].score != list[b].score {
|
||||
return list[a].score > list[b].score // higher relevance first
|
||||
}
|
||||
return list[a].idx < list[b].idx // stable: preserve engine/discovery order on ties
|
||||
})
|
||||
|
||||
out := make([]webSource, 0, limit)
|
||||
for _, s := range list {
|
||||
out = append(out, s.src)
|
||||
if len(out) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// relevanceScore weights a title term hit 3× a content hit, and adds a whole-phrase
|
||||
// bonus (title 5, content 2) so an exact-subject page rises to the top.
|
||||
func relevanceScore(terms []string, phrase, title, content string) int {
|
||||
lt, lc := strings.ToLower(title), strings.ToLower(content)
|
||||
score := 0
|
||||
for _, t := range terms {
|
||||
if strings.Contains(lt, t) {
|
||||
score += 3
|
||||
}
|
||||
if strings.Contains(lc, t) {
|
||||
score++
|
||||
}
|
||||
}
|
||||
if phrase != "" {
|
||||
if strings.Contains(lt, phrase) {
|
||||
score += 5
|
||||
}
|
||||
if strings.Contains(lc, phrase) {
|
||||
score += 2
|
||||
}
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
// queryTerms lowercases the query and returns its distinct content terms (≥2 chars,
|
||||
// stopwords dropped) — the tokens relevance is scored against.
|
||||
func queryTerms(query string) []string {
|
||||
fields := strings.FieldsFunc(strings.ToLower(query), func(r rune) bool {
|
||||
return !(r >= 'a' && r <= 'z' || r >= '0' && r <= '9')
|
||||
})
|
||||
seen := make(map[string]bool, len(fields))
|
||||
out := make([]string, 0, len(fields))
|
||||
for _, f := range fields {
|
||||
if len(f) < 2 || stopwords[f] || seen[f] {
|
||||
continue
|
||||
}
|
||||
seen[f] = true
|
||||
out = append(out, f)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var stopwords = map[string]bool{
|
||||
"the": true, "and": true, "for": true, "are": true, "was": true, "who": true,
|
||||
"what": true, "why": true, "how": true, "when": true, "where": true, "which": true,
|
||||
"with": true, "from": true, "did": true, "does": true, "his": true, "her": true,
|
||||
"you": true, "your": true, "that": true, "this": true, "into": true, "about": true,
|
||||
"is": true, "of": true, "to": true, "in": true, "on": true, "at": true, "by": true,
|
||||
"or": true, "an": true, "as": true, "be": true, "it": true, "its": true, "has": true,
|
||||
"have": true, "had": true, "not": true, "but": true, "can": true, "will": true,
|
||||
}
|
||||
|
||||
// hostOf returns the lowercased host of a URL, www-stripped, or "" if unparseable.
|
||||
func hostOf(raw string) string {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Host == "" {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimPrefix(strings.ToLower(u.Host), "www.")
|
||||
}
|
||||
|
||||
// favicon derives the Google s2 favicon for a host (matches the SDK's SearchSource).
|
||||
func favicon(host string) string {
|
||||
if host == "" {
|
||||
return ""
|
||||
}
|
||||
return "https://www.google.com/s2/favicons?domain=" + host + "&sz=64"
|
||||
}
|
||||
|
||||
func orHost(title, host string) string {
|
||||
if t := strings.TrimSpace(title); t != "" {
|
||||
return t
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
// clip truncates s to at most n runes (no partial-rune corruption).
|
||||
func clip(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return string(r[:n])
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package ask
|
||||
|
||||
// stream.go — the WEB domain's streamed envelope and its two sinks. The event
|
||||
// shapes match the @hanzo/ai SearchEvent union EXACTLY (type-discriminated), so a
|
||||
// client that consumes the SDK's search()/deepResearch() stream today consumes
|
||||
// /v1/ask (web mode) unchanged: sources → status → text → follow_ups → done (or
|
||||
// error). This is additive: the advisor's figure path still replies with a single
|
||||
// AskResponse JSON; only web mode can stream.
|
||||
//
|
||||
// ONE loop, two deliveries: runWeb() drives a `sink`; sseSink writes SSE frames for
|
||||
// a streaming client, bufferSink accumulates the final answer/sources/follow-ups
|
||||
// for a single JSON reply. Neither leaks into the loop — runWeb() only calls sink
|
||||
// methods.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// maxAnswerChunk caps a single streamed text delta (runes) so a long answer is
|
||||
// delivered progressively at word boundaries rather than one giant frame.
|
||||
const maxAnswerChunk = 200
|
||||
|
||||
// sink receives the loop's envelope events. Implemented by sseSink (streaming) and
|
||||
// bufferSink (JSON). One method per SearchEvent variant keeps the loop declarative.
|
||||
// There is no fail() — the loop never hard-fails: a down model degrades to an honest
|
||||
// answer + a done frame (and is not billed), so the client always gets a terminal frame.
|
||||
type sink interface {
|
||||
status(stage, detail string)
|
||||
sources(s []webSource)
|
||||
text(delta string)
|
||||
followUps(qs []string)
|
||||
done(answer string, s []webSource)
|
||||
}
|
||||
|
||||
// ── SSE sink ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// sseSink writes each event as an SSE frame (`data: <json>\n\n`) and flushes, so
|
||||
// the browser/SDK renders sources, progress, and the answer as they arrive. The
|
||||
// JSON self-describes via `type`, so a data-only SSE reader needs no `event:` line.
|
||||
type sseSink struct{ w *bufio.Writer }
|
||||
|
||||
func (s *sseSink) frame(v any) {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := fmt.Fprintf(s.w, "data: %s\n\n", b); err != nil {
|
||||
return
|
||||
}
|
||||
_ = s.w.Flush()
|
||||
}
|
||||
|
||||
func (s *sseSink) status(stage, detail string) {
|
||||
e := map[string]any{"type": "status", "stage": stage}
|
||||
if detail != "" {
|
||||
e["detail"] = detail
|
||||
}
|
||||
s.frame(e)
|
||||
}
|
||||
func (s *sseSink) sources(src []webSource) {
|
||||
s.frame(map[string]any{"type": "sources", "sources": nonNilSrc(src)})
|
||||
}
|
||||
func (s *sseSink) text(delta string) { s.frame(map[string]any{"type": "text", "delta": delta}) }
|
||||
func (s *sseSink) followUps(qs []string) {
|
||||
s.frame(map[string]any{"type": "follow_ups", "questions": nonNilStr(qs)})
|
||||
}
|
||||
func (s *sseSink) done(answer string, src []webSource) {
|
||||
s.frame(map[string]any{"type": "done", "answer": answer, "sources": nonNilSrc(src)})
|
||||
// A terminal [DONE] sentinel mirrors the OpenAI SSE convention so a generic
|
||||
// reader knows the stream is complete even if it ignores the typed done frame.
|
||||
_, _ = s.w.WriteString("data: [DONE]\n\n")
|
||||
_ = s.w.Flush()
|
||||
}
|
||||
|
||||
// ── buffer sink ──────────────────────────────────────────────────────────────
|
||||
|
||||
// bufferSink accumulates the terminal result for the non-stream JSON reply. It
|
||||
// keeps the last sources and the final answer (from done) and the follow-ups —
|
||||
// status/text deltas are progress-only and not retained.
|
||||
type bufferSink struct {
|
||||
answer string
|
||||
srcs []webSource
|
||||
follow []string
|
||||
}
|
||||
|
||||
func (b *bufferSink) status(string, string) {}
|
||||
func (b *bufferSink) sources(s []webSource) { b.srcs = s }
|
||||
func (b *bufferSink) text(string) {}
|
||||
func (b *bufferSink) followUps(qs []string) { b.follow = qs }
|
||||
func (b *bufferSink) done(answer string, s []webSource) {
|
||||
b.answer = answer
|
||||
if s != nil {
|
||||
b.srcs = s
|
||||
}
|
||||
}
|
||||
|
||||
// ── stream negotiation ───────────────────────────────────────────────────────
|
||||
|
||||
// wantsStream reports whether to stream SSE. Explicit `stream` in the body wins;
|
||||
// otherwise an `Accept: text/event-stream` (the SDK) or `?stream=1` opts in. Default
|
||||
// is a single JSON reply — friendlier for curl and simple clients.
|
||||
func wantsStream(c *zip.Ctx, req AskRequest) bool {
|
||||
if req.Stream != nil {
|
||||
return *req.Stream
|
||||
}
|
||||
if strings.Contains(c.Header("Accept"), "text/event-stream") {
|
||||
return true
|
||||
}
|
||||
return strings.TrimSpace(c.Query("stream")) == "1"
|
||||
}
|
||||
|
||||
// setStreamHeaders writes the SSE response headers (never buffer at a proxy).
|
||||
func setStreamHeaders(c *zip.Ctx) {
|
||||
c.SetHeader("Content-Type", "text/event-stream")
|
||||
c.SetHeader("Cache-Control", "no-cache")
|
||||
c.SetHeader("Connection", "keep-alive")
|
||||
c.SetHeader("X-Accel-Buffering", "no")
|
||||
}
|
||||
|
||||
// ── text helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
// chunkText splits s into pieces of at most size runes, breaking at the last
|
||||
// whitespace before the limit so words and markdown links stay intact. A short
|
||||
// string yields one chunk; empty yields none.
|
||||
func chunkText(s string, size int) []string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return nil
|
||||
}
|
||||
r := []rune(s)
|
||||
if size <= 0 || len(r) <= size {
|
||||
return []string{s}
|
||||
}
|
||||
var out []string
|
||||
for len(r) > 0 {
|
||||
if len(r) <= size {
|
||||
out = append(out, string(r))
|
||||
break
|
||||
}
|
||||
cut := size
|
||||
for cut > 0 && !isSpace(r[cut]) {
|
||||
cut--
|
||||
}
|
||||
if cut == 0 { // no space in the window — hard split
|
||||
cut = size
|
||||
}
|
||||
out = append(out, string(r[:cut]))
|
||||
// skip the boundary space so it is not duplicated at the next chunk's head
|
||||
for cut < len(r) && isSpace(r[cut]) {
|
||||
cut++
|
||||
}
|
||||
r = r[cut:]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func isSpace(r rune) bool { return r == ' ' || r == '\n' || r == '\t' || r == '\r' }
|
||||
|
||||
// webSourcesBlock renders the numbered grounding context the model synthesizes over.
|
||||
func webSourcesBlock(src []webSource) string {
|
||||
if len(src) == 0 {
|
||||
return "(no web sources were found — answer from general knowledge and say so)"
|
||||
}
|
||||
var b strings.Builder
|
||||
for i, s := range src {
|
||||
fmt.Fprintf(&b, "[%d] %s\n%s\n%s\n\n", i+1, s.Title, s.URL, s.Snippet)
|
||||
}
|
||||
return strings.TrimRight(b.String(), "\n")
|
||||
}
|
||||
|
||||
func nonNilSrc(s []webSource) []webSource {
|
||||
if s == nil {
|
||||
return []webSource{}
|
||||
}
|
||||
return s
|
||||
}
|
||||
func nonNilStr(s []string) []string {
|
||||
if s == nil {
|
||||
return []string{}
|
||||
}
|
||||
return s
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user