feat(deploy): the second cloud app that is not linked in

After o11y, deploy is the largest EXCLUSIVE contributor to cmd/cloud's package
graph: 253 packages nothing else pulls, because it is the ONLY importer of
k8s.io/kubernetes/pkg (100) and k8s.io/kubectl/pkg (38). Measured exclusive
contribution, not total — every other candidate shares its weight with
hanzoai/ai or hanzoai/commerce, so unlinking one of THOSE frees single digits.
clients/deploy was imported by apps/apps.go and by nothing else, so this is a
pure subtraction: 3250 -> 2997 packages, 421MB -> 405MB.

cmd/deploy also mounts paas, and that is the interesting part. deploy's
rollback delegates the CR patch to cloud.OnServiceRelease — a package-global
func pointer clients/paas installs at mount — and a func pointer does not cross
a process boundary. paas cannot simply move out with it: the HOST needs the
same seam (clients/platform's release path) and the fleet observer paas
publishes (clients/admin's god-view). So it is linked in both, and that seam is
what has to become a call before this coupling is actually gone.

Two gaps this closes on the way, both of which would have shipped:

  - The image never built the plugin binaries. A missing plugin is not a
    degraded feature, it is fork/exec failing inside MountAll — the host
    refuses to boot. Dockerfile and Makefile now build every plugin beside
    /cloud, reading WHICH from apps.go (the single source; a Dockerfile cannot
    link Go to ask Wire()), and apps.TestPluginBinaries proves that pattern
    yields exactly cloud.PluginNames(apps.Wire()).
  - Nothing asserted a declared prefix actually reaches its child.
    TestEveryPluginPrefixRoutesToItsChild does, on the ROUTE TABLE rather than
    a status code: on the previous branch /v1/sentry/health did not 404, it
    fell through to the /v1/* AI catch-all and answered 503. A plausible
    response from the wrong place is exactly what a status-code test misses.

Co-authored-by: Hanzo Dev <dev@hanzo.ai>

Dockerfile

@@ -150,15 +150,37 @@ RUN --mount=type=cache,id=cloud-gomod-v4,target=/go/pkg/mod,sharing=locked \
 RUN --mount=type=cache,id=cloud-gomod-v4,target=/go/pkg/mod,sharing=locked \
     --mount=type=cache,id=cloud-gobuild-v4,target=/root/.cache/go-build,sharing=locked \
     CGO_ENABLED=1 go build -tags "libsqlite3 sqlite_fts5" -ldflags="-s -w" -o /cloud ./cmd/cloud
+# The PLUGIN binaries — the subsystems that are no longer linked into /cloud and
+# therefore have to ship next to it, or the host fails to mount them and refuses
+# to boot. Same toolchain, same tags, same codec as /cloud: a plugin owns a store
+# too (o11y's annotation queues), so a pure-Go plugin beside a sqlcipher host
+# would be a second, weaker storage posture in one image.
+#
+# The list is READ FROM apps/apps.go, the single source — a Dockerfile cannot link
+# Go to ask Wire(), but it can read the same text, and apps.TestPluginBinaries
+# proves this exact pattern yields exactly cloud.PluginNames(apps.Wire()). A
+# second hand-maintained list here is how "it built, it just does not start" ships.
+RUN --mount=type=cache,id=cloud-gomod-v4,target=/go/pkg/mod,sharing=locked \
+    --mount=type=cache,id=cloud-gobuild-v4,target=/root/.cache/go-build,sharing=locked \
+    set -eu; mkdir -p /plugins; \
+    for p in $(grep -oE 'PluginSpec."[a-z0-9-]+"' apps/apps.go | cut -d'"' -f2); do \
+      echo ">> building plugin $p"; \
+      CGO_ENABLED=1 go build -tags "libsqlite3 sqlite_fts5" -ldflags="-s -w" -o "/plugins/$p" "./cmd/$p"; \
+    done; \
+    test -n "$(ls -A /plugins)" || { echo "FATAL: no plugin binaries built — apps.go declares none, or the pattern drifted"; exit 1; }
 # The functional smoke prober (cmd/smoke) — a stdlib-only, static binary shipped
 # alongside /cloud so the release gate can `docker exec` it against the freshly-built
 # image (and any deployment can be smoked via `docker run --entrypoint /smoke ...`).
 RUN --mount=type=cache,id=cloud-gomod-v4,target=/go/pkg/mod,sharing=locked \
     --mount=type=cache,id=cloud-gobuild-v4,target=/root/.cache/go-build,sharing=locked \
     CGO_ENABLED=0 go build -ldflags="-s -w" -o /smoke ./cmd/smoke
-# Prove the SHIPPED binary binds sqlite3_* to libsqlcipher, not a plaintext libsqlite3.
-RUN readelf -d /cloud | grep -qE 'NEEDED.*(sqlcipher|sqlite3)' || { echo "FATAL: /cloud links no sqlite/sqlcipher .so"; exit 1; }; \
-    ! ldd /cloud 2>/dev/null | grep -E 'libsqlite3' | grep -vq 'libsqlcipher' || { echo "FATAL: /cloud resolves a NON-sqlcipher libsqlite3 (plaintext risk)"; exit 1; }
+# Prove every SHIPPED binary binds sqlite3_* to libsqlcipher, not a plaintext
+# libsqlite3 — the plugins too: an unencrypted store is not less of a problem for
+# being in a child process.
+RUN set -eu; for b in /cloud /plugins/*; do \
+      readelf -d "$b" | grep -qE 'NEEDED.*(sqlcipher|sqlite3)' || { echo "FATAL: $b links no sqlite/sqlcipher .so"; exit 1; }; \
+      ! ldd "$b" 2>/dev/null | grep -E 'libsqlite3' | grep -vq 'libsqlcipher' || { echo "FATAL: $b resolves a NON-sqlcipher libsqlite3 (plaintext risk)"; exit 1; }; \
+    done
 
 # ── final image (alpine, NOT scratch — CGO needs libc + libsqlcipher) ─────────
 FROM ghcr.io/hanzoai/mirror/alpine:3.22@sha256:7c8cb692ae09657cbc4a3f3cbd0e8d5a2690ba38386aaaf252dbb060bf5eb2e6
@@ -198,6 +220,10 @@ COPY --from=build /usr/share/zoneinfo /usr/share/zoneinfo
 COPY --from=build /etc/passwd /etc/passwd
 COPY --from=build /etc/group /etc/group
 COPY --from=build /cloud /cloud
+# The plugin binaries land BESIDE /cloud, which is exactly where apps.pluginAt
+# looks (os.Executable's dir, not $PATH) — so a host always starts the plugin it
+# was built and shipped with. Still one image, still one artifact to ship.
+COPY --from=build /plugins/ /
 COPY --from=build /smoke /smoke
 EXPOSE 8080 9090 9653
 USER 65532:65532

Makefile

@@ -81,9 +81,26 @@ agentskills: ## Regenerate the FULL agent-skills catalog into clients/agentskill
 	python3 "$(OPENAPI_DIR)/skills.py" --no-services --out clients/agentskills/catalog
 	@echo ">> embedded FULL agent-skills catalog ($$(jq -r .skill_count clients/agentskills/catalog/hanzo/index.json) skills/brand)"
 
-build: ## Build the unified cloud binary into ./bin/cloud (embeds whatever webui/dist holds — run `webui` first for the real console).
+# PLUGINS are the subsystems that are NOT linked into cloud and ship as their own
+# binary (cloud.PluginSpec in the composition root). Read from apps/apps.go — the
+# single source — because make cannot link Go to ask Wire(); apps.TestPluginBinaries
+# proves this exact pattern yields exactly cloud.PluginNames(apps.Wire()).
+#
+# They must sit BESIDE bin/cloud: apps.pluginAt resolves a plugin from
+# os.Executable's directory, so a host loads the plugin it was built with rather
+# than whatever a $PATH happens to find. Without them `make build && ./bin/cloud`
+# fails to mount and refuses to boot.
+# (The '(' is matched by '.' rather than written literally: make counts parens
+# inside $(shell ...) and an escaped one ends the call early.)
+PLUGINS := $(shell grep -oE 'PluginSpec."[a-z0-9-]+"' apps/apps.go | cut -d'"' -f2)
+
+build: ## Build the unified cloud binary + its plugin binaries into ./bin (embeds whatever webui/dist holds — run `webui` first for the real console).
 	@mkdir -p bin
 	CGO_ENABLED=$(CGO_ENABLED) $(GO) build -ldflags="$(LDFLAGS)" -o bin/$(BIN) $(PKG)
+	@for p in $(PLUGINS); do \
+	  echo "CGO_ENABLED=$(CGO_ENABLED) $(GO) build -o bin/$$p ./cmd/$$p"; \
+	  CGO_ENABLED=$(CGO_ENABLED) $(GO) build -ldflags="$(LDFLAGS)" -o bin/$$p ./cmd/$$p || exit 1; \
+	done
 
 build-standalone: webui build ## Build the REAL 1-binary console: console build:embed → webui/dist → go build.
 

apps/apps.go

@@ -84,7 +84,11 @@ import (
 	"github.com/hanzoai/cloud/clients/content"
 	"github.com/hanzoai/cloud/clients/crm"
 	"github.com/hanzoai/cloud/clients/dataroom"
-	"github.com/hanzoai/cloud/clients/deploy"
+	// NOTE: clients/deploy is deliberately NOT imported — it is loaded at run
+	// time as a plugin (see the deploy entry in Wire). Deleting this import is
+	// the whole reason that works: an import here keeps its 253 exclusive
+	// packages — k8s.io/kubernetes/pkg, k8s.io/kubectl/pkg — linked into cloud
+	// whether or not any Wire entry referenced it.
 	"github.com/hanzoai/cloud/clients/destinations"
 	"github.com/hanzoai/cloud/clients/dns"
 	"github.com/hanzoai/cloud/clients/do"
@@ -254,11 +258,10 @@ func Wire() []cloud.MountSpec {
 		// otel-collector, prometheus and gonum are here and nowhere else — and it is
 		// imported by NOTHING but this line, so unlinking it is a pure subtraction.
 		//
-		// KNOWN GAP, see the branch report: o11y also owns /v1/sentry/* (mountSentry),
-		// which is a SECOND public prefix. zip.Load takes one, so /v1/sentry/* is not
-		// mounted on the host by this line and 404s until zip.Plugin can name more than
-		// one prefix. Do not merge this to main before that is closed.
-		cloud.PluginSpec("o11y", "/v1/o11y", o11yPlugin()),
+		// It owns TWO prefixes: the /v1/o11y read plane and the /v1/sentry/* wildcard
+		// mountSentry registers. Both are named here because zip.Load is variadic —
+		// a prefix left out is not an error, it is a silent 404 on that subtree.
+		cloud.PluginSpec("o11y", pluginAt("o11y"), "/v1/o11y", "/v1/sentry"),
 		{Name: "authz", Mount: cloud.Global(authz.Mount), Global: true},
 		// Embedded commerce plane /v1/commerce/*, /_/commerce/* — the hanzoai/commerce
 		// MODULE via the adapter in commerce.go (un-forked; the in-process
@@ -305,9 +308,24 @@ func Wire() []cloud.MountSpec {
 		{Name: "x402", Mount: x402.Mount, Shutdown: ctxShutdown(x402.Shutdown)},
 		{Name: "paas", Mount: paas.Mount, OwnsHealth: true},
 		// GitOps deploy dashboard /v1/deploy/* (the ArgoCD-grade fleet view over the
-		// operator App CRs). After paas so the release seam paas installs is registered
-		// before a gitops rollback delegates to it; owns its own /v1/deploy/health.
-		{Name: "deploy", Mount: deploy.Mount, OwnsHealth: true},
+		// operator App CRs) — the second app that is NOT linked in. It runs as its own
+		// binary (cmd/deploy) and mounts at /v1/deploy over a private unix socket.
+		//
+		// Why this one next: after o11y it is the largest EXCLUSIVE contributor to the
+		// graph — 253 packages nothing else pulls, because it is the ONLY importer of
+		// k8s.io/kubernetes/pkg and k8s.io/kubectl/pkg. Every other candidate shares
+		// its weight with hanzoai/ai or hanzoai/commerce, so unlinking one frees single
+		// digits. clients/deploy was imported by this file and nothing else.
+		//
+		// It stays AFTER paas: the position is now cosmetic (the plugin carries its own
+		// paas mount, because the release seam is a func pointer and does not cross a
+		// process boundary — see cmd/deploy), but the two are still read as a pair.
+		//
+		// Not OwnsHealth: the plugin serves /v1/deploy/health itself, and the generic
+		// always-ok route Serve registers before MountAll still wins — the same
+		// behaviour o11y has. No Shutdown: zip.Load registers its own OnShutdown that
+		// stops the child.
+		cloud.PluginSpec("deploy", pluginAt("deploy"), "/v1/deploy"),
 		{Name: "functions", Mount: functions.Mount},
 		{Name: "tracker", Mount: tracker.Mount},
 		{Name: "templates", Mount: templates.Mount},
@@ -568,27 +586,30 @@ func ServeSingle(name string) error {
 	return fmt.Errorf("ServeSingle: unknown app %q — run `hanzo code ls`/`hanzo` for the list", name)
 }
 
-// o11yPlugin says where to find the o11y binary. Its two knobs map 1:1 onto
-// zip.Plugin's own fields, so there is no third notion of "where a plugin is"
-// and nothing to translate:
+// pluginAt says where to find the binary for the plugin named name. Its two knobs
+// are DERIVED from the name, so a plugin's configuration cannot disagree with its
+// identity and adding one invents no new env var to document:
 //
-//	CLOUD_O11Y_ADDR — already listening there; start nothing, just mount it.
-//	CLOUD_O11Y_BIN  — the binary's path on disk.
+//	CLOUD_<NAME>_ADDR — already listening there; start nothing, just mount it.
+//	CLOUD_<NAME>_BIN  — the binary's path on disk.
 //
-// The default is a file named "o11y" beside the running cloud binary, which is
-// the container layout: both binaries in the image, still one artifact to ship.
-// Resolving it from os.Executable rather than $PATH means a host always loads
-// the o11y it was built and shipped with, not whichever one a PATH happens to
-// find.
-func o11yPlugin() zip.Plugin {
-	if addr := strings.TrimSpace(os.Getenv("CLOUD_O11Y_ADDR")); addr != "" {
+// They map 1:1 onto zip.Plugin's own fields, so there is no third notion of "where
+// a plugin is" and nothing to translate.
+//
+// The default is a file named <name> beside the running cloud binary, which is the
+// container layout: every binary in the image, still one artifact to ship.
+// Resolving it from os.Executable rather than $PATH means a host always loads the
+// plugin it was built and shipped with, not whichever one a PATH happens to find.
+func pluginAt(name string) zip.Plugin {
+	env := "CLOUD_" + strings.ToUpper(strings.ReplaceAll(name, "-", "_"))
+	if addr := strings.TrimSpace(os.Getenv(env + "_ADDR")); addr != "" {
 		return zip.Plugin{Addr: addr}
 	}
-	path := strings.TrimSpace(os.Getenv("CLOUD_O11Y_BIN"))
+	path := strings.TrimSpace(os.Getenv(env + "_BIN"))
 	if path == "" {
-		path = "o11y"
+		path = name
 		if self, err := os.Executable(); err == nil {
-			path = filepath.Join(filepath.Dir(self), "o11y")
+			path = filepath.Join(filepath.Dir(self), name)
 		}
 	}
 	return zip.Plugin{Path: path}

apps/plugin_test.go

@@ -0,0 +1,52 @@
+package apps
+
+import (
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/hanzoai/cloud"
+)
+
+// shellPluginRE is the pattern the Makefile and the Dockerfile use to learn which
+// plugin binaries to build (there, paren-free — `PluginSpec."[a-z0-9-]+"` — because
+// make counts parens inside $(shell ...); here it takes a capture group and so
+// writes the '(' out). They cannot link this package to ask Wire(), so they read
+// the same text, and this test is what stops the text and the compiled list from
+// drifting apart. Keep the three in lockstep.
+var shellPluginRE = regexp.MustCompile(`PluginSpec.("[a-z0-9-]+")`)
+
+// The failure this guards is not a build error, it is a boot error: a plugin the
+// image forgot to build makes zip.Load's fork/exec fail, MountAll returns, and
+// cloud refuses to start. Nothing about compiling cloud notices.
+func TestPluginBinaries(t *testing.T) {
+	src, err := os.ReadFile("apps.go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var fromText []string
+	for _, m := range shellPluginRE.FindAllStringSubmatch(string(src), -1) {
+		fromText = append(fromText, strings.Trim(m[1], `"`)) // the shell's `cut -d'"' -f2`
+	}
+	compiled := cloud.PluginNames(Wire())
+
+	if len(compiled) == 0 {
+		t.Fatal("Wire() declares no plugin — if that is deliberate, delete this test with the last PluginSpec")
+	}
+	if len(fromText) != len(compiled) {
+		t.Fatalf("the shell sees %v, Wire() says %v — Makefile/Dockerfile would build the wrong set", fromText, compiled)
+	}
+	for i, name := range compiled {
+		if fromText[i] != name {
+			t.Fatalf("plugin %d: shell sees %q, Wire() says %q", i, fromText[i], name)
+		}
+		// The entrypoint has to exist, or `go build ./cmd/<name>` fails in the
+		// image build rather than here, where the reason is legible.
+		dir := filepath.Join("..", "cmd", name)
+		if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
+			t.Fatalf("plugin %q has no cmd/%s — an extracted app with no binary cannot be mounted", name, name)
+		}
+	}
+}

apps/wire_test.go

@@ -63,7 +63,15 @@ var frozen = []struct {
 	{"wallets", false, true, false},         // was order 127
 	{"x402", false, true, false},            // new: x402 pay-per-use settlement (after wallets)
 	{"paas", true, false, false},            // was order 128
-	{"deploy", true, false, false},          // after paas (release seam), before functions
+	// OwnsHealth flipped true->false and Global false->true when deploy became a
+	// PLUGIN (cloud.PluginSpec, its own cmd/deploy binary) — the same two flips
+	// o11y took, for the same two reasons. Global: zip.Load registers the prefix
+	// itself, so it needs the bare *zip.App; a scoped Router would nest /v1/deploy
+	// under the subsystem name. OwnsHealth: the plugin still serves
+	// /v1/deploy/health, but the generic always-ok route Serve registers before
+	// MountAll now answers first, so claiming ownership here would be a lie about
+	// which route wins. Position and name are UNCHANGED.
+	{"deploy", false, false, true},          // after paas (release seam), before functions; out-of-process
 	{"functions", false, false, false},      // was order 128
 	{"tracker", false, false, false},        // was order 129
 	{"templates", false, false, false},      // was order 129

build.go

@@ -1002,6 +1002,27 @@ type MountSpec struct {
 	// Today it is held only by linked modules whose own Mount still takes *zip.App
 	// (see cloud.Global) — none of which installs middleware.
 	Global bool
+
+	// Plugin says this subsystem is NOT linked into this binary: it is served by
+	// a separate binary named <Name>, mounted at run time (see PluginSpec, which
+	// is the only thing that sets it). Routing, health and shutdown are identical
+	// either way — this exists so the ONE list that already says what the binary
+	// is composed of can also answer "and which of those ship as their own
+	// executable?", instead of a second list drifting alongside Wire(). The
+	// Dockerfile and the cmd/cloud tests both need that answer.
+	Plugin bool
+}
+
+// PluginNames returns the Name of every spec served by its own binary, in
+// Wire() order. Derived from the list, never a copy of it.
+func PluginNames(specs []MountSpec) []string {
+	var out []string
+	for _, s := range specs {
+		if s.Plugin {
+			out = append(out, s.Name)
+		}
+	}
+	return out
 }
 
 // MountAll mounts every ENABLED subsystem in specs, in slice order — the order is

clients/websearch/websearch.go

@@ -315,7 +315,7 @@ func Mount(app cloud.Router, deps cloud.Deps) error {
 		return searchKeyed(c)
 	})
 
-	scrape := zip.AdaptNetHTTPFunc(scrapeHandler)
+	scrape := zip.AdaptNetHTTP(http.HandlerFunc(scrapeHandler))
 	// Firecrawl builds {apiUrl}/{version}/scrape; pin firecrawlVersion:v1 so the
 	// client POSTs /v1/websearch/v1/scrape. Also accept the bare /scrape.
 	g.Post("/v1/scrape", scrape)

cmd/cloud/main_test.go

@@ -12,6 +12,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"testing"
 
 	"github.com/hanzoai/cloud"
@@ -28,12 +29,17 @@ import (
 // (pricing, o11y's annotation queues) refuse to open a data plane unencrypted,
 // which is correct, and a test box has no KMS. One dev-only key, one pattern.
 //
-// The o11y binary is new, and is the cost of a plugin subsystem: o11y is no
-// longer linked in, so the composition root can only mount it by starting the
-// real binary. Building it here is deliberate — the alternative, a stub, would
-// let these tests pass against a route table no deployment ever serves. Cached
-// after the first run. If it cannot be built the dependent tests fail with
-// zip's own fork/exec message, which names the missing file.
+// The plugin binaries are the cost of a plugin subsystem: they are no longer
+// linked in, so the composition root can only mount one by starting the real
+// binary. Building them here is deliberate — the alternative, a stub, would let
+// these tests pass against a route table no deployment ever serves. Cached after
+// the first run. If one cannot be built the dependent tests fail with zip's own
+// fork/exec message, which names the missing file.
+//
+// WHICH binaries is asked of the composition root (cloud.PluginNames over
+// apps.Wire()), never listed here: a test with its own list would keep passing
+// after someone extracts the next app and forgets to add it, which is precisely
+// the failure this harness exists to catch.
 func TestMain(m *testing.M) { os.Exit(runTests(m)) }
 
 // runTests exists so the temp dirs are removed on the way out: os.Exit does not
@@ -52,22 +58,26 @@ func runTests(m *testing.M) int {
 		defer os.RemoveAll(dir)
 		_ = os.Setenv("CLOUD_DATA_DIR", dir)
 	}
-	if os.Getenv("CLOUD_O11Y_BIN") == "" {
-		dir, err := os.MkdirTemp("", "cloud-test-plugins-")
-		if err != nil {
-			return 1
+	dir, err := os.MkdirTemp("", "cloud-test-plugins-")
+	if err != nil {
+		return 1
+	}
+	defer os.RemoveAll(dir)
+	for _, name := range cloud.PluginNames(apps.Wire()) {
+		env := "CLOUD_" + strings.ToUpper(strings.ReplaceAll(name, "-", "_")) + "_BIN"
+		if os.Getenv(env) != "" {
+			continue
 		}
-		defer os.RemoveAll(dir)
-		bin := filepath.Join(dir, "o11y")
+		bin := filepath.Join(dir, name)
 		// GOROOT/bin/go, not "go": the toolchain that is running this test is the
 		// one that must build the plugin, and it is not always on PATH.
-		cmd := exec.Command(goTool(), "build", "-o", bin, "./cmd/o11y")
+		cmd := exec.Command(goTool(), "build", "-o", bin, "./cmd/"+name)
 		cmd.Dir = "../.." // the package dir is cmd/cloud
 		cmd.Stderr = os.Stderr
 		if err := cmd.Run(); err != nil {
-			os.Stderr.WriteString("TestMain: building the o11y plugin failed: " + err.Error() + "\n")
+			os.Stderr.WriteString("TestMain: building the " + name + " plugin failed: " + err.Error() + "\n")
 		} else {
-			_ = os.Setenv("CLOUD_O11Y_BIN", bin)
+			_ = os.Setenv(env, bin)
 		}
 	}
 	code := m.Run()

cmd/cloud/plugin_route_test.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"net/http/httptest"
+	"testing"
+
+	"github.com/hanzoai/cloud/apps"
+)
+
+// Every prefix a plugin declares must actually reach the child process.
+//
+// This is the guard the o11y extraction shipped without, and it is why /v1/sentry
+// silently stopped reaching o11y for a whole branch: a plugin declared with SOME
+// of its prefixes compiles, mounts, boots and passes every route-table test — the
+// subtrees nobody named just quietly go elsewhere. Measured on that branch,
+// /v1/sentry/health did not even 404: it fell through to the /v1/* AI catch-all
+// and answered 503, which is why this asserts on the ROUTE TABLE and not on a
+// status code. A response can be plausible and still come from the wrong place.
+//
+// zip.Load registers exactly two routes per prefix, `<prefix>` and `<prefix>/*`
+// (App.mountVia), so their presence IS the host's promise to forward that subtree.
+// Add a prefix to a PluginSpec and this test covers it with no edit here.
+func TestEveryPluginPrefixRoutesToItsChild(t *testing.T) {
+	if testing.Short() {
+		t.Skip("mounts every subsystem in apps.Wire(); slow by construction")
+	}
+	app := fullyMountedApp(t)
+
+	var prefixes []string
+	for _, s := range apps.Wire() {
+		if s.Plugin {
+			prefixes = append(prefixes, s.Prefixes...)
+		}
+	}
+	if len(prefixes) == 0 {
+		t.Fatal("no plugin subsystems — this guard proves nothing")
+	}
+
+	// The host's own route table: what it will forward, independent of what any
+	// handler chooses to answer.
+	registered := map[string]bool{}
+	for _, r := range app.Fiber().GetRoutes(true) {
+		registered[r.Path] = true
+	}
+
+	for _, prefix := range prefixes {
+		t.Run(prefix, func(t *testing.T) {
+			for _, want := range []string{prefix, prefix + "/*"} {
+				if !registered[want] {
+					t.Fatalf("no host route %q — this subtree is NOT forwarded to the plugin. "+
+						"Name every prefix the subsystem owns in its cloud.PluginSpec (it is variadic).", want)
+				}
+			}
+			// And it answers over the socket, not merely in the table: a route that
+			// resolves to a dead child is a 502/503 from mountVia, never a 200.
+			res, err := app.Fiber().Test(httptest.NewRequest("GET", prefix+"/health", nil))
+			if err != nil {
+				t.Fatalf("%s: %v", prefix, err)
+			}
+			defer res.Body.Close()
+			if res.StatusCode == 502 || res.StatusCode == 503 {
+				t.Fatalf("%s/health -> %d: the route exists but the child did not answer it", prefix, res.StatusCode)
+			}
+			t.Logf("%s and %s/* forwarded; GET %s/health -> %d from the child", prefix, prefix, prefix, res.StatusCode)
+		})
+	}
+}

cmd/deploy/main.go

@@ -0,0 +1,83 @@
+// deploy is the GitOps deploy control plane built as its OWN binary.
+//
+// It is an ordinary zip app. There is no SDK, no schema and nothing
+// plugin-specific in here except zip.Addr — which is the whole plugin contract:
+// serve on the socket a host handed us, or on our own port when run directly.
+// The same binary therefore covers both deployments without a second code path.
+//
+// It mounts EXACTLY what apps.Wire() used to mount in-process, by calling the
+// same deploy.Mount. The subsystem's code did not move and did not fork; only
+// the process it runs in changed.
+//
+// Why this one: after o11y, deploy is the largest EXCLUSIVE contributor to the
+// unified binary's package graph — 253 packages that NOTHING else in cloud
+// pulls, because it is the only importer of k8s.io/kubernetes/pkg (100) and
+// k8s.io/kubectl/pkg (38). Every other candidate shares its heavy deps with
+// hanzoai/ai or hanzoai/commerce, so unlinking it frees almost nothing.
+// clients/deploy is imported by apps/apps.go and by NOTHING else, so unlinking
+// it is a pure subtraction.
+//
+// paas is mounted HERE TOO, and that is not incidental. deploy's rollback
+// delegates the CR patch to the process-global release seam
+// (cloud.OnServiceRelease), and clients/paas is what installs it — a func
+// pointer, which does not cross a process boundary. Without this line the
+// rollback route stops reaching releaseService and degrades to
+// "release plane not available (paas subsystem not co-resident)". paas cannot
+// simply move out with deploy: the HOST needs it too (clients/platform's
+// release path reads the same seam, and clients/admin's god-view reads the
+// fleet observer paas publishes), so it stays linked in both. This is the seam
+// that has to become an HTTP call before the coupling is actually gone.
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/hanzoai/cloud"
+	"github.com/hanzoai/cloud/clients/deploy"
+	"github.com/hanzoai/cloud/clients/paas"
+	"github.com/zap-proto/zip"
+)
+
+// listenEnv names the address to serve on when this binary is run DIRECTLY
+// rather than by a host. Under a host, zip.Addr ignores it and uses the private
+// unix socket the host created. The default deliberately is not cloud's own
+// :9653, so running both on one box does not collide.
+const (
+	listenEnv     = "DEPLOY_LISTEN"
+	defaultListen = ":9655"
+)
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Fprintf(os.Stderr, "deploy: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	// The same config and the same Deps the unified binary builds — building it
+	// the one canonical way keeps this entrypoint honest about what a subsystem
+	// may reach for.
+	deps := cloud.BuildDeps(cloud.LoadConfig())
+
+	app := zip.New(zip.Config{AppName: "deploy", Logger: deps.Logger})
+
+	// The release seam first: paas.Mount is what calls RegisterServiceReleaser,
+	// and deploy's rollback resolves it per request. Its /v1/paas routes are
+	// registered on THIS app, which the host does not forward to — the host
+	// serves /v1/paas from its own linked-in paas. Running this binary directly
+	// therefore yields the same board from the same code, never a second one.
+	if err := paas.Mount(app, deps); err != nil {
+		return fmt.Errorf("mount paas (release seam): %w", err)
+	}
+	if err := deploy.Mount(app, deps); err != nil {
+		return fmt.Errorf("mount: %w", err)
+	}
+
+	addr := os.Getenv(listenEnv)
+	if addr == "" {
+		addr = defaultListen
+	}
+	return app.Listen(zip.Addr(addr))
+}

go.mod

@@ -1,6 +1,6 @@
 module github.com/hanzoai/cloud
 
-go 1.26.4
+go 1.26.5
 
 // Dependencies will be added as subsystems are mounted per HIP-0106.
 
@@ -43,7 +43,7 @@ require (
 	github.com/zap-proto/fiber/v3 v3.2.1
 	github.com/zap-proto/go v1.3.0
 	github.com/zap-proto/md v0.1.0
-	github.com/zap-proto/zip v1.10.0
+	github.com/zap-proto/zip v1.16.0
 	go.opentelemetry.io/collector/component v1.54.0
 	go.opentelemetry.io/collector/confmap v1.54.0
 	go.opentelemetry.io/collector/confmap/provider/envprovider v1.50.0
@@ -104,7 +104,7 @@ require (
 	github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
 	github.com/vultr/govultr/v3 v3.30.0 // indirect
-	github.com/zap-proto/http v0.3.0 // indirect
+	github.com/zap-proto/http v0.3.1 // indirect
 	github.com/zap-proto/zap2pb v0.2.0 // indirect
 	go.mongodb.org/mongo-driver v1.17.9 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect
@@ -733,7 +733,7 @@ require (
 	github.com/hanzoai/base v1.5.7
 	github.com/hanzoai/licensing v0.1.5
 	github.com/hanzoai/metrics v1.110.2
-	github.com/hanzoai/o11y v1.5.32
+	github.com/hanzoai/o11y v1.5.33
 	github.com/hanzoai/thinking v0.1.1 // indirect
 	github.com/hanzoai/vfs v0.6.6
 	github.com/hanzoai/zen v1.4.4

go.sum

@@ -1093,6 +1093,8 @@ github.com/hanzoai/o11y v1.5.31 h1:cq0oay92IbqCaRzazxoG7fMifhhVXF4zP1QfGR2wRpM=
 github.com/hanzoai/o11y v1.5.31/go.mod h1:npZ7y0k6+uP6CnKHGtiL+dgIuqI6e3WCV/+6IYNImw4=
 github.com/hanzoai/o11y v1.5.32 h1:XI9LEjNuzAsg27s5zNCTx71Rhw1pCv8gRy/ILvPiY0s=
 github.com/hanzoai/o11y v1.5.32/go.mod h1:tsrIwTJJEEy+AnOGXLM4RSrkhUyM1ja+bkstzFkc0Hw=
+github.com/hanzoai/o11y v1.5.33 h1:99NO2O/MB5czh1dsqGPaCbk1Lyf+QeFy1VJ1cc+PppI=
+github.com/hanzoai/o11y v1.5.33/go.mod h1:bdqSdqagTAQfEliZttOujk5VjR41OwUBn1yrCWx06wE=
 github.com/hanzoai/orm v0.6.16 h1:w3UXH65huahNJ8RgC88ffUeicAbHoUpQW8oLuDCojK8=
 github.com/hanzoai/orm v0.6.16/go.mod h1:KpbP5UwQ8BBNGVM3tku9rgs7PADB+UG8fqh8Nol0X/s=
 github.com/hanzoai/otel-collector v1.2.0 h1:lBDL5lKotq89JaqchcM+/oxEnjrjazEI2JJfDhotudc=
@@ -2103,12 +2105,16 @@ github.com/zap-proto/go v1.3.0 h1:S3rMoawwhH/BbSZ4G8zG05hJoQnMSMDPzIq75diCTqE=
 github.com/zap-proto/go v1.3.0/go.mod h1:914SNGTH6Rv3Yu1MweWJBPEN8FZlo5C39QyhaB0C7Q0=
 github.com/zap-proto/http v0.3.0 h1:l7DvlngiYqmzNY6fzyRYw2ZIAhF35FqwOe6mvAOqpMg=
 github.com/zap-proto/http v0.3.0/go.mod h1:UYfGhDDCetgxs65XSev8Lpf65COg5vKQK+cWwZGh4zQ=
+github.com/zap-proto/http v0.3.1 h1:A2rCPWYCX866eAsdiWuns0dvWnBmViZtGm4pwX7jwlY=
+github.com/zap-proto/http v0.3.1/go.mod h1:UYfGhDDCetgxs65XSev8Lpf65COg5vKQK+cWwZGh4zQ=
 github.com/zap-proto/md v0.1.0 h1:1R6w/i1FYAdGIIiOvNggKO0RjikzhWWRodQUOgzEEpc=
 github.com/zap-proto/md v0.1.0/go.mod h1:pmMx2F4Dwj1H48PIuLzRZxB2R5qVvqeg8ZScKQIntyQ=
 github.com/zap-proto/zap2pb v0.2.0 h1:sos6HnayhGMGLRO54px1InzimDzTZ2o5TSMEatYBjzs=
 github.com/zap-proto/zap2pb v0.2.0/go.mod h1:wD97Z2VTPabDq/4AMNL++PWnQ0YwEtajiuNkLGg3/18=
 github.com/zap-proto/zip v1.10.0 h1:0Swzr+SNr+4VeO8pUw+Umdkh3z/lnD0hJ4N1kObdP60=
 github.com/zap-proto/zip v1.10.0/go.mod h1:9R3FOq2ItZa7G+9QilsB/punEpVSHtdXiQji0P84LSE=
+github.com/zap-proto/zip v1.16.0 h1:Bb3StSa9xMHzWUOVDxeMUZFTrgRsBGyrm1tCJMvKRxw=
+github.com/zap-proto/zip v1.16.0/go.mod h1:BxFNqjnAVhArMJ+s7VnXdwAtfOVy47GAi62wwqyu8go=
 github.com/zeebo/assert v1.3.1 h1:vukIABvugfNMZMQO1ABsyQDJDTVQbn+LWSMy1ol1h6A=
 github.com/zeebo/assert v1.3.1/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
 github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI=

plugin_spec.go

@@ -8,7 +8,7 @@ import (
 	"github.com/zap-proto/zip"
 )
 
-// PluginSpec returns a MountSpec that serves prefix from a SEPARATE binary
+// PluginSpec returns a MountSpec that serves prefixes from a SEPARATE binary
 // instead of code linked into this one.
 //
 // It exists so that where a subsystem runs stops being a property of the source.
@@ -18,26 +18,41 @@ import (
 // nothing downstream (routing, health, shutdown ordering) can tell the difference.
 //
 // The plugin names exactly one of Addr (already listening), Bin (the binary,
-// normally go:embed'd) or Path. For Bin and Path, zip starts it as a child on a
-// private unix socket and mounts the routes onto it; the child is stopped when
-// Shutdown runs, so a plugin subsystem tears down with the rest.
+// normally go:embed'd), Path, or URL+Sum. For all but Addr, zip starts it as a
+// child on a private unix socket and mounts the routes onto it; the child is
+// stopped when Shutdown runs, so a plugin subsystem tears down with the rest.
 //
-// Global is set because zip.Load registers under the prefix it was given. Handing
-// it a scoped Router would nest that prefix under the subsystem name and the
-// routes would answer somewhere nobody is asking.
-func PluginSpec(name, prefix string, p zip.Plugin) MountSpec {
+// Pass EVERY prefix the subsystem owns. A subsystem routinely owns more than one
+// route subtree — o11y answers /v1/o11y AND /v1/sentry — and a prefix left out is
+// not an error, it is a silent 404 on that subtree, which is the worst way for
+// this to fail. Grep the subsystem's Mount for every path it registers before
+// converting it. Naming none at all is refused rather than mounted inert.
+//
+// Global is set because zip.Load registers under the prefixes it was given.
+// Handing it a scoped Router would nest those prefixes under the subsystem name
+// and the routes would answer somewhere nobody is asking.
+func PluginSpec(name string, p zip.Plugin, prefixes ...string) MountSpec {
 	if p.Name == "" {
 		p.Name = name
 	}
 	return MountSpec{
 		Name:   name,
 		Global: true,
+		Plugin: true,
+		// The subtrees this subsystem owns, recorded on the spec rather than
+		// captured only in the closure. MountAll ignores Prefixes for a Global
+		// spec, so this costs nothing at mount — and it means the ONE list can be
+		// asked what a plugin serves without starting it.
+		Prefixes: prefixes,
 		Mount: func(router Router, _ Deps) error {
+			if len(prefixes) == 0 {
+				return fmt.Errorf("pluginspec %q: no prefix — a plugin that owns nothing serves nothing", name)
+			}
 			app, ok := router.(*zip.App)
 			if !ok {
 				return fmt.Errorf("pluginspec %q: needs the root app, got %T — Global must stay set", name, router)
 			}
-			return zip.Load(prefix, p)(app)
+			return zip.Load(p, prefixes...)(app)
 		},
 	}
 }

plugin_spec_test.go

@@ -13,7 +13,7 @@ import (
 // same MountSpec type, so Wire() can swap in-process for out-of-process by
 // editing one line.
 func TestPluginSpec_IsAnOrdinaryMountSpec(t *testing.T) {
-	s := PluginSpec("search", "/v1/search", zip.Plugin{Addr: "127.0.0.1:1"})
+	s := PluginSpec("search", zip.Plugin{Addr: "127.0.0.1:1"}, "/v1/search")
 	if s.Name != "search" {
 		t.Fatalf("name = %q, want search", s.Name)
 	}
@@ -28,7 +28,7 @@ func TestPluginSpec_IsAnOrdinaryMountSpec(t *testing.T) {
 // Mounting onto a scoped Router is a wiring mistake, not something to paper
 // over: the routes would answer under a doubled prefix. Fail loudly.
 func TestPluginSpec_RefusesAScopedRouter(t *testing.T) {
-	s := PluginSpec("bad", "/v1/bad", zip.Plugin{Addr: "127.0.0.1:1"})
+	s := PluginSpec("bad", zip.Plugin{Addr: "127.0.0.1:1"}, "/v1/bad")
 	err := s.Mount(scopedStub{}, Deps{})
 	if err == nil {
 		t.Fatal("mounting on a non-root Router must fail")
@@ -38,4 +38,20 @@ func TestPluginSpec_RefusesAScopedRouter(t *testing.T) {
 	}
 }
 
+// A subsystem that owns several route subtrees must be able to say so in ONE
+// call — a plugin declared with one of its prefixes silently 404s the rest,
+// which is how the o11y extraction lost /v1/sentry.
+func TestPluginSpec_TakesEveryPrefixTheSubsystemOwns(t *testing.T) {
+	s := PluginSpec("o11y", zip.Plugin{Addr: "127.0.0.1:1"}, "/v1/o11y", "/v1/sentry")
+	if s.Mount == nil {
+		t.Fatal("Mount is nil")
+	}
+	// Naming NO prefix is the failure mode this guards: it would mount a child
+	// nothing can reach. Refuse it rather than start a process for no routes.
+	none := PluginSpec("void", zip.Plugin{Addr: "127.0.0.1:1"})
+	if err := none.Mount(&zip.App{}, Deps{}); err == nil {
+		t.Fatal("a plugin with no prefix must be refused, not mounted inert")
+	}
+}
+
 type scopedStub struct{ Router }