admin: what this host is ACTUALLY running
Every board under /v1/admin/* aggregates an upstream. GET /v1/admin/plugins does not — it asks the process itself. zip v1.15 gave a host App.Plugins(): which subsystems it loaded out-of-binary, at which artifact digest, whether they are still up, and what each costs from /proc. Nothing read it. The gap it closes is not cosmetic. A deployment manifest answers what was INTENDED. Only the process knows what is TRUE, and during a rolling upgrade the two disagree by design. The version reported here is the artifact's SHA-256 — the one version identifier that cannot drift from the bits running, because it IS the bits. Restarts is the field the board exists for. Reloads are deliberate (someone swapped the binary); Restarts is the supervisor bringing a plugin back after it died on its own. Nonzero Restarts is a crash, not a deploy, and a climbing one is a crash loop. That verdict is computed once, server-side (Crashed), so no reader re-derives the policy and no two readers can disagree about it. /v1/admin/plugins, core.Guard, registered beside /o11y and /aimetrics in routes() — SuperAdmin only, like every platform read. Artifact digests, pids and RSS name what is deployed and where the memory went; that is not a customer-visible fact. Honest in the two ways that matter. It is THIS replica's answer, so the board stamps Host (Deps.Self, the same id the durability ring elects on) — one pod's restarts presented as the fleet's would be worse than no board. And a host with no plugins returns an empty list, never an error: "everything is linked in" is true and expected, since cloud composes a linked-in service and a plugin as the same type. Router gains Plugins() next to Fiber(): a second named, read-only hole, and strictly the weaker of the two — it registers nothing and mutates nothing. The alternative was making admin Global, granting app-wide middleware to buy a status field. Carried along, because the bump requires them: zip v1.11.0 -> v1.16.1 (Load takes the Plugin first + variadic prefixes; AdaptNetHTTPFunc is gone — http.HandlerFunc IS an http.Handler) hanzoai/o11y v1.5.32 -> v1.5.33 (same adapter removal, already fixed there) selfID(cfg) is now the ONE self-id resolver; durability and Deps.Self read the same expression rather than two that could drift. Co-authored-by: Hanzo Dev <dev@hanzo.ai>
This commit is contained in:
@@ -76,6 +76,7 @@ func BuildDeps(cfg *Config) Deps {
|
||||
Env: cfg.Env,
|
||||
Domain: cfg.Domain,
|
||||
IAMIssuer: cfg.IAMIssuer,
|
||||
Self: selfID(cfg),
|
||||
DataDir: cfg.DataDir,
|
||||
MasterKey: masterKeyBytes(cfg),
|
||||
AIDefaultModel: cfg.AIDefaultModel,
|
||||
@@ -817,7 +818,7 @@ func buildDurability(cfg *Config, log luxlog.Logger) (*Durability, func() []ha.M
|
||||
// membership_k8s.go). A single-pod deployment with no peers is its own sole writer.
|
||||
// The 2s refresh keeps a drained pod out of every peer's election within a bound the
|
||||
// terminationGracePeriod covers, so a rolling handoff loses no request.
|
||||
self := firstNonEmptyStr(strings.TrimSpace(cfg.ShardSelf), hostnameOr("cloud-0"))
|
||||
self := selfID(cfg)
|
||||
peers := parsePeers(cfg.ShardPeers)
|
||||
if len(peers) == 0 {
|
||||
peers = []org.Member{{ID: self, Addr: self}}
|
||||
@@ -935,6 +936,15 @@ func durableCipher(cfg *Config, log luxlog.Logger) *org.Cipher {
|
||||
return c
|
||||
}
|
||||
|
||||
// selfID is THIS process's stable id: the StatefulSet ordinal (CLOUD_POD_NAME /
|
||||
// POD_NAME, already resolved onto cfg.ShardSelf) falling back to the OS hostname.
|
||||
// ONE resolver — the durability membership elects on it and Deps.Self reports it, so
|
||||
// the id an operator reads in a status IS the id the ring routes by. Two resolutions
|
||||
// that drifted would name the same pod two different things at the worst moment.
|
||||
func selfID(cfg *Config) string {
|
||||
return firstNonEmptyStr(strings.TrimSpace(cfg.ShardSelf), hostnameOr("cloud-0"))
|
||||
}
|
||||
|
||||
// hostnameOr returns the OS hostname, or def when unavailable — a stable self id for
|
||||
// a single-pod deployment that sets no CLOUD_POD_NAME.
|
||||
func hostnameOr(def string) string {
|
||||
|
||||
@@ -76,7 +76,7 @@ func Mount(app cloud.Router, deps cloud.Deps) error {
|
||||
},
|
||||
}
|
||||
|
||||
routes(app, s)
|
||||
routes(app, s, deps.Self)
|
||||
|
||||
b.Log.Info("admin surface mounted",
|
||||
"prefix", "/v1/admin",
|
||||
@@ -92,7 +92,10 @@ func Mount(app cloud.Router, deps cloud.Deps) error {
|
||||
// through the two-tier gate: org-scoped panels behind core.GuardScoped, the platform
|
||||
// control plane behind core.Guard. Each carved-out domain (audit/customer/revenue/finance)
|
||||
// owns its own route registration.
|
||||
func routes(app cloud.Router, s *cloud.Service[core.State]) {
|
||||
//
|
||||
// self is this replica's id (Deps.Self), threaded through for the ONE read that reports
|
||||
// on the host itself rather than an upstream (/plugins).
|
||||
func routes(app cloud.Router, s *cloud.Service[core.State], self string) {
|
||||
g := app.Group("/v1/admin")
|
||||
// Org-scoped panels — GuardScoped. Cross-tenant reads are impossible for a non-super
|
||||
// caller.
|
||||
@@ -109,6 +112,10 @@ func routes(app cloud.Router, s *cloud.Service[core.State]) {
|
||||
g.Get("/block-storage", core.Guard(s, blockStorage))
|
||||
g.Get("/o11y", core.Guard(s, o11y))
|
||||
g.Get("/aimetrics", core.Guard(s, aimetrics))
|
||||
// What this HOST is actually running (plugins.go). The only read here that asks
|
||||
// the process rather than an upstream — so it takes the Router, whose Plugins()
|
||||
// is the read-only window onto the app. Self names the replica answering.
|
||||
g.Get("/plugins", core.Guard(s, pluginsBoard(app, self)))
|
||||
g.Post("/sync", core.Guard(s, syncNow))
|
||||
|
||||
// Credit grants — the ONE admin mint surface (SuperAdmin only). Thin, audited
|
||||
|
||||
@@ -55,7 +55,9 @@ func mountService(t *testing.T, iamURL, commerceURL, healthURL string) (func(met
|
||||
// (org-scoped panels behind GuardScoped, the platform control plane behind Guard,
|
||||
// each domain owning its own routes), so the harness stays authoritative for the
|
||||
// two-tier gate + every surface.
|
||||
routes(app, s)
|
||||
// `self` is the replica id Mount threads from Deps.Self; the harness pins it so the
|
||||
// /plugins board's Host is asserted against a known value rather than a hostname.
|
||||
routes(app, s, testSelf)
|
||||
fa := app.Fiber()
|
||||
|
||||
return func(method, path string, hdr map[string]string) (*http.Response, []byte) {
|
||||
@@ -111,6 +113,7 @@ var platformAdminRoutes = []adminRoute{
|
||||
{"POST", "/v1/admin/infra/volumes/v1/snapshot"},
|
||||
{"DELETE", "/v1/admin/infra/volumes/v1"},
|
||||
{"POST", "/v1/admin/infra/nodes/1/cordon"},
|
||||
{"GET", "/v1/admin/plugins"},
|
||||
}
|
||||
|
||||
// adminRoutes is the full surface (both tiers) — the fail-closed gate test denies an
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package admin
|
||||
|
||||
// plugins — GET /v1/admin/plugins, what this host is ACTUALLY running.
|
||||
//
|
||||
// Every other board here aggregates an upstream. This one does not: it asks the
|
||||
// process itself (zip's App.Plugins()) which subsystems it loaded out-of-binary, at
|
||||
// which artifact digest, whether they are still up, and what each costs — pid, uptime,
|
||||
// reloads, restarts, and CPU/RSS/threads/FDs read from /proc.
|
||||
//
|
||||
// It exists because a deployment manifest answers what was INTENDED. Only the process
|
||||
// knows what is TRUE, and during a rolling upgrade the two disagree BY DESIGN. The
|
||||
// version here is the artifact's SHA-256 — the one version identifier that cannot drift
|
||||
// from the bits running, because it IS the bits.
|
||||
//
|
||||
// SUPERADMIN ONLY (core.Guard, like every platform /v1/admin/* read). Artifact digests,
|
||||
// pids and resource usage are operational detail: they name what is deployed and where
|
||||
// the memory went, which is not a customer-visible fact.
|
||||
//
|
||||
// HONEST BY CONSTRUCTION, in two ways that matter:
|
||||
//
|
||||
// - THIS REPLICA, NOT THE FLEET. App.Plugins() is one process's answer. The board
|
||||
// stamps Host (Deps.Self — the same id the durability ring elects on) so a reader
|
||||
// knows which pod answered. A board that presented one pod's restarts as the
|
||||
// fleet's would be worse than no board.
|
||||
// - A HOST WITH NO PLUGINS IS AN HONEST EMPTY LIST, never an error. "Everything is
|
||||
// linked into this binary" is a true and expected answer — cloud composes a
|
||||
// linked-in service and a plugin as the SAME type, so which one a subsystem is
|
||||
// today is a Wire() decision, not a property of the code.
|
||||
//
|
||||
// Restarts is the field this board exists for. Reloads are deliberate (an operator
|
||||
// swapped the binary); Restarts is the supervisor bringing a plugin back after it died
|
||||
// on its own. Nonzero Restarts is a CRASH, and a climbing one is a crash loop. The
|
||||
// verdict is computed HERE, once, so no reader has to re-derive the policy — and no
|
||||
// two readers can disagree about what "unhealthy" means.
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/hanzoai/cloud"
|
||||
"github.com/hanzoai/cloud/clients/admin/core"
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// pluginRow is one plugin as this host currently sees it — zip.PluginStatus projected
|
||||
// onto the operator's wire contract, plus the two derived facts every reader would
|
||||
// otherwise re-derive (uptime, crashed).
|
||||
type pluginRow struct {
|
||||
Name string `json:"name"`
|
||||
// Prefixes is EVERY route subtree this plugin answers. It is the blast radius:
|
||||
// what stops answering when this plugin does.
|
||||
Prefixes []string `json:"prefixes"`
|
||||
// Source is where the binary came from: embedded | path | url | remote.
|
||||
Source string `json:"source"`
|
||||
// Version is the artifact's SHA-256 when installed from a URL — empty for the
|
||||
// other sources, which have no digest to report. Empty means "no version to
|
||||
// report", NEVER "version unknown but probably fine".
|
||||
Version string `json:"version,omitempty"`
|
||||
Addr string `json:"addr,omitempty"`
|
||||
PID int `json:"pid,omitempty"`
|
||||
// Running is false after an Unload, or after a child died with no Reload to
|
||||
// replace it. Its routes stay registered and answer 503 — so false here is the
|
||||
// difference between "not deployed" and "deployed but down".
|
||||
Running bool `json:"running"`
|
||||
// Since is when the CURRENT instance started; it resets on Reload, so UptimeSec
|
||||
// is the age of what is running, not of the mount.
|
||||
Since time.Time `json:"since,omitzero"`
|
||||
UptimeSec int64 `json:"uptimeSec"`
|
||||
// Reloads are DELIBERATE swaps. Restarts are the supervisor resurrecting a plugin
|
||||
// that died. Never fold them: one is a deploy, the other is an outage.
|
||||
Reloads int `json:"reloads"`
|
||||
Restarts int `json:"restarts"`
|
||||
// Crashed is the ONE derived verdict: Restarts > 0. Computed server-side so the
|
||||
// rule lives in exactly one place and no two readers can disagree about it.
|
||||
Crashed bool `json:"crashed"`
|
||||
|
||||
// Kernel-measured cost. Zero means NOT MEASURED (no /proc, or nothing running),
|
||||
// not measured-as-zero — which is why Running is the field to read first.
|
||||
CPUSec float64 `json:"cpuSec,omitzero"`
|
||||
RSSBytes int64 `json:"rssBytes,omitzero"`
|
||||
Threads int `json:"threads,omitzero"`
|
||||
FDs int `json:"fds,omitzero"`
|
||||
}
|
||||
|
||||
// pluginBoard is the whole read: which replica answered, when, the rollup the operator's
|
||||
// KPI band renders, and the rows.
|
||||
type pluginBoard struct {
|
||||
// Host is THIS replica's id (Deps.Self). The rows below are its answer alone.
|
||||
Host string `json:"host"`
|
||||
At time.Time `json:"at"`
|
||||
// Total/Running/Down/Crashed are the KPI band. Down counts loaded-but-not-running
|
||||
// (routes answering 503); Crashed counts plugins the supervisor has had to
|
||||
// resurrect at least once, whether or not they are up right now.
|
||||
Total int `json:"total"`
|
||||
Running int `json:"running"`
|
||||
Down int `json:"down"`
|
||||
Crashed int `json:"crashed"`
|
||||
Plugins []pluginRow `json:"plugins"`
|
||||
}
|
||||
|
||||
// pluginsBoard binds the handler to the host it reports on. Router is the ONLY thing a
|
||||
// subsystem is handed, and Plugins() is its read-only window onto the process — so the
|
||||
// board closes over the router rather than admin holding a second reference to the app.
|
||||
// Deps.Self is captured at mount: it is fixed for the life of the process.
|
||||
func pluginsBoard(app cloud.Router, self string) core.Handler {
|
||||
return func(_ *cloud.Service[core.State], c *zip.Ctx) error {
|
||||
return core.OK(c, readPlugins(app.Plugins(), self, time.Now().UTC()))
|
||||
}
|
||||
}
|
||||
|
||||
// readPlugins is the pure projection: zip's status → the board. Pure so the rollup and
|
||||
// the crash verdict are testable without a process to supervise.
|
||||
func readPlugins(ps []zip.PluginStatus, self string, now time.Time) pluginBoard {
|
||||
b := pluginBoard{Host: self, At: now, Total: len(ps), Plugins: make([]pluginRow, 0, len(ps))}
|
||||
for _, p := range ps {
|
||||
r := pluginRow{
|
||||
Name: p.Name,
|
||||
Prefixes: prefixesOf(p),
|
||||
Source: p.Source,
|
||||
Version: p.Version,
|
||||
Addr: p.Addr,
|
||||
PID: p.PID,
|
||||
Running: p.Running,
|
||||
Since: p.Since,
|
||||
Reloads: p.Reloads,
|
||||
Restarts: p.Restarts,
|
||||
Crashed: p.Restarts > 0,
|
||||
CPUSec: p.Usage.CPU.Seconds(),
|
||||
RSSBytes: p.Usage.RSS,
|
||||
Threads: p.Usage.Threads,
|
||||
FDs: p.Usage.FDs,
|
||||
}
|
||||
// Uptime is derived from Since, which resets on Reload. A zero/future Since
|
||||
// yields 0 rather than a negative or absurd age — an unknown uptime reads as
|
||||
// unknown, never as a number someone might trust.
|
||||
if !p.Since.IsZero() && now.After(p.Since) {
|
||||
r.UptimeSec = int64(now.Sub(p.Since) / time.Second)
|
||||
}
|
||||
if r.Running {
|
||||
b.Running++
|
||||
} else {
|
||||
b.Down++
|
||||
}
|
||||
if r.Crashed {
|
||||
b.Crashed++
|
||||
}
|
||||
b.Plugins = append(b.Plugins, r)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// prefixesOf returns every subtree a plugin answers. zip reports Prefixes plus Prefix
|
||||
// (the first, the one log lines name it by); older hosts carry only Prefix, so fall back
|
||||
// to it rather than reporting an empty blast radius.
|
||||
func prefixesOf(p zip.PluginStatus) []string {
|
||||
if len(p.Prefixes) > 0 {
|
||||
return p.Prefixes
|
||||
}
|
||||
if p.Prefix != "" {
|
||||
return []string{p.Prefix}
|
||||
}
|
||||
return []string{}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
// Copyright 2023-2026 Hanzo AI Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package admin
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// testSelf is the replica id the harness mounts admin with, so the board's Host is
|
||||
// asserted against a known value instead of whatever hostname the runner has.
|
||||
const testSelf = "cloud-0"
|
||||
|
||||
// decodeBoard unwraps the /v1 envelope { status, data } into the plugin board.
|
||||
func decodeBoard(t *testing.T, body []byte) pluginBoard {
|
||||
t.Helper()
|
||||
var env struct {
|
||||
Status string `json:"status"`
|
||||
Msg string `json:"msg"`
|
||||
Data pluginBoard `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &env); err != nil {
|
||||
t.Fatalf("decode plugins envelope: %v (body=%s)", err, body)
|
||||
}
|
||||
if env.Status != "ok" {
|
||||
t.Fatalf("status = %q msg=%q, want ok (body=%s)", env.Status, env.Msg, body)
|
||||
}
|
||||
return env.Data
|
||||
}
|
||||
|
||||
// TestPlugins_SuperAdminReadsHostTruth proves the route is admitted for a SuperAdmin,
|
||||
// answers the /v1 envelope, and names the replica that answered. The harness mounts no
|
||||
// plugins, so this also pins the honest-empty contract: a host that links everything in
|
||||
// returns an empty list and zeroed counts — never an error, and never a fabricated row.
|
||||
func TestPlugins_SuperAdminReadsHostTruth(t *testing.T) {
|
||||
do := mount(t, "http://127.0.0.1:0", "http://127.0.0.1:0", "http://127.0.0.1:0")
|
||||
|
||||
resp, body := do("GET", "/v1/admin/plugins", adminHdr())
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("SuperAdmin GET /v1/admin/plugins: got %d, want 200 (body=%s)", resp.StatusCode, body)
|
||||
}
|
||||
b := decodeBoard(t, body)
|
||||
if b.Host != testSelf {
|
||||
t.Errorf("Host = %q, want %q — a one-replica answer must name its replica", b.Host, testSelf)
|
||||
}
|
||||
if b.At.IsZero() {
|
||||
t.Error("At is zero: the board must date its own read")
|
||||
}
|
||||
if b.Total != 0 || b.Running != 0 || b.Down != 0 || b.Crashed != 0 {
|
||||
t.Errorf("rollup = %+v, want all zero on a host with no plugins", b)
|
||||
}
|
||||
// json.Unmarshal of `[]` yields a non-nil empty slice; of `null` it yields nil.
|
||||
// The contract is `[]` — a UI that maps over the field must never see null.
|
||||
if b.Plugins == nil {
|
||||
t.Error("plugins encoded as null; the honest empty answer is []")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPlugins_DeniedWithoutSuperAdmin proves the gate. Plugin digests, pids and RSS are
|
||||
// operational detail, so this route is SuperAdmin-only like every other platform read —
|
||||
// an anonymous caller and a validated org admin are both refused, and the org admin is
|
||||
// refused even though they pass the org-scoped tier used by /overview and friends.
|
||||
//
|
||||
// (TestGate_DeniesEveryRoute covers the same two callers across the whole surface via
|
||||
// platformAdminRoutes; this states it for THIS route, at THIS route's threat model.)
|
||||
func TestPlugins_DeniedWithoutSuperAdmin(t *testing.T) {
|
||||
do := mount(t, "http://127.0.0.1:0", "http://127.0.0.1:0", "http://127.0.0.1:0")
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
hdr map[string]string
|
||||
}{
|
||||
{"anonymous", nil},
|
||||
// A client that forges an org header still has no validated principal.
|
||||
// (A forged X-User-IsAdmin is not tested here because it cannot reach this
|
||||
// code: SanitizeIdentity strips it at ingress and re-mints it only for
|
||||
// owner == AdminOrg, so the c.IsAdmin() read the gate makes is authoritative.
|
||||
// Asserting it at this layer would test the harness, not the gate.)
|
||||
{"forged X-Org-Id, no validated user", map[string]string{"X-Org-Id": "victim"}},
|
||||
{"validated org admin of an ENABLED white-label tenant", map[string]string{
|
||||
"X-Org-Id": "maxpower", "X-User-Id": "maxpower/max",
|
||||
"X-User-Email": "max@maxpower.test", "X-User-IsOrgAdmin": "true",
|
||||
}},
|
||||
} {
|
||||
resp, body := do("GET", "/v1/admin/plugins", tc.hdr)
|
||||
if resp.StatusCode != http.StatusForbidden {
|
||||
t.Errorf("%s: got %d, want 403 (body=%s)", tc.name, resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadPlugins_ProjectsHostStatus drives the pure projection with the statuses zip
|
||||
// actually reports, and pins every field the operator board renders — including the two
|
||||
// derived ones (uptime from Since, crashed from Restarts).
|
||||
func TestReadPlugins_ProjectsHostStatus(t *testing.T) {
|
||||
now := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC)
|
||||
ps := []zip.PluginStatus{
|
||||
{
|
||||
Name: "o11y", Prefix: "/v1/o11y", Prefixes: []string{"/v1/o11y", "/v1/sentry"},
|
||||
Source: "url", Version: "cafebabe", Addr: "/run/zip/o11y.sock", PID: 4242,
|
||||
Running: true, Since: now.Add(-90 * time.Minute), Reloads: 2, Restarts: 0,
|
||||
Usage: zip.Usage{CPU: 30 * time.Second, RSS: 128 << 20, Threads: 9, FDs: 41},
|
||||
},
|
||||
{
|
||||
Name: "search", Prefix: "/v1/search", Prefixes: []string{"/v1/search"},
|
||||
Source: "embedded", Running: true, Since: now.Add(-30 * time.Second),
|
||||
Reloads: 0, Restarts: 3, PID: 4243,
|
||||
Usage: zip.Usage{CPU: 2 * time.Second, RSS: 16 << 20, Threads: 4, FDs: 12},
|
||||
},
|
||||
{
|
||||
// Unloaded, or died with no reload to replace it: its routes stay
|
||||
// registered and answer 503. "Deployed but down" is not "not deployed".
|
||||
Name: "billing", Prefix: "/v1/billing", Prefixes: []string{"/v1/billing"},
|
||||
Source: "path", Running: false, Restarts: 1,
|
||||
},
|
||||
}
|
||||
|
||||
b := readPlugins(ps, "cloud-2", now)
|
||||
|
||||
if b.Host != "cloud-2" || !b.At.Equal(now) {
|
||||
t.Fatalf("host/at = %q/%v, want cloud-2/%v", b.Host, b.At, now)
|
||||
}
|
||||
if b.Total != 3 || b.Running != 2 || b.Down != 1 || b.Crashed != 2 {
|
||||
t.Fatalf("rollup total/running/down/crashed = %d/%d/%d/%d, want 3/2/1/2",
|
||||
b.Total, b.Running, b.Down, b.Crashed)
|
||||
}
|
||||
|
||||
o := b.Plugins[0]
|
||||
if !slices.Equal(o.Prefixes, []string{"/v1/o11y", "/v1/sentry"}) {
|
||||
t.Errorf("o11y prefixes = %v, want both subtrees — the blast radius, not just the first", o.Prefixes)
|
||||
}
|
||||
if o.Version != "cafebabe" || o.Source != "url" || o.PID != 4242 || o.Addr != "/run/zip/o11y.sock" {
|
||||
t.Errorf("o11y identity mis-projected: %+v", o)
|
||||
}
|
||||
if o.UptimeSec != 5400 {
|
||||
t.Errorf("o11y uptime = %ds, want 5400 (Since resets on reload — age of what RUNS)", o.UptimeSec)
|
||||
}
|
||||
if o.CPUSec != 30 || o.RSSBytes != 128<<20 || o.Threads != 9 || o.FDs != 41 {
|
||||
t.Errorf("o11y usage mis-projected: %+v", o)
|
||||
}
|
||||
// Two reloads are DELIBERATE swaps. That is a deploy, not a crash.
|
||||
if o.Reloads != 2 || o.Crashed {
|
||||
t.Errorf("o11y reloads=%d crashed=%v — reloads must never read as a crash", o.Reloads, o.Crashed)
|
||||
}
|
||||
|
||||
s := b.Plugins[1]
|
||||
if s.Restarts != 3 || !s.Crashed {
|
||||
t.Errorf("search restarts=%d crashed=%v — a supervised resurrection IS a crash", s.Restarts, s.Crashed)
|
||||
}
|
||||
if s.Version != "" {
|
||||
t.Errorf("search version = %q; an embedded plugin has no artifact digest and must report none", s.Version)
|
||||
}
|
||||
|
||||
// Down, but still crashed: the plugin died and no reload replaced it. Counting it
|
||||
// as healthy because it is not running would hide exactly the outage that matters.
|
||||
d := b.Plugins[2]
|
||||
if d.Running || !d.Crashed || d.UptimeSec != 0 || d.PID != 0 {
|
||||
t.Errorf("billing row mis-projected: %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReadPlugins_UnknownUptimeIsZero proves an unknown or nonsensical Since never
|
||||
// becomes a number a reader might trust: a zero Since (nothing running) and a
|
||||
// future Since (clock skew) both yield 0, never a negative or absurd age.
|
||||
func TestReadPlugins_UnknownUptimeIsZero(t *testing.T) {
|
||||
now := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC)
|
||||
b := readPlugins([]zip.PluginStatus{
|
||||
{Name: "zero", Prefix: "/v1/zero"},
|
||||
{Name: "future", Prefix: "/v1/future", Running: true, Since: now.Add(time.Hour)},
|
||||
}, "cloud-0", now)
|
||||
for _, r := range b.Plugins {
|
||||
if r.UptimeSec != 0 {
|
||||
t.Errorf("%s uptime = %d, want 0 for an unknowable age", r.Name, r.UptimeSec)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPrefixesOf_NeverEmptyBlastRadius proves the fallback: a host that reports only
|
||||
// the legacy single Prefix still yields that one subtree, and a plugin with neither
|
||||
// yields [] rather than null. Reporting an empty blast radius for a plugin that
|
||||
// clearly serves something would understate exactly what an operator is deciding on.
|
||||
func TestPrefixesOf_NeverEmptyBlastRadius(t *testing.T) {
|
||||
if got := prefixesOf(zip.PluginStatus{Prefix: "/v1/only"}); !slices.Equal(got, []string{"/v1/only"}) {
|
||||
t.Errorf("legacy single-prefix status = %v, want [/v1/only]", got)
|
||||
}
|
||||
both := zip.PluginStatus{Prefix: "/v1/a", Prefixes: []string{"/v1/a", "/v1/b"}}
|
||||
if got := prefixesOf(both); !slices.Equal(got, []string{"/v1/a", "/v1/b"}) {
|
||||
t.Errorf("multi-prefix status = %v, want both", got)
|
||||
}
|
||||
if got := prefixesOf(zip.PluginStatus{}); got == nil || len(got) != 0 {
|
||||
t.Errorf("empty status = %v, want a non-nil empty slice (never null on the wire)", got)
|
||||
}
|
||||
}
|
||||
@@ -209,7 +209,7 @@ func Mount(app cloud.Router, deps cloud.Deps) error {
|
||||
return searchKeyed(c)
|
||||
})
|
||||
|
||||
scrape := zip.AdaptNetHTTPFunc(scrapeHandler)
|
||||
scrape := zip.AdaptNetHTTP(http.HandlerFunc(scrapeHandler))
|
||||
// Firecrawl builds {apiUrl}/{version}/scrape; pin firecrawlVersion:v1 so the
|
||||
// client POSTs /v1/websearch/v1/scrape. Also accept the bare /scrape.
|
||||
g.Post("/v1/scrape", scrape)
|
||||
|
||||
@@ -44,6 +44,13 @@ type Deps struct {
|
||||
// bypasses billing (every env bills against its own commerce ledger).
|
||||
Env string
|
||||
|
||||
// Self is THIS process's stable id — the StatefulSet ordinal (CLOUD_POD_NAME /
|
||||
// POD_NAME) or the OS hostname. It is the SAME id the durability membership
|
||||
// elects on (selfID), so a status a subsystem reports names the replica the ring
|
||||
// already knows by that name. Any read that is one replica's answer rather than
|
||||
// the fleet's must say WHICH replica, or "restarts: 3" is unactionable.
|
||||
Self string
|
||||
|
||||
// Domain is the deployment's primary domain (e.g. "api.hanzo.ai",
|
||||
// "api.osage.cloud"). Subsystems use this to scope URLs in responses.
|
||||
Domain string
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module github.com/hanzoai/cloud
|
||||
|
||||
go 1.26.4
|
||||
go 1.26.5
|
||||
|
||||
// Dependencies will be added as subsystems are mounted per HIP-0106.
|
||||
|
||||
@@ -43,7 +43,7 @@ require (
|
||||
github.com/zap-proto/fiber/v3 v3.2.1
|
||||
github.com/zap-proto/go v1.3.0
|
||||
github.com/zap-proto/md v0.1.0
|
||||
github.com/zap-proto/zip v1.11.0
|
||||
github.com/zap-proto/zip v1.16.1
|
||||
go.opentelemetry.io/collector/component v1.54.0
|
||||
go.opentelemetry.io/collector/confmap v1.54.0
|
||||
go.opentelemetry.io/collector/confmap/provider/envprovider v1.50.0
|
||||
@@ -111,7 +111,7 @@ require (
|
||||
github.com/sendgrid/sendgrid-go v3.16.1+incompatible // indirect
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect
|
||||
github.com/vultr/govultr/v3 v3.30.0 // indirect
|
||||
github.com/zap-proto/http v0.3.0 // indirect
|
||||
github.com/zap-proto/http v0.3.1 // indirect
|
||||
github.com/zap-proto/zap2pb v0.2.0 // indirect
|
||||
go.mongodb.org/mongo-driver v1.17.9 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect
|
||||
@@ -738,7 +738,7 @@ require (
|
||||
github.com/hanzoai/base v1.5.7
|
||||
github.com/hanzoai/licensing v0.1.5
|
||||
github.com/hanzoai/metrics v1.110.2
|
||||
github.com/hanzoai/o11y v1.5.32
|
||||
github.com/hanzoai/o11y v1.5.33
|
||||
github.com/hanzoai/thinking v0.1.1 // indirect
|
||||
github.com/hanzoai/vfs v0.6.6
|
||||
github.com/hanzoai/zen v1.4.4
|
||||
|
||||
@@ -1074,6 +1074,8 @@ github.com/hanzoai/notify v1.6.18 h1:YLIKheJSMhGqRuo7NRsMicHjAWSVFV6j6ZGqm2H+IBM
|
||||
github.com/hanzoai/notify v1.6.18/go.mod h1:O8OZj1cfUAIY39ROTPpiaVH8jv947VNfAGor2AZ/ebQ=
|
||||
github.com/hanzoai/o11y v1.5.32 h1:XI9LEjNuzAsg27s5zNCTx71Rhw1pCv8gRy/ILvPiY0s=
|
||||
github.com/hanzoai/o11y v1.5.32/go.mod h1:tsrIwTJJEEy+AnOGXLM4RSrkhUyM1ja+bkstzFkc0Hw=
|
||||
github.com/hanzoai/o11y v1.5.33 h1:99NO2O/MB5czh1dsqGPaCbk1Lyf+QeFy1VJ1cc+PppI=
|
||||
github.com/hanzoai/o11y v1.5.33/go.mod h1:bdqSdqagTAQfEliZttOujk5VjR41OwUBn1yrCWx06wE=
|
||||
github.com/hanzoai/orm v0.6.16 h1:w3UXH65huahNJ8RgC88ffUeicAbHoUpQW8oLuDCojK8=
|
||||
github.com/hanzoai/orm v0.6.16/go.mod h1:KpbP5UwQ8BBNGVM3tku9rgs7PADB+UG8fqh8Nol0X/s=
|
||||
github.com/hanzoai/otel-collector v1.2.0 h1:lBDL5lKotq89JaqchcM+/oxEnjrjazEI2JJfDhotudc=
|
||||
@@ -2071,12 +2073,16 @@ github.com/zap-proto/go v1.3.0 h1:S3rMoawwhH/BbSZ4G8zG05hJoQnMSMDPzIq75diCTqE=
|
||||
github.com/zap-proto/go v1.3.0/go.mod h1:914SNGTH6Rv3Yu1MweWJBPEN8FZlo5C39QyhaB0C7Q0=
|
||||
github.com/zap-proto/http v0.3.0 h1:l7DvlngiYqmzNY6fzyRYw2ZIAhF35FqwOe6mvAOqpMg=
|
||||
github.com/zap-proto/http v0.3.0/go.mod h1:UYfGhDDCetgxs65XSev8Lpf65COg5vKQK+cWwZGh4zQ=
|
||||
github.com/zap-proto/http v0.3.1 h1:A2rCPWYCX866eAsdiWuns0dvWnBmViZtGm4pwX7jwlY=
|
||||
github.com/zap-proto/http v0.3.1/go.mod h1:UYfGhDDCetgxs65XSev8Lpf65COg5vKQK+cWwZGh4zQ=
|
||||
github.com/zap-proto/md v0.1.0 h1:1R6w/i1FYAdGIIiOvNggKO0RjikzhWWRodQUOgzEEpc=
|
||||
github.com/zap-proto/md v0.1.0/go.mod h1:pmMx2F4Dwj1H48PIuLzRZxB2R5qVvqeg8ZScKQIntyQ=
|
||||
github.com/zap-proto/zap2pb v0.2.0 h1:sos6HnayhGMGLRO54px1InzimDzTZ2o5TSMEatYBjzs=
|
||||
github.com/zap-proto/zap2pb v0.2.0/go.mod h1:wD97Z2VTPabDq/4AMNL++PWnQ0YwEtajiuNkLGg3/18=
|
||||
github.com/zap-proto/zip v1.11.0 h1:jUke0MbopJECzJMOeaAq+oM0G+EF8UyWCr41+Vl/J2Q=
|
||||
github.com/zap-proto/zip v1.11.0/go.mod h1:9R3FOq2ItZa7G+9QilsB/punEpVSHtdXiQji0P84LSE=
|
||||
github.com/zap-proto/zip v1.16.1 h1:CBKN1zTxyzU4qS0XFPn9jJdduvBwcvcs8xt+oFx2T4w=
|
||||
github.com/zap-proto/zip v1.16.1/go.mod h1:BxFNqjnAVhArMJ+s7VnXdwAtfOVy47GAi62wwqyu8go=
|
||||
github.com/zeebo/assert v1.3.1 h1:vukIABvugfNMZMQO1ABsyQDJDTVQbn+LWSMy1ol1h6A=
|
||||
github.com/zeebo/assert v1.3.1/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI=
|
||||
|
||||
+1
-1
@@ -37,7 +37,7 @@ func PluginSpec(name, prefix string, p zip.Plugin) MountSpec {
|
||||
if !ok {
|
||||
return fmt.Errorf("pluginspec %q: needs the root app, got %T — Global must stay set", name, router)
|
||||
}
|
||||
return zip.Load(prefix, p)(app)
|
||||
return zip.Load(p, prefix)(app)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,16 +28,24 @@ import (
|
||||
"github.com/zap-proto/zip"
|
||||
)
|
||||
|
||||
// Router is the surface a subsystem mounts on: zip's routing methods, plus the
|
||||
// *fiber.App escape the in-process dispatchers need (fiber.Test, GetRoutes,
|
||||
// adaptor.FiberApp). *zip.App satisfies it as-is, so Serve can hand the bare app
|
||||
// to a Global subsystem and tests can pass a raw app.
|
||||
// Router is the surface a subsystem mounts on: zip's routing methods, plus two
|
||||
// named, read-only holes onto the host — the *fiber.App the in-process
|
||||
// dispatchers need (fiber.Test, GetRoutes, adaptor.FiberApp) and Plugins(), what
|
||||
// this process is actually running. *zip.App satisfies it as-is, so Serve can
|
||||
// hand the bare app to a Global subsystem and tests can pass a raw app.
|
||||
//
|
||||
// Fiber() is a deliberate, named hole: it is the concrete engine, and middleware
|
||||
// installed through it is app-wide. It is promoted onto the scoped Router rather
|
||||
// than granting those subsystems Global — the alternative was four more Globals for
|
||||
// four read-only uses. Its callers are greppable and none of them registers
|
||||
// middleware.
|
||||
//
|
||||
// Plugins() is promoted for the same reason and is strictly weaker: it registers
|
||||
// nothing and mutates nothing. It exists because the ONE thing config cannot tell
|
||||
// you is what a live process actually loaded — deployment manifests answer what was
|
||||
// INTENDED, and during a rolling upgrade the two disagree by design. The admin
|
||||
// fleet board (/v1/admin/plugins) is the reader; making it Global to ask a
|
||||
// read-only question would have granted app-wide middleware to buy a status field.
|
||||
type Router interface {
|
||||
Use(handlers ...zip.Handler) zip.Router
|
||||
|
||||
@@ -53,6 +61,12 @@ type Router interface {
|
||||
Group(prefix string, handlers ...zip.Handler) zip.Router
|
||||
|
||||
Fiber() *fiber.App
|
||||
|
||||
// Plugins reports every plugin this HOST has loaded — name, prefixes, source,
|
||||
// artifact digest, pid, running, uptime, reloads, restarts and kernel-measured
|
||||
// usage. It is this replica's answer, never the fleet's: a reader that presents
|
||||
// it as fleet-wide is lying about the other pods.
|
||||
Plugins() []zip.PluginStatus
|
||||
}
|
||||
|
||||
// scope is the Router a non-Global subsystem mounts on. It holds the subsystem's
|
||||
@@ -121,6 +135,7 @@ func (s *scope) Head(p string, h ...zip.Handler) zip.Router { return s.app.He
|
||||
func (s *scope) Options(p string, h ...zip.Handler) zip.Router { return s.app.Options(p, h...) }
|
||||
func (s *scope) All(p string, h ...zip.Handler) zip.Router { return s.app.All(p, h...) }
|
||||
func (s *scope) Fiber() *fiber.App { return s.app.Fiber() }
|
||||
func (s *scope) Plugins() []zip.PluginStatus { return s.app.Plugins() }
|
||||
|
||||
// err reports the middleware the subsystem tried to install outside its prefixes.
|
||||
func (s *scope) err() error {
|
||||
|
||||
@@ -196,3 +196,36 @@ func TestGlobalMountNeedsTheGlobalFlag(t *testing.T) {
|
||||
t.Fatal("MountAll succeeded — cloud.Global ran without Global: true")
|
||||
}
|
||||
}
|
||||
|
||||
// TestScopedRouterReportsHostPlugins pins the read-only hole Plugins() opens. A
|
||||
// non-Global subsystem is handed a scope, not the app — so without this passthrough
|
||||
// the admin fleet board (/v1/admin/plugins) would have had to take Global, granting
|
||||
// app-wide middleware to buy a status field. The scope reports the host's plugins
|
||||
// verbatim: same names, same count, no filtering by the subsystem's own prefixes.
|
||||
func TestScopedRouterReportsHostPlugins(t *testing.T) {
|
||||
app := newApp()
|
||||
if err := app.Add(zip.Load(
|
||||
zip.Plugin{Name: "remote", Addr: "127.0.0.1:1"}, "/v1/remote", "/v1/legacy",
|
||||
)); err != nil {
|
||||
t.Fatalf("Add(Load): %v", err)
|
||||
}
|
||||
|
||||
var seen []zip.PluginStatus
|
||||
err := mountAll(t, app, []cloud.MountSpec{
|
||||
{Name: "board", Mount: func(r cloud.Router, _ cloud.Deps) error {
|
||||
seen = r.Plugins() // a scope, not the bare app
|
||||
return nil
|
||||
}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("MountAll: %v", err)
|
||||
}
|
||||
if len(seen) != 1 || seen[0].Name != "remote" {
|
||||
t.Fatalf("scoped Plugins() = %+v, want the host's one plugin", seen)
|
||||
}
|
||||
// The whole surface, not the subsystem's own slice of it: a board that reported
|
||||
// only prefixes its own subsystem owns would report nothing at all.
|
||||
if len(seen[0].Prefixes) != 2 {
|
||||
t.Errorf("prefixes = %v, want both subtrees the host mounted", seen[0].Prefixes)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user