mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
The signing cert every bearer is validated against was fetched in a package init(), over the network, and the process panicked when it could not be read. That was the right answer to the fail-open it replaced — warn-and-serve left the service Running, green, and answering 401 to everything including free routes for a day — and the wrong answer to everything else. It has now taken production down twice. 2026-07-27: a one-word manifest typo killed IAM's pods. 2026-08-02: an IAM build answered no /healthz, so its Service held zero endpoints for 25 minutes. Both times the blast radius was every route in the process, and both times it could not recover once IAM came back — the dial had already happened, before any subsystem mounted, and the only way forward was a restart the crashloop kept undoing. Three behaviours are possible and only one is right. Serving without a cert is fail-open. Dying without a cert makes identity a hard, boot-ordered dependency of everything. So: resolve LAZILY, retry the failure, and fail CLOSED at the request. Unresolved is NOT "no principal" — that is precisely the 401-everything fail-open, and at a token-parsing call site the two are indistinguishable. It is 503 at the door, in ONE filter ahead of every filter that reads a token, so no downstream code ever has to tell them apart. 503 rather than 401 because 401 blames the caller for our outage, invites a client to discard a good token, and reads as ordinary traffic on every dashboard. The retry is rate-limited to one attempt per 5s per process. Without that, every request during an IAM outage becomes an IAM request, and the recovering service is met with the full inbound rate the moment it starts answering. The security property is unchanged: a request is never served with authentication silently switched off. What changes is the cost of an IAM blip — the requests during it, and the first request after IAM returns re-establishes the cert with no restart. The fail-closed tests move with the code rather than being deleted; their fixtures are responses captured from the live IAM and are still the point. The one asserting a panic is replaced by one asserting the failure is REPORTED and stays reported — what must hold is that it is never swallowed, not that it is fatal. Mutation-checked: failing open, dropping the rate limit, re-fetching a resolved cert, and answering 401 instead of 503 each fail a test. The memoization test did NOT fail at first — the retry window was answering for it — so it now asserts on the far side of that window. Co-authored-by: Hanzo Dev <[email protected]>
147 lines
5.8 KiB
Go
147 lines
5.8 KiB
Go
// Copyright 2026 Hanzo AI Inc. All Rights Reserved.
|
|
|
|
package object
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
iam "github.com/hanzoai/ai/internal/iam"
|
|
|
|
"github.com/hanzoai/ai/conf"
|
|
)
|
|
|
|
// Establishing the IAM signing cert every bearer token is validated against.
|
|
//
|
|
// THREE behaviours are possible here and only one of them is right.
|
|
//
|
|
// It used to WARN AND SERVE. That is a fail-open on the security boundary of the
|
|
// whole platform, and it cost a full day: IAM tightened authorization on the
|
|
// application read, GetApplication began answering 403, this printed "(auth
|
|
// features disabled)" and the process served anyway. With no cert every bearer
|
|
// failed to validate, so EVERY authenticated route answered 401 — including free
|
|
// ones. The pod stayed Running and passed its probes, so nothing caught it.
|
|
//
|
|
// It then PANICKED IN AN init(). That fixed the fail-open and bought a worse
|
|
// failure: the cert was fetched over the network before any subsystem mounted, so
|
|
// a momentary IAM outage was a hard, self-heal-less outage of everything in the
|
|
// process. It has now happened twice in production — 2026-07-27 (a one-word
|
|
// manifest typo took IAM's pods down) and 2026-08-02 (an IAM build answered no
|
|
// /healthz, so its Service held zero endpoints for 25 minutes). Both times the
|
|
// blast radius was the entire API, and both times the process could not recover on
|
|
// its own after IAM came back.
|
|
//
|
|
// So: resolve it LAZILY, retry a failure, and fail CLOSED at the request.
|
|
// Unresolved is not "no principal" — that is precisely the 401-everything
|
|
// fail-open — it is 503 SERVICE UNAVAILABLE at the door (AuthAvailableFilter),
|
|
// before any filter reads a token. The security property is unchanged: a request
|
|
// is never served with authentication silently switched off. What changes is that
|
|
// an IAM blip costs the requests during the blip instead of the process, and the
|
|
// first request after IAM returns re-establishes the cert with no restart.
|
|
//
|
|
// A deployment with no IAM_URL is a legitimate no-auth configuration (dev, tests)
|
|
// and resolves cleanly. That is the ONLY path that leaves auth unconfigured.
|
|
|
|
// retryAfter bounds how often an unresolved cert is re-fetched. Without it, every
|
|
// request during an IAM outage becomes an IAM request, so the recovering service
|
|
// is met with the full inbound rate the moment it starts answering — the retry
|
|
// storm that turns a blip into an outage. One attempt per interval per process is
|
|
// enough to recover within a probe cycle.
|
|
const retryAfter = 5 * time.Second
|
|
|
|
var authCfg struct {
|
|
mu sync.Mutex
|
|
ready bool // the cert is established; never re-fetched
|
|
lastErr error // why the last attempt failed, for the 503 body
|
|
lastTry time.Time // when that attempt was, for retryAfter
|
|
attempts int // attempts since process start, for the log line
|
|
}
|
|
|
|
// AuthReady reports whether this process can validate a bearer token, resolving
|
|
// the signing cert on first use and retrying a previous failure at most once per
|
|
// retryAfter.
|
|
//
|
|
// nil means either "the cert is established" or "this deployment runs no IAM".
|
|
// A non-nil error means requests that carry authentication MUST be refused with
|
|
// 503 — never served, and never treated as anonymous.
|
|
func AuthReady() error {
|
|
authCfg.mu.Lock()
|
|
defer authCfg.mu.Unlock()
|
|
if authCfg.ready {
|
|
return nil
|
|
}
|
|
// A failure inside the retry window answers with the SAME error rather than a
|
|
// fresh attempt, so the 503 a caller sees names the real cause for the whole
|
|
// window instead of alternating between causes.
|
|
if !authCfg.lastTry.IsZero() && time.Since(authCfg.lastTry) < retryAfter {
|
|
return authCfg.lastErr
|
|
}
|
|
authCfg.lastTry = time.Now()
|
|
authCfg.attempts++
|
|
if err := resolveAuthConfig(); err != nil {
|
|
authCfg.lastErr = err
|
|
return err
|
|
}
|
|
authCfg.ready = true
|
|
authCfg.lastErr = nil
|
|
return nil
|
|
}
|
|
|
|
// ResetAuthReady drops the resolved state so the next AuthReady re-resolves. It
|
|
// exists for tests, which run several configurations in one process.
|
|
func ResetAuthReady() {
|
|
authCfg.mu.Lock()
|
|
defer authCfg.mu.Unlock()
|
|
authCfg.ready, authCfg.lastErr, authCfg.lastTry, authCfg.attempts = false, nil, time.Time{}, 0
|
|
}
|
|
|
|
// AuthAttempts reports how many times the cert has been fetched since this process
|
|
// started. A test asserts the retry window holds by watching it not move.
|
|
func AuthAttempts() int {
|
|
authCfg.mu.Lock()
|
|
defer authCfg.mu.Unlock()
|
|
return authCfg.attempts
|
|
}
|
|
|
|
// resolveAuthConfig performs one attempt: read the application, read its cert, and
|
|
// install the cert on the IAM SDK's client. Every failure is returned rather than
|
|
// logged-and-swallowed, so the caller decides what it costs.
|
|
func resolveAuthConfig() error {
|
|
endpoint := conf.GetConfigString("IAM_URL")
|
|
clientID := conf.GetConfigString("IAM_CLIENT_ID")
|
|
clientSecret := conf.GetConfigString("IAM_CLIENT_SECRET")
|
|
organization := conf.GetConfigString("IAM_ORG")
|
|
application := conf.GetConfigString("IAM_APP_NAME")
|
|
|
|
// No IAM configured: this deployment is deliberately running without auth.
|
|
if endpoint == "" {
|
|
return nil
|
|
}
|
|
|
|
iam.InitConfig(endpoint, clientID, clientSecret, "", organization, application)
|
|
app, err := iam.GetApplication(application)
|
|
if err != nil {
|
|
return fmt.Errorf("read IAM application %q from %s: %w", application, endpoint, err)
|
|
}
|
|
if app == nil {
|
|
return fmt.Errorf("IAM application %q does not exist at %s", application, endpoint)
|
|
}
|
|
|
|
cert, err := iam.GetCert(app.Cert)
|
|
if err != nil {
|
|
return fmt.Errorf("read cert %q for application %q: %w", app.Cert, application, err)
|
|
}
|
|
if cert == nil {
|
|
return fmt.Errorf("cert %q for application %q does not exist", app.Cert, application)
|
|
}
|
|
if strings.TrimSpace(cert.Certificate) == "" {
|
|
return fmt.Errorf("cert %q for application %q is empty — every bearer would fail to validate",
|
|
app.Cert, application)
|
|
}
|
|
|
|
iam.InitConfig(endpoint, clientID, clientSecret, cert.Certificate, organization, application)
|
|
return nil
|
|
}
|