Files
ai/object/provider_state_test.go
zeekayandHanzo Dev 3534c5252b refactor(kms): read secrets from the EMBEDDED store in-process, not over HTTP to a dead standalone
ai runs inside the unified cloud binary, which embeds luxfi/kms in-process
(hanzoai/cloud apps/kms). It was already being handed that client as deps.KMS
and throwing it away, then dialling the STANDALONE KMS over HTTP to read a value
one map lookup away.

That client had never worked in production. Measured from the running pod:

  $KMS_ENDPOINT/v1/kms/auth/login  ->  404   (KMS_ENDPOINT carries an /api prefix,
                                              so the call went to /api/v1/kms/...)
  https://kms.hanzo.ai/v1/kms/auth/login -> 401   (correct path, credentials rejected)

Both doors shut — and the deployment it aimed at reports "secrets plane disabled"
at boot with its ZAP port closed, while the embedded KMS in THIS binary answers
{"ready":true}.

Nothing noticed because resolution fell back to the environment and every working
provider key happened to be an env var from a K8s Secret. So every kms:// ref was
decorative: it meant "read an env var". That is why a key can be stored in KMS and
still be unavailable to the service that needs it.

Order is now KMS-FIRST, env-fallback. A non-empty value from the store wins;
absent/empty/error — or no store bound at all (standalone cmd/aid) — falls through
to the env var, so the providers still env-fed keep serving while their keys
migrate. When the last key has moved, the fallback goes.

An unresolvable reference is now an ERROR rather than a pass-through: returning
"kms://NAME" verbatim authenticates upstream AS that string, a guaranteed 401 that
also puts the reference on the wire. unsealConnectionKey absorbs it to "" because
for a connections read "cannot unseal" and "not connected" are the same
user-visible fact — logged, since a store that cannot open a sealed key is an
operator problem even when the UI degrades gracefully.

Drops 186 lines: the HTTP client, its token cache, universal-auth login, and the
KMS_ENDPOINT/KMS_CLIENT_ID/KMS_SERVICE_TOKEN/KMS_PROJECT_ID env surface. No
hostname, no /api, no second KMS.

9 tests cover the seam (store-beats-env, env fallback, store-down non-fatal,
unresolvable never emits the ref, BYOK round trip, keyPresent agrees with
resolution, caching). NOTE: object's TestMain exits 0 without a seeded database,
so these run in CI and are skipped locally — verified here by bypassing that gate
temporarily; all 9 pass.

Co-authored-by: Hanzo Dev <[email protected]>
2026-08-03 17:28:39 -07:00

173 lines
7.1 KiB
Go

// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package object
import (
"testing"
"time"
)
// ── ModelProviderUsable: the single-point routing State gate ────────────────
//
// This is also the LOW-2 predicate: SetPrimaryAdminProvider rejects making a
// provider primary unless ModelProviderUsable(provider) is true (State=="Active"),
// so a disabled provider can never become primary. The "disabled model → false"
// and "paused model → false" cases below are that guard's regression coverage.
func TestModelProviderUsable(t *testing.T) {
cases := []struct {
name string
p *Provider
want bool
}{
{"nil provider", nil, false},
{"active model", &Provider{Category: "Model", State: "Active"}, true},
{"disabled model", &Provider{Category: "Model", State: "Disabled"}, false},
{"paused model (empty state)", &Provider{Category: "Model", State: ""}, false},
{"arbitrary non-active state", &Provider{Category: "Model", State: "Paused"}, false},
// Non-Model providers are not governed by this gate — always usable here.
{"storage provider ignores state", &Provider{Category: "Storage", State: "Disabled"}, true},
{"embedding provider ignores state", &Provider{Category: "Embedding", State: ""}, true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := ModelProviderUsable(tc.p); got != tc.want {
t.Errorf("ModelProviderUsable(%+v) = %v, want %v", tc.p, got, tc.want)
}
})
}
}
// ── ProviderKeyPresent: boolean-only key signal, never leaks the value ──────
func TestProviderKeyPresent_PlaintextValue(t *testing.T) {
// A non-kms (dev) plaintext secret is reported present when non-empty. This
// path does not touch KMS.
p := &Provider{Name: "dev", Category: "Model", ClientSecret: "sk-plaintext-dev-key"}
if !ProviderKeyPresent(p) {
t.Error("ProviderKeyPresent = false for a non-empty plaintext secret, want true")
}
// The helper must NOT mutate the caller's provider (no resolved/masked value
// bleeding back into a response object).
if p.ClientSecret != "sk-plaintext-dev-key" {
t.Errorf("ProviderKeyPresent mutated ClientSecret to %q, want it unchanged", p.ClientSecret)
}
}
func TestProviderKeyPresent_EmptyAbsent(t *testing.T) {
if ProviderKeyPresent(&Provider{Name: "empty", Category: "Model", ClientSecret: ""}) {
t.Error("ProviderKeyPresent = true for an empty secret, want false")
}
if ProviderKeyPresent(nil) {
t.Error("ProviderKeyPresent(nil) = true, want false")
}
}
func TestProviderKeyPresent_KmsRefWithoutKMSIsAbsent(t *testing.T) {
// With no secret store bound (the unit env binds none) and no matching env
// var, a "kms://…" reference cannot be confirmed and is fail-closed reported
// as NOT present — the operator sees the key is unavailable rather than a
// false positive. It also must not leak the ref.
p := &Provider{Name: "do-ai", Category: "Model", ClientSecret: "kms://DO_AI_API_KEY_TEST_ABSENT"}
if ProviderKeyPresent(p) {
t.Error("ProviderKeyPresent = true for an unresolved kms:// ref (KMS disabled), want false")
}
if p.ClientSecret != "kms://DO_AI_API_KEY_TEST_ABSENT" {
t.Errorf("ProviderKeyPresent mutated the kms:// ref to %q, want unchanged", p.ClientSecret)
}
}
// ── InvalidateProviderNameCache: toggles take effect immediately ────────────
func TestInvalidateProviderNameCache_ByName(t *testing.T) {
// Seed the hot-path caches directly (no DB) and confirm a named invalidation
// drops the global entry AND every per-org BYOK entry for that name, while
// leaving unrelated providers cached.
providerByNameCacheMu.Lock()
providerByNameCache["do-ai"] = &providerByNameEntry{provider: &Provider{Name: "do-ai"}, fetchedAt: time.Now()}
providerByNameCache["fireworks"] = &providerByNameEntry{provider: &Provider{Name: "fireworks"}, fetchedAt: time.Now()}
providerByNameCacheMu.Unlock()
orgProviderCacheMu.Lock()
orgProviderCache["maxpower/do-ai"] = &providerByNameEntry{provider: &Provider{Name: "do-ai"}, fetchedAt: time.Now()}
orgProviderCache["acme/do-ai"] = &providerByNameEntry{provider: &Provider{Name: "do-ai"}, fetchedAt: time.Now()}
orgProviderCache["maxpower/fireworks"] = &providerByNameEntry{provider: &Provider{Name: "fireworks"}, fetchedAt: time.Now()}
orgProviderCacheMu.Unlock()
InvalidateProviderNameCache("do-ai")
providerByNameCacheMu.RLock()
_, doai := providerByNameCache["do-ai"]
_, fw := providerByNameCache["fireworks"]
providerByNameCacheMu.RUnlock()
if doai {
t.Error("global do-ai entry should be evicted")
}
if !fw {
t.Error("unrelated fireworks entry should remain cached")
}
orgProviderCacheMu.RLock()
_, mpDoai := orgProviderCache["maxpower/do-ai"]
_, acmeDoai := orgProviderCache["acme/do-ai"]
_, mpFw := orgProviderCache["maxpower/fireworks"]
orgProviderCacheMu.RUnlock()
if mpDoai || acmeDoai {
t.Error("all per-org do-ai overrides should be evicted")
}
if !mpFw {
t.Error("unrelated per-org fireworks override should remain cached")
}
// Cleanup so we don't leak state into other tests.
InvalidateProviderNameCache("")
}
func TestInvalidateProviderNameCache_FlushAll(t *testing.T) {
providerByNameCacheMu.Lock()
providerByNameCache["do-ai"] = &providerByNameEntry{provider: &Provider{Name: "do-ai"}, fetchedAt: time.Now()}
providerByNameCacheMu.Unlock()
orgProviderCacheMu.Lock()
orgProviderCache["maxpower/do-ai"] = &providerByNameEntry{provider: &Provider{Name: "do-ai"}, fetchedAt: time.Now()}
orgProviderCacheMu.Unlock()
InvalidateProviderNameCache("") // flush everything
providerByNameCacheMu.RLock()
nGlobal := len(providerByNameCache)
providerByNameCacheMu.RUnlock()
orgProviderCacheMu.RLock()
nOrg := len(orgProviderCache)
orgProviderCacheMu.RUnlock()
if nGlobal != 0 || nOrg != 0 {
t.Errorf("flush-all left entries: global=%d org=%d, want 0/0", nGlobal, nOrg)
}
}
func TestProviderKeyPresent_KmsRefResolvedFromEnv(t *testing.T) {
// ResolveProviderSecret is env-var-first: a matching env var makes a "kms://…"
// reference resolve to a real value → present. (This is the working prod hot
// path — no live KMS dependency.) The env var name is the kms:// suffix.
t.Setenv("DO_AI_API_KEY_TEST_PRESENT", "resolved-secret-value")
p := &Provider{Name: "do-ai", Category: "Model", ClientSecret: "kms://DO_AI_API_KEY_TEST_PRESENT"}
if !ProviderKeyPresent(p) {
t.Error("ProviderKeyPresent = false for a kms:// ref resolvable via env, want true")
}
// Caller's record is untouched — the resolved value must never persist onto it.
if p.ClientSecret != "kms://DO_AI_API_KEY_TEST_PRESENT" {
t.Errorf("ProviderKeyPresent leaked the resolved value onto the record: %q", p.ClientSecret)
}
}