Files
z fa658d41e3 openrouter: the pool also walks OPENROUTER_API_KEYS
ai tried only OPENROUTER_API_KEY and _2 through _8. zen and enso walk the list
OPENROUTER_API_KEYS, so when every numbered account ran dry ai answered 503 on
every OpenRouter model while enso kept serving from a funded account only the
list held. The list now follows the numbered names, a key already named is tried
once, and a list account is named by its line (OPENROUTER_API_KEYS#n).
2026-10-18 10:44:15 -07:00

328 lines
12 KiB
Go

// Copyright 2026 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package object
import (
"context"
"errors"
"strings"
"testing"
)
// fakeStore is an in-memory SecretStore — the same shape cloud hands over.
type fakeStore struct {
vals map[string]string
err error
gets int
}
func (f *fakeStore) GetSecret(_ context.Context, ref string) ([]byte, error) {
f.gets++
if f.err != nil {
return nil, f.err
}
v, ok := f.vals[ref]
if !ok {
return nil, nil
}
return []byte(v), nil
}
func (f *fakeStore) PutSecret(_ context.Context, ref string, value []byte) error {
if f.err != nil {
return f.err
}
if f.vals == nil {
f.vals = map[string]string{}
}
f.vals[ref] = string(value)
return nil
}
// bind installs a store and clears the read cache, which is process-global.
func bind(t *testing.T, s SecretStore) {
t.Helper()
SetSecretStore(s)
kmsSecMu.Lock()
kmsSecrets = map[string]*kmsSecretEntry{}
kmsSecMu.Unlock()
t.Cleanup(func() {
SetSecretStore(nil)
kmsSecMu.Lock()
kmsSecrets = map[string]*kmsSecretEntry{}
kmsSecMu.Unlock()
})
}
// TestResolve_StoreBeatsEnv is the ordering that matters: the embedded KMS is
// authoritative. The env var is the LEGACY K8s-Secret path being retired, so a
// key present in both must resolve to the store's value — otherwise migrating a
// secret into KMS would silently change nothing.
func TestResolve_StoreBeatsEnv(t *testing.T) {
bind(t, &fakeStore{vals: map[string]string{"OPENROUTER_API_KEY": "from-kms"}})
t.Setenv("OPENROUTER_API_KEY", "from-env")
p := &Provider{Name: "openrouter", ClientSecret: "kms://OPENROUTER_API_KEY"}
if err := ResolveProviderSecret(p); err != nil {
t.Fatalf("resolve: %v", err)
}
if p.ClientSecret != "from-kms" {
t.Errorf("ClientSecret = %q, want the STORE value %q", p.ClientSecret, "from-kms")
}
}
// TestResolve_FallsBackToEnv: a provider whose key has not migrated yet keeps
// serving. Removing this fallback before every key is in KMS would take the
// env-fed providers (do-ai, openai-direct, anthropic, fireworks) offline.
func TestResolve_FallsBackToEnv(t *testing.T) {
bind(t, &fakeStore{vals: map[string]string{}})
t.Setenv("DO_AI_API_KEY", "env-only")
p := &Provider{Name: "do-ai", ClientSecret: "kms://DO_AI_API_KEY"}
if err := ResolveProviderSecret(p); err != nil {
t.Fatalf("resolve: %v", err)
}
if p.ClientSecret != "env-only" {
t.Errorf("ClientSecret = %q, want %q", p.ClientSecret, "env-only")
}
}
// TestResolve_StoreErrorFallsBackNotFatal: a store that is down must not take
// down providers whose keys are still env-fed.
func TestResolve_StoreErrorFallsBackNotFatal(t *testing.T) {
bind(t, &fakeStore{err: errors.New("store down")})
t.Setenv("DO_AI_API_KEY", "env-only")
p := &Provider{Name: "do-ai", ClientSecret: "kms://DO_AI_API_KEY"}
if err := ResolveProviderSecret(p); err != nil {
t.Fatalf("resolve: %v", err)
}
if p.ClientSecret != "env-only" {
t.Errorf("ClientSecret = %q, want %q", p.ClientSecret, "env-only")
}
}
// TestResolve_UnresolvableRefErrorsAndNeverEmitsTheRef is the invariant that
// keeps a reference off the wire. Returning "kms://NAME" verbatim would
// authenticate upstream AS that string — a guaranteed 401 that also discloses
// the reference.
func TestResolve_UnresolvableRefErrorsAndNeverEmitsTheRef(t *testing.T) {
bind(t, &fakeStore{vals: map[string]string{}})
p := &Provider{Name: "openrouter", ClientSecret: "kms://NOPE_ABSENT"}
err := ResolveProviderSecret(p)
if err == nil {
t.Fatal("want an error for an unresolvable reference")
}
if strings.Contains(err.Error(), "kms://") {
t.Errorf("error echoes the ref scheme: %v", err)
}
if p.ClientSecret != "kms://NOPE_ABSENT" {
t.Errorf("source mutated on failure: %q", p.ClientSecret)
}
}
// TestResolve_NonRefUntouched: a key set directly on the admin row is a
// deliberate operator choice and is passed through unchanged.
func TestResolve_NonRefUntouched(t *testing.T) {
bind(t, &fakeStore{vals: map[string]string{}})
p := &Provider{Name: "openrouter", ClientSecret: "sk-or-v1-literal"}
if err := ResolveProviderSecret(p); err != nil {
t.Fatalf("resolve: %v", err)
}
if p.ClientSecret != "sk-or-v1-literal" {
t.Errorf("ClientSecret = %q, want unchanged", p.ClientSecret)
}
}
// TestStoreProviderSecret_FailsClosedWithoutStore: with nothing bound, sealing
// must ERROR rather than let the caller persist a raw key.
func TestStoreProviderSecret_FailsClosedWithoutStore(t *testing.T) {
bind(t, nil)
if _, err := StoreProviderSecret("byok_org_openai", "sk-secret"); err == nil {
t.Fatal("want an error with no store bound (never store plaintext)")
}
}
// TestStoreProviderSecret_RoundTrip: seal returns the ref, and resolving that
// ref returns the value — the BYOK loop, entirely in-process.
func TestStoreProviderSecret_RoundTrip(t *testing.T) {
fs := &fakeStore{vals: map[string]string{}}
bind(t, fs)
ref, err := StoreProviderSecret("byok_acme_openai", "sk-acme")
if err != nil {
t.Fatalf("store: %v", err)
}
if ref != "kms://byok_acme_openai" {
t.Fatalf("ref = %q", ref)
}
p := &Provider{Name: "openai", ClientSecret: ref}
if err := ResolveProviderSecret(p); err != nil {
t.Fatalf("resolve: %v", err)
}
if p.ClientSecret != "sk-acme" {
t.Errorf("round trip = %q, want %q", p.ClientSecret, "sk-acme")
}
}
// TestKeyPresent_AgreesWithResolution: the admin view's keyPresent must not
// claim a key the completion path would fail to find, so both go through the
// one precedence rule.
func TestKeyPresent_AgreesWithResolution(t *testing.T) {
bind(t, &fakeStore{vals: map[string]string{"HAVE_IT": "v"}})
if !ProviderKeyPresent(&Provider{Name: "a", ClientSecret: "kms://HAVE_IT"}) {
t.Error("keyPresent = false for a key the store holds")
}
absent := &Provider{Name: "b", ClientSecret: "kms://MISSING"}
if ProviderKeyPresent(absent) {
t.Error("keyPresent = true for a key nothing holds")
}
if absent.ClientSecret != "kms://MISSING" {
t.Errorf("keyPresent mutated the row: %q", absent.ClientSecret)
}
}
// TestKmsGet_AbsentIsNotAFailure: the store reports a missing secret as an
// error, and during the migration off env vars MOST secrets are missing. Absence
// must therefore be silent and cache as an ordinary empty value — treating it as
// a fault would warn about the expected state of the world on every request, and
// would make a real store outage indistinguishable from a key that simply has not
// moved yet.
func TestKmsGet_AbsentIsNotAFailure(t *testing.T) {
fs := &fakeStore{err: errors.New("store: secret not found")}
bind(t, fs)
v, err := kmsGet("MISSING")
if err != nil {
t.Fatalf("absent secret returned an error: %v", err)
}
if v != "" {
t.Fatalf("value = %q, want empty", v)
}
// Cached as a value, not as a failure: a second read does not re-probe.
if _, err := kmsGet("MISSING"); err != nil {
t.Fatalf("second read errored: %v", err)
}
if fs.gets != 1 {
t.Errorf("store probes = %d, want 1 (absence cached as a value)", fs.gets)
}
}
// TestResolve_AbsentFallsBackToEnvSilently is the migration path itself: a key
// that has not moved into KMS yet resolves from the environment, with the store
// bound and answering "not found".
func TestResolve_AbsentFallsBackToEnvSilently(t *testing.T) {
bind(t, &fakeStore{err: errors.New("store: secret not found")})
t.Setenv("DO_AI_API_KEY", "env-only")
p := &Provider{Name: "do-ai", ClientSecret: "kms://DO_AI_API_KEY"}
if err := ResolveProviderSecret(p); err != nil {
t.Fatalf("resolve: %v", err)
}
if p.ClientSecret != "env-only" {
t.Errorf("ClientSecret = %q, want %q", p.ClientSecret, "env-only")
}
}
// TestKmsGet_CachesFailures is a hot-path guard, not a nicety. cloud ALWAYS
// hands over a non-nil client — when KMS is not mounted it is a fail-closed stub
// whose every read errors — so an uncached failure would re-probe and re-log on
// every provider resolution, i.e. every completion request. The failure is cached
// briefly instead, which bounds both.
func TestKmsGet_CachesFailures(t *testing.T) {
fs := &fakeStore{err: errors.New("store down")}
bind(t, fs)
for range 5 {
if _, err := kmsGet("K"); err == nil {
t.Fatal("want an error from a failing store")
}
}
if fs.gets != 1 {
t.Errorf("store probes = %d, want 1 (failure cached)", fs.gets)
}
}
// TestResolve_FailureCacheStillFallsBack: caching the failure must not turn into
// caching an empty answer — the env fallback has to keep working while the store
// is down, or an outage takes the env-fed providers with it.
func TestResolve_FailureCacheStillFallsBack(t *testing.T) {
bind(t, &fakeStore{err: errors.New("store down")})
t.Setenv("DO_AI_API_KEY", "env-only")
for i := range 3 {
p := &Provider{Name: "do-ai", ClientSecret: "kms://DO_AI_API_KEY"}
if err := ResolveProviderSecret(p); err != nil {
t.Fatalf("resolve %d: %v", i, err)
}
if p.ClientSecret != "env-only" {
t.Fatalf("resolve %d = %q, want %q", i, p.ClientSecret, "env-only")
}
}
}
// TestKmsGet_CachesReads keeps the hot path off the store for the TTL.
func TestKmsGet_CachesReads(t *testing.T) {
fs := &fakeStore{vals: map[string]string{"K": "v"}}
bind(t, fs)
for range 3 {
if v, err := kmsGet("K"); err != nil || v != "v" {
t.Fatalf("kmsGet = %q, %v", v, err)
}
}
if fs.gets != 1 {
t.Errorf("store reads = %d, want 1 (cached)", fs.gets)
}
}
// TestProviderKeysReadTheNamedPath: with PROVIDER_KEYS_PATH set, a provider key
// is read from that KMS path first — the record the kms-operator syncs to the
// family services — and a name the path does not hold falls back to the store's
// bare name and then the environment.
func TestProviderKeysReadTheNamedPath(t *testing.T) {
bind(t, &fakeStore{vals: map[string]string{
"orgs/hanzo/ai/OPENROUTER_API_KEY@prod": "k-path",
"orgs/hanzo/ai/OPENROUTER_API_KEY_3@prod": "k-path-3",
"orgs/hanzo/ai/OPENROUTER_API_KEY_5@prod": "k-path-5",
"OPENROUTER_API_KEY": "k-bare",
}})
t.Setenv("OPENROUTER_API_KEY_2", "k-env-2")
t.Setenv(ProviderKeysPath, "orgs/hanzo/ai@prod")
// A key written under a later name joins the pool; an unset name between is skipped.
if got := FamilyKeys("openrouter", OpenRouterKeys, ""); strings.Join(got, ",") != "k-path,k-env-2,k-path-3,k-path-5" {
t.Fatalf("keys = %v, want [k-path k-env-2 k-path-3 k-path-5]", got)
}
// The list zen and enso walk joins the pool after the numbered names: an account
// only the list holds is tried, one the names already hold is tried once, and each
// is named by its line.
t.Setenv(OpenRouterKeyList, "k-path-3\nk-list-only\n\n")
if got := FamilyKeys("openrouter", OpenRouterKeys, OpenRouterKeyList); strings.Join(got, ",") != "k-path,k-env-2,k-path-3,k-path-5,k-list-only" {
t.Fatalf("keys with the list = %v, want the four names then k-list-only", got)
}
if got := AccountOf(OpenRouterKeys, OpenRouterKeyList, "k-list-only"); got != OpenRouterKeyList+"#2" {
t.Fatalf("account of the list-only key = %q, want %s#2", got, OpenRouterKeyList)
}
t.Setenv(ProviderKeysPath, "")
if got := resolveKey("OPENROUTER_API_KEY"); got != "k-bare" {
t.Fatalf("unset path: key = %q, want the bare name's k-bare", got)
}
}