mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
- opener_eaten (models.yaml, per route and fallback): content is reasoning</think>answer, so everything to the first </think> is reasoning; no </think> is the answer, whole. Every other route keeps the strict leading-<think> rule. - /v1/messages (HTTP and ZAP) strips the reasoning, then lifts: one order (readAnswer); the translator no longer strips. A cut upstream ends in an error event, not end_turn. - ZAP gateway frames: X-Hanzo-Strict refused as the HTTP twin refuses it; anthropic-version and anthropic-beta reach a native vendor. - markup: a repeated JSON name in any form is no call; a blockquote is never lifted from; prose after a coder, DSML or calls opener is let go inside 256 bytes. Co-authored-by: Hanzo Dev <[email protected]>
294 lines
13 KiB
Go
294 lines
13 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package ai
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"sync"
|
|
|
|
"github.com/hanzoai/ai/log"
|
|
|
|
webtools "github.com/hanzoai/ai/agent/builtin_tool/web"
|
|
"github.com/hanzoai/ai/conf"
|
|
"github.com/hanzoai/ai/controllers"
|
|
"github.com/hanzoai/ai/model"
|
|
"github.com/hanzoai/ai/object"
|
|
"github.com/hanzoai/ai/proxy"
|
|
"github.com/hanzoai/ai/routers"
|
|
"github.com/hanzoai/ai/util"
|
|
)
|
|
|
|
// Bootstrap performs the AI runtime initialization shared by BOTH
|
|
// entrypoints — the standalone server (cmd/aid) and the embedded unified
|
|
// cloud binary (Mount, per HIP-0106). It is the SINGLE source of the
|
|
// runtime boot sequence: DB + adapter + tables, model/pricing config, the
|
|
// HTTP client, GeoIP/parser, the background maintenance tasks, the
|
|
// Commerce-backed balance gate + tier cache + per-key rate limiter, the
|
|
// full BeforeRouter/AfterExec filter chain, the session config, and
|
|
// the billing usage queue. After wiring those it publishes the
|
|
// fully-configured native router via SetHandler so the unified
|
|
// binary's /v1/ai/* adapter stops returning 503 "ai runtime not
|
|
// initialized".
|
|
//
|
|
// What Bootstrap deliberately does NOT do (those are standalone-only
|
|
// concerns that the embedded binary owns differently, and several would
|
|
// break or collide when co-resident in the unified process):
|
|
//
|
|
// - It binds NO listeners. The native ZAP inference node (port 9999),
|
|
// the inter-service ZAP transport (CLOUD_ZAP_PORT, default 9320) and
|
|
// the standalone HTTP listener (:httpport) stay in cmd/aid. The unified
|
|
// binary serves routers.App through zip on its own :8080 and runs its own
|
|
// ZAP at :9653; starting the legacy nodes here would collide.
|
|
// - It calls NO util.StopOldInstance — that races on the legacy standalone
|
|
// port and is meaningless when the embedded binary never listens there.
|
|
// - It installs NO signal handler and never calls os.Exit. cloud.Serve
|
|
// owns graceful shutdown for the unified binary; cmd/aid keeps its own
|
|
// drain goroutine, wired to the handles returned here.
|
|
//
|
|
// Bootstrap is guarded by sync.Once: it is safe to call more than once, so
|
|
// Mount can call it at mount time and the standalone can call it explicitly
|
|
// without double-initializing global runtime state. The cached result
|
|
// (including the error) is returned on every call.
|
|
//
|
|
// It returns an error rather than panicking: several init steps (notably
|
|
// the DB open) panic deep inside on a missing/unreachable backend, which
|
|
// would take down the whole multi-subsystem cloud binary. Bootstrap
|
|
// recovers those into a precise error so Mount can surface
|
|
// "mount ai: bootstrap: ..." and the operator sees exactly what failed.
|
|
//
|
|
// The rate limiter and billing queue created here are exposed via
|
|
// RateLimiter() and BillingQueue() for the standalone's graceful-drain
|
|
// wiring. Either may be nil (e.g. the billing queue is nil when no Commerce
|
|
// endpoint is configured).
|
|
func Bootstrap() error {
|
|
bootstrapOnce.Do(func() {
|
|
bootErr = doBootstrap()
|
|
})
|
|
return bootErr
|
|
}
|
|
|
|
// RateLimiter returns the per-key rate limiter created by Bootstrap, or nil
|
|
// if Bootstrap has not run (or failed before creating it).
|
|
func RateLimiter() *routers.RateLimiter { return bootRateLimiter }
|
|
|
|
// BillingQueue returns the Commerce billing usage queue created by
|
|
// Bootstrap, or nil if none was configured / Bootstrap did not reach it.
|
|
func BillingQueue() *util.BillingQueue { return bootBillingQueue }
|
|
|
|
var (
|
|
bootstrapOnce sync.Once
|
|
bootErr error
|
|
bootRateLimiter *routers.RateLimiter
|
|
bootBillingQueue *util.BillingQueue
|
|
)
|
|
|
|
func doBootstrap() (err error) {
|
|
// Convert the deep DB-open panic (and any other init panic) into a
|
|
// precise error so a missing backend fails the AI mount cleanly instead
|
|
// of crashing the entire cloud process.
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
err = fmt.Errorf("ai: bootstrap: %v", r)
|
|
}
|
|
}()
|
|
|
|
// Where each Kai version that runs as its own service lives (KAI_URLS). Read
|
|
// first: a malformed pair refuses the boot before anything else is opened.
|
|
if err := controllers.InitKaiURLs(); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Persistence: flags, ORM adapter, schema, DB connection.
|
|
object.InitFlag()
|
|
object.InitAdapter()
|
|
object.CreateTables()
|
|
object.InitDb()
|
|
|
|
// Analytics datastore (datastore): the OLAP ledger behind the usage/spend
|
|
// Overview (hanzo.cloud_usage) and the per-tenant trace ledger
|
|
// (hanzo.observations). Opt-in via DATASTORE_ADDR; connects in the background
|
|
// and is a no-op when unset, so boot never blocks on the warehouse. This runs
|
|
// in BOTH cmd/aid and the unified cloud binary (the ai ZAP node does not, so
|
|
// the datastore is reached directly, not via a ZAP peer). See object.InitDatastore.
|
|
object.InitDatastore()
|
|
|
|
// OTel GenAI telemetry: one gen_ai span per LLM call → o11y OTLP
|
|
// collector. Opt-in via OTEL_EXPORTER_OTLP_ENDPOINT; connects in the
|
|
// background and is a no-op when unset, so boot never blocks on the
|
|
// collector. Runs in BOTH cmd/aid and the unified cloud binary, mirroring
|
|
// InitDatastore. See object.InitTelemetry.
|
|
object.InitTelemetry()
|
|
|
|
// Cross-pod cache-invalidation bus: broadcast org-settings (router policy /
|
|
// savings↔quality dial) writes over Valkey pub/sub so a console change converges
|
|
// sub-second fleet-wide. Opt-in via KV_URL; connects in the background and is a
|
|
// no-op when unset, so boot never blocks on Valkey and the per-pod TTL still
|
|
// backstops. Mirrors InitDatastore/InitTelemetry. See object.InitCacheBus.
|
|
object.InitCacheBus()
|
|
|
|
// Model routing/pricing config from YAML. Non-fatal: static fallback.
|
|
configPath := conf.GetConfigString("modelConfigPath")
|
|
if configPath == "" {
|
|
configPath = "conf/models.yaml"
|
|
}
|
|
if err := controllers.InitModelConfig(configPath); err != nil {
|
|
log.Warn("Model config: %v (using static fallback)", err)
|
|
}
|
|
|
|
// Wire the YAML-backed context-window resolver: models.yaml context_window
|
|
// becomes the per-model source of truth for max context, overriding the
|
|
// static getContextLength table. Without this, glm-5.2 (1M context) fell
|
|
// through the table to a 16K fallback and dead-ended long prompts + /compact.
|
|
// Nil-safe: if ModelConfig failed to load, the table-only path stays.
|
|
if mc := controllers.GetModelConfig(); mc != nil {
|
|
model.SetContextWindowResolver(mc.ContextWindow)
|
|
model.SetOpenerEatenResolver(func(upstream string) bool { return controllers.OpenerEaten("", upstream) })
|
|
}
|
|
|
|
// Provider HTTP client, GeoIP, request parsing, maintenance tasks.
|
|
proxy.InitHttpClient()
|
|
util.InitMaxmindFiles()
|
|
util.InitIpDb()
|
|
object.InitCleanupChats()
|
|
object.InitStoreCount()
|
|
object.InitCommitRecordsTask()
|
|
object.InitScanJobProcessor()
|
|
object.InitMessageTransactionRetry()
|
|
|
|
// The in-pod balance ledger (object.GlobalBalanceLedger) reserves/settles in
|
|
// process memory — correct ONLY at a SINGLE replica (deploy strategy=Recreate
|
|
// + HPA min=max=1). If the deployment states CLOUD_REPLICAS > 1, refuse to
|
|
// start rather than silently double-spend across pods; scaling out requires a
|
|
// Commerce-atomic conditional reserve first.
|
|
if n, ok := object.SinglePodReplicaHint(); ok && n > 1 {
|
|
log.Error("FATAL: CLOUD_REPLICAS=%d — the in-pod balance ledger double-spends across pods. Refusing to start; implement a Commerce-atomic reserve before scaling out.", n)
|
|
panic("cloud-api: in-pod balance ledger requires a single replica (CLOUD_REPLICAS>1)")
|
|
}
|
|
log.Info("balance ledger: single-pod invariant active (reservation ledger requires replicas=1, strategy=Recreate, HPA min=max=1)")
|
|
|
|
// The page fetcher behind the fetch_url builtin tool. Installed HERE rather than
|
|
// called directly because agent/builtin_tool/web must stay a leaf: `object`
|
|
// imports `agent`, and `agent` imports the tool registry, so a tool reaching into
|
|
// object for Crawl closes the loop and the module stops compiling. The root
|
|
// package imports both and can hand one to the other.
|
|
//
|
|
// Search and deep research are the HOST's (hanzoai/cloud serves /v1/websearch and
|
|
// /v1/ask); it installs those at mount time. Until it does, those two tools answer
|
|
// that they are unavailable in this deployment — never an empty result, which an
|
|
// agent would read as "the web holds nothing on this" and answer from memory.
|
|
webtools.SetFetch(func(_ context.Context, urls []string) ([]webtools.FetchResult, error) {
|
|
results, err := object.Crawl(urls)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := make([]webtools.FetchResult, 0, len(results))
|
|
for _, r := range results {
|
|
out = append(out, webtools.FetchResult{
|
|
URL: r.URL, Title: r.Title, Markdown: r.Markdown, Success: r.Success,
|
|
})
|
|
}
|
|
return out, nil
|
|
})
|
|
|
|
// Commerce balance gate (pre-request balance enforcement).
|
|
routers.InitBalanceGate()
|
|
|
|
// Commerce-backed tier cache — must precede InitRateLimiter so
|
|
// DefaultTierFunc can resolve tiers through it.
|
|
routers.InitTierCache()
|
|
|
|
// Per-key rate limiting (env override → tier cache → zen-free). It logs the
|
|
// tiers it armed, from the table it armed them from.
|
|
bootRateLimiter = routers.InitRateLimiter(routers.DefaultTierFunc)
|
|
|
|
// The filter chain is installed with the ROUTES, by routers.Register, because
|
|
// the router walks its stack in registration order and a filter added after a
|
|
// route never runs before it. Boot is runtime init — the database, the
|
|
// providers, the queues — and no longer wires the surface.
|
|
//
|
|
// There is no session store. It was a per-pod in-memory store backing a cookie
|
|
// flow, written only by the auto-signin filter that is gone; identity is the
|
|
// verified IAM token, read where it is needed.
|
|
|
|
// Optional log adapter reconfig. Guarded: in the embedded binary there
|
|
// is no conf/app.conf, so logConfig is empty — json.Unmarshal("") would
|
|
// panic. Skip when unset and keep the default logger.
|
|
if raw := conf.GetConfigString("logConfig"); raw != "" {
|
|
logConfigMap := make(map[string]any)
|
|
if err := json.Unmarshal([]byte(raw), &logConfigMap); err != nil {
|
|
log.Warn("logConfig parse failed: %v (keeping default logger)", err)
|
|
} else {
|
|
logAdapter := "file"
|
|
if v, ok := logConfigMap["adapter"].(string); ok {
|
|
logAdapter = v
|
|
}
|
|
if logAdapter == "console" {
|
|
log.Reset()
|
|
}
|
|
if err := log.SetLogger(logAdapter, raw); err != nil {
|
|
log.Warn("logConfig SetLogger failed: %v (keeping default logger)", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Commerce-backed billing usage queue (retried with backoff). nil when
|
|
// no Commerce endpoint is configured.
|
|
bootBillingQueue = controllers.InitBillingQueue()
|
|
if bootBillingQueue != nil {
|
|
log.Info("Billing queue started (Commerce endpoint configured)")
|
|
}
|
|
|
|
// Nothing to publish. The app IS the handler: ai.App builds it, registers every
|
|
// address on it, and stores it — so the socket and the ZAP transports reach one
|
|
// route table with one filter chain, rather than a handler being set here and
|
|
// found by whoever asks.
|
|
|
|
// Enso router flywheel (HIP-510): the learned-routing loop that trains on real
|
|
// traffic. Started HERE — in the SINGLE sync.Once boot sequence BOTH cmd/aid and
|
|
// ai.Mount run — so it is the ONE launch site (DRY): the deployed unified binary
|
|
// mounts via cloud.Wire → ai.Mount → Bootstrap, so wiring it here (not in a caller)
|
|
// is what makes the flywheel actually run in production, where it previously never
|
|
// did (retrains: [], rewarded_events: 0). Both Start* self-gate on env and default
|
|
// OFF (ROUTER_TRAIN_ENABLED for the trainer; ROUTER_PROBE_RPH+ROUTER_PROBE_TOKEN for
|
|
// the probe), so this is dark-by-default and turns real only via deployment config.
|
|
// sync.Once guarantees the goroutines are launched exactly once per process.
|
|
// StartRouterJudge arms the LLM-as-a-judge dense quality reward (openai_api.go's
|
|
// post-response hook): it turns served (prompt, response) pairs into 0..1 quality
|
|
// rewards for the flywheel — anonymous, consent-gated, content-transient. Unlike
|
|
// the probe/trainer it is DB-backed and ON by default: it reads its config live
|
|
// from the "*" GlobalDefaultOwner OrgSettings row (judge.enabled/models/url/sample),
|
|
// tunable at admin.hanzo.ai with no restart, and runs the Mean-Field Judge Panel on
|
|
// ~10% of eligible (opted-in, non-EU-protected) traffic out of the box. Point
|
|
// judge.url at a TEE endpoint for confidential inference (controllers/router_judge.go).
|
|
// SeedInternalTrainingConsent first flips our OWN reserved orgs to explicit opt-in
|
|
// so the EU per-request guard never suppresses our internal traffic.
|
|
controllers.SeedInternalTrainingConsent()
|
|
// Read each model family's catalog once at boot, so the first request after a
|
|
// restart is priced from discovery (controllers.WarmFamilies).
|
|
controllers.WarmFamilies()
|
|
// Finish, keep and bill every video job a process before this one left
|
|
// unfinished (controllers.ResumeVideos): the vendor finishes it either way.
|
|
go controllers.ResumeVideos()
|
|
// Serve each family's newest routing version, and keep serving it (routing.go).
|
|
controllers.StartRoutingConvergence()
|
|
controllers.StartRouterProbe()
|
|
controllers.StartRouterTrainer()
|
|
controllers.StartRouterJudge()
|
|
|
|
return nil
|
|
}
|