mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
The module reads and writes the store from the call, not from the next InitAdapter; UseDB(nil) takes it back. Co-authored-by: Hanzo Dev <[email protected]>
317 lines
11 KiB
Go
317 lines
11 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
// Portions Copyright 2023 The OpenAgent Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
package object
|
|
|
|
import (
|
|
"database/sql"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"runtime"
|
|
|
|
_ "github.com/go-sql-driver/mysql" // mysql
|
|
"github.com/hanzoai/ai/conf"
|
|
"github.com/hanzoai/dbx"
|
|
_ "github.com/hanzoai/sqlite" // sqlite: the ONE Hanzo driver — registers "sqlite" exactly once (cgo→mattn/SQLCipher, !cgo→modernc). Never import modernc directly, or a cgo build double-registers "sqlite" and panics at init.
|
|
_ "github.com/lib/pq" // postgres
|
|
_ "github.com/microsoft/go-mssqldb" // mssql (maintained successor to denisenkom; one mssql driver registration across the stack)
|
|
)
|
|
|
|
var (
|
|
adapter *Adapter = nil
|
|
hosted *dbx.DB = nil
|
|
providerAdapter *Adapter = nil
|
|
isCreateDatabaseDefined = false
|
|
createDatabase = true
|
|
)
|
|
|
|
func InitFlag() {
|
|
if !isCreateDatabaseDefined {
|
|
isCreateDatabaseDefined = true
|
|
createDatabase = getCreateDatabaseFlag()
|
|
}
|
|
}
|
|
|
|
func getCreateDatabaseFlag() bool {
|
|
res := flag.Bool("createDatabase", false, "true if you need to create database")
|
|
flag.Parse()
|
|
return *res
|
|
}
|
|
|
|
func InitConfig() {
|
|
err := conf.LoadAppConfig("ini", "../conf/app.conf")
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
InitAdapter()
|
|
CreateTables()
|
|
}
|
|
|
|
func InitAdapter() {
|
|
if hosted != nil {
|
|
// The host handed over the database (UseDB): it owns the connection, its
|
|
// pool, its dialect and its close, so nothing here is read from config.
|
|
adapter = hostedAdapter(hosted)
|
|
providerAdapter = nil
|
|
return
|
|
}
|
|
adapter = NewAdapter(conf.GetConfigString("driverName"), conf.GetConfigDataSourceName())
|
|
providerDbName := conf.GetConfigString("providerDbName")
|
|
if adapter.DbName == providerDbName {
|
|
providerDbName = ""
|
|
}
|
|
if providerDbName != "" {
|
|
providerAdapter = NewAdapterWithDbName(conf.GetConfigString("driverName"), conf.GetConfigDataSourceName(), providerDbName)
|
|
}
|
|
}
|
|
|
|
func CreateTables() {
|
|
if createDatabase && hosted == nil {
|
|
err := adapter.CreateDatabase()
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
adapter.createTable()
|
|
}
|
|
|
|
// Adapter represents the database adapter for storage.
|
|
type Adapter struct {
|
|
driverName string
|
|
dataSourceName string
|
|
DbName string
|
|
db *dbx.DB
|
|
}
|
|
|
|
// finalizer is the destructor for Adapter.
|
|
func finalizer(a *Adapter) {
|
|
err := a.db.Close()
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
// NewAdapter is the constructor for Adapter.
|
|
func NewAdapter(driverName string, dataSourceName string) *Adapter {
|
|
a := &Adapter{}
|
|
a.driverName = driverName
|
|
a.dataSourceName = dataSourceName
|
|
a.DbName = conf.GetConfigString("dbName")
|
|
// Open the DB, create it if not existed.
|
|
a.open()
|
|
// Call the destructor when the object is released.
|
|
runtime.SetFinalizer(a, finalizer)
|
|
return a
|
|
}
|
|
|
|
func NewAdapterWithDbName(driverName string, dataSourceName string, dbName string) *Adapter {
|
|
a := &Adapter{}
|
|
a.driverName = driverName
|
|
a.dataSourceName = dataSourceName
|
|
a.DbName = dbName
|
|
// Open the DB, create it if not existed.
|
|
a.open()
|
|
// Call the destructor when the object is released.
|
|
runtime.SetFinalizer(a, finalizer)
|
|
return a
|
|
}
|
|
|
|
func (a *Adapter) CreateDatabase() error {
|
|
db, err := dbx.Open(a.driverName, a.dataSourceName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer db.Close()
|
|
var stmt string
|
|
switch a.driverName {
|
|
case "sqlite", "sqlite3":
|
|
// SQLite has no CREATE DATABASE. The database IS the file, and it
|
|
// exists the moment it is opened — there is nothing to create, so the
|
|
// correct statement is no statement.
|
|
//
|
|
// Without this branch sqlite fell through to default, which emits
|
|
// MySQL DDL, and every caller got `near "DATABASE": syntax error`. A
|
|
// default that assumes one dialect is wrong for every dialect it has
|
|
// not met; the ones that need a database created now say so by name.
|
|
return nil
|
|
case "postgres":
|
|
stmt = fmt.Sprintf(`
|
|
DO $$
|
|
BEGIN
|
|
IF NOT EXISTS (SELECT FROM pg_database WHERE datname = '%s') THEN
|
|
CREATE DATABASE %s WITH ENCODING='UTF8' LC_COLLATE='en_US.utf8' LC_CTYPE='en_US.utf8' TEMPLATE=template0;
|
|
END IF;
|
|
END
|
|
$$`,
|
|
a.DbName, a.DbName)
|
|
default:
|
|
stmt = fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s default charset utf8mb4 COLLATE utf8mb4_general_ci", a.DbName)
|
|
}
|
|
_, err = db.NewQuery(stmt).Execute()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (a *Adapter) open() {
|
|
dataSourceName := a.dataSourceName + a.DbName
|
|
if a.driverName != "mysql" {
|
|
dataSourceName = a.dataSourceName
|
|
}
|
|
db, err := dbx.MustOpen(a.driverName, dataSourceName)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
a.db = db
|
|
}
|
|
|
|
func (a *Adapter) close() {
|
|
a.db.Close()
|
|
a.db = nil
|
|
}
|
|
|
|
func (a *Adapter) createTable() {
|
|
if err := SyncSchema(a.db); err != nil {
|
|
slog.Warn("schema not synced", "err", err)
|
|
}
|
|
}
|
|
|
|
// UseDB hands ai the database its host opened, and ai reads and writes it from this
|
|
// call on. The host owns the connection, so ai opens none from config (InitAdapter,
|
|
// CreateTables) and closes none. Call it before Bootstrap; db is in whatever dialect
|
|
// the host runs (sqlite, postgres or pgx). Pass nil to take it back.
|
|
func UseDB(db *dbx.DB) {
|
|
if db == nil {
|
|
if adapter != nil && adapter.db == hosted {
|
|
adapter = nil
|
|
}
|
|
hosted = nil
|
|
return
|
|
}
|
|
hosted = db
|
|
adapter = hostedAdapter(db)
|
|
}
|
|
|
|
func hostedAdapter(db *dbx.DB) *Adapter {
|
|
return &Adapter{driverName: db.DriverName(), DbName: conf.GetConfigString("dbName"), db: db}
|
|
}
|
|
|
|
// SyncSchema creates the tables ai keeps on db, and grows those that exist by the
|
|
// columns added since, in db's own dialect. It is idempotent and is what a host
|
|
// runs as the open of ai's store, so a move between engines finds the same tables
|
|
// on both sides. Every model is tried; the error joins those that failed.
|
|
func SyncSchema(db *dbx.DB) error {
|
|
// The Go structs are the single source of truth for the schema. dbx.Sync is
|
|
// idempotent — CREATE TABLE IF NOT EXISTS for new tables and ALTER TABLE ADD
|
|
// COLUMN for fields added since the table was created — so this is safe on
|
|
// both a fresh embedded SQLite store and an existing DB. No external SQL.
|
|
models := []any{
|
|
&Application{}, &Article{}, &Asset{}, &Chat{}, &Connection{},
|
|
&File{}, &Form{}, &Graph{}, &Message{}, &ModelRoute{}, &Node{},
|
|
&Provider{}, &Record{}, &Scale{}, &Scan{}, &Session{}, &Store{},
|
|
&Task{}, &Template{}, &Vector{}, &Video{}, &Workflow{},
|
|
&Memory{}, // cloud memory backend (per-user scoped)
|
|
&OrgSettings{}, // per-org feature overrides (auto-routing, …)
|
|
&RoutingEvent{}, // privacy-preserving auto-routing decision ledger (training)
|
|
&RouterArtifactMeta{}, // latest retrain outcome per scope (upsert, one row/owner)
|
|
&RouterTrainingLog{}, // append-only retrain timeline (one immutable row per fit)
|
|
&FinetuneJob{}, // fine-tuning / training runs brokered to the cluster trainer
|
|
&ModelAccess{}, // per-(org,user,model) grants for gated SKUs (enso limited preview)
|
|
&RoutingVersion{}, // applied Zen/Enso routing catalogs, one row per edit (routing.go)
|
|
&Listing{}, // every model a vendor lists, as the last sync read it (controllers' listing.go)
|
|
&ListingEvent{}, // each change a sync read: new, gone, back, repriced
|
|
&Lease{}, // jobs the fleet runs once an interval
|
|
&Served{}, // what each vendor account did with each model, by hour
|
|
&MediaJob{}, // generations a vendor runs for a caller, until billed (media_job.go)
|
|
}
|
|
var errs []error
|
|
for _, m := range models {
|
|
if err := syncModel(db, m); err != nil {
|
|
// Keep going so one bad model can't block the rest; a table that
|
|
// genuinely failed to create surfaces at first use, and in the error.
|
|
errs = append(errs, fmt.Errorf("%T: %w", m, err))
|
|
}
|
|
}
|
|
// dbx.Sync's ALTER TABLE ADD COLUMN carries no default, so rows that predate a
|
|
// newly-added column hold SQL NULL there — which a full struct scan cannot read
|
|
// into a Go scalar ("converting NULL to string is unsupported"). Repair the one
|
|
// table whose per-org router/judge config the trainer reads at boot and on
|
|
// cadence; idempotent, so it is a no-op once the rows are clean.
|
|
backfillNullScalars(db, "org_settings", &OrgSettings{})
|
|
// Routes are configuration — tens of rows, not millions — so the whole-model
|
|
// sweep is the cheap option here rather than the expensive one, and every column
|
|
// added to a route from now on is repaired without another line. Priced is the
|
|
// one that needs it today: a bool cannot take NULL, so without this every read
|
|
// of a route written before the column fails outright rather than reading false.
|
|
backfillNullScalars(db, "model_route", &ModelRoute{})
|
|
// A version written before Boot holds NULL there, which a scan into the string
|
|
// cannot read; the empty Boot it repairs to is a run no deployment matches, so
|
|
// convergence leaves whatever serves standing.
|
|
backfillNullScalars(db, "routing_version", &RoutingVersion{})
|
|
// The message table is the largest here and a whole-model sweep costs a scan per
|
|
// column, so it is repaired one field at a time. These are the columns added since
|
|
// message rows existed: a NULL claimed_time would make its row unclaimable — a
|
|
// message nobody can answer — and a NULL in either fails the scan into a string
|
|
// field before any handler sees the row.
|
|
backfillNullField(db, "message", &Message{}, "ClaimedTime")
|
|
backfillNullField(db, "message", &Message{}, "AnsweredTime")
|
|
return errors.Join(errs...)
|
|
}
|
|
|
|
// RawDB returns the underlying *sql.DB for direct access when needed.
|
|
func (a *Adapter) RawDB() *sql.DB {
|
|
return a.db.DB()
|
|
}
|
|
|
|
// UseMemoryDB points the package adapter at a fresh in-memory SQLite database,
|
|
// syncs the given models, and returns a restore func that reinstates the previous
|
|
// adapter and closes the DB. It is the ONE hermetic in-memory-DB seam shared by
|
|
// every package's tests — object's own unit tests and the controllers' HTTP-status
|
|
// tests — so no test hand-rolls adapter wiring. SQLite is the canonical embedded
|
|
// store (the same driver prod uses), so this exercises the real query path, not a
|
|
// fake. Never call it from non-test code: it swaps the process-wide adapter.
|
|
func UseMemoryDB(dsn string, models ...any) (restore func(), err error) {
|
|
var (
|
|
db *dbx.DB
|
|
drop = func() {}
|
|
drv = "sqlite"
|
|
close = func() {}
|
|
)
|
|
if pg := os.Getenv(TestSQLEnv); pg != "" {
|
|
// The suite on the server hosts run ai on: one schema a call, dropped on restore.
|
|
db, drop, err = postgresDB(pg)
|
|
drv = "pgx"
|
|
} else {
|
|
db, err = dbx.Open("sqlite", dsn)
|
|
close = func() { _ = db.Close() }
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, m := range models {
|
|
if err := syncModel(db, m); err != nil {
|
|
close()
|
|
drop()
|
|
return nil, err
|
|
}
|
|
}
|
|
prev := adapter
|
|
adapter = &Adapter{driverName: drv, dataSourceName: dsn, db: db}
|
|
return func() { adapter = prev; close(); drop() }, nil
|
|
}
|