Files
zeekay e71dfbc4f3 SuperAdmin is the host boundary's verdict, never the org a token names
util.IsSuperAdmin compared the resolved user's owner to "admin", a
second predicate beside the one the host boundary decides, and the two
disagreed. An admin-org application (no orgs, owner claim admin), an
admin-org person's sk- key, and an org spelled "admin " (the parse
trimmed orgs[0]) were each SuperAdmin here and refused by the boundary.

The host now states the verdict: object.SetSuper, which hanzo-inc/cloud
fills with principal.Super, the X-User-IsAdmin its boundary deletes from
every request and re-mints from authz.Claims.Sudo. Every HTTP seam that
turns a token into a principal goes through object.Verify, which stamps
it on iam.User.Sudo (never decoded); IsSuperAdmin and ScopeOwner read
that and compare no org. util.AdminOrg is gone; the rows that are the
platform's own are named by authz.AdminOrg.

With no host there is no SuperAdmin, and a ZAP door is not the host:
target deletes X-User-IsAdmin from a ZAP-built request, and zapPrincipal
carries a verdict only when the HTTP bridge vouched for the request it
was handed. A key is never the SuperAdmin, as at the host.
2026-10-18 22:21:43 -07:00

468 lines
15 KiB
Go

// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
import (
"encoding/json"
"fmt"
"os"
"strconv"
"strings"
"github.com/hanzoai/ai/cluster"
"github.com/hanzoai/ai/object"
"github.com/hanzoai/ai/util"
)
// finetune.go is the `/v1/ai/finetune/*` broker surface — the ONE API the console's
// Training page drives. It owns the whole lifecycle against the cluster training
// operator (hanzo-ml/trainer) and inference (hanzo-ml/kserve): browse/search
// HuggingFace, recommend efficient defaults, submit a real TrainJob, stream status,
// meter GPU-hours to commerce, and deploy a finished checkpoint to inference. All
// business logic lives in object/finetune_*.go; these handlers are the thin controller
// MVC layer (mirror controllers/model_route.go): read the request, resolve the
// authed org, delegate, return the {status,msg,data} envelope.
//
// Org scoping is authoritative: GetOrg() honors X-Org-Id only for the
// principal's own org or a SuperAdmin, so a caller can only ever touch their own
// org's jobs.
// createFinetuneRequest is the new-training-job payload from the console.
type createFinetuneRequest struct {
DisplayName string `json:"displayName"`
BaseModel string `json:"baseModel"` // HF repo id or hf://… / s3://…
Method string `json:"method"` // lora | qlora | full
Task string `json:"task"` // instruct | chat | completion
Dataset string `json:"dataset"` // HF dataset id or hf://… / s3://…
Preset string `json:"preset"` // recommended | balanced | aggressive | custom
Hyperparams *object.Hyperparams `json:"hyperparams"`
GpuType string `json:"gpuType"`
GpuCount int `json:"gpuCount"`
NumNodes int `json:"numNodes"`
DatasetExamples int `json:"datasetExamples"` // optional size hint for the time estimate
}
// finetuneOutputBase is the S3 root for checkpoint output (org's Space). Override
// with FINETUNE_OUTPUT_BASE.
func finetuneOutputBase() string {
if v := strings.TrimRight(os.Getenv("FINETUNE_OUTPUT_BASE"), "/"); v != "" {
return v
}
return "s3://hanzo-finetunes"
}
// slugify makes a DNS-1123-safe label segment from arbitrary text.
func slugify(s string) string {
s = strings.ToLower(strings.TrimSpace(s))
var b strings.Builder
for _, r := range s {
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
b.WriteRune(r)
case r == '-', r == '/', r == '_', r == '.', r == ' ':
b.WriteByte('-')
}
}
out := strings.Trim(b.String(), "-")
for strings.Contains(out, "--") {
out = strings.ReplaceAll(out, "--", "-")
}
if len(out) > 40 {
out = strings.Trim(out[:40], "-")
}
return out
}
// lastSegment returns the trailing path segment of an id (the model/repo name).
func lastSegment(id string) string {
id = strings.TrimRight(id, "/")
if i := strings.LastIndex(id, "/"); i >= 0 {
return id[i+1:]
}
return id
}
// orgKMSProject returns the org's KMS project id for org-scoped HF tokens, or ""
// to fall back to env + system KMS. The mapping convention is the org-scoped KMS
// project; absent a per-org mapping this stays empty (system token still resolves).
func (c *ApiController) orgKMSProject() string {
return strings.TrimSpace(os.Getenv("FINETUNE_HF_KMS_PROJECT"))
}
// ── HuggingFace discovery ────────────────────────────────────────────────────
// SearchHfModels proxies a HuggingFace model search (base-model picker).
func (c *ApiController) SearchHfModels() {
if _, ok := c.RequireSignedInUser(); !ok {
return
}
query := c.Input().Get("q")
task := c.Input().Get("task")
limit, _ := strconv.Atoi(c.Input().Get("limit"))
token := object.ResolveHfToken(c.orgKMSProject())
models, err := object.SearchHfModels(query, task, limit, token)
if err != nil {
c.ResponseError(err.Error())
return
}
c.ResponseOk(models)
}
// SearchHfDatasets proxies a HuggingFace dataset search (dataset picker).
func (c *ApiController) SearchHfDatasets() {
if _, ok := c.RequireSignedInUser(); !ok {
return
}
query := c.Input().Get("q")
limit, _ := strconv.Atoi(c.Input().Get("limit"))
token := object.ResolveHfToken(c.orgKMSProject())
datasets, err := object.SearchHfDatasets(query, limit, token)
if err != nil {
c.ResponseError(err.Error())
return
}
c.ResponseOk(datasets)
}
// GetHfRepo returns a repo's detail (files, gated/private state). ?id=&kind=model|dataset
func (c *ApiController) GetHfRepo() {
if _, ok := c.RequireSignedInUser(); !ok {
return
}
id := c.Input().Get("id")
kind := c.Input().Get("kind")
if kind == "" {
kind = "model"
}
token := object.ResolveHfToken(c.orgKMSProject())
info, err := object.GetHfRepoInfo(id, kind, token)
if err != nil {
c.ResponseError(err.Error())
return
}
c.ResponseOk(info)
}
// ── Presets / recommendations ────────────────────────────────────────────────
// GetFinetunePresets returns the new-job catalog plus, when a selection is passed
// (?baseModel&method&task&preset[&datasetExamples]), the recommended config so the
// console can render "Recommended" as a one-click, ready-to-run default.
func (c *ApiController) GetFinetunePresets() {
if _, ok := c.RequireSignedInUser(); !ok {
return
}
catalog := object.GetFinetuneCatalog()
baseModel := c.Input().Get("baseModel")
if baseModel == "" {
c.ResponseOk(map[string]any{"catalog": catalog})
return
}
method := c.Input().Get("method")
task := c.Input().Get("task")
preset := c.Input().Get("preset")
examples, _ := strconv.Atoi(c.Input().Get("datasetExamples"))
rec := object.Recommend(baseModel, method, task, preset, examples)
c.ResponseOk(map[string]any{"catalog": catalog, "recommendation": rec})
}
// ── Job lifecycle ────────────────────────────────────────────────────────────
// ListFinetuneJobs returns the org's jobs, refreshing live status for active ones.
func (c *ApiController) ListFinetuneJobs() {
if _, ok := c.RequireSignedInUser(); !ok {
return
}
org := c.GetOrg()
jobs, err := object.GetFinetuneJobs(org)
if err != nil {
c.ResponseError(err.Error())
return
}
lang := c.GetAcceptLanguage()
for _, j := range jobs {
if !j.IsTerminal() && j.CrName != "" {
c.refreshFinetuneJob(j, lang)
}
}
c.ResponseOk(jobs)
}
// GetFinetuneJob returns one job with refreshed live status. ?id=owner/name or ?name=
func (c *ApiController) GetFinetuneJob() {
if _, ok := c.RequireSignedInUser(); !ok {
return
}
org := c.GetOrg()
name := c.jobNameParam(org)
job, err := object.GetFinetuneJob(org, name)
if err != nil {
c.ResponseError(err.Error())
return
}
if job == nil {
c.ResponseError(fmt.Sprintf("fine-tune job %q not found", name))
return
}
if !job.IsTerminal() && job.CrName != "" {
c.refreshFinetuneJob(job, c.GetAcceptLanguage())
}
c.ResponseOk(job)
}
// jobNameParam pulls the job name from ?name= or the owner/name form of ?id=.
func (c *ApiController) jobNameParam(org string) string {
if n := c.Input().Get("name"); n != "" {
return n
}
id := c.Input().Get("id")
if strings.Contains(id, "/") {
return id[strings.LastIndex(id, "/")+1:]
}
return id
}
// CreateFinetuneJob validates the request, resolves efficient defaults, persists the
// job, and submits a real TrainJob CR. A submit failure (e.g. no cluster wired) is
// surfaced honestly: the job is saved with status "failed" + the reason, never faked.
func (c *ApiController) CreateFinetuneJob() {
user, ok := c.RequireSignedInUser()
if !ok {
return
}
org := c.GetOrg()
var req createFinetuneRequest
if err := json.Unmarshal(c.Body(), &req); err != nil {
c.ResponseError(err.Error())
return
}
req.BaseModel = strings.TrimSpace(req.BaseModel)
req.Dataset = strings.TrimSpace(req.Dataset)
if req.BaseModel == "" {
c.ResponseError("a base model is required")
return
}
if req.Dataset == "" {
c.ResponseError("a dataset is required")
return
}
// Resolve efficient defaults (the "Recommended" brain), then apply any explicit
// overrides from the request.
rec := object.Recommend(req.BaseModel, req.Method, req.Task, req.Preset, req.DatasetExamples)
hp := rec.Hyperparams
if req.Hyperparams != nil {
hp = *req.Hyperparams
}
hpJSON, _ := json.Marshal(hp)
gpuType := firstNonEmptyStr(req.GpuType, rec.GpuType)
gpuCount := req.GpuCount
if gpuCount <= 0 {
gpuCount = rec.GpuCount
}
numNodes := req.NumNodes
if numNodes <= 0 {
numNodes = rec.NumNodes
}
method := req.Method
if method == "" {
method = "qlora"
}
name := slugify(lastSegment(req.BaseModel)) + "-" + strings.ToLower(util.GetRandomName())
name = strings.Trim(name, "-")
ns := cluster.FinetuneNamespace(org)
outputUri := fmt.Sprintf("%s/%s/%s/", finetuneOutputBase(), slugify(org), name)
job := &object.FinetuneJob{
Owner: org,
Name: name,
CreatedBy: user.Name,
DisplayName: firstNonEmptyStr(req.DisplayName, req.BaseModel),
BaseModel: req.BaseModel,
Method: method,
Task: firstNonEmptyStr(req.Task, "instruct"),
Dataset: req.Dataset,
Preset: firstNonEmptyStr(req.Preset, "recommended"),
Hyperparams: string(hpJSON),
Runtime: rec.Runtime,
GpuType: gpuType,
GpuCount: gpuCount,
NumNodes: numNodes,
Namespace: ns,
CrName: name,
OutputUri: outputUri,
Status: "pending",
}
if _, err := object.AddFinetuneJob(job); err != nil {
c.ResponseError(err.Error())
return
}
// Submit the real CR. Resolve an HF token (KMS, never the browser) so private
// base models/datasets pull.
token := object.ResolveHfToken(c.orgKMSProject())
lang := c.GetAcceptLanguage()
if err := cluster.SubmitTrainJob(job, hp, token, lang); err != nil {
job.Status = "failed"
job.Error = err.Error()
_, _ = object.UpdateFinetuneJob(job.Owner, job.Name, job)
// Return the saved job (with the honest failure) rather than a bare error,
// so the console shows the run row + the precise reason.
c.ResponseOk(job)
return
}
// Queued — StartedTime is set later from the CR's Running condition (in
// refreshFinetuneJob), so GPU-hour metering never counts queue time.
job.Status = "queued"
_, _ = object.UpdateFinetuneJob(job.Owner, job.Name, job)
c.ResponseOk(job)
}
// CancelFinetuneJob deletes the TrainJob CR, meters the GPU-hours used so far, and
// marks the job cancelled. ?id= or ?name=
func (c *ApiController) CancelFinetuneJob() {
if _, ok := c.RequireSignedInUser(); !ok {
return
}
org := c.GetOrg()
name := c.jobNameParam(org)
job, err := object.GetFinetuneJob(org, name)
if err != nil || job == nil {
c.ResponseError(fmt.Sprintf("fine-tune job %q not found", name))
return
}
lang := c.GetAcceptLanguage()
if job.CrName != "" {
if derr := cluster.DeleteTrainJob(job, lang); derr != nil {
c.ResponseError(derr.Error())
return
}
}
job.FinishedTime = util.GetCurrentTime()
job.Status = "cancelled"
c.meterFinetuneTerminal(job)
if _, err := object.UpdateFinetuneJob(job.Owner, job.Name, job); err != nil {
c.ResponseError(err.Error())
return
}
c.ResponseOk(job)
}
// DeployFinetuneJob serves a completed job's checkpoints and registers the result
// as a routable model on api.hanzo.ai. ?id= or ?name=
func (c *ApiController) DeployFinetuneJob() {
if _, ok := c.RequireSignedInUser(); !ok {
return
}
org := c.GetOrg()
name := c.jobNameParam(org)
job, err := object.GetFinetuneJob(org, name)
if err != nil || job == nil {
c.ResponseError(fmt.Sprintf("fine-tune job %q not found", name))
return
}
res, derr := cluster.DeployFinetune(job, c.GetAcceptLanguage())
if derr != nil {
c.ResponseError(derr.Error())
return
}
job.DeployedModel = res.ModelId
job.DeployUrl = res.Url
if _, err := object.UpdateFinetuneJob(job.Owner, job.Name, job); err != nil {
c.ResponseError(err.Error())
return
}
c.ResponseOk(res)
}
// ── helpers ──────────────────────────────────────────────────────────────────
// refreshFinetuneJob reads live TrainJob status into the job, meters GPU-hours on
// the first transition to a terminal state, and persists any change. Best-effort:
// a cluster error leaves the last-known status untouched (no fabrication).
func (c *ApiController) refreshFinetuneJob(job *object.FinetuneJob, lang string) {
st, err := cluster.TrainJobStatus(job, lang)
if err != nil || st == nil {
return
}
changed := false
if !st.Found {
// CR gone but our record is still active → it was deleted out-of-band.
if !job.IsTerminal() {
job.Status = "failed"
job.Error = firstNonEmptyStr(job.Error, "training job no longer exists in the cluster")
changed = true
}
} else {
if st.Status != "" && st.Status != job.Status {
job.Status = st.Status
changed = true
}
if st.Progress > job.Progress {
job.Progress = st.Progress
changed = true
}
if st.Message != "" && st.Message != job.Message {
job.Message = st.Message
changed = true
}
if st.Started != "" && job.StartedTime == "" {
job.StartedTime = st.Started
changed = true
}
if st.Finished != "" && job.FinishedTime == "" {
job.FinishedTime = st.Finished
changed = true
}
}
if job.IsTerminal() && !job.Metered {
if c.meterFinetuneTerminal(job) {
changed = true
}
}
if changed {
_, _ = object.UpdateFinetuneJob(job.Owner, job.Name, job)
}
}
// meterFinetuneTerminal bills the GPU-hours a finished run consumed, once. Returns
// true if it mutated the job. A commerce error leaves Metered=false so a later
// refresh retries (mirrors transaction.go's retry posture).
func (c *ApiController) meterFinetuneTerminal(job *object.FinetuneJob) bool {
if job.Metered {
return false
}
start := firstNonEmptyStr(job.StartedTime, job.CreatedTime)
finish := firstNonEmptyStr(job.FinishedTime, util.GetCurrentTime())
gpuSeconds := object.GpuSecondsForRun(start, finish, job.GpuCount, job.NumNodes)
if gpuSeconds <= 0 {
job.Metered = true
return true
}
subject := job.Owner + "/" + firstNonEmptyStr(job.CreatedBy, "admin")
cents, err := object.MeterFinetuneGpuHours(subject, gpuSeconds, job.GpuType)
if err != nil {
// Leave Metered=false to retry on a later refresh.
return false
}
job.GpuSeconds += gpuSeconds
job.CostCents += cents
job.Metered = true
return true
}