Files
ai/controllers/model_config_live.go

172 lines
4.6 KiB
Go

// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package controllers
import (
"encoding/json"
"fmt"
"net/http"
"strings"
"time"
"github.com/hanzoai/ai/log"
)
// pricingRetry is how soon a failed refresh is tried again, doubling up to the TTL.
// The first fetch runs as the pod boots, when the service it asks may not be
// answering yet; waiting the whole TTL left every model without a local row on the
// default price until then.
var pricingRetry = 10 * time.Second
// backgroundRefresh is a long-running goroutine that periodically refreshes
// pricing data from pricing.hanzo.ai. It runs only when live_mode is true.
func (mc *ModelConfig) backgroundRefresh() {
mc.mu.RLock()
ttl := mc.pricingTTL
mc.mu.RUnlock()
if ttl <= 0 {
ttl = 6 * time.Hour
}
var wait time.Duration
first := pricingRetry
retry := first
for {
select {
case <-time.After(wait):
case <-mc.stopCh:
return
}
if mc.fetchLivePricing() {
wait, retry = ttl, first
continue
}
wait, retry = retry, min(2*retry, ttl)
}
}
// livePricingResponse is the expected response from pricing.hanzo.ai.
type livePricingResponse struct {
Models []livePricingModel `json:"models"`
}
type livePricingModel struct {
Name string `json:"name"`
Pricing livePricingEntry `json:"pricing"`
}
type livePricingEntry struct {
Input float64 `json:"input"`
Output float64 `json:"output"`
CostIn float64 `json:"cost_in,omitempty"` // provider COGS $/1M input (optional; 0 ⇒ cost defaults to Input)
CostOut float64 `json:"cost_out,omitempty"` // provider COGS $/1M output (optional; 0 ⇒ cost defaults to Output)
}
// fetchLivePricing fetches current pricing from the pricing service and
// merges it into the runtime config. Only overwrites pricing for models
// that exist in the response — never removes existing entries. It reports
// whether the prices were read.
func (mc *ModelConfig) fetchLivePricing() bool {
mc.mu.RLock()
url := mc.pricingURL
mc.mu.RUnlock()
if url == "" {
return false
}
url = strings.TrimRight(url, "/") + "/v1/pricing/models"
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Get(url)
if err != nil {
log.Warn("Live pricing fetch failed: %v", err)
return false
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Warn("Live pricing returned status %d", resp.StatusCode)
return false
}
var result livePricingResponse
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
log.Warn("Live pricing parse failed: %v", err)
return false
}
if len(result.Models) == 0 {
log.Info("Live pricing: no models in response, keeping current data")
return false
}
// Merge live pricing into existing map
mc.mu.Lock()
updated := 0
for _, m := range result.Models {
key := strings.ToLower(m.Name)
if m.Pricing.Input > 0 || m.Pricing.Output > 0 {
mc.pricing[key] = modelPrice{
InputPerMillion: m.Pricing.Input,
OutputPerMillion: m.Pricing.Output,
CostInPerMillion: m.Pricing.CostIn,
CostOutPerMillion: m.Pricing.CostOut,
}
updated++
}
}
mc.lastPricingAt = time.Now()
mc.changedAt = mc.lastPricingAt
mc.mu.Unlock()
log.Info("Live pricing refreshed: %d models updated from %s", updated, url)
return true
}
// LastPricingRefresh returns when pricing was last refreshed from live source.
func (mc *ModelConfig) LastPricingRefresh() time.Time {
mc.mu.RLock()
defer mc.mu.RUnlock()
return mc.lastPricingAt
}
// Stop signals the background refresh goroutine to exit.
func (mc *ModelConfig) Stop() {
select {
case mc.stopCh <- struct{}{}:
default:
}
}
// Status returns a human-readable status string for diagnostics.
func (mc *ModelConfig) Status() string {
mc.mu.RLock()
defer mc.mu.RUnlock()
liveStr := "disabled"
if mc.features.LiveMode {
if mc.lastPricingAt.IsZero() {
liveStr = "enabled (never fetched)"
} else {
liveStr = fmt.Sprintf("enabled (last: %s)", mc.lastPricingAt.Format(time.RFC3339))
}
}
return fmt.Sprintf("routes=%d pricing=%d live=%s",
len(mc.routes), len(mc.pricing), liveStr)
}