mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
172 lines
4.6 KiB
Go
172 lines
4.6 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/hanzoai/ai/log"
|
|
)
|
|
|
|
// pricingRetry is how soon a failed refresh is tried again, doubling up to the TTL.
|
|
// The first fetch runs as the pod boots, when the service it asks may not be
|
|
// answering yet; waiting the whole TTL left every model without a local row on the
|
|
// default price until then.
|
|
var pricingRetry = 10 * time.Second
|
|
|
|
// backgroundRefresh is a long-running goroutine that periodically refreshes
|
|
// pricing data from pricing.hanzo.ai. It runs only when live_mode is true.
|
|
func (mc *ModelConfig) backgroundRefresh() {
|
|
mc.mu.RLock()
|
|
ttl := mc.pricingTTL
|
|
mc.mu.RUnlock()
|
|
|
|
if ttl <= 0 {
|
|
ttl = 6 * time.Hour
|
|
}
|
|
|
|
var wait time.Duration
|
|
first := pricingRetry
|
|
retry := first
|
|
for {
|
|
select {
|
|
case <-time.After(wait):
|
|
case <-mc.stopCh:
|
|
return
|
|
}
|
|
if mc.fetchLivePricing() {
|
|
wait, retry = ttl, first
|
|
continue
|
|
}
|
|
wait, retry = retry, min(2*retry, ttl)
|
|
}
|
|
}
|
|
|
|
// livePricingResponse is the expected response from pricing.hanzo.ai.
|
|
type livePricingResponse struct {
|
|
Models []livePricingModel `json:"models"`
|
|
}
|
|
|
|
type livePricingModel struct {
|
|
Name string `json:"name"`
|
|
Pricing livePricingEntry `json:"pricing"`
|
|
}
|
|
|
|
type livePricingEntry struct {
|
|
Input float64 `json:"input"`
|
|
Output float64 `json:"output"`
|
|
CostIn float64 `json:"cost_in,omitempty"` // provider COGS $/1M input (optional; 0 ⇒ cost defaults to Input)
|
|
CostOut float64 `json:"cost_out,omitempty"` // provider COGS $/1M output (optional; 0 ⇒ cost defaults to Output)
|
|
}
|
|
|
|
// fetchLivePricing fetches current pricing from the pricing service and
|
|
// merges it into the runtime config. Only overwrites pricing for models
|
|
// that exist in the response — never removes existing entries. It reports
|
|
// whether the prices were read.
|
|
func (mc *ModelConfig) fetchLivePricing() bool {
|
|
mc.mu.RLock()
|
|
url := mc.pricingURL
|
|
mc.mu.RUnlock()
|
|
|
|
if url == "" {
|
|
return false
|
|
}
|
|
|
|
url = strings.TrimRight(url, "/") + "/v1/pricing/models"
|
|
|
|
client := &http.Client{Timeout: 30 * time.Second}
|
|
resp, err := client.Get(url)
|
|
if err != nil {
|
|
log.Warn("Live pricing fetch failed: %v", err)
|
|
return false
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
log.Warn("Live pricing returned status %d", resp.StatusCode)
|
|
return false
|
|
}
|
|
|
|
var result livePricingResponse
|
|
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
|
log.Warn("Live pricing parse failed: %v", err)
|
|
return false
|
|
}
|
|
|
|
if len(result.Models) == 0 {
|
|
log.Info("Live pricing: no models in response, keeping current data")
|
|
return false
|
|
}
|
|
|
|
// Merge live pricing into existing map
|
|
mc.mu.Lock()
|
|
updated := 0
|
|
for _, m := range result.Models {
|
|
key := strings.ToLower(m.Name)
|
|
if m.Pricing.Input > 0 || m.Pricing.Output > 0 {
|
|
mc.pricing[key] = modelPrice{
|
|
InputPerMillion: m.Pricing.Input,
|
|
OutputPerMillion: m.Pricing.Output,
|
|
CostInPerMillion: m.Pricing.CostIn,
|
|
CostOutPerMillion: m.Pricing.CostOut,
|
|
}
|
|
updated++
|
|
}
|
|
}
|
|
mc.lastPricingAt = time.Now()
|
|
mc.changedAt = mc.lastPricingAt
|
|
mc.mu.Unlock()
|
|
|
|
log.Info("Live pricing refreshed: %d models updated from %s", updated, url)
|
|
return true
|
|
}
|
|
|
|
// LastPricingRefresh returns when pricing was last refreshed from live source.
|
|
func (mc *ModelConfig) LastPricingRefresh() time.Time {
|
|
mc.mu.RLock()
|
|
defer mc.mu.RUnlock()
|
|
return mc.lastPricingAt
|
|
}
|
|
|
|
// Stop signals the background refresh goroutine to exit.
|
|
func (mc *ModelConfig) Stop() {
|
|
select {
|
|
case mc.stopCh <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// Status returns a human-readable status string for diagnostics.
|
|
func (mc *ModelConfig) Status() string {
|
|
mc.mu.RLock()
|
|
defer mc.mu.RUnlock()
|
|
|
|
liveStr := "disabled"
|
|
if mc.features.LiveMode {
|
|
if mc.lastPricingAt.IsZero() {
|
|
liveStr = "enabled (never fetched)"
|
|
} else {
|
|
liveStr = fmt.Sprintf("enabled (last: %s)", mc.lastPricingAt.Format(time.RFC3339))
|
|
}
|
|
}
|
|
|
|
return fmt.Sprintf("routes=%d pricing=%d live=%s",
|
|
len(mc.routes), len(mc.pricing), liveStr)
|
|
}
|