mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
Finished reads stop/eos/end_turn/tool_calls/function_call in any case. An open think block at tool_calls/function_call is thinking, dropped like a cut. QueryText on openrouter strips like local. The lift gating moves to markup.RulesFor so in-process callers apply the same rules. Co-authored-by: Hanzo Dev <[email protected]>
246 lines
9.3 KiB
Go
246 lines
9.3 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
// Portions Copyright 2023 The OpenAgent Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package model
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
|
|
"github.com/hanzoai/ai/i18n"
|
|
"github.com/hanzoai/ai/proxy"
|
|
"github.com/hanzoai/go-openrouter"
|
|
)
|
|
|
|
type OpenRouterModelProvider struct {
|
|
subType string
|
|
secretKey string
|
|
siteName string
|
|
siteUrl string
|
|
temperature *float32
|
|
topP *float32
|
|
}
|
|
|
|
func NewOpenRouterModelProvider(subType string, secretKey string, temperature float32, topP float32) (*OpenRouterModelProvider, error) {
|
|
p := &OpenRouterModelProvider{
|
|
subType: subType,
|
|
secretKey: secretKey,
|
|
siteName: "Hanzo Cloud",
|
|
siteUrl: "https://hanzo.ai",
|
|
temperature: &temperature,
|
|
topP: &topP,
|
|
}
|
|
return p, nil
|
|
}
|
|
|
|
func (p *OpenRouterModelProvider) GetPricing() string {
|
|
return `URL:
|
|
https://openrouter.ai/docs#models
|
|
|
|
| Model Name | Prompt cost ($ per 1k tokens) | Completion cost ($ per 1k tokens) | Context (tokens) | Moderation |
|
|
|------------------------------|-------------------------------|-----------------------------------|------------------|------------|
|
|
| google/palm-2-codechat-bison | $0.00025 | $0.0005 | 28,672 | None |
|
|
| google/palm-2-chat-bison | $0.00025 | $0.0005 | 36,864 | None |
|
|
| openai/gpt-3.5-turbo | $0.001 | $0.002 | 4,095 | Moderated |
|
|
| openai/gpt-3.5-turbo-16k | $0.0005 | $0.0015 | 16,385 | Moderated |
|
|
| openai/gpt-4 | $0.03 | $0.06 | 8,191 | Moderated |
|
|
| openai/gpt-4-32k | $0.06 | $0.12 | 32,767 | Moderated |
|
|
| anthropic/claude-2 | $0.008 | $0.024 | 200,000 | Moderated |
|
|
| anthropic/claude-instant-v1 | $0.0008 | $0.0024 | 100,000 | Moderated |
|
|
| meta-llama/llama-2-13b-chat | $0.0007 | $0.0009 | 4,096 | None |
|
|
| meta-llama/llama-2-70b-chat | $0.0007 | $0.0009 | 4,096 | None |
|
|
| palm-2-codechat-bison | $0.00025 | $0.0005 | 28,672 | None |
|
|
| palm-2-chat-bison | $0.00025 | $0.0005 | 36,864 | None |
|
|
| gpt-3.5-turbo | $0.001 | $0.002 | 4,095 | Moderated |
|
|
| gpt-3.5-turbo-16k | $0.0005 | $0.0015 | 16,385 | Moderated |
|
|
| gpt-4 | $0.03 | $0.06 | 8,191 | Moderated |
|
|
| gpt-4-32k | $0.06 | $0.12 | 32,767 | Moderated |
|
|
| claude-2 | $0.008 | $0.024 | 200,000 | Moderated |
|
|
| claude-instant-v1 | $0.0008 | $0.0024 | 100,000 | Moderated |
|
|
| llama-2-13b-chat | $0.0007 | $0.0009 | 4,096 | None |
|
|
| llama-2-70b-chat | $0.0007 | $0.0009 | 4,096 | None |
|
|
`
|
|
}
|
|
|
|
func (p *OpenRouterModelProvider) calculatePrice(modelResult *ModelResult, lang string) error {
|
|
var inputPricePerThousandTokens, outputPricePerThousandTokens float64
|
|
priceTable := map[string][]float64{
|
|
"google/palm-2-codechat-bison": {0.00025, 0.0005},
|
|
"google/palm-2-chat-bison": {0.00025, 0.0005},
|
|
"openai/gpt-3.5-turbo": {0.001, 0.002},
|
|
"openai/gpt-3.5-turbo-16k": {0.0005, 0.0015},
|
|
"openai/gpt-4": {0.03, 0.06},
|
|
"openai/gpt-4-32k": {0.06, 0.12},
|
|
"anthropic/claude-2": {0.008, 0.024},
|
|
"anthropic/claude-instant-v1": {0.0008, 0.0024},
|
|
"meta-llama/llama-2-13b-chat": {0.0007, 0.0009},
|
|
"meta-llama/llama-2-70b-chat": {0.0007, 0.0009},
|
|
"palm-2-codechat-bison": {0.00025, 0.0005},
|
|
"palm-2-chat-bison": {0.00025, 0.0005},
|
|
"gpt-3.5-turbo": {0.001, 0.002},
|
|
"gpt-3.5-turbo-16k": {0.0005, 0.0015},
|
|
"gpt-4": {0.03, 0.06},
|
|
"gpt-4-32k": {0.06, 0.12},
|
|
"claude-2": {0.008, 0.024},
|
|
"claude-instant-v1": {0.0008, 0.0024},
|
|
"llama-2-13b-chat": {0.0007, 0.0009},
|
|
"llama-2-70b-chat": {0.0007, 0.0009},
|
|
}
|
|
|
|
if priceItem, ok := priceTable[p.subType]; ok {
|
|
inputPricePerThousandTokens = priceItem[0]
|
|
outputPricePerThousandTokens = priceItem[1]
|
|
} else {
|
|
return fmt.Errorf("%s", fmt.Sprintf(i18n.Translate(lang, "embedding:calculatePrice() error: unknown model type: %s"), p.subType))
|
|
}
|
|
|
|
inputPrice := getPrice(modelResult.PromptTokenCount, inputPricePerThousandTokens)
|
|
outputPrice := getPrice(modelResult.ResponseTokenCount, outputPricePerThousandTokens)
|
|
modelResult.TotalPrice = AddPrices(inputPrice, outputPrice)
|
|
modelResult.Currency = "USD"
|
|
return nil
|
|
}
|
|
|
|
func (p *OpenRouterModelProvider) getProxyClientFromToken() *openrouter.Client {
|
|
config, err := openrouter.DefaultConfig(p.secretKey, p.siteName, p.siteUrl)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
config.HTTPClient = proxy.ProxyHttpClient
|
|
|
|
c := openrouter.NewClientWithConfig(config)
|
|
return c
|
|
}
|
|
|
|
func (p *OpenRouterModelProvider) QueryText(question string, writer io.Writer, history []*RawMessage, prompt string, knowledgeMessages []*RawMessage, agentInfo *AgentInfo, lang string) (*ModelResult, error) {
|
|
client := p.getProxyClientFromToken()
|
|
|
|
ctx := context.Background()
|
|
flusher, ok := writer.(http.Flusher)
|
|
if !ok {
|
|
return nil, fmt.Errorf("%s", i18n.Translate(lang, "model:writer does not implement http.Flusher"))
|
|
}
|
|
|
|
model := p.subType
|
|
if model == "" {
|
|
model = openrouter.Gpt35Turbo
|
|
}
|
|
|
|
tokenCount, err := GetTokenSize(model, question)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
contextLength := GetContextLength(p.subType)
|
|
|
|
if strings.HasPrefix(question, "$CloudDryRun$") {
|
|
modelResult, err := getDefaultModelResult(model, question, "")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s", i18n.Translate(lang, "model:cannot calculate tokens"))
|
|
}
|
|
if contextLength > modelResult.TotalTokenCount {
|
|
return modelResult, nil
|
|
} else {
|
|
return nil, fmt.Errorf("%s", i18n.Translate(lang, "model:exceed max tokens"))
|
|
}
|
|
}
|
|
|
|
maxTokens := contextLength - tokenCount
|
|
if maxTokens < 0 {
|
|
return nil, fmt.Errorf("%s", fmt.Sprintf(i18n.Translate(lang, "model:The token count: [%d] exceeds the model: [%s]'s maximum token count: [%d]"), tokenCount, model, contextLength))
|
|
}
|
|
|
|
temperature := p.temperature
|
|
topP := p.topP
|
|
|
|
respStream, err := client.CreateChatCompletionStream(
|
|
ctx,
|
|
&openrouter.ChatCompletionRequest{
|
|
Model: p.subType,
|
|
Messages: []openrouter.ChatCompletionMessage{
|
|
{
|
|
Role: openrouter.ChatMessageRoleSystem,
|
|
Content: "You are a helpful assistant.",
|
|
},
|
|
{
|
|
Role: openrouter.ChatMessageRoleUser,
|
|
Content: question,
|
|
},
|
|
},
|
|
Stream: false,
|
|
Temperature: temperature,
|
|
TopP: topP,
|
|
MaxTokens: maxTokens,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer respStream.Close()
|
|
|
|
responseStringBuilder := strings.Builder{}
|
|
|
|
isLeadingReturn := true
|
|
// A reasoning-inlining upstream writes its <think></think> block into content;
|
|
// it is stripped here as on every other path.
|
|
strip := NewReasoningStripper(p.subType, OpenerEaten(p.subType))
|
|
for {
|
|
completion, streamErr := respStream.Recv()
|
|
if streamErr != nil {
|
|
if streamErr == io.EOF {
|
|
break
|
|
}
|
|
return nil, streamErr
|
|
}
|
|
if len(completion.Choices) == 0 {
|
|
continue
|
|
}
|
|
|
|
data := strip.Answer(completion.Choices[0].Message.Content, false, completion.Choices[0].FinishReason)
|
|
if isLeadingReturn && len(data) != 0 {
|
|
if strings.Count(data, "\n") == len(data) {
|
|
continue
|
|
} else {
|
|
isLeadingReturn = false
|
|
}
|
|
}
|
|
|
|
if _, err = fmt.Fprintf(writer, "event: message\ndata: %s\n\n", data); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// save the response for token count
|
|
_, _ = responseStringBuilder.WriteString(data)
|
|
|
|
flusher.Flush()
|
|
}
|
|
|
|
modelResult, err := getDefaultModelResult(p.subType, question, responseStringBuilder.String())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
err = p.calculatePrice(modelResult, lang)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return modelResult, nil
|
|
}
|