Files
ai/object/record.go
zeekay bedc76a3d9 The audit trail keeps a prompt's digest, never the prompt
Every org writes a record on every call, and the Record filter stored each
model request's body in it, redacted only of credentials: every customer's
prompts, kept whether or not their org had asked for them to be, and handed on
to the record commit. A request to a model surface is now recorded as the
sha256 and size of its body. The trail still says that a model was called and
with which body; what the caller said is kept only where their org asked for it.

Claude-Session: https://claude.ai/code/session_013cvpQYtJPnTNfWzbqwtBJZ
2026-09-29 21:49:03 -07:00

463 lines
15 KiB
Go

// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
// Portions Copyright 2023 The OpenAgent Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package object
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"net/http"
"strings"
"github.com/google/uuid"
"github.com/hanzoai/ai/conf"
"github.com/hanzoai/ai/i18n"
"github.com/hanzoai/ai/util"
"github.com/hanzoai/dbx"
"github.com/zap-proto/zip"
)
var logPostOnly bool
func init() {
logPostOnly = conf.GetConfigBool("logPostOnly")
}
// Recorded reports whether a request with this method is kept. It is the one place
// the logPostOnly rule is read, so a caller can ask before building a record rather
// than after: under logPostOnly a GET is discarded, and composing one costs a geo-IP
// lookup and a provider query that nothing then reads.
//
// It also answers no when there is nowhere to keep one. A record is written to the
// store and the store is opened by boot, so a request served before boot — or in a
// deployment configured without a database, which is a supported mode that answers
// fail-closed — has nothing to write to. AddRecord queries the providers through
// adapter.db on its way, so asking afterwards is a nil dereference on the way to
// discovering there was no point: the request dies in a filter, after it was served,
// over a log row.
func Recorded(method string) bool {
if adapter == nil || adapter.db == nil {
return false
}
return !(logPostOnly && method == "GET")
}
type Record struct {
Id int `db:"pk" json:"id"`
Owner string `json:"owner"`
Name string `json:"name"`
CreatedTime string `json:"createdTime"`
Organization string `json:"organization"`
ClientIp string `json:"clientIp"`
UserAgent string `json:"userAgent"`
User string `json:"user"`
Method string `json:"method"`
RequestUri string `json:"requestUri"`
Action string `json:"action"`
Language string `json:"language"`
Query string `json:"query"`
Region string `json:"region"`
City string `json:"city"`
Unit string `json:"unit"`
Section string `json:"section"`
Object string `json:"object"`
Response string `json:"response"`
ErrorText string `json:"errorText"`
// ExtendedUser *User `db:"-" json:"extendedUser"`
Provider string `json:"provider"`
Block string `json:"block"`
BlockHash string `json:"blockHash"`
Transaction string `json:"transaction"`
Provider2 string `json:"provider2"`
Block2 string `json:"block2"`
BlockHash2 string `json:"blockHash2"`
Transaction2 string `json:"transaction2"`
// For cross-chain records
Count int `json:"count"`
IsTriggered bool `json:"isTriggered"`
// NO `db:"index"`. dbx's `db` tag is the COLUMN NAME, not a directive —
// parseTag (dbx struct.go:229) special-cases only "pk" and "pk,<name>" and
// returns everything else verbatim as the name. So `db:"index"` did not
// index anything; it named this column `index` (a reserved SQL word), while
// ScanNeedCommitRecords queries `need_commit`. Live effect: every 5 minutes,
// forever, `no such column: need_commit` — the record-chain commit task has
// never committed a record. Untagged, the name is DefaultFieldMapFunc's
// snake_case, which is the `need_commit` the query already asks for.
NeedCommit bool `json:"needCommit"`
}
type Response struct {
Status string `json:"status"`
Msg string `json:"msg"`
}
func GetRecordCount(owner, field, value string) (int64, error) {
return rowCount("record", owner, field, value)
}
func GetRecords(owner string) ([]*Record, error) {
records := []*Record{}
err := findAll(adapter.db, "record", &records, dbx.HashExp{"owner": owner}, "id DESC")
if err != nil {
return records, err
}
return records, nil
}
func getAllRecords() ([]*Record, error) {
records := []*Record{}
err := findAll(adapter.db, "record", &records, dbx.HashExp{}, "id DESC")
if err != nil {
return records, err
}
return records, nil
}
func getValidAndNeedCommitRecords(records []*Record) ([]*Record, []int, []any, error) {
providerFirst, providerSecond, err := GetTwoActiveBlockchainProvider("admin")
if err != nil {
return nil, nil, nil, err
}
var validRecords []*Record
var needCommitIdx []int
var data []any
recordTime := util.GetCurrentTimeWithMilli()
for _, record := range records {
ok, err := prepareRecord(record, providerFirst, providerSecond)
if err != nil {
return nil, nil, nil, err
}
if !ok {
continue
}
record.CreatedTime = util.GetCurrentTimeBasedOnLastMilli(recordTime)
recordTime = record.CreatedTime
validRecords = append(validRecords, record)
data = append(data, map[string]any{"name": record.Name})
if record.NeedCommit {
// validRecords and data are appended in lockstep, and the commit results
// come back one per VALID record — so this index addresses those, not the
// raw input slice a dropped record makes shorter.
needCommitIdx = append(needCommitIdx, len(validRecords)-1)
}
}
return validRecords, needCommitIdx, data, nil
}
func GetPaginationRecords(owner string, offset, limit int, field, value, sortField, sortOrder string) ([]*Record, error) {
return rowsPage[Record]("record", owner, offset, limit, field, value, sortField, sortOrder)
}
// GetRecord retrieves a record by its ID or owner/name format.
func GetRecord(id string, lang string) (*Record, error) {
record := &Record{}
owner, name, err := util.GetOwnerAndNameFromIdWithError(id)
if err != nil {
return nil, fmt.Errorf("%s", fmt.Sprintf(i18n.Translate(lang, "object:failed to parse record identifier '%s': neither a valid owner/[id|name] format"), id))
}
// Try to parse as integer ID first
if recordId, err := util.ParseIntWithError(name); err == nil && recordId > 0 {
// Valid integer ID
record.Id = recordId
} else {
record.Owner = owner
record.Name = name
}
var existed bool
if record.Id != 0 {
existed, err = getOne(adapter.db, "record", record, pkID(record.Id))
} else {
existed, err = getOne(adapter.db, "record", record, pk2(record.Owner, record.Name))
}
if err != nil {
return nil, fmt.Errorf("%s", fmt.Sprintf(i18n.Translate(lang, "object:failed to get record with id '%s': %w"), id, err))
}
if existed {
return record, nil
}
return nil, nil
}
func prepareRecord(record *Record, providerFirst, providerSecond *Provider) (bool, error) {
if !Recorded(record.Method) {
return false, nil
}
if strings.HasSuffix(record.Action, "-record") {
return false, nil
}
if strings.HasSuffix(record.Action, "-record-second") {
return false, nil
}
if strings.HasSuffix(record.Action, "-records") {
return false, nil
}
if record.Provider == "" {
if providerFirst != nil {
record.Provider = providerFirst.Name
}
if providerSecond != nil {
record.Provider2 = providerSecond.Name
}
}
record.Id = 0
record.Name = uuid.NewString()
record.Owner = record.Organization
// Set default count to 1 if not set
if record.Count == 0 {
record.Count = 1
}
return true, nil
}
func UpdateRecord(id string, record *Record, lang string) (bool, error) {
p, err := GetRecord(id, lang)
if err != nil {
return false, err
} else if p == nil {
return false, nil
}
// Update provider
if record.Provider != p.Provider {
record.Block = ""
record.BlockHash = ""
record.Transaction = ""
}
if record.Provider2 != p.Provider2 {
record.Block2 = ""
record.BlockHash2 = ""
record.Transaction2 = ""
}
record.Id = int(p.Id)
return updated(record)
}
func UpdateRecordInternal(id int, record Record) error {
err := adapter.db.Model(&record).Update()
if err != nil {
return err
}
return nil
}
func UpdateRecordFields(id string, fields map[string]any, lang string) (bool, error) {
p, err := GetRecord(id, lang)
if err != nil {
return false, err
} else if p == nil {
return false, nil
}
affected, err := updateByPK(adapter.db, "record", pkID(int(p.Id)), dbx.Params(fields))
if err != nil {
return false, err
}
return affected != 0, nil
}
func NewRecord(ctx *zip.Ctx) (*Record, error) {
ip := strings.Replace(util.GetIPInfo(ctx.Fiber().IP()), ": ", "", -1)
action := strings.TrimPrefix(ctx.Path(), "/v1/")
// The query is redacted by the same rule the body is, rather than by a list of
// one name: a credential in a URL is the worst place for one, and a record is
// read by more surfaces than the one that wrote it. Redaction keeps the shape
// of the request, which is what an audit trail is for.
requestUri := RedactQuery(string(ctx.Fiber().Request().RequestURI()))
if len(requestUri) > 1000 {
requestUri = requestUri[0:1000]
}
// The body is redacted BEFORE it is stored, not on the way out. A record is
// read by more surfaces than the one that wrote it, and a secret that was
// never written cannot be served by any of them. An admin saving a provider
// pastes an upstream key into this body on its way to being sealed, so the
// unredacted form would keep a plaintext copy of the very key that sealing
// exists to remove.
//
// A prompt is not kept at all. The trail records that a model was called and
// with what, never what the caller said to it: a customer's prompts are theirs,
// kept only where their org has asked for them to be (the capture store), and an
// audit row every org writes on every call is not that place.
object := ""
if body := ctx.Body(); len(body) != 0 {
if promptBearing(ctx.Path()) {
object = bodyDigest(body)
} else {
object = RedactBody(string(body))
}
}
status, msg := "ok", ""
if code := ctx.Fiber().Response().StatusCode(); code >= 400 {
status, msg = "error", http.StatusText(code)
}
language := ctx.Header("Accept-Language")
if len(language) > 2 {
language = language[0:2]
}
languageCode := conf.GetLanguage(language)
// get location info from client ip
locationInfo, err := util.GetInfoFromIP(ip)
if err != nil {
return nil, err
}
region := locationInfo.Country
city := locationInfo.City
record := Record{
Name: uuid.NewString(),
CreatedTime: util.GetCurrentTimeWithMilli(),
ClientIp: ip,
User: "",
Method: ctx.Method(),
RequestUri: requestUri,
Action: action,
Language: languageCode,
Region: region,
City: city,
Object: object,
Response: fmt.Sprintf("{\"status\":\"%s\",\"msg\":\"%s\"}", status, msg),
Count: 1,
IsTriggered: false,
}
return &record, nil
}
func AddRecord(record *Record, lang string) (bool, any, error) {
providerFirst, providerSecond, err := GetTwoActiveBlockchainProvider(record.Owner)
if err != nil {
return false, nil, err
}
ok, err := prepareRecord(record, providerFirst, providerSecond)
if err != nil {
return false, nil, err
}
if !ok {
return false, nil, nil
}
record.CreatedTime = util.GetCurrentTimeWithMilli()
// Set default count to 1 if not set
if record.Count == 0 {
record.Count = 1
}
err = insertRow(adapter.db, record)
if err != nil {
return false, nil, err
}
data := map[string]any{"name": record.Name}
if record.NeedCommit {
_, commitResult, err := CommitRecord(record, lang)
if err != nil {
data["error_text"] = err.Error()
} else {
data = commitResult
}
}
return true, data, nil
}
func AddRecords(records []*Record, syncEnabled bool, lang string) (bool, any, error) {
if len(records) == 0 {
return false, nil, nil
}
validRecords, needCommitRecordsIdx, data, err := getValidAndNeedCommitRecords(records)
if err != nil {
return false, nil, err
}
if len(validRecords) == 0 {
return false, nil, nil
}
totalAffected := int64(0)
err = adapter.db.Transactional(func(tx *dbx.Tx) error {
batchSize := 150
for i := 0; i < len(validRecords); i += batchSize {
end := min(i+batchSize, len(validRecords))
batch := validRecords[i:end]
for _, r := range batch {
if err := tx.Model(r).Insert(); err != nil {
return err
}
totalAffected++
}
}
return nil
})
if err != nil {
return false, nil, err
}
// Send commit event for records that need to be committed
if len(needCommitRecordsIdx) > 0 {
if syncEnabled {
var needCommitRecords []*Record
for _, idx := range needCommitRecordsIdx {
needCommitRecords = append(needCommitRecords, validRecords[idx])
}
_, commitResults := CommitRecords(needCommitRecords, lang)
for i, idx := range needCommitRecordsIdx {
data[idx] = commitResults[i]
}
} else {
go ScanNeedCommitRecords()
}
}
return totalAffected != 0, data, nil
}
func DeleteRecord(record *Record) (bool, error) {
affected, err := deleteByPK(adapter.db, "record", pkID(int(record.Id)))
if err != nil {
return false, err
}
return affected != 0, nil
}
func (record *Record) getUniqueId() string {
return fmt.Sprintf("%s/%d", record.Owner, record.Id)
}
func (record *Record) getId() string {
return fmt.Sprintf("%s/%s", record.Owner, record.Name)
}
func (r *Record) updateErrorText(errText string, lang string) (bool, error) {
r.ErrorText = errText
if r.Id != 0 {
affected, err := updateCols(adapter.db, "record", pk2(r.Owner, r.Name), dbx.Params{"error_text": r.ErrorText})
if err != nil {
return affected > 0, fmt.Errorf("%s", fmt.Sprintf(i18n.Translate(lang, "object:failed to update error text for record %s: %s"), r.getId(), err))
}
return affected > 0, nil
} else {
affected, err := updateCols(adapter.db, "record", pkID(r.Id), dbx.Params{"error_text": r.ErrorText})
if err != nil {
return affected > 0, fmt.Errorf("%s", fmt.Sprintf(i18n.Translate(lang, "object:failed to update error text for record %s: %s"), r.getUniqueId(), err))
}
return affected > 0, nil
}
}
// promptBearing reports a path whose request body is a customer's prompt: the model
// surfaces, which carry what the caller said to a model and nothing an audit needs.
func promptBearing(path string) bool {
p := strings.TrimSuffix(strings.ToLower(path), "/")
for _, surface := range []string{"/v1/chat", "/v1/completions", "/v1/responses", "/v1/messages",
"/v1/embeddings", "/v1/rerank", "/v1/images", "/v1/audio", "/v1/videos"} {
if p == surface || strings.HasPrefix(p, surface+"/") {
return true
}
}
return false
}
// bodyDigest is what the trail keeps of a prompt: which body it was, and how large.
func bodyDigest(body []byte) string {
sum := sha256.Sum256(body)
return fmt.Sprintf(`{"sha256":"%s","bytes":%d}`, hex.EncodeToString(sum[:]), len(body))
}