mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
dbx's parseTag (struct.go:229) special-cases exactly "pk" and "pk,<name>" and returns everything else VERBATIM as the column name. It has no notion of an index directive — that is xorm's vocabulary, carried over in the port. So `NeedCommit bool `db:"index"`` indexed nothing. It named the column `index` (a reserved SQL word), while ScanNeedCommitRecords queries `need_commit`. Measured in production: every five minutes, forever, ScanNeedCommitRecords() failed to scan records that need to be committed: no such column: need_commit The record-chain commit task has never committed a record. The error is also pure noise on a five-minute cadence, which is what a real error has to compete with to be noticed. Untagged, the column is DefaultFieldMapFunc's snake_case — the `need_commit` the query already asks for. The orphaned `index` column holds a transient "needs committing" flag, so nothing durable is stranded by the rename. The class matters more than the instance, because this failure is nearly invisible: the ORM writes and reads the SAME wrong column and stays self-consistent, so it only surfaces where raw SQL names the column the Go field implies. routers/db_tag_test.go parses object/*.go and rejects any db tag that is xorm vocabulary or contains spaces/parens. It lives in routers, not object, on purpose: object's TestMain os.Exit(0)s the whole package when no seeded database is present, so a guard placed there would have reported "ok" without ever running — which is exactly how it passed the first time I wrote it. A schema-shape invariant needs no database and must not be gated behind one. Two tags predate this and are listed rather than fixed: object/connection.go Operations `db:"json varchar(1000)"` and object/template.go BasicConfigOptions `db:"json"`. dbx QUOTES the column name, so those are real created columns that may hold real rows; dropping the tag creates a fresh empty column and the old values stop being read. That is a data migration, not a tag edit. The guard logs them by name so the debt is visible, and still fails on anything new. Verified: reintroducing `db:"index"` fails both tests; removing it passes. Co-authored-by: Hanzo Dev <[email protected]>
425 lines
12 KiB
Go
425 lines
12 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
// Portions Copyright 2023 The OpenAgent Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
package object
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/hanzoai/ai/conf"
|
|
"github.com/hanzoai/ai/i18n"
|
|
"github.com/hanzoai/ai/util"
|
|
"github.com/hanzoai/ai/web"
|
|
"github.com/hanzoai/dbx"
|
|
)
|
|
|
|
var logPostOnly bool
|
|
|
|
func init() {
|
|
logPostOnly = conf.GetConfigBool("logPostOnly")
|
|
}
|
|
|
|
type Record struct {
|
|
Id int `db:"pk" json:"id"`
|
|
Owner string `json:"owner"`
|
|
Name string `json:"name"`
|
|
CreatedTime string `json:"createdTime"`
|
|
Organization string `json:"organization"`
|
|
ClientIp string `json:"clientIp"`
|
|
UserAgent string `json:"userAgent"`
|
|
User string `json:"user"`
|
|
Method string `json:"method"`
|
|
RequestUri string `json:"requestUri"`
|
|
Action string `json:"action"`
|
|
Language string `json:"language"`
|
|
Query string `json:"query"`
|
|
Region string `json:"region"`
|
|
City string `json:"city"`
|
|
Unit string `json:"unit"`
|
|
Section string `json:"section"`
|
|
Object string `json:"object"`
|
|
Response string `json:"response"`
|
|
ErrorText string `json:"errorText"`
|
|
// ExtendedUser *User `db:"-" json:"extendedUser"`
|
|
Provider string `json:"provider"`
|
|
Block string `json:"block"`
|
|
BlockHash string `json:"blockHash"`
|
|
Transaction string `json:"transaction"`
|
|
Provider2 string `json:"provider2"`
|
|
Block2 string `json:"block2"`
|
|
BlockHash2 string `json:"blockHash2"`
|
|
Transaction2 string `json:"transaction2"`
|
|
// For cross-chain records
|
|
Count int `json:"count"`
|
|
IsTriggered bool `json:"isTriggered"`
|
|
// NO `db:"index"`. dbx's `db` tag is the COLUMN NAME, not a directive —
|
|
// parseTag (dbx struct.go:229) special-cases only "pk" and "pk,<name>" and
|
|
// returns everything else verbatim as the name. So `db:"index"` did not
|
|
// index anything; it named this column `index` (a reserved SQL word), while
|
|
// ScanNeedCommitRecords queries `need_commit`. Live effect: every 5 minutes,
|
|
// forever, `no such column: need_commit` — the record-chain commit task has
|
|
// never committed a record. Untagged, the name is DefaultFieldMapFunc's
|
|
// snake_case, which is the `need_commit` the query already asks for.
|
|
NeedCommit bool `json:"needCommit"`
|
|
}
|
|
type Response struct {
|
|
Status string `json:"status"`
|
|
Msg string `json:"msg"`
|
|
}
|
|
|
|
func GetRecordCount(owner, field, value string) (int64, error) {
|
|
session := GetDbQuery(owner, -1, -1, field, value, "", "")
|
|
return queryCount(session, "record")
|
|
}
|
|
|
|
func GetRecords(owner string) ([]*Record, error) {
|
|
records := []*Record{}
|
|
err := findAll(adapter.db, "record", &records, dbx.HashExp{"owner": owner}, "id DESC")
|
|
if err != nil {
|
|
return records, err
|
|
}
|
|
return records, nil
|
|
}
|
|
|
|
func getAllRecords() ([]*Record, error) {
|
|
records := []*Record{}
|
|
err := findAll(adapter.db, "record", &records, dbx.HashExp{}, "id DESC")
|
|
if err != nil {
|
|
return records, err
|
|
}
|
|
return records, nil
|
|
}
|
|
|
|
func getValidAndNeedCommitRecords(records []*Record) ([]*Record, []int, []interface{}, error) {
|
|
providerFirst, providerSecond, err := GetTwoActiveBlockchainProvider("admin")
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
var validRecords []*Record
|
|
var needCommitIdx []int
|
|
var data []interface{}
|
|
recordTime := util.GetCurrentTimeWithMilli()
|
|
for i, record := range records {
|
|
ok, err := prepareRecord(record, providerFirst, providerSecond)
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
if !ok {
|
|
continue
|
|
}
|
|
record.CreatedTime = util.GetCurrentTimeBasedOnLastMilli(recordTime)
|
|
recordTime = record.CreatedTime
|
|
validRecords = append(validRecords, record)
|
|
data = append(data, map[string]interface{}{"name": record.Name})
|
|
if record.NeedCommit {
|
|
needCommitIdx = append(needCommitIdx, i)
|
|
}
|
|
}
|
|
return validRecords, needCommitIdx, data, nil
|
|
}
|
|
|
|
func GetPaginationRecords(owner string, offset, limit int, field, value, sortField, sortOrder string) ([]*Record, error) {
|
|
records := []*Record{}
|
|
session := GetDbQuery(owner, offset, limit, field, value, sortField, sortOrder)
|
|
err := queryFind(session, "record", &records)
|
|
if err != nil {
|
|
return records, err
|
|
}
|
|
return records, nil
|
|
}
|
|
|
|
// GetRecord retrieves a record by its ID or owner/name format.
|
|
func GetRecord(id string, lang string) (*Record, error) {
|
|
record := &Record{}
|
|
owner, name, err := util.GetOwnerAndNameFromIdWithError(id)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s", fmt.Sprintf(i18n.Translate(lang, "object:failed to parse record identifier '%s': neither a valid owner/[id|name] format"), id))
|
|
}
|
|
// Try to parse as integer ID first
|
|
if recordId, err := util.ParseIntWithError(name); err == nil && recordId > 0 {
|
|
// Valid integer ID
|
|
record.Id = recordId
|
|
} else {
|
|
record.Owner = owner
|
|
record.Name = name
|
|
}
|
|
var existed bool
|
|
if record.Id != 0 {
|
|
existed, err = getOne(adapter.db, "record", record, pkID(record.Id))
|
|
} else {
|
|
existed, err = getOne(adapter.db, "record", record, pk2(record.Owner, record.Name))
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s", fmt.Sprintf(i18n.Translate(lang, "object:failed to get record with id '%s': %w"), id, err))
|
|
}
|
|
if existed {
|
|
return record, nil
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func prepareRecord(record *Record, providerFirst, providerSecond *Provider) (bool, error) {
|
|
if logPostOnly && record.Method == "GET" {
|
|
return false, nil
|
|
}
|
|
if strings.HasSuffix(record.Action, "-record") {
|
|
return false, nil
|
|
}
|
|
if strings.HasSuffix(record.Action, "-record-second") {
|
|
return false, nil
|
|
}
|
|
if strings.HasSuffix(record.Action, "-records") {
|
|
return false, nil
|
|
}
|
|
if record.Provider == "" {
|
|
if providerFirst != nil {
|
|
record.Provider = providerFirst.Name
|
|
}
|
|
if providerSecond != nil {
|
|
record.Provider2 = providerSecond.Name
|
|
}
|
|
}
|
|
record.Id = 0
|
|
record.Name = uuid.NewString()
|
|
record.Owner = record.Organization
|
|
// Set default count to 1 if not set
|
|
if record.Count == 0 {
|
|
record.Count = 1
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
func UpdateRecord(id string, record *Record, lang string) (bool, error) {
|
|
p, err := GetRecord(id, lang)
|
|
if err != nil {
|
|
return false, err
|
|
} else if p == nil {
|
|
return false, nil
|
|
}
|
|
// Update provider
|
|
if record.Provider != p.Provider {
|
|
record.Block = ""
|
|
record.BlockHash = ""
|
|
record.Transaction = ""
|
|
}
|
|
if record.Provider2 != p.Provider2 {
|
|
record.Block2 = ""
|
|
record.BlockHash2 = ""
|
|
record.Transaction2 = ""
|
|
}
|
|
record.Id = int(p.Id)
|
|
err = adapter.db.Model(record).Update()
|
|
affected := int64(1)
|
|
if err != nil {
|
|
affected = 0
|
|
}
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return affected != 0, nil
|
|
}
|
|
|
|
func UpdateRecordInternal(id int, record Record) error {
|
|
err := adapter.db.Model(&record).Update()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func UpdateRecordFields(id string, fields map[string]interface{}, lang string) (bool, error) {
|
|
p, err := GetRecord(id, lang)
|
|
if err != nil {
|
|
return false, err
|
|
} else if p == nil {
|
|
return false, nil
|
|
}
|
|
affected, err := updateByPK(adapter.db, "record", pkID(int(p.Id)), dbx.Params(fields))
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return affected != 0, nil
|
|
}
|
|
|
|
func NewRecord(ctx *web.Context) (*Record, error) {
|
|
ip := strings.Replace(util.GetIPFromRequest(ctx.Request), ": ", "", -1)
|
|
action := strings.TrimPrefix(ctx.Request.URL.Path, "/v1/")
|
|
requestUri := util.FilterQuery(ctx.Request.RequestURI, []string{"accessToken"})
|
|
if len(requestUri) > 1000 {
|
|
requestUri = requestUri[0:1000]
|
|
}
|
|
object := ""
|
|
if len(ctx.Input.RequestBody) != 0 {
|
|
object = string(ctx.Input.RequestBody)
|
|
}
|
|
respBytes, err := json.Marshal(ctx.Input.Data()["json"])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var resp Response
|
|
err = json.Unmarshal(respBytes, &resp)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
language := ctx.Request.Header.Get("Accept-Language")
|
|
if len(language) > 2 {
|
|
language = language[0:2]
|
|
}
|
|
languageCode := conf.GetLanguage(language)
|
|
// get location info from client ip
|
|
locationInfo, err := util.GetInfoFromIP(ip)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
region := locationInfo.Country
|
|
city := locationInfo.City
|
|
record := Record{
|
|
Name: uuid.NewString(),
|
|
CreatedTime: util.GetCurrentTimeWithMilli(),
|
|
ClientIp: ip,
|
|
User: "",
|
|
Method: ctx.Request.Method,
|
|
RequestUri: requestUri,
|
|
Action: action,
|
|
Language: languageCode,
|
|
Region: region,
|
|
City: city,
|
|
Object: object,
|
|
Response: fmt.Sprintf("{\"status\":\"%s\",\"msg\":\"%s\"}", resp.Status, resp.Msg),
|
|
Count: 1,
|
|
IsTriggered: false,
|
|
}
|
|
return &record, nil
|
|
}
|
|
|
|
func AddRecord(record *Record, lang string) (bool, interface{}, error) {
|
|
providerFirst, providerSecond, err := GetTwoActiveBlockchainProvider(record.Owner)
|
|
if err != nil {
|
|
return false, nil, err
|
|
}
|
|
ok, err := prepareRecord(record, providerFirst, providerSecond)
|
|
if err != nil {
|
|
return false, nil, err
|
|
}
|
|
if !ok {
|
|
return false, nil, nil
|
|
}
|
|
record.CreatedTime = util.GetCurrentTimeWithMilli()
|
|
// Set default count to 1 if not set
|
|
if record.Count == 0 {
|
|
record.Count = 1
|
|
}
|
|
err = insertRow(adapter.db, record)
|
|
affected := int64(1)
|
|
if err != nil {
|
|
affected = 0
|
|
}
|
|
if err != nil {
|
|
return false, nil, err
|
|
}
|
|
data := map[string]interface{}{"name": record.Name}
|
|
if record.NeedCommit {
|
|
_, commitResult, err := CommitRecord(record, lang)
|
|
if err != nil {
|
|
data["error_text"] = err.Error()
|
|
} else {
|
|
data = commitResult
|
|
}
|
|
}
|
|
return affected != 0, data, nil
|
|
}
|
|
|
|
func AddRecords(records []*Record, syncEnabled bool, lang string) (bool, interface{}, error) {
|
|
if len(records) == 0 {
|
|
return false, nil, nil
|
|
}
|
|
validRecords, needCommitRecordsIdx, data, err := getValidAndNeedCommitRecords(records)
|
|
if err != nil {
|
|
return false, nil, err
|
|
}
|
|
if len(validRecords) == 0 {
|
|
return false, nil, nil
|
|
}
|
|
totalAffected := int64(0)
|
|
err = adapter.db.Transactional(func(tx *dbx.Tx) error {
|
|
batchSize := 150
|
|
for i := 0; i < len(validRecords); i += batchSize {
|
|
end := min(i+batchSize, len(validRecords))
|
|
batch := validRecords[i:end]
|
|
for _, r := range batch {
|
|
if err := tx.Model(r).Insert(); err != nil {
|
|
return err
|
|
}
|
|
totalAffected++
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return false, nil, err
|
|
}
|
|
// Send commit event for records that need to be committed
|
|
if len(needCommitRecordsIdx) > 0 {
|
|
if syncEnabled {
|
|
var needCommitRecords []*Record
|
|
for _, idx := range needCommitRecordsIdx {
|
|
needCommitRecords = append(needCommitRecords, records[idx])
|
|
}
|
|
_, commitResults := CommitRecords(needCommitRecords, lang)
|
|
for i, idx := range needCommitRecordsIdx {
|
|
data[idx] = commitResults[i]
|
|
}
|
|
} else {
|
|
go ScanNeedCommitRecords()
|
|
}
|
|
}
|
|
return totalAffected != 0, data, nil
|
|
}
|
|
|
|
func DeleteRecord(record *Record) (bool, error) {
|
|
affected, err := deleteByPK(adapter.db, "record", pkID(int(record.Id)))
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return affected != 0, nil
|
|
}
|
|
|
|
func (record *Record) getUniqueId() string {
|
|
return fmt.Sprintf("%s/%d", record.Owner, record.Id)
|
|
}
|
|
|
|
func (record *Record) getId() string {
|
|
return fmt.Sprintf("%s/%s", record.Owner, record.Name)
|
|
}
|
|
|
|
func (r *Record) updateErrorText(errText string, lang string) (bool, error) {
|
|
r.ErrorText = errText
|
|
if r.Id != 0 {
|
|
affected, err := updateCols(adapter.db, "record", pk2(r.Owner, r.Name), dbx.Params{"error_text": r.ErrorText})
|
|
if err != nil {
|
|
return affected > 0, fmt.Errorf("%s", fmt.Sprintf(i18n.Translate(lang, "object:failed to update error text for record %s: %s"), r.getId(), err))
|
|
}
|
|
return affected > 0, nil
|
|
} else {
|
|
affected, err := updateCols(adapter.db, "record", pkID(r.Id), dbx.Params{"error_text": r.ErrorText})
|
|
if err != nil {
|
|
return affected > 0, fmt.Errorf("%s", fmt.Sprintf(i18n.Translate(lang, "object:failed to update error text for record %s: %s"), r.getUniqueId(), err))
|
|
}
|
|
return affected > 0, nil
|
|
}
|
|
}
|