mirror of
https://github.com/hanzoai/ai.git
synced 2026-11-20 14:13:23 +00:00
go test ./... in this repo failed in five packages, so nobody ran it, so
nothing in it was read. Underneath were four genuine defects:
1. CREATE DATABASE against SQLite. object/adapter.go switched on driverName
with a postgres case and a DEFAULT that emits MySQL DDL, so sqlite got
`CREATE DATABASE IF NOT EXISTS … charset utf8mb4` and every caller took
`near "DATABASE": syntax error`. SQLite has no CREATE DATABASE at all — the
database IS the file. A default that assumes one dialect is wrong for every
dialect it has not met.
2. Markdown chunks came out in RANDOM ORDER. SplitText ranged a map, and Go
randomises map iteration, so the same document produced the same chunks in a
different sequence every run. Chunks are reassembled into context
downstream; document order is the only order that means anything. Now sorted
by position in the source (not alphabetically — "Section 10" must not sort
before "Section 9").
3. Tables lost their last row. Every row pattern ends in \\n, so a table running
to end-of-input dropped its final row into the remainder, where it came back
out glued to the neighbouring prose — a table silently split across two
chunks. Sections are sliced per heading, so ending without a newline is the
COMMON case.
4. GetVideoCoverUrl("") panicked. VodClient is package-level and nil until
SetVodClient runs, so on any deployment without the Aliyun VOD provider all
four entry points called through nil, inside request handlers. An
unconfigured OPTIONAL provider took the process down. Plus resp.Body.Video
was three unguarded dereferences on a response we do not control.
Also: ExtractMarkdownTree emitted result["root"] = "" for documents opening
with a heading — a key for content that does not exist — and the flush logic
was written twice.
Why they were invisible: 14 files are operator SCRIPTS wearing func Test —
69 panics, ZERO assertions, needing a populated database, live credentials or
fixtures not in this repo. A panicking test aborts the whole binary, so one
missing mp3 took every real test in its package with it. They are now behind
`-tags ai_scripts`; several WRITE to whatever database they find, which
`go test ./...` must never do. TestListGeminiModels had `apiKey := ""`
hardcoded and could only ever panic; it and the txt fixture test now skip,
matching TestGenerateImageDOAI_Live which already did this correctly.
The split tests also said only "did not get the expected result" and "Expected
6 headings, got 7". They now name which section and which heading.
Whole suite green.
Co-authored-by: Hanzo Dev <[email protected]>
228 lines
7.3 KiB
Go
228 lines
7.3 KiB
Go
// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
|
|
// Portions Copyright 2025 The OpenAgent Authors. All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package split
|
|
|
|
import (
|
|
"fmt"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
type MarkdownSplitProvider struct{}
|
|
|
|
func NewMarkdownSplitProvider() (*MarkdownSplitProvider, error) {
|
|
return &MarkdownSplitProvider{}, nil
|
|
}
|
|
|
|
func ExtractMarkdownTree(markdownText string) map[string]string {
|
|
numberedHeadingPattern := regexp.MustCompile(`^(\d+(\.\d+)*\.)\s+(.+)$`)
|
|
hashHeadingPattern := regexp.MustCompile(`^(#{1,6})\s+(.+)$`)
|
|
|
|
lines := strings.Split(markdownText, "\n")
|
|
result := make(map[string]string)
|
|
|
|
var currentKey string
|
|
var currentContent []string
|
|
var path []string
|
|
|
|
// flush records the section that just ended.
|
|
//
|
|
// A document opening with a heading has no preamble, and the old code still
|
|
// wrote result["root"] = "" — a key for content that does not exist. It
|
|
// showed up as a phantom section with no text and made the heading count
|
|
// wrong. A named heading is kept even when its body is empty, because the
|
|
// heading itself is content; "root" is not a heading, it is the absence of
|
|
// one, so with no body there is nothing to record.
|
|
//
|
|
// Also the only copy: these five lines were written twice, once in the loop
|
|
// and once after it.
|
|
flush := func() {
|
|
body := strings.TrimSpace(strings.Join(currentContent, "\n"))
|
|
if currentKey != "" {
|
|
result[currentKey] = body
|
|
return
|
|
}
|
|
if body != "" {
|
|
result["root"] = body
|
|
}
|
|
}
|
|
|
|
for _, line := range lines {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
|
|
var isHeading bool
|
|
var level int
|
|
var title string
|
|
|
|
if m := hashHeadingPattern.FindStringSubmatch(line); m != nil {
|
|
title = fmt.Sprintf("%s %s", m[1], m[2])
|
|
level = len(m[1])
|
|
isHeading = true
|
|
} else if m := numberedHeadingPattern.FindStringSubmatch(line); m != nil {
|
|
title = fmt.Sprintf("%s %s", m[1], m[3])
|
|
level = strings.Count(m[1], ".")
|
|
isHeading = true
|
|
}
|
|
|
|
if isHeading {
|
|
flush()
|
|
|
|
// update path by level
|
|
if level == len(path)+1 {
|
|
// normal level up
|
|
path = append(path, title)
|
|
} else if level == len(path) {
|
|
// same level, replace the last layer
|
|
path[len(path)-1] = title
|
|
} else if level < len(path) {
|
|
// return to parent level
|
|
path = path[:level-1]
|
|
path = append(path, title)
|
|
} else {
|
|
path = append(path, title)
|
|
}
|
|
|
|
currentKey = strings.Join(path, " > ")
|
|
currentContent = []string{}
|
|
} else {
|
|
currentContent = append(currentContent, line)
|
|
}
|
|
}
|
|
|
|
flush()
|
|
|
|
return result
|
|
}
|
|
|
|
func ExtractTablesAndRemainder(markdownText string) (string, []string, error) {
|
|
tables := []string{}
|
|
|
|
// Every row pattern below ends in \n, so a table that runs to the end of
|
|
// the input loses its LAST ROW: the row fails to match, stays in the
|
|
// remainder, and comes back out glued to the neighbouring prose chunk. That
|
|
// is how "| A2 | B2 | C2 |" ended up appended to "Here is a standard
|
|
// Markdown table:" — a table silently split across two chunks, which is
|
|
// worse than not splitting it at all.
|
|
//
|
|
// Sections are sliced per heading, so ending without a trailing newline is
|
|
// the common case, not the edge case. One newline makes the last row look
|
|
// like every other row.
|
|
if !strings.HasSuffix(markdownText, "\n") {
|
|
markdownText += "\n"
|
|
}
|
|
remainder := markdownText
|
|
|
|
if strings.Contains(markdownText, "|") {
|
|
// Standard Markdown table matching pattern
|
|
borderTablePattern := regexp.MustCompile(`(?m)(?:\n|^)(?:\|.*?\|.*?\|.*?\n)(?:\|(?:\s*[:-]+[-| :]*\s*)\|.*?\n)(?:\|.*?\|.*?\|.*?\n)+`)
|
|
borderTables := borderTablePattern.FindAllString(markdownText, -1)
|
|
tables = append(tables, borderTables...)
|
|
remainder = borderTablePattern.ReplaceAllString(remainder, "\n")
|
|
|
|
// Borderless Markdown Table Matching Mode
|
|
noBorderTablePattern := regexp.MustCompile(`(?m)(?:\n|^)(?:\S.*?\|.*?\n)(?:(?:\s*[:-]+[-| :]*\s*).*?\n)(?:\S.*?\|.*?\n)+`)
|
|
noBorderTables := noBorderTablePattern.FindAllString(remainder, -1)
|
|
tables = append(tables, noBorderTables...)
|
|
remainder = noBorderTablePattern.ReplaceAllString(remainder, "\n")
|
|
}
|
|
|
|
// If the remaining text contains'<table>'(case ignored), try extracting the HTML table
|
|
if strings.Contains(strings.ToLower(remainder), "<table>") {
|
|
htmlTablePattern := regexp.MustCompile(`(?i)(?:\n|^)\s*(?:(?:<html[^>]*>\s*<body[^>]*>\s*<table[^>]*>[\s\S]*?</table>\s*</body>\s*</html>)|(?:<body[^>]*>\s*<table[^>]*>[\s\S]*?</table>\s*</body>)|(?:<table[^>]*>[\s\S]*?</table>))\s*(?:\n|$)`)
|
|
htmlTables := htmlTablePattern.FindAllString(remainder, -1)
|
|
tables = append(tables, htmlTables...)
|
|
remainder = htmlTablePattern.ReplaceAllString(remainder, "\n")
|
|
}
|
|
return remainder, tables, nil
|
|
}
|
|
|
|
func ExtractTablesWithContext(markdownText string, contextKey string) (string, []string, error) {
|
|
remainder, tables, err := ExtractTablesAndRemainder(markdownText)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
|
|
tablesWithContext := make([]string, len(tables))
|
|
for i, table := range tables {
|
|
// Trim before joining: the table patterns capture their leading
|
|
// newline, so joining with "\n\n" produced three in a row.
|
|
tablesWithContext[i] = contextKey + "\n\n" + strings.TrimSpace(table)
|
|
}
|
|
|
|
return remainder, tablesWithContext, nil
|
|
}
|
|
|
|
func (p *MarkdownSplitProvider) SplitText(text string) ([]string, error) {
|
|
headingsMap := ExtractMarkdownTree(text)
|
|
|
|
// Ranging the map directly made the section ORDER random between runs —
|
|
// Go randomises map iteration, so the same document produced the same
|
|
// chunks in a different sequence every time. For a splitter that is not
|
|
// cosmetic: chunks are reassembled into context downstream, and document
|
|
// order is the only order that means anything.
|
|
//
|
|
// Sorted by where each heading appears in the source, so the result is the
|
|
// document's own order rather than the map's or the alphabet's ("Section
|
|
// 10" must not sort before "Section 9").
|
|
keys := make([]string, 0, len(headingsMap))
|
|
for key := range headingsMap {
|
|
keys = append(keys, key)
|
|
}
|
|
sort.Slice(keys, func(i, j int) bool {
|
|
return strings.Index(text, keys[i]) < strings.Index(text, keys[j])
|
|
})
|
|
|
|
var sections []string
|
|
|
|
for _, key := range keys {
|
|
content := headingsMap[key]
|
|
remainder, tables, err := ExtractTablesWithContext(content, key)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// add tables to sections
|
|
for _, table := range tables {
|
|
sections = append(sections, strings.TrimSpace(table))
|
|
}
|
|
|
|
// add text to sections
|
|
if strings.TrimSpace(remainder) != "" {
|
|
textSplitter, err := NewDefaultSplitProvider("markdown")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
textSections, err := textSplitter.SplitText(remainder)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
for _, section := range textSections {
|
|
if strings.TrimSpace(section) != "" {
|
|
sections = append(sections, key+"\n\n"+strings.TrimSpace(section))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return sections, nil
|
|
}
|