mirror of
https://github.com/hanzoai/cloud.git
synced 2026-11-17 16:36:24 +00:00
The single Go binary that runs the compute/backend control plane: KMS secrets, object storage, DNS/provisioning, PaaS, code + exec, zero-trust sharing, audit, gateway edge, feature flags, security, and the plugin runtime — every subsystem mounted from one composition root. Open-core by construction, not by deletion. The proprietary planes — identity, commerce, AI inference, observability, the message bus, billing — are injected through the cloud/types extension interfaces, so this binary links only public infrastructure libraries and a missing plane mounts fail-closed and says so rather than pretending to work. Published from a fresh root rather than a filtered history: the extraction it came from began life without git, and a squashed root is the only way to be sure no earlier blob travels with it. Runs on a laptop with no Kubernetes, no cluster and no external services — `CLOUD_DEV_UNENCRYPTED=1 cloud -data-dir /tmp/cloud` boots ~32 subsystems on HTTP and ZAP. Encryption at rest stays fatal-by-default on a capable build; the dev flag relaxes only the no-key case, loudly. Co-authored-by: Hanzo Dev <[email protected]>
164 lines
4.9 KiB
Go
164 lines
4.9 KiB
Go
package code
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"strings"
|
|
"unicode"
|
|
)
|
|
|
|
// operators are the multi- and single-char code operators the tokenizer keeps as
|
|
// standalone terms (the SOTA "keep operators" rule) so a search for `->` or `::`
|
|
// is a real token, not lost to whitespace splitting. Longest-first so `==` is one
|
|
// token, not two `=`.
|
|
var operators = []string{
|
|
"<<=", ">>=", "...", "===", "!==",
|
|
"->", "=>", "::", "==", "!=", "<=", ">=", "&&", "||", "++", "--",
|
|
":=", "+=", "-=", "*=", "/=", "%=", "|=", "&=", "^=", "<<", ">>", "**",
|
|
"+", "-", "*", "/", "%", "<", ">", "=", "&", "|", "^", "!", "~", "?",
|
|
}
|
|
|
|
// codeTokens splits source text into search terms the way a code-search index
|
|
// should: identifiers are lowercased and ALSO split on camelCase and snake_case
|
|
// boundaries (getUserName → getusername, get, user, name; MAX_RETRIES →
|
|
// max_retries, max, retries), numbers are kept, and operators survive as their
|
|
// own tokens. The original (joined) identifier is always emitted alongside its
|
|
// subwords so both an exact identifier search and a natural-language "user name"
|
|
// search hit the same chunk.
|
|
func codeTokens(s string) []string {
|
|
var out []string
|
|
seen := map[string]bool{}
|
|
emit := func(t string) {
|
|
if t == "" || seen[t] {
|
|
return
|
|
}
|
|
seen[t] = true
|
|
out = append(out, t)
|
|
}
|
|
runes := []rune(s)
|
|
for i := 0; i < len(runes); {
|
|
r := runes[i]
|
|
switch {
|
|
case unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_':
|
|
j := i
|
|
for j < len(runes) && (unicode.IsLetter(runes[j]) || unicode.IsDigit(runes[j]) || runes[j] == '_') {
|
|
j++
|
|
}
|
|
word := string(runes[i:j])
|
|
lw := strings.ToLower(word)
|
|
emit(lw)
|
|
for _, sub := range splitIdent(word) {
|
|
emit(strings.ToLower(sub))
|
|
}
|
|
i = j
|
|
case unicode.IsSpace(r):
|
|
i++
|
|
default:
|
|
if op := matchOperator(runes[i:]); op != "" {
|
|
emit(op)
|
|
i += len([]rune(op))
|
|
} else {
|
|
i++
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// splitIdent breaks one identifier on snake_case and camelCase boundaries,
|
|
// returning its subwords (never the whole identifier — the caller emits that).
|
|
// Handles acronym runs: HTTPServer → HTTP, Server; parseURLPath → parse, URL, Path.
|
|
func splitIdent(word string) []string {
|
|
var parts []string
|
|
for _, seg := range strings.FieldsFunc(word, func(r rune) bool { return r == '_' }) {
|
|
parts = append(parts, splitCamel(seg)...)
|
|
}
|
|
if len(parts) <= 1 {
|
|
return nil // single word — no additional subword beyond the whole identifier
|
|
}
|
|
return parts
|
|
}
|
|
|
|
func splitCamel(seg string) []string {
|
|
runes := []rune(seg)
|
|
if len(runes) == 0 {
|
|
return nil
|
|
}
|
|
var words []string
|
|
start := 0
|
|
for i := 1; i < len(runes); i++ {
|
|
prev, cur := runes[i-1], runes[i]
|
|
var next rune
|
|
if i+1 < len(runes) {
|
|
next = runes[i+1]
|
|
}
|
|
lowerToUpper := unicode.IsLower(prev) && unicode.IsUpper(cur)
|
|
digitBoundary := (unicode.IsDigit(prev) != unicode.IsDigit(cur))
|
|
acronymEnd := unicode.IsUpper(prev) && unicode.IsUpper(cur) && next != 0 && unicode.IsLower(next)
|
|
if lowerToUpper || acronymEnd || digitBoundary {
|
|
words = append(words, string(runes[start:i]))
|
|
start = i
|
|
}
|
|
}
|
|
words = append(words, string(runes[start:]))
|
|
return words
|
|
}
|
|
|
|
func matchOperator(runes []rune) string {
|
|
for _, op := range operators {
|
|
or := []rune(op)
|
|
if len(runes) < len(or) {
|
|
continue
|
|
}
|
|
if string(runes[:len(or)]) == op {
|
|
return op
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// ftsBody renders chunk text into the string stored in the FTS `body` column: the
|
|
// code-tokenized terms joined by spaces. Feeding tokenized text to the trigram
|
|
// tokenizer makes both split subwords and original identifiers substring-matchable.
|
|
func ftsBody(text string) string {
|
|
return strings.Join(codeTokens(text), " ")
|
|
}
|
|
|
|
// buildFTSMatch turns a user query into an FTS5 trigram MATCH expression: an OR of
|
|
// the query's code-tokens (each quoted so punctuation is literal). Tokens shorter
|
|
// than 3 chars are dropped — the trigram tokenizer cannot index them. Returns
|
|
// ok=false when nothing usable remains (the caller then degrades honestly).
|
|
func buildFTSMatch(query string) (string, bool) {
|
|
var terms []string
|
|
seen := map[string]bool{}
|
|
for _, t := range codeTokens(query) {
|
|
if len(t) < 3 || seen[t] {
|
|
continue
|
|
}
|
|
seen[t] = true
|
|
terms = append(terms, `"`+strings.ReplaceAll(t, `"`, `""`)+`"`)
|
|
}
|
|
if len(terms) == 0 {
|
|
return "", false
|
|
}
|
|
return strings.Join(terms, " OR "), true
|
|
}
|
|
|
|
func escapeLike(s string) string {
|
|
r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`)
|
|
return r.Replace(s)
|
|
}
|
|
|
|
func sha256Hex(s string) string {
|
|
sum := sha256.Sum256([]byte(s))
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
// estimateTokens approximates the LLM token count of text with the ~4-chars-per-
|
|
// token industry heuristic (plus 1 so empty is non-zero). It is deliberately
|
|
// cheap and provider-agnostic — the /context budget packer only needs a stable
|
|
// monotone estimate to pack against, not exact tokenization.
|
|
func estimateTokens(text string) int {
|
|
return len(text)/4 + 1
|
|
}
|