mirror of
https://github.com/hanzoai/semantic.git
synced 2026-10-30 15:04:19 +00:00
The first commit was the shape. This is the implementation: every stage of the pipeline, the graph and its measures, the type system over it, entailment, deduplication and conflict resolution, the decision layer, the stores and the export formats. Twenty-six thousand lines of Go and sixteen thousand of tests, against no dependency but the standard library. Where the Python reaches for torch, spacy or a database driver, the Go states an interface and reports what is missing rather than pretending. A caller can tell "this build cannot serve that" from "nobody has heard of it", which is the difference between a port and a stub. The deterministic paths are complete and are what the tests exercise; the model-shaped ones are one adapter away. Porting a bug is not parity, so several places now disagree with the original on purpose. The JSON repair that closed a truncated reply counted each bracket kind separately and closed them backwards on the one input it exists for; it now works off a stack and ignores brackets inside strings. The entity patterns joined words with \s, so a heading ran into the sentence beneath it and became one name while the real relation was lost; a name is separated by horizontal space now. Eigenvector centrality ran plain power iteration, which on a bipartite graph swings between the two ends of a sign-paired eigenvalue forever and reported every node in a star as equally central; it iterates with A+I, which moves no eigenvector and settles. Similarity summed its weighted parts in map order, and float addition is not associative, so the same comparison scored differently between runs and duplicate detection returned a different answer each time. An inferred ontology named object-property endpoints by raw source type and data-property endpoints by class, so the schema referred to classes it did not contain. A property with no IRI of its own was minted as a class and came back Pascal-cased, renaming the term on the way out of the document. The command built a pipeline with every stage nil and panicked on each invocation. It is now the composition the package documents, and Run reports a missing ingester instead of dereferencing it. Co-authored-by: Hanzo Dev <[email protected]>
243 lines
5.7 KiB
Go
243 lines
5.7 KiB
Go
package store
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
)
|
|
|
|
// Op is how a condition compares a metadata field to a value.
|
|
type Op string
|
|
|
|
const (
|
|
Eq Op = "eq" // field equals value
|
|
Ne Op = "ne" // field differs from value
|
|
Gt Op = "gt" // field is greater than value
|
|
Ge Op = "ge" // field is greater than or equal to value
|
|
Lt Op = "lt" // field is less than value
|
|
Le Op = "le" // field is less than or equal to value
|
|
In Op = "in" // field is one of the values in a list
|
|
Has Op = "has" // field contains value: a substring, or a member of a list
|
|
)
|
|
|
|
// Cond is one comparison against one metadata field.
|
|
type Cond struct {
|
|
Field string
|
|
Op Op
|
|
Value any
|
|
}
|
|
|
|
// Filter is a conjunction: metadata passes when every condition holds. The
|
|
// zero value passes everything, so a query with no filter is not a special
|
|
// case.
|
|
//
|
|
// Conditions are added by the builder methods, each returning the extended
|
|
// filter, so a filter reads as the question it asks:
|
|
//
|
|
// f := store.Filter{}.Eq("lang", "en").Ge("year", 2020)
|
|
type Filter []Cond
|
|
|
|
// Add appends one condition.
|
|
func (f Filter) Add(field string, op Op, v any) Filter {
|
|
return append(f, Cond{Field: field, Op: op, Value: v})
|
|
}
|
|
|
|
// Eq requires the field to equal v.
|
|
func (f Filter) Eq(field string, v any) Filter { return f.Add(field, Eq, v) }
|
|
|
|
// Ne requires the field to be present and differ from v.
|
|
func (f Filter) Ne(field string, v any) Filter { return f.Add(field, Ne, v) }
|
|
|
|
// Gt requires the field to be greater than v.
|
|
func (f Filter) Gt(field string, v any) Filter { return f.Add(field, Gt, v) }
|
|
|
|
// Ge requires the field to be greater than or equal to v.
|
|
func (f Filter) Ge(field string, v any) Filter { return f.Add(field, Ge, v) }
|
|
|
|
// Lt requires the field to be less than v.
|
|
func (f Filter) Lt(field string, v any) Filter { return f.Add(field, Lt, v) }
|
|
|
|
// Le requires the field to be less than or equal to v.
|
|
func (f Filter) Le(field string, v any) Filter { return f.Add(field, Le, v) }
|
|
|
|
// In requires the field to equal one of the values in list.
|
|
func (f Filter) In(field string, list ...any) Filter { return f.Add(field, In, list) }
|
|
|
|
// Has requires the field to contain v: a substring of a string field, or a
|
|
// member of a list field.
|
|
func (f Filter) Has(field string, v any) Filter { return f.Add(field, Has, v) }
|
|
|
|
// Match reports whether metadata satisfies every condition. A field the
|
|
// metadata does not carry fails, whatever the operator — an absent field is
|
|
// not evidence of inequality — so Ne is "present and different", not "not
|
|
// equal or missing".
|
|
func (f Filter) Match(meta map[string]any) bool {
|
|
for _, c := range f {
|
|
got, ok := meta[c.Field]
|
|
if !ok || !c.holds(got) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (c Cond) holds(got any) bool {
|
|
switch c.Op {
|
|
case Eq:
|
|
return same(got, c.Value)
|
|
case Ne:
|
|
return !same(got, c.Value)
|
|
case Gt, Ge, Lt, Le:
|
|
d, ok := order(got, c.Value)
|
|
if !ok {
|
|
return false
|
|
}
|
|
switch c.Op {
|
|
case Gt:
|
|
return d > 0
|
|
case Ge:
|
|
return d >= 0
|
|
case Lt:
|
|
return d < 0
|
|
}
|
|
return d <= 0
|
|
case In:
|
|
for _, v := range list(c.Value) {
|
|
if same(got, v) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
case Has:
|
|
if s, ok := got.(string); ok {
|
|
want, ok := c.Value.(string)
|
|
return ok && strings.Contains(s, want)
|
|
}
|
|
for _, v := range list(got) {
|
|
if same(v, c.Value) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
return false
|
|
}
|
|
|
|
// same compares two metadata values. Numbers compare as numbers whatever
|
|
// their Go type, because a value that has been through JSON comes back as a
|
|
// float64 and is still the same value.
|
|
func same(a, b any) bool {
|
|
if x, ok := num(a); ok {
|
|
if y, ok := num(b); ok {
|
|
return x == y
|
|
}
|
|
return false
|
|
}
|
|
if x, ok := a.(string); ok {
|
|
y, ok := b.(string)
|
|
return ok && x == y
|
|
}
|
|
if x, ok := a.(bool); ok {
|
|
y, ok := b.(bool)
|
|
return ok && x == y
|
|
}
|
|
xs, ys := list(a), list(b)
|
|
if xs == nil || ys == nil || len(xs) != len(ys) {
|
|
return a == nil && b == nil
|
|
}
|
|
for i := range xs {
|
|
if !same(xs[i], ys[i]) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// order compares two values, reporting -1, 0 or 1, and whether they are
|
|
// comparable at all. Numbers order numerically and strings lexically;
|
|
// anything else has no order.
|
|
func order(a, b any) (int, bool) {
|
|
if x, ok := num(a); ok {
|
|
y, ok := num(b)
|
|
if !ok {
|
|
return 0, false
|
|
}
|
|
switch {
|
|
case x < y:
|
|
return -1, true
|
|
case x > y:
|
|
return 1, true
|
|
}
|
|
return 0, true
|
|
}
|
|
if x, ok := a.(string); ok {
|
|
y, ok := b.(string)
|
|
if !ok {
|
|
return 0, false
|
|
}
|
|
return strings.Compare(x, y), true
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
// num reads any Go numeric value, and a json.Number, as a float64.
|
|
func num(v any) (float64, bool) {
|
|
switch n := v.(type) {
|
|
case int:
|
|
return float64(n), true
|
|
case int8:
|
|
return float64(n), true
|
|
case int16:
|
|
return float64(n), true
|
|
case int32:
|
|
return float64(n), true
|
|
case int64:
|
|
return float64(n), true
|
|
case uint:
|
|
return float64(n), true
|
|
case uint8:
|
|
return float64(n), true
|
|
case uint16:
|
|
return float64(n), true
|
|
case uint32:
|
|
return float64(n), true
|
|
case uint64:
|
|
return float64(n), true
|
|
case float32:
|
|
return float64(n), true
|
|
case float64:
|
|
return n, true
|
|
case json.Number:
|
|
f, err := n.Float64()
|
|
return f, err == nil
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
// list reads a value as a sequence, so a []string written by a caller and
|
|
// the []any that comes back from JSON are the same list.
|
|
func list(v any) []any {
|
|
switch xs := v.(type) {
|
|
case []any:
|
|
return xs
|
|
case []string:
|
|
out := make([]any, len(xs))
|
|
for i, x := range xs {
|
|
out[i] = x
|
|
}
|
|
return out
|
|
case []int:
|
|
out := make([]any, len(xs))
|
|
for i, x := range xs {
|
|
out[i] = x
|
|
}
|
|
return out
|
|
case []float64:
|
|
out := make([]any, len(xs))
|
|
for i, x := range xs {
|
|
out[i] = x
|
|
}
|
|
return out
|
|
}
|
|
return nil
|
|
}
|