mirror of
https://github.com/hanzoai/graph.git
synced 2026-10-30 18:35:37 +00:00
zeekay
f5aaeb712e
Derived from Dgraph (github.com/dgraph-io/dgraph) at commit 64804bd2, Apache-2.0. See NOTICE. WHAT THIS IS. The graph engine as a Go library: posting lists, the DQL parser, the type and tokenizer system, the index and schema layers, and the wire types. 39,170 lines of the upstream's 166,337. THE STORE is github.com/luxfi/zapdb, a Badger fork that kept the package name, the subpackages (options, pb, y) and the API — Open, OpenManaged, DB, Options — so the 74 import sites are aliased rather than rewritten. THE CODEC is ZAP. zapdb serializes with ZAP, not protobuf: its pb package builds the ZAP types by default and the protobuf ones only under `-tags grpc`, a path that does not compile (table/builder.go calls pb.Marshal, which that tag removes). One call site in posting needed converting; every other type here is Hanzo's own protobuf and is untouched. NO TRANSPORT. An embedded engine dials nothing, so nothing here imports google.golang.org/grpc. Removed: 20 transport functions in x, the generated service stubs for Raft, Zero and Worker (1,657 lines), and the JWT helper — identity belongs to IAM. The core called none of them. NOT KEPT: worker, conn, raftwal, the cluster commands, the GraphQL layer and the test harnesses. `query`, the DQL execution engine, went with them because it depends on worker; parsing DQL is available here, executing it is not. The history of the upstream project lives at github.com/dgraph-io/dgraph. This repository begins at the derivation because three quarters of that history describes code this fork does not carry.
77 lines
2.3 KiB
Go
77 lines
2.3 KiB
Go
/*
|
|
* SPDX-FileCopyrightText: © 2017-2026 Istari Digital, Inc.
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
package tok
|
|
|
|
import (
|
|
"github.com/pkg/errors"
|
|
"golang.org/x/text/collate"
|
|
"golang.org/x/text/language"
|
|
)
|
|
|
|
var (
|
|
enLangTag, _ = language.Parse("en")
|
|
)
|
|
|
|
// GetTokenizerForLang returns the correct full-text tokenizer for the given language.
|
|
func GetTokenizerForLang(t Tokenizer, lang string) Tokenizer {
|
|
if lang == "" {
|
|
return t
|
|
}
|
|
switch t.(type) {
|
|
case FullTextTokenizer:
|
|
// We must return a new instance because another goroutine might be calling this
|
|
// with a different lang.
|
|
return FullTextTokenizer{lang: lang}
|
|
case TermTokenizer:
|
|
return TermTokenizer{lang: lang}
|
|
case ExactTokenizer:
|
|
langTag, err := language.Parse(lang)
|
|
// We default to english if the language is not supported.
|
|
if err != nil {
|
|
langTag = enLangTag
|
|
}
|
|
// If this gets expensive memory-vise, then convert it to sync.Pool.
|
|
return ExactTokenizer{langBase: LangBase(lang), cl: collate.New(langTag),
|
|
buffer: &collate.Buffer{}}
|
|
default:
|
|
return t
|
|
}
|
|
}
|
|
|
|
// GetTokens returns the tokens for the given tokenizer ID and value.
|
|
// funcArgs should only have one element which is the value that needs to be tokenized.
|
|
func GetTokens(id byte, funcArgs ...string) ([]string, error) {
|
|
if l := len(funcArgs); l != 1 {
|
|
return nil, errors.Errorf("Function requires 1 arguments, but got %d", l)
|
|
}
|
|
tokenizer, ok := GetTokenizerByID(id)
|
|
if !ok {
|
|
return nil, errors.Errorf("No tokenizer was found with id %v", id)
|
|
}
|
|
return BuildTokens(funcArgs[0], tokenizer)
|
|
}
|
|
|
|
// GetTermTokens returns the term tokens for the given value.
|
|
func GetTermTokens(funcArgs []string) ([]string, error) {
|
|
return GetTokens(IdentTerm, funcArgs...)
|
|
}
|
|
|
|
func GetNGramTokens(funcArgs []string, lang string) ([]string, error) {
|
|
return BuildTokens(funcArgs[0], NGramTokenizer{lang: lang})
|
|
}
|
|
|
|
func GetNGramQueryTokens(funcArgs []string, lang string) ([]string, error) {
|
|
return BuildNGramQueryTokens(funcArgs[0], NGramTokenizer{lang: lang})
|
|
}
|
|
|
|
// GetFullTextTokens returns the full-text tokens for the given value.
|
|
func GetFullTextTokens(funcArgs []string, lang string) ([]string, error) {
|
|
if l := len(funcArgs); l != 1 {
|
|
return nil, errors.Errorf("Function requires 1 arguments, but got %d", l)
|
|
}
|
|
return BuildTokens(funcArgs[0], FullTextTokenizer{lang: lang})
|
|
}
|