mirror of
https://github.com/hanzoai/graph.git
synced 2026-10-30 18:35:37 +00:00
zeekay
f5aaeb712e
Derived from Dgraph (github.com/dgraph-io/dgraph) at commit 64804bd2, Apache-2.0. See NOTICE. WHAT THIS IS. The graph engine as a Go library: posting lists, the DQL parser, the type and tokenizer system, the index and schema layers, and the wire types. 39,170 lines of the upstream's 166,337. THE STORE is github.com/luxfi/zapdb, a Badger fork that kept the package name, the subpackages (options, pb, y) and the API — Open, OpenManaged, DB, Options — so the 74 import sites are aliased rather than rewritten. THE CODEC is ZAP. zapdb serializes with ZAP, not protobuf: its pb package builds the ZAP types by default and the protobuf ones only under `-tags grpc`, a path that does not compile (table/builder.go calls pb.Marshal, which that tag removes). One call site in posting needed converting; every other type here is Hanzo's own protobuf and is untouched. NO TRANSPORT. An embedded engine dials nothing, so nothing here imports google.golang.org/grpc. Removed: 20 transport functions in x, the generated service stubs for Raft, Zero and Worker (1,657 lines), and the JWT helper — identity belongs to IAM. The core called none of them. NOT KEPT: worker, conn, raftwal, the cluster commands, the GraphQL layer and the test harnesses. `query`, the DQL execution engine, went with them because it depends on worker; parsing DQL is available here, executing it is not. The history of the upstream project lives at github.com/dgraph-io/dgraph. This repository begins at the derivation because three quarters of that history describes code this fork does not carry.
71 lines
1.9 KiB
Go
71 lines
1.9 KiB
Go
/*
|
|
* SPDX-FileCopyrightText: © 2017-2026 Istari Digital, Inc.
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
package tok
|
|
|
|
import (
|
|
"github.com/blevesearch/bleve/v2/analysis"
|
|
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
|
|
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
|
|
"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
|
|
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
|
|
"github.com/blevesearch/bleve/v2/registry"
|
|
|
|
"github.com/hanzoai/graph/x"
|
|
)
|
|
|
|
const unicodenormName = "unicodenorm_nfkc"
|
|
|
|
var (
|
|
bleveCache = registry.NewCache()
|
|
termAnalyzer, fulltextAnalyzer analysis.Analyzer
|
|
)
|
|
|
|
// setupBleve creates bleve filters and analyzers that we use for term and fulltext tokenizers.
|
|
func setupBleve() {
|
|
// unicode normalizer filter - simplifies unicode words using Normalization Form KC (NFKC)
|
|
// See: http://unicode.org/reports/tr15/#Norm_Forms
|
|
_, err := bleveCache.DefineTokenFilter(unicodenormName,
|
|
map[string]interface{}{
|
|
"type": unicodenorm.Name,
|
|
"form": unicodenorm.NFKC,
|
|
})
|
|
x.Check(err)
|
|
|
|
// term analyzer - splits on word boundaries, lowercase and normalize tokens.
|
|
termAnalyzer, err = bleveCache.DefineAnalyzer("term",
|
|
map[string]interface{}{
|
|
"type": custom.Name,
|
|
"tokenizer": unicode.Name,
|
|
"token_filters": []string{
|
|
lowercase.Name,
|
|
unicodenormName,
|
|
},
|
|
})
|
|
x.Check(err)
|
|
|
|
// fulltext analyzer - does language stop-words removal and stemming.
|
|
fulltextAnalyzer, err = bleveCache.DefineAnalyzer("fulltext",
|
|
map[string]interface{}{
|
|
"type": custom.Name,
|
|
"tokenizer": unicode.Name,
|
|
"token_filters": []string{
|
|
lowercase.Name,
|
|
unicodenormName,
|
|
},
|
|
})
|
|
x.Check(err)
|
|
}
|
|
|
|
// uniqueTerms takes a token stream and returns a string slice of unique terms.
|
|
func uniqueTerms(tokens analysis.TokenStream) []string {
|
|
var terms []string
|
|
for i := range tokens {
|
|
terms = append(terms, string(tokens[i].Term))
|
|
}
|
|
terms = x.RemoveDuplicates(terms)
|
|
return terms
|
|
}
|