mirror of
https://github.com/hanzoai/graph.git
synced 2026-10-30 18:35:37 +00:00
zeekay
f5aaeb712e
Derived from Dgraph (github.com/dgraph-io/dgraph) at commit 64804bd2, Apache-2.0. See NOTICE. WHAT THIS IS. The graph engine as a Go library: posting lists, the DQL parser, the type and tokenizer system, the index and schema layers, and the wire types. 39,170 lines of the upstream's 166,337. THE STORE is github.com/luxfi/zapdb, a Badger fork that kept the package name, the subpackages (options, pb, y) and the API — Open, OpenManaged, DB, Options — so the 74 import sites are aliased rather than rewritten. THE CODEC is ZAP. zapdb serializes with ZAP, not protobuf: its pb package builds the ZAP types by default and the protobuf ones only under `-tags grpc`, a path that does not compile (table/builder.go calls pb.Marshal, which that tag removes). One call site in posting needed converting; every other type here is Hanzo's own protobuf and is untouched. NO TRANSPORT. An embedded engine dials nothing, so nothing here imports google.golang.org/grpc. Removed: 20 transport functions in x, the generated service stubs for Raft, Zero and Worker (1,657 lines), and the JWT helper — identity belongs to IAM. The core called none of them. NOT KEPT: worker, conn, raftwal, the cluster commands, the GraphQL layer and the test harnesses. `query`, the DQL execution engine, went with them because it depends on worker; parsing DQL is available here, executing it is not. The history of the upstream project lives at github.com/dgraph-io/dgraph. This repository begins at the derivation because three quarters of that history describes code this fork does not carry.
79 lines
2.6 KiB
Go
79 lines
2.6 KiB
Go
/*
|
|
* SPDX-FileCopyrightText: © 2017-2026 Istari Digital, Inc.
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
package tok
|
|
|
|
import (
|
|
"github.com/blevesearch/bleve/v2/analysis"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/ar" // Needed for bleve language support.
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/cjk"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/ckb"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/da"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/de"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/es"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/fi"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/fr"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/hi"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/hu"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/it"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/nl"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/no"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/pt"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/ro"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/ru"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/sv"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/lang/tr"
|
|
_ "github.com/blevesearch/bleve/v2/analysis/token/porter"
|
|
"github.com/golang/glog"
|
|
)
|
|
|
|
var langStemmers = map[string]string{
|
|
"ar": "stemmer_ar",
|
|
"ckb": "stemmer_ckb",
|
|
"da": "stemmer_da_snowball",
|
|
"de": "stemmer_de_light",
|
|
"en": "stemmer_porter",
|
|
"es": "stemmer_es_light",
|
|
"fi": "stemmer_fi_snowball",
|
|
"fr": "stemmer_fr_light",
|
|
"hi": "stemmer_hi",
|
|
"hu": "stemmer_hu_snowball",
|
|
"it": "stemmer_it_light",
|
|
"ja": "cjk_bigram",
|
|
"ko": "cjk_bigram",
|
|
"nl": "stemmer_nl_snowball",
|
|
"no": "stemmer_no_snowball",
|
|
"pt": "stemmer_pt_light",
|
|
"ro": "stemmer_ro_snowball",
|
|
"ru": "stemmer_ru_snowball",
|
|
"sv": "stemmer_sv_snowball",
|
|
"tr": "stemmer_tr_snowball",
|
|
"zh": "cjk_bigram",
|
|
}
|
|
|
|
// filterStemmers filters stems using an existing filter, imported here.
|
|
// If the lang filter is found, the we will forward requests to it.
|
|
// Returns filtered tokens if filter is found, otherwise returns tokens unmodified.
|
|
func filterStemmers(lang string, input analysis.TokenStream) analysis.TokenStream {
|
|
if len(input) == 0 {
|
|
return input
|
|
}
|
|
// check if we have stemmer filter for this lang.
|
|
name, ok := langStemmers[lang]
|
|
if !ok {
|
|
return input
|
|
}
|
|
// get filter from concurrent cache so we dont recreate.
|
|
filter, err := bleveCache.TokenFilterNamed(name)
|
|
if err != nil {
|
|
glog.Errorf("Error while filtering %q stems: %s", lang, err)
|
|
return input
|
|
}
|
|
if filter != nil {
|
|
return filter.Filter(input)
|
|
}
|
|
return input
|
|
}
|