mirror of
https://github.com/hanzoai/graph.git
synced 2026-10-30 18:35:37 +00:00
zeekay
f5aaeb712e
Derived from Dgraph (github.com/dgraph-io/dgraph) at commit 64804bd2, Apache-2.0. See NOTICE. WHAT THIS IS. The graph engine as a Go library: posting lists, the DQL parser, the type and tokenizer system, the index and schema layers, and the wire types. 39,170 lines of the upstream's 166,337. THE STORE is github.com/luxfi/zapdb, a Badger fork that kept the package name, the subpackages (options, pb, y) and the API — Open, OpenManaged, DB, Options — so the 74 import sites are aliased rather than rewritten. THE CODEC is ZAP. zapdb serializes with ZAP, not protobuf: its pb package builds the ZAP types by default and the protobuf ones only under `-tags grpc`, a path that does not compile (table/builder.go calls pb.Marshal, which that tag removes). One call site in posting needed converting; every other type here is Hanzo's own protobuf and is untouched. NO TRANSPORT. An embedded engine dials nothing, so nothing here imports google.golang.org/grpc. Removed: 20 transport functions in x, the generated service stubs for Raft, Zero and Worker (1,657 lines), and the JWT helper — identity belongs to IAM. The core called none of them. NOT KEPT: worker, conn, raftwal, the cluster commands, the GraphQL layer and the test harnesses. `query`, the DQL execution engine, went with them because it depends on worker; parsing DQL is available here, executing it is not. The history of the upstream project lives at github.com/dgraph-io/dgraph. This repository begins at the derivation because three quarters of that history describes code this fork does not carry.
108 lines
2.8 KiB
Go
108 lines
2.8 KiB
Go
/*
|
|
* SPDX-FileCopyrightText: © 2017-2026 Istari Digital, Inc.
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
package tok
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/blevesearch/bleve/v2/analysis"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestFilterStopwords(t *testing.T) {
|
|
tests := []struct {
|
|
lang string
|
|
in analysis.TokenStream
|
|
out analysis.TokenStream
|
|
}{
|
|
{lang: "en",
|
|
in: analysis.TokenStream{
|
|
&analysis.Token{Term: []byte("the")},
|
|
&analysis.Token{Term: []byte("quick")},
|
|
&analysis.Token{Term: []byte("brown")},
|
|
&analysis.Token{Term: []byte("foxes")},
|
|
&analysis.Token{Term: []byte("jump")},
|
|
&analysis.Token{Term: []byte("over")},
|
|
&analysis.Token{Term: []byte("the")},
|
|
&analysis.Token{Term: []byte("big")},
|
|
&analysis.Token{Term: []byte("dogs")},
|
|
},
|
|
out: analysis.TokenStream{
|
|
&analysis.Token{Term: []byte("quick")},
|
|
&analysis.Token{Term: []byte("brown")},
|
|
&analysis.Token{Term: []byte("foxes")},
|
|
&analysis.Token{Term: []byte("jump")},
|
|
&analysis.Token{Term: []byte("big")},
|
|
&analysis.Token{Term: []byte("dogs")},
|
|
},
|
|
},
|
|
{lang: "es",
|
|
in: analysis.TokenStream{
|
|
&analysis.Token{Term: []byte("deseándoles")},
|
|
&analysis.Token{Term: []byte("muchas")},
|
|
&analysis.Token{Term: []byte("alegrías")},
|
|
&analysis.Token{Term: []byte("a")},
|
|
&analysis.Token{Term: []byte("las")},
|
|
&analysis.Token{Term: []byte("señoritas")},
|
|
&analysis.Token{Term: []byte("y")},
|
|
&analysis.Token{Term: []byte("los")},
|
|
&analysis.Token{Term: []byte("señores")},
|
|
&analysis.Token{Term: []byte("programadores")},
|
|
&analysis.Token{Term: []byte("de")},
|
|
&analysis.Token{Term: []byte("Dgraph")},
|
|
},
|
|
out: analysis.TokenStream{
|
|
&analysis.Token{Term: []byte("deseándoles")},
|
|
&analysis.Token{Term: []byte("muchas")},
|
|
&analysis.Token{Term: []byte("alegrías")},
|
|
&analysis.Token{Term: []byte("señoritas")},
|
|
&analysis.Token{Term: []byte("señores")},
|
|
&analysis.Token{Term: []byte("programadores")},
|
|
&analysis.Token{Term: []byte("Dgraph")},
|
|
},
|
|
},
|
|
{lang: "x-klingon",
|
|
in: analysis.TokenStream{
|
|
&analysis.Token{Term: []byte("tlhIngan")},
|
|
&analysis.Token{Term: []byte("maH!")},
|
|
},
|
|
out: analysis.TokenStream{
|
|
&analysis.Token{Term: []byte("tlhIngan")},
|
|
&analysis.Token{Term: []byte("maH!")},
|
|
},
|
|
},
|
|
{lang: "en",
|
|
in: analysis.TokenStream{
|
|
&analysis.Token{
|
|
Term: []byte("same"),
|
|
},
|
|
},
|
|
out: analysis.TokenStream{},
|
|
},
|
|
{lang: "en",
|
|
in: analysis.TokenStream{},
|
|
out: analysis.TokenStream{},
|
|
},
|
|
{lang: "",
|
|
in: analysis.TokenStream{
|
|
&analysis.Token{
|
|
Term: []byte(""),
|
|
},
|
|
},
|
|
out: analysis.TokenStream{
|
|
&analysis.Token{
|
|
Term: []byte(""),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
out := filterStopwords(tc.lang, tc.in)
|
|
require.Equal(t, tc.out, out)
|
|
}
|
|
}
|