mirror of
https://github.com/hanzoai/graph.git
synced 2026-10-30 18:35:37 +00:00
zeekay
f5aaeb712e
Derived from Dgraph (github.com/dgraph-io/dgraph) at commit 64804bd2, Apache-2.0. See NOTICE. WHAT THIS IS. The graph engine as a Go library: posting lists, the DQL parser, the type and tokenizer system, the index and schema layers, and the wire types. 39,170 lines of the upstream's 166,337. THE STORE is github.com/luxfi/zapdb, a Badger fork that kept the package name, the subpackages (options, pb, y) and the API — Open, OpenManaged, DB, Options — so the 74 import sites are aliased rather than rewritten. THE CODEC is ZAP. zapdb serializes with ZAP, not protobuf: its pb package builds the ZAP types by default and the protobuf ones only under `-tags grpc`, a path that does not compile (table/builder.go calls pb.Marshal, which that tag removes). One call site in posting needed converting; every other type here is Hanzo's own protobuf and is untouched. NO TRANSPORT. An embedded engine dials nothing, so nothing here imports google.golang.org/grpc. Removed: 20 transport functions in x, the generated service stubs for Raft, Zero and Worker (1,657 lines), and the JWT helper — identity belongs to IAM. The core called none of them. NOT KEPT: worker, conn, raftwal, the cluster commands, the GraphQL layer and the test harnesses. `query`, the DQL execution engine, went with them because it depends on worker; parsing DQL is available here, executing it is not. The history of the upstream project lives at github.com/dgraph-io/dgraph. This repository begins at the derivation because three quarters of that history describes code this fork does not carry.
78 lines
2.1 KiB
Go
78 lines
2.1 KiB
Go
/*
|
|
* SPDX-FileCopyrightText: © 2017-2026 Istari Digital, Inc.
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
package tok
|
|
|
|
import (
|
|
"sync"
|
|
|
|
"github.com/golang/glog"
|
|
"golang.org/x/text/language"
|
|
)
|
|
|
|
const enBase = "en"
|
|
|
|
// langBaseCache keeps a copy of lang -> base conversions.
|
|
var langBaseCache struct {
|
|
sync.RWMutex
|
|
m map[string]string
|
|
}
|
|
|
|
// LangBase returns the BCP47 base of a language.
|
|
// If the confidence of the matching is better than none, we return that base.
|
|
// Otherwise, we return "en" (English) which is a good default.
|
|
func LangBase(lang string) string {
|
|
if lang == "" {
|
|
return enBase // default to this
|
|
}
|
|
|
|
// Acquire the read lock and lookup for the lang in cache.
|
|
langBaseCache.RLock()
|
|
|
|
// check if we already have this
|
|
if s, found := langBaseCache.m[lang]; found {
|
|
langBaseCache.RUnlock()
|
|
return s
|
|
}
|
|
|
|
// Upgrade the lock only since the lang is not found in cache.
|
|
langBaseCache.RUnlock()
|
|
langBaseCache.Lock()
|
|
defer langBaseCache.Unlock()
|
|
|
|
// Recheck if the lang is added to the cache.
|
|
if s, found := langBaseCache.m[lang]; found {
|
|
return s
|
|
}
|
|
|
|
// Parse will return the best guess for a language tag.
|
|
// It will return undefined, or 'language.Und', if it gives up. That means the language
|
|
// tag is either new (to the standard) or simply invalid.
|
|
// We ignore errors from Parse because to Dgraph they aren't fatal.
|
|
tag, err := language.Parse(lang)
|
|
if err != nil {
|
|
glog.Errorf("While trying to parse lang %q. Error: %v", lang, err)
|
|
|
|
} else if tag != language.Und {
|
|
// Found a not undefined, i.e. valid language.
|
|
// The tag value returned will have a 'confidence' value attached.
|
|
// The confidence will be one of: No, Low, High, Exact.
|
|
// Low confidence is close to being undefined (see above) so we treat it as such.
|
|
// Any other confidence values are good enough for us.
|
|
// e.g., A lang tag like "x-klingon" should retag to "en"
|
|
if base, conf := tag.Base(); conf > language.No {
|
|
langBaseCache.m[lang] = base.String()
|
|
return base.String()
|
|
}
|
|
}
|
|
glog.Warningf("Unable to find lang %q. Reverting to English.", lang)
|
|
langBaseCache.m[lang] = enBase
|
|
return enBase
|
|
}
|
|
|
|
func init() {
|
|
langBaseCache.m = make(map[string]string)
|
|
}
|