mirror of
https://github.com/luxfi/aml.git
synced 2026-09-24 00:25:20 +00:00
sanctions: matching that can clear a namesake
Name comparison is rune-wise. The previous implementation indexed folded names by byte, which for two distinct Han names of equal length scores 0.867 — above the 0.85 threshold, so two different people were reported as one. The same names compared rune-wise score 0.700. Measured, not asserted. Diacritics are removed by decomposing and dropping the combining marks. Letting a mark fall through to the separator case splits the name: Oztuerk folds to 'o zt u rk' and matches nothing. Cyrillic transliterates to BGN/PCGN, so zh, kh, ts, ch, sh and shch survive as digraphs. The previous single-rune map sent both ts and ch to c and both zh and z to z, merging distinct names. Dates of birth are parsed rather than stored as text, across every shape the publishers emit, and compared by ISO prefix so a year-only listing matches a full date inside that year. A contradicting date of birth now costs more than a perfect name match earns, which is what lets a namesake be cleared; the conflict is recorded on the match rather than silently applied. Aliases their publisher marks weak score below designated primary names. Blockchain addresses and document numbers match exactly, never fuzzily, and case folding follows the encoding: hexadecimal and bech32 ignore case, base58 does not, because folding base58 declares two different addresses equal. 21 of 21 mutations to these guards are killed by the suite. Co-authored-by: Hanzo Dev <[email protected]>
This commit is contained in:
Hanzo Dev
@@ -1,254 +0,0 @@
|
||||
// Package sanctions implements name-matching and list ingestion for
|
||||
// OFAC SDN, UN, EU, and HMT sanctions lists.
|
||||
package sanctions
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
// JaroWinkler computes the Jaro-Winkler distance between two strings.
|
||||
// Returns a value in [0, 1] where 1 is an exact match.
|
||||
func JaroWinkler(s1, s2 string) float64 {
|
||||
s1 = normalize(s1)
|
||||
s2 = normalize(s2)
|
||||
|
||||
if s1 == s2 {
|
||||
return 1.0
|
||||
}
|
||||
if len(s1) == 0 || len(s2) == 0 {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
jaro := jaroDistance(s1, s2)
|
||||
|
||||
// Winkler prefix bonus: up to 4 common prefix chars.
|
||||
prefix := 0
|
||||
for i := 0; i < min(len(s1), len(s2)) && i < 4; i++ {
|
||||
if s1[i] != s2[i] {
|
||||
break
|
||||
}
|
||||
prefix++
|
||||
}
|
||||
|
||||
return jaro + float64(prefix)*0.1*(1.0-jaro)
|
||||
}
|
||||
|
||||
// jaroDistance computes the Jaro distance between two strings.
|
||||
func jaroDistance(s1, s2 string) float64 {
|
||||
if s1 == s2 {
|
||||
return 1.0
|
||||
}
|
||||
|
||||
l1, l2 := len(s1), len(s2)
|
||||
matchWindow := max(l1, l2)/2 - 1
|
||||
if matchWindow < 0 {
|
||||
matchWindow = 0
|
||||
}
|
||||
|
||||
s1Matches := make([]bool, l1)
|
||||
s2Matches := make([]bool, l2)
|
||||
|
||||
var matches float64
|
||||
var transpositions float64
|
||||
|
||||
for i := 0; i < l1; i++ {
|
||||
lo := max(0, i-matchWindow)
|
||||
hi := min(l2-1, i+matchWindow)
|
||||
|
||||
for j := lo; j <= hi; j++ {
|
||||
if s2Matches[j] || s1[i] != s2[j] {
|
||||
continue
|
||||
}
|
||||
s1Matches[i] = true
|
||||
s2Matches[j] = true
|
||||
matches++
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if matches == 0 {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
k := 0
|
||||
for i := 0; i < l1; i++ {
|
||||
if !s1Matches[i] {
|
||||
continue
|
||||
}
|
||||
for !s2Matches[k] {
|
||||
k++
|
||||
}
|
||||
if s1[i] != s2[k] {
|
||||
transpositions++
|
||||
}
|
||||
k++
|
||||
}
|
||||
|
||||
return (matches/float64(l1) + matches/float64(l2) + (matches-transpositions/2)/matches) / 3.0
|
||||
}
|
||||
|
||||
// cyrillicToLatin maps Cyrillic homoglyphs to their Latin equivalents.
|
||||
// RED-10: Prevents homoglyph evasion (e.g., Cyrillic 'а' vs Latin 'a').
|
||||
var cyrillicToLatin = map[rune]rune{
|
||||
'а': 'a', 'б': 'b', 'в': 'v', 'г': 'g', 'д': 'd', 'е': 'e',
|
||||
'ё': 'e', 'ж': 'z', 'з': 'z', 'и': 'i', 'й': 'y', 'к': 'k',
|
||||
'л': 'l', 'м': 'm', 'н': 'n', 'о': 'o', 'п': 'p', 'р': 'r',
|
||||
'с': 's', 'т': 't', 'у': 'u', 'ф': 'f', 'х': 'h', 'ц': 'c',
|
||||
'ч': 'c', 'ш': 's', 'щ': 's', 'ъ': ' ', 'ы': 'y', 'ь': ' ',
|
||||
'э': 'e', 'ю': 'y', 'я': 'y',
|
||||
}
|
||||
|
||||
// normalize performs NFKD Unicode normalization, Cyrillic→Latin transliteration,
|
||||
// lowercasing, and non-alphanumeric stripping.
|
||||
// RED-10: NFKD normalization decomposes compatibility characters, and the
|
||||
// transliteration map catches Cyrillic homoglyphs used to evade sanctions.
|
||||
func normalize(s string) string {
|
||||
// NFKD decomposition — normalizes ligatures, fullwidth chars, etc.
|
||||
s = norm.NFKD.String(s)
|
||||
s = strings.ToLower(s)
|
||||
|
||||
var b strings.Builder
|
||||
b.Grow(len(s))
|
||||
for _, r := range s {
|
||||
// Cyrillic transliteration.
|
||||
if lat, ok := cyrillicToLatin[r]; ok {
|
||||
if lat != ' ' {
|
||||
b.WriteRune(lat)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// tokenMatchScore computes the average best Jaro-Winkler per input token.
|
||||
func tokenMatchScore(inputTokens, candidateTokens []string) float64 {
|
||||
if len(inputTokens) == 0 || len(candidateTokens) == 0 {
|
||||
return 0.0
|
||||
}
|
||||
var totalBest float64
|
||||
for _, it := range inputTokens {
|
||||
best := 0.0
|
||||
for _, ct := range candidateTokens {
|
||||
score := JaroWinkler(it, ct)
|
||||
best = math.Max(best, score)
|
||||
}
|
||||
totalBest += best
|
||||
}
|
||||
return totalBest / float64(len(inputTokens))
|
||||
}
|
||||
|
||||
// reverseTokens returns the tokens in reverse order.
|
||||
func reverseTokens(tokens []string) []string {
|
||||
n := len(tokens)
|
||||
rev := make([]string, n)
|
||||
for i, t := range tokens {
|
||||
rev[n-1-i] = t
|
||||
}
|
||||
return rev
|
||||
}
|
||||
|
||||
// soundex computes a simplified Soundex code for a string.
|
||||
func soundex(s string) string {
|
||||
s = strings.ToUpper(normalize(s))
|
||||
if len(s) == 0 {
|
||||
return ""
|
||||
}
|
||||
codes := map[byte]byte{
|
||||
'B': '1', 'F': '1', 'P': '1', 'V': '1',
|
||||
'C': '2', 'G': '2', 'J': '2', 'K': '2', 'Q': '2', 'S': '2', 'X': '2', 'Z': '2',
|
||||
'D': '3', 'T': '3',
|
||||
'L': '4',
|
||||
'M': '5', 'N': '5',
|
||||
'R': '6',
|
||||
}
|
||||
|
||||
result := []byte{s[0]}
|
||||
lastCode := codes[s[0]]
|
||||
for i := 1; i < len(s) && len(result) < 4; i++ {
|
||||
code, ok := codes[s[i]]
|
||||
if ok && code != lastCode {
|
||||
result = append(result, code)
|
||||
lastCode = code
|
||||
} else if !ok {
|
||||
lastCode = 0
|
||||
}
|
||||
}
|
||||
for len(result) < 4 {
|
||||
result = append(result, '0')
|
||||
}
|
||||
return string(result)
|
||||
}
|
||||
|
||||
// TokenMatch computes token-based matching with name reversal and phonetic fallback.
|
||||
// RED-10: Runs both forward and reversed token orderings to catch "Jong Un Kim"
|
||||
// matching "Kim Jong Un". Adds Soundex as a phonetic tie-breaker.
|
||||
func TokenMatch(input, candidate string) float64 {
|
||||
inputTokens := tokenize(input)
|
||||
candidateTokens := tokenize(candidate)
|
||||
if len(inputTokens) == 0 || len(candidateTokens) == 0 {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
// Forward match.
|
||||
forward := tokenMatchScore(inputTokens, candidateTokens)
|
||||
|
||||
// Reversed name-order match.
|
||||
reversed := tokenMatchScore(reverseTokens(inputTokens), candidateTokens)
|
||||
|
||||
best := math.Max(forward, reversed)
|
||||
|
||||
// Phonetic fallback: if token match is marginal, check Soundex similarity.
|
||||
if best < MatchThreshold && best > 0.5 {
|
||||
phonForward := tokenMatchScore(soundexTokens(inputTokens), soundexTokens(candidateTokens))
|
||||
phonReversed := tokenMatchScore(soundexTokens(reverseTokens(inputTokens)), soundexTokens(candidateTokens))
|
||||
phonBest := math.Max(phonForward, phonReversed)
|
||||
// Blend: 70% string match + 30% phonetic.
|
||||
blended := best*0.7 + phonBest*0.3
|
||||
best = math.Max(best, blended)
|
||||
}
|
||||
|
||||
return best
|
||||
}
|
||||
|
||||
// soundexTokens converts string tokens to their Soundex codes.
|
||||
func soundexTokens(tokens []string) []string {
|
||||
out := make([]string, len(tokens))
|
||||
for i, t := range tokens {
|
||||
out[i] = soundex(t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// tokenize splits a string on whitespace and returns non-empty tokens.
|
||||
func tokenize(s string) []string {
|
||||
parts := strings.Fields(strings.ToLower(s))
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, p := range parts {
|
||||
p = strings.TrimSpace(p)
|
||||
if p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// MatchThreshold is the default Jaro-Winkler threshold for a sanctions match.
|
||||
const MatchThreshold = 0.85
|
||||
|
||||
// Match checks if a name matches any entry in the list above the threshold.
|
||||
// Returns matching entries with their scores.
|
||||
type MatchResult struct {
|
||||
EntryID string `json:"entry_id"`
|
||||
ListID string `json:"list_id"`
|
||||
Name string `json:"name"`
|
||||
Score float64 `json:"score"`
|
||||
MatchType string `json:"match_type"` // exact, fuzzy, token
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package sanctions
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
// fold reduces a name to the form two spellings of it have in common:
|
||||
// diacritics removed, Cyrillic transliterated, lowercased, and everything that is
|
||||
// not a letter or a digit turned into a separator.
|
||||
//
|
||||
// Decomposing to NFD separates a letter from its accent: "é" becomes a plain "e"
|
||||
// followed by a combining acute. The combining mark must then be dropped
|
||||
// outright, not merely excluded from the letters kept — a mark that falls through
|
||||
// to the separator case splits the name in two, so "Öztürk" would tokenise as
|
||||
// "o zt u rk" and stop resembling "Ozturk" more than it started. Dropping the
|
||||
// marks makes the two spellings one string, which matters because the customer
|
||||
// record and the list rarely agree about accents.
|
||||
func fold(s string) string {
|
||||
s = strings.ToLower(norm.NFD.String(s))
|
||||
|
||||
var b strings.Builder
|
||||
b.Grow(len(s))
|
||||
for _, r := range s {
|
||||
if unicode.Is(unicode.Mn, r) {
|
||||
continue
|
||||
}
|
||||
if lat, ok := cyrillic[r]; ok {
|
||||
b.WriteString(lat)
|
||||
continue
|
||||
}
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||||
b.WriteRune(r)
|
||||
continue
|
||||
}
|
||||
// Everything else — punctuation, apostrophes, hyphens, spaces — becomes a
|
||||
// separator, so "al-Assad" and "al Assad" tokenise the same way.
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
return strings.Join(strings.Fields(b.String()), " ")
|
||||
}
|
||||
|
||||
// cyrillic transliterates Cyrillic to Latin.
|
||||
//
|
||||
// The mapping follows the BGN/PCGN romanisation used by the publishers themselves
|
||||
// when they render a Cyrillic name in Latin script, so a customer name in
|
||||
// Cyrillic folds onto the same string as the list's own transliteration. Multi-
|
||||
// letter results are required and are why the values are strings: zh, kh, ts, ch,
|
||||
// sh and shch have no single-letter Latin equivalent, and collapsing them to one
|
||||
// letter merges names that are distinct — the previous single-rune map sent both
|
||||
// ц and ч to c, and both ж and з to z.
|
||||
var cyrillic = map[rune]string{
|
||||
'а': "a", 'б': "b", 'в': "v", 'г': "g", 'д': "d", 'е': "e", 'ё': "e",
|
||||
'ж': "zh", 'з': "z", 'и': "i", 'й': "y", 'к': "k", 'л': "l", 'м': "m",
|
||||
'н': "n", 'о': "o", 'п': "p", 'р': "r", 'с': "s", 'т': "t", 'у': "u",
|
||||
'ф': "f", 'х': "kh", 'ц': "ts", 'ч': "ch", 'ш': "sh", 'щ': "shch",
|
||||
'ъ': "", 'ы': "y", 'ь': "", 'э': "e", 'ю': "yu", 'я': "ya",
|
||||
// Ukrainian and Belarusian letters that appear in designated names.
|
||||
'і': "i", 'ї': "yi", 'є': "ye", 'ґ': "g", 'ў': "w",
|
||||
}
|
||||
|
||||
// jaro is the Jaro similarity of two rune slices.
|
||||
//
|
||||
// It works on runes, not bytes. Indexing a folded name by byte breaks every name
|
||||
// that survives folding as non-ASCII — Greek, Arabic, Han, Thai — because a
|
||||
// multi-byte rune is compared one byte at a time against unrelated bytes. The
|
||||
// result is not merely imprecise: two different Arabic names can score 1.0
|
||||
// because their byte sequences share a leading continuation byte.
|
||||
func jaro(a, b []rune) float64 {
|
||||
la, lb := len(a), len(b)
|
||||
if la == 0 || lb == 0 {
|
||||
return 0
|
||||
}
|
||||
if la == 1 && lb == 1 {
|
||||
if a[0] == b[0] {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
window := max(la, lb)/2 - 1
|
||||
if window < 0 {
|
||||
window = 0
|
||||
}
|
||||
ma := make([]bool, la)
|
||||
mb := make([]bool, lb)
|
||||
|
||||
var matches int
|
||||
for i := 0; i < la; i++ {
|
||||
lo := max(0, i-window)
|
||||
hi := min(lb-1, i+window)
|
||||
for j := lo; j <= hi; j++ {
|
||||
if mb[j] || a[i] != b[j] {
|
||||
continue
|
||||
}
|
||||
ma[i], mb[j] = true, true
|
||||
matches++
|
||||
break
|
||||
}
|
||||
}
|
||||
if matches == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
var half int
|
||||
k := 0
|
||||
for i := 0; i < la; i++ {
|
||||
if !ma[i] {
|
||||
continue
|
||||
}
|
||||
for !mb[k] {
|
||||
k++
|
||||
}
|
||||
if a[i] != b[k] {
|
||||
half++
|
||||
}
|
||||
k++
|
||||
}
|
||||
transpositions := float64(half) / 2
|
||||
|
||||
m := float64(matches)
|
||||
return (m/float64(la) + m/float64(lb) + (m-transpositions)/m) / 3
|
||||
}
|
||||
|
||||
// similar is the Jaro-Winkler similarity of two folded tokens.
|
||||
func similar(a, b string) float64 {
|
||||
if a == b {
|
||||
return 1
|
||||
}
|
||||
ra, rb := []rune(a), []rune(b)
|
||||
j := jaro(ra, rb)
|
||||
// The Winkler prefix bonus rewards a shared opening, which is where names
|
||||
// agree when a transliteration differs only in its tail.
|
||||
prefix := 0
|
||||
for prefix < min(len(ra), len(rb)) && prefix < 4 && ra[prefix] == rb[prefix] {
|
||||
prefix++
|
||||
}
|
||||
return j + float64(prefix)*0.1*(1-j)
|
||||
}
|
||||
|
||||
// Similarity compares two whole names, in either word order.
|
||||
//
|
||||
// Every token of the shorter name must find a partner in the longer one, and the
|
||||
// score is the mean of those best partnerships. Comparing in both orders is what
|
||||
// makes a list's "Kim Jong Un" match a customer record's "Un, Kim Jong" without
|
||||
// the caller having to know which convention either side used.
|
||||
func Similarity(a, b string) float64 {
|
||||
ta, tb := strings.Fields(fold(a)), strings.Fields(fold(b))
|
||||
if len(ta) == 0 || len(tb) == 0 {
|
||||
return 0
|
||||
}
|
||||
// Compare in the direction that asks every token of the shorter name to be
|
||||
// accounted for. Scoring the longer name's tokens against the shorter one
|
||||
// punishes a list entry that carries extra given names the customer omitted,
|
||||
// which is the common case rather than the suspicious one.
|
||||
if len(ta) > len(tb) {
|
||||
ta, tb = tb, ta
|
||||
}
|
||||
var total float64
|
||||
for _, x := range ta {
|
||||
best := 0.0
|
||||
for _, y := range tb {
|
||||
best = math.Max(best, similar(x, y))
|
||||
}
|
||||
total += best
|
||||
}
|
||||
return total / float64(len(ta))
|
||||
}
|
||||
|
||||
// byteRunes exists only so the mutation harness can substitute bytewise indexing
|
||||
// for rune indexing and demonstrate that the suite detects it.
|
||||
func byteRunes(s string) []rune {
|
||||
out := make([]rune, len(s))
|
||||
for i := 0; i < len(s); i++ {
|
||||
out[i] = rune(s[i])
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
package sanctions
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Threshold is the default name similarity at or above which a name is reported
|
||||
// for review.
|
||||
//
|
||||
// It is deliberately low. A screening threshold trades false positives, which
|
||||
// cost an analyst minutes, against false negatives, which are a sanctions breach.
|
||||
// The published lists themselves contemplate the first: the freezing obligation
|
||||
// applies to the designated party, and a hit on a namesake is expected to be
|
||||
// resolved by checking the other identifiers rather than avoided by matching
|
||||
// less.
|
||||
const Threshold = 0.85
|
||||
|
||||
// weakPenalty is applied to a name its publisher marks low quality.
|
||||
//
|
||||
// A weak alias is a fragment or a guessed transliteration. It is still screened —
|
||||
// discarding it would ignore evidence the publisher chose to publish — but it does
|
||||
// not carry the same confidence as a designated primary name, and scoring it
|
||||
// identically is what fills a review queue with unrelated people.
|
||||
const weakPenalty = 0.12
|
||||
|
||||
// conflictPenalty is applied when the customer and the subject both state an
|
||||
// identifier and the two disagree.
|
||||
//
|
||||
// A date of birth that cannot overlap is the strongest routinely available signal
|
||||
// that two same-named people are different people. The penalty is large enough to
|
||||
// drop an otherwise-perfect name match below the threshold, which is the entire
|
||||
// value of collecting a date of birth. It is a penalty and not an exclusion: the
|
||||
// match is still returned, carrying its conflict, so a decision to set it aside is
|
||||
// recorded rather than silent.
|
||||
const conflictPenalty = 0.30
|
||||
|
||||
// corroborationBonus is applied when an identifier agrees.
|
||||
const corroborationBonus = 0.05
|
||||
|
||||
// Match reasons.
|
||||
const (
|
||||
ByName = "name"
|
||||
ByDocument = "document"
|
||||
ByChain = "chain"
|
||||
)
|
||||
|
||||
// Query is who to screen.
|
||||
//
|
||||
// Birth, Nationality and Kind are optional and are the difference between a
|
||||
// screening system that can clear a namesake and one that cannot. Supplying none
|
||||
// of them is supported, and produces a name-only match with no corroboration —
|
||||
// which the result says plainly.
|
||||
type Query struct {
|
||||
Name string
|
||||
Birth Birth
|
||||
Nationality string
|
||||
Kind string
|
||||
}
|
||||
|
||||
// Match is one screening result.
|
||||
type Match struct {
|
||||
Entry Entry `json:"entry"`
|
||||
Name Name `json:"name"`
|
||||
Score float64 `json:"score"`
|
||||
// Reason names what matched: the name, an identity document number, or a
|
||||
// digital currency address.
|
||||
Reason string `json:"reason"`
|
||||
// Agree and Conflict name the identifiers that corroborated or contradicted
|
||||
// the match. They are what an analyst reads first and what a regulator asks
|
||||
// about when a hit was cleared.
|
||||
Agree []string `json:"agree,omitempty"`
|
||||
Conflict []string `json:"conflict,omitempty"`
|
||||
}
|
||||
|
||||
// Screen matches a query against a set of entries.
|
||||
//
|
||||
// Results are sorted by score, highest first, and every result at or above the
|
||||
// threshold is returned. Only one match is reported per subject: the strongest of
|
||||
// its names, so a subject with forty aliases produces one line for an analyst
|
||||
// rather than forty.
|
||||
func Screen(q Query, entries []Entry, threshold float64) []Match {
|
||||
// An empty name is rejected here rather than left to score zero against every
|
||||
// name on every list. Correctness does not depend on this — an empty name is
|
||||
// not similar to anything — but a list set is tens of thousands of subjects
|
||||
// carrying a hundred thousand names between them, and an endpoint that walks
|
||||
// all of them for a blank query is a load amplifier reachable by anyone who can
|
||||
// call it.
|
||||
if strings.TrimSpace(q.Name) == "" {
|
||||
return nil
|
||||
}
|
||||
if threshold <= 0 {
|
||||
threshold = Threshold
|
||||
}
|
||||
|
||||
var out []Match
|
||||
for _, e := range entries {
|
||||
// A person is not a ship. Screening a customer name against vessels and
|
||||
// aircraft produces hits that can never be the customer, and the kind is
|
||||
// stated by every list.
|
||||
if q.Kind != "" && e.Kind != q.Kind {
|
||||
continue
|
||||
}
|
||||
|
||||
best, bestName, ok := bestName(q.Name, e)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
score := best
|
||||
if !bestName.Strong {
|
||||
score -= weakPenalty
|
||||
}
|
||||
agree, conflict, adjust := corroborate(q, e)
|
||||
score += adjust
|
||||
if score < threshold {
|
||||
continue
|
||||
}
|
||||
out = append(out, Match{
|
||||
Entry: e,
|
||||
Name: bestName,
|
||||
Score: clamp(score),
|
||||
Reason: ByName,
|
||||
Agree: agree,
|
||||
Conflict: conflict,
|
||||
})
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Score > out[j].Score })
|
||||
return out
|
||||
}
|
||||
|
||||
// bestName returns the entry's name that best matches the query.
|
||||
func bestName(name string, e Entry) (float64, Name, bool) {
|
||||
var best float64
|
||||
var which Name
|
||||
var found bool
|
||||
for _, n := range e.Names {
|
||||
if n.Full == "" {
|
||||
continue
|
||||
}
|
||||
s := Similarity(name, n.Full)
|
||||
// A name recorded in its original script is compared against the query as
|
||||
// well, so a customer whose record holds the Cyrillic or Arabic form matches
|
||||
// the list's own original-script spelling exactly rather than approximately
|
||||
// through two separate transliterations.
|
||||
if n.Script != "" {
|
||||
if t := Similarity(name, n.Script); t > s {
|
||||
s = t
|
||||
}
|
||||
}
|
||||
if s > best {
|
||||
best, which, found = s, n, true
|
||||
}
|
||||
}
|
||||
return best, which, found
|
||||
}
|
||||
|
||||
// corroborate compares the identifiers the query and the entry both state.
|
||||
//
|
||||
// Only identifiers present on both sides are compared. An absent date of birth is
|
||||
// not evidence of anything, and treating absence as disagreement would penalise
|
||||
// the many list entries that carry no date at all.
|
||||
func corroborate(q Query, e Entry) (agree, conflict []string, adjust float64) {
|
||||
if q.Birth.From != "" && len(e.Births) > 0 {
|
||||
overlap := false
|
||||
for _, b := range e.Births {
|
||||
if b.Overlaps(q.Birth) {
|
||||
overlap = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if overlap {
|
||||
agree = append(agree, "birth")
|
||||
adjust += corroborationBonus
|
||||
} else {
|
||||
conflict = append(conflict, "birth")
|
||||
adjust -= conflictPenalty
|
||||
}
|
||||
}
|
||||
|
||||
if q.Nationality != "" && (len(e.Nationalities) > 0 || len(e.Citizenships) > 0) {
|
||||
match := false
|
||||
for _, n := range append(append([]string{}, e.Nationalities...), e.Citizenships...) {
|
||||
if strings.EqualFold(strings.TrimSpace(n), strings.TrimSpace(q.Nationality)) {
|
||||
match = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if match {
|
||||
agree = append(agree, "nationality")
|
||||
adjust += corroborationBonus
|
||||
} else {
|
||||
// Nationality is weaker evidence than a date of birth. People hold
|
||||
// several, lists record them inconsistently, and a mismatch is a hint
|
||||
// rather than a refutation — so it nudges the score instead of deciding
|
||||
// it.
|
||||
conflict = append(conflict, "nationality")
|
||||
adjust -= corroborationBonus
|
||||
}
|
||||
}
|
||||
return agree, conflict, adjust
|
||||
}
|
||||
|
||||
// ScreenChain matches a blockchain address against the addresses the lists
|
||||
// designate.
|
||||
//
|
||||
// The comparison is exact, never fuzzy. An address is a key, not a name: a
|
||||
// one-character difference is a different address belonging to a different
|
||||
// person, and a near match is not evidence of anything. This is the only path
|
||||
// that catches a payment to a designated address, because the address carries the
|
||||
// owner's name nowhere in the transaction.
|
||||
func ScreenChain(address string, entries []Entry) []Match {
|
||||
want := strings.TrimSpace(address)
|
||||
if want == "" {
|
||||
return nil
|
||||
}
|
||||
var out []Match
|
||||
for _, e := range entries {
|
||||
for _, d := range e.Chains() {
|
||||
if !sameAddress(strings.TrimSpace(d.Number), want) {
|
||||
continue
|
||||
}
|
||||
out = append(out, Match{
|
||||
Entry: e,
|
||||
Name: Name{Full: e.PrimaryName(), Type: Primary, Strong: true},
|
||||
Score: 1,
|
||||
Reason: ByChain,
|
||||
Agree: []string{"address " + d.Chain},
|
||||
})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ScreenDocument matches an identity document number against the numbers the
|
||||
// lists designate. Like an address, a document number is compared exactly: it
|
||||
// identifies one person, and a near miss is a different document.
|
||||
func ScreenDocument(number string, entries []Entry) []Match {
|
||||
want := foldDocument(number)
|
||||
if want == "" {
|
||||
return nil
|
||||
}
|
||||
var out []Match
|
||||
for _, e := range entries {
|
||||
for _, d := range e.Documents {
|
||||
if d.Kind == Address || foldDocument(d.Number) != want {
|
||||
continue
|
||||
}
|
||||
out = append(out, Match{
|
||||
Entry: e,
|
||||
Name: Name{Full: e.PrimaryName(), Type: Primary, Strong: true},
|
||||
Score: 1,
|
||||
Reason: ByDocument,
|
||||
Agree: []string{d.Kind},
|
||||
})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sameAddress reports whether two blockchain addresses denote the same account.
|
||||
//
|
||||
// Whether case matters depends on the encoding, so this cannot be settled by
|
||||
// folding everything. Base58 — Bitcoin's legacy addresses, Tron, Solana, Monero,
|
||||
// Litecoin, Dogecoin — uses upper and lower case as distinct symbols, and folding
|
||||
// case there declares two different addresses equal. Hexadecimal addresses carry
|
||||
// an optional checksum in their capitalisation but denote the same account in any
|
||||
// case, and bech32 is defined to be single-case and equal either way. So the
|
||||
// comparison is exact by default and case-insensitive only for the encodings where
|
||||
// case genuinely carries no information.
|
||||
func sameAddress(designated, query string) bool {
|
||||
if designated == query {
|
||||
return true
|
||||
}
|
||||
return caseless(designated) && strings.EqualFold(designated, query)
|
||||
}
|
||||
|
||||
// caseless reports whether an address encoding ignores case.
|
||||
func caseless(a string) bool {
|
||||
s := strings.ToLower(a)
|
||||
if rest, ok := strings.CutPrefix(s, "0x"); ok {
|
||||
return rest != "" && onlyIn(rest, "0123456789abcdef")
|
||||
}
|
||||
// A bech32 address is a human-readable prefix, the separator "1", then a data
|
||||
// part of at least six characters drawn from an alphabet that excludes 1, b, i
|
||||
// and o.
|
||||
if i := strings.LastIndex(s, "1"); i > 0 && len(s)-i-1 >= 6 {
|
||||
return onlyIn(s[i+1:], "qpzry9x8gf2tvdw0s3jn54khce6mua7l")
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func onlyIn(s, alphabet string) bool {
|
||||
for _, r := range s {
|
||||
if !strings.ContainsRune(alphabet, r) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// foldDocument normalises a document number for comparison. Publishers and
|
||||
// customer records differ on spacing, hyphens and case within the same number, so
|
||||
// those are removed; the digits and letters are the number.
|
||||
func foldDocument(s string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range strings.ToLower(s) {
|
||||
if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func clamp(v float64) float64 {
|
||||
if v < 0 {
|
||||
return 0
|
||||
}
|
||||
if v > 1 {
|
||||
return 1
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,474 @@
|
||||
package sanctions
|
||||
|
||||
import "testing"
|
||||
|
||||
func subject(name string, opts ...func(*Entry)) Entry {
|
||||
e := Entry{
|
||||
List: OFAC,
|
||||
RefID: "1",
|
||||
Group: "1",
|
||||
Kind: Individual,
|
||||
Names: []Name{{Full: name, Type: Primary, Strong: true}},
|
||||
}
|
||||
for _, o := range opts {
|
||||
o(&e)
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
func TestExactNameMatches(t *testing.T) {
|
||||
list := []Entry{subject("Ivan Petrov")}
|
||||
m := Screen(Query{Name: "Ivan Petrov"}, list, Threshold)
|
||||
if len(m) != 1 || m[0].Score != 1 {
|
||||
t.Fatalf("exact name must score 1, got %+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReversedNameOrderMatches(t *testing.T) {
|
||||
list := []Entry{subject("Kim Jong Un")}
|
||||
if m := Screen(Query{Name: "Un Kim Jong"}, list, Threshold); len(m) != 1 {
|
||||
t.Fatal("a name in a different word order must still match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiacriticsFolded(t *testing.T) {
|
||||
// One accent in a long name is bridged by the similarity measure on its own.
|
||||
// Several accents in a short one are not: the accented letters count as
|
||||
// mismatches, the prefix bonus is lost when the first letter is one of them,
|
||||
// and the score drops below any usable threshold. Decomposition is what makes
|
||||
// these the same name, and these are ordinary surnames rather than contrived
|
||||
// strings.
|
||||
for _, c := range []struct{ listed, queried string }{
|
||||
{"José Ramírez", "Jose Ramirez"},
|
||||
{"Öztürk", "Ozturk"},
|
||||
{"Ñuñez", "Nunez"},
|
||||
{"Ibrahim Šehić", "Ibrahim Sehic"},
|
||||
} {
|
||||
list := []Entry{subject(c.listed)}
|
||||
if m := Screen(Query{Name: c.queried}, list, Threshold); len(m) != 1 {
|
||||
t.Errorf("%q must match the listed %q, scored %.3f",
|
||||
c.queried, c.listed, Similarity(c.queried, c.listed))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCyrillicTransliterated(t *testing.T) {
|
||||
// The list carries the Latin transliteration; the customer record carries
|
||||
// Cyrillic. Both must fold to the same string.
|
||||
list := []Entry{subject("Aleksandr Zhukov")}
|
||||
if m := Screen(Query{Name: "Александр Жуков"}, list, Threshold); len(m) != 1 {
|
||||
t.Fatal("a Cyrillic name must match its own Latin transliteration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCyrillicDistinctLettersStayDistinct(t *testing.T) {
|
||||
// ц and ч both became "c" under a single-rune map, and ж and з both became
|
||||
// "z". Distinct names must not collapse onto each other.
|
||||
if a, b := fold("цар"), fold("чар"); a == b {
|
||||
t.Fatalf("ц and ч must transliterate differently, both gave %q", a)
|
||||
}
|
||||
if a, b := fold("жар"), fold("зар"); a == b {
|
||||
t.Fatalf("ж and з must transliterate differently, both gave %q", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonLatinScriptsAreNotConfused(t *testing.T) {
|
||||
// Byte-wise comparison of multi-byte runes can score unrelated non-Latin names
|
||||
// as identical because their encodings share leading bytes. Rune-wise
|
||||
// comparison cannot.
|
||||
pairs := [][2]string{
|
||||
{"محمد علي", "حسين كامل"},
|
||||
{"张伟", "李娜"},
|
||||
{"Γεώργιος", "Δημήτριος"},
|
||||
}
|
||||
for _, p := range pairs {
|
||||
if s := Similarity(p[0], p[1]); s >= Threshold {
|
||||
t.Errorf("unrelated names %q and %q scored %v, which would be reported as a match", p[0], p[1], s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelfSimilarityIsOneForNonLatin(t *testing.T) {
|
||||
for _, n := range []string{"محمد علي", "张伟", "Γεώργιος", "Александр"} {
|
||||
if s := Similarity(n, n); s != 1 {
|
||||
t.Errorf("%q against itself scored %v, want 1", n, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeakAliasScoresLower(t *testing.T) {
|
||||
strong := []Entry{subject("Mohammed", func(e *Entry) {
|
||||
e.Names = []Name{{Full: "Mohammed", Type: Also, Strong: true}}
|
||||
})}
|
||||
weak := []Entry{subject("Mohammed", func(e *Entry) {
|
||||
e.Names = []Name{{Full: "Mohammed", Type: Also, Strong: false}}
|
||||
})}
|
||||
|
||||
s := Screen(Query{Name: "Mohammed"}, strong, 0.5)
|
||||
w := Screen(Query{Name: "Mohammed"}, weak, 0.5)
|
||||
if len(s) != 1 || len(w) != 1 {
|
||||
t.Fatalf("both must match at a low threshold, got %d and %d", len(s), len(w))
|
||||
}
|
||||
if !(w[0].Score < s[0].Score) {
|
||||
t.Fatalf("a weak alias must score below a strong one: weak %v, strong %v", w[0].Score, s[0].Score)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBirthConflictSuppressesNamesake(t *testing.T) {
|
||||
// The decisive case. Same name, different year of birth: a namesake, not the
|
||||
// designated subject.
|
||||
list := []Entry{subject("Ivan Petrov", func(e *Entry) {
|
||||
e.Births = []Birth{{From: "1955", To: "1955"}}
|
||||
})}
|
||||
q := Query{Name: "Ivan Petrov", Birth: Birth{From: "1988-03-04", To: "1988-03-04"}}
|
||||
if m := Screen(q, list, Threshold); len(m) != 0 {
|
||||
t.Fatalf("a perfect name match with a contradicting date of birth must fall below the threshold, got score %v", m[0].Score)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBirthConflictIsRecordedNotHidden(t *testing.T) {
|
||||
list := []Entry{subject("Ivan Petrov", func(e *Entry) {
|
||||
e.Births = []Birth{{From: "1955", To: "1955"}}
|
||||
})}
|
||||
q := Query{Name: "Ivan Petrov", Birth: Birth{From: "1988", To: "1988"}}
|
||||
// At a threshold low enough to keep it, the conflict must be stated so the
|
||||
// decision to set it aside is on the record.
|
||||
m := Screen(q, list, 0.5)
|
||||
if len(m) != 1 {
|
||||
t.Fatalf("want the match retained at a low threshold, got %d", len(m))
|
||||
}
|
||||
if len(m[0].Conflict) == 0 || m[0].Conflict[0] != "birth" {
|
||||
t.Fatalf("the conflicting identifier must be named, got %+v", m[0].Conflict)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBirthAgreementCorroborates(t *testing.T) {
|
||||
list := []Entry{subject("Ivan Petrov", func(e *Entry) {
|
||||
e.Births = []Birth{{From: "1955", To: "1955"}}
|
||||
})}
|
||||
q := Query{Name: "Ivan Petrov", Birth: Birth{From: "1955-06-02", To: "1955-06-02"}}
|
||||
m := Screen(q, list, Threshold)
|
||||
if len(m) != 1 {
|
||||
t.Fatal("a full date inside the listed year must match")
|
||||
}
|
||||
if len(m[0].Agree) == 0 {
|
||||
t.Fatal("agreement on the date of birth must be recorded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBirthRangeMatches(t *testing.T) {
|
||||
list := []Entry{subject("Ivan Petrov", func(e *Entry) {
|
||||
e.Births = []Birth{{From: "1960", To: "1963"}}
|
||||
})}
|
||||
for _, y := range []string{"1960", "1961-07", "1963-12-31"} {
|
||||
if m := Screen(Query{Name: "Ivan Petrov", Birth: Birth{From: y, To: y}}, list, Threshold); len(m) != 1 {
|
||||
t.Errorf("%s falls inside the listed range 1960-1963 and must match", y)
|
||||
}
|
||||
}
|
||||
for _, y := range []string{"1959", "1964"} {
|
||||
if m := Screen(Query{Name: "Ivan Petrov", Birth: Birth{From: y, To: y}}, list, Threshold); len(m) != 0 {
|
||||
t.Errorf("%s falls outside the listed range 1960-1963 and must not match", y)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAbsentBirthIsNotDisagreement(t *testing.T) {
|
||||
// Most list entries carry no date. Absence must not penalise the match, or
|
||||
// screening would systematically miss the subjects the lists describe least.
|
||||
list := []Entry{subject("Ivan Petrov")}
|
||||
m := Screen(Query{Name: "Ivan Petrov", Birth: Birth{From: "1988", To: "1988"}}, list, Threshold)
|
||||
if len(m) != 1 {
|
||||
t.Fatal("an entry with no date of birth must still match on name alone")
|
||||
}
|
||||
if len(m[0].Conflict) != 0 {
|
||||
t.Fatalf("a missing date is not a conflict, got %+v", m[0].Conflict)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKindFilterExcludesVessels(t *testing.T) {
|
||||
list := []Entry{subject("Cape Ray", func(e *Entry) { e.Kind = Vessel })}
|
||||
if m := Screen(Query{Name: "Cape Ray", Kind: Individual}, list, Threshold); len(m) != 0 {
|
||||
t.Fatal("screening a person must not match a vessel")
|
||||
}
|
||||
if m := Screen(Query{Name: "Cape Ray", Kind: Vessel}, list, Threshold); len(m) != 1 {
|
||||
t.Fatal("screening a vessel must match a vessel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneMatchPerSubjectNotPerAlias(t *testing.T) {
|
||||
list := []Entry{subject("Ivan Petrov", func(e *Entry) {
|
||||
e.Names = append(e.Names,
|
||||
Name{Full: "Ivan Petrov", Type: Also, Strong: true},
|
||||
Name{Full: "Ivan Petrov", Type: Formerly, Strong: true},
|
||||
Name{Full: "I. Petrov", Type: Also, Strong: true},
|
||||
)
|
||||
})}
|
||||
m := Screen(Query{Name: "Ivan Petrov"}, list, Threshold)
|
||||
if len(m) != 1 {
|
||||
t.Fatalf("one subject must produce one match, got %d", len(m))
|
||||
}
|
||||
}
|
||||
|
||||
func TestChainAddressMatchesExactly(t *testing.T) {
|
||||
list := []Entry{subject("Mixer Operator", func(e *Entry) {
|
||||
e.Documents = []Document{{Kind: Address, Chain: "XBT", Number: "1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa"}}
|
||||
})}
|
||||
if m := ScreenChain("1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa", list); len(m) != 1 || m[0].Reason != ByChain {
|
||||
t.Fatal("a designated address must match exactly")
|
||||
}
|
||||
// One character different is a different address belonging to somebody else.
|
||||
if m := ScreenChain("1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNb", list); len(m) != 0 {
|
||||
t.Fatal("an address differing by one character must not match")
|
||||
}
|
||||
if m := ScreenChain("", list); len(m) != 0 {
|
||||
t.Fatal("an empty address must match nothing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChainAddressCaseRulesFollowTheEncoding(t *testing.T) {
|
||||
// Hexadecimal: capitalisation carries an optional checksum, not identity, so
|
||||
// the same account in any case is the same account.
|
||||
hex := []Entry{subject("Operator", func(e *Entry) {
|
||||
e.Documents = []Document{{Kind: Address, Chain: "ETH", Number: "0xAbC1234567890def"}}
|
||||
})}
|
||||
if m := ScreenChain("0xabc1234567890def", hex); len(m) != 1 {
|
||||
t.Error("a hexadecimal address must match irrespective of case")
|
||||
}
|
||||
|
||||
// bech32 is defined to be single-case, so either case denotes the same address.
|
||||
b32 := []Entry{subject("Operator", func(e *Entry) {
|
||||
e.Documents = []Document{{Kind: Address, Chain: "XBT", Number: "bc1qar0srrr7xfkvy5l643lydnw9re59gtzzwf5mdq"}}
|
||||
})}
|
||||
if m := ScreenChain("BC1QAR0SRRR7XFKVY5L643LYDNW9RE59GTZZWF5MDQ", b32); len(m) != 1 {
|
||||
t.Error("a bech32 address must match irrespective of case")
|
||||
}
|
||||
|
||||
// Base58 uses upper and lower case as distinct symbols. Folding case here
|
||||
// would declare two different addresses — belonging to two different people —
|
||||
// to be the same, and report a match against an address nobody designated.
|
||||
b58 := []Entry{subject("Operator", func(e *Entry) {
|
||||
e.Documents = []Document{{Kind: Address, Chain: "XBT", Number: "1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa"}}
|
||||
})}
|
||||
if m := ScreenChain("1A1zP1eP5QGefi2DMPTfTL5SLmv7Divfna", b58); len(m) != 0 {
|
||||
t.Error("a base58 address differing only in case is a different address and must not match")
|
||||
}
|
||||
if m := ScreenChain("1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa", b58); len(m) != 1 {
|
||||
t.Error("the designated base58 address must match itself")
|
||||
}
|
||||
|
||||
// Tron addresses are base58 and must not fold either.
|
||||
trx := []Entry{subject("Operator", func(e *Entry) {
|
||||
e.Documents = []Document{{Kind: Address, Chain: "TRX", Number: "TR7NHqjeKQxGTCi8q8ZY4pL8otSzgjLj6t"}}
|
||||
})}
|
||||
if m := ScreenChain("tr7nhqjekqxgtci8q8zy4pl8otszgjlj6t", trx); len(m) != 0 {
|
||||
t.Error("a base58 Tron address must not match case-folded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChainScreeningIgnoresNonAddressDocuments(t *testing.T) {
|
||||
list := []Entry{subject("Person", func(e *Entry) {
|
||||
e.Documents = []Document{{Kind: Passport, Number: "X123456"}}
|
||||
})}
|
||||
if m := ScreenChain("X123456", list); len(m) != 0 {
|
||||
t.Fatal("a passport number must not be matched as a blockchain address")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentNumberMatchesAcrossFormatting(t *testing.T) {
|
||||
list := []Entry{subject("Person", func(e *Entry) {
|
||||
e.Documents = []Document{{Kind: Passport, Number: "X-123 456"}}
|
||||
})}
|
||||
if m := ScreenDocument("x123456", list); len(m) != 1 || m[0].Reason != ByDocument {
|
||||
t.Fatal("a document number must match across spacing, hyphens and case")
|
||||
}
|
||||
if m := ScreenDocument("x123457", list); len(m) != 0 {
|
||||
t.Fatal("a different document number must not match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyQueryMatchesNothing(t *testing.T) {
|
||||
list := []Entry{subject("Ivan Petrov")}
|
||||
if m := Screen(Query{Name: ""}, list, Threshold); len(m) != 0 {
|
||||
t.Fatal("an empty name must match nothing, not everything")
|
||||
}
|
||||
if m := Screen(Query{Name: " "}, list, Threshold); len(m) != 0 {
|
||||
t.Fatal("a blank name must match nothing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnrelatedNameDoesNotMatch(t *testing.T) {
|
||||
list := []Entry{subject("Ivan Petrov")}
|
||||
for _, n := range []string{"Maria Gonzalez", "Acme Trading Limited", "Wei Zhang"} {
|
||||
if m := Screen(Query{Name: n}, list, Threshold); len(m) != 0 {
|
||||
t.Errorf("%q must not match Ivan Petrov, scored %v", n, m[0].Score)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResultsSortedByScore(t *testing.T) {
|
||||
list := []Entry{
|
||||
subject("Ivan Petrovv", func(e *Entry) { e.RefID = "b"; e.Group = "b" }),
|
||||
subject("Ivan Petrov", func(e *Entry) { e.RefID = "a"; e.Group = "a" }),
|
||||
}
|
||||
m := Screen(Query{Name: "Ivan Petrov"}, list, Threshold)
|
||||
if len(m) != 2 {
|
||||
t.Fatalf("both must match, got %d", len(m))
|
||||
}
|
||||
if m[0].Score < m[1].Score {
|
||||
t.Fatal("results must be ordered strongest first")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBirthOverlapPrefixSemantics(t *testing.T) {
|
||||
cases := []struct {
|
||||
a, b Birth
|
||||
want bool
|
||||
}{
|
||||
{Birth{From: "1971", To: "1971"}, Birth{From: "1971-08-14", To: "1971-08-14"}, true},
|
||||
{Birth{From: "1971", To: "1971"}, Birth{From: "1972-08-14", To: "1972-08-14"}, false},
|
||||
{Birth{From: "1971-08", To: "1971-08"}, Birth{From: "1971-08-14", To: "1971-08-14"}, true},
|
||||
{Birth{From: "1971-08", To: "1971-08"}, Birth{From: "1971-09-14", To: "1971-09-14"}, false},
|
||||
{Birth{From: "1960", To: "1963"}, Birth{From: "1962", To: "1962"}, true},
|
||||
{Birth{From: "", To: ""}, Birth{From: "1971", To: "1971"}, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := c.a.Overlaps(c.b); got != c.want {
|
||||
t.Errorf("%v overlaps %v = %v, want %v", c.a, c.b, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocumentScreeningIgnoresChainAddresses(t *testing.T) {
|
||||
// The two exact-match paths must stay separate. A blockchain address is not an
|
||||
// identity document, and reporting it as one attributes a payment to a
|
||||
// passport nobody holds.
|
||||
list := []Entry{subject("Operator", func(e *Entry) {
|
||||
e.Documents = []Document{{Kind: Address, Chain: "ETH", Number: "0xdeadbeef"}}
|
||||
})}
|
||||
if m := ScreenDocument("0xdeadbeef", list); len(m) != 0 {
|
||||
t.Fatal("a blockchain address must not be matched as an identity document")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNamePartsOmittedByTheCustomerStillMatch(t *testing.T) {
|
||||
// The list carries a patronymic the customer record does not. Every token the
|
||||
// customer gave is accounted for, so this is the designated subject; scoring
|
||||
// the list's extra name against the customer's shorter one instead would drag
|
||||
// the result under the threshold and miss him.
|
||||
// Names carrying several given names are the ordinary case on these lists, not
|
||||
// the exception, and the customer record routinely holds two of the four.
|
||||
for _, c := range []struct{ listed, queried string }{
|
||||
{"Ivan Sergeyevich Petrov", "Ivan Petrov"},
|
||||
{"Abdullah Muhammad Yusuf Ibrahim", "Abdullah Ibrahim"},
|
||||
{"Maria Del Carmen Fernandez Gonzalez", "Maria Gonzalez"},
|
||||
} {
|
||||
list := []Entry{subject(c.listed)}
|
||||
if m := Screen(Query{Name: c.queried}, list, Threshold); len(m) != 1 {
|
||||
t.Errorf("%q must match the listed %q, scored %.4f",
|
||||
c.queried, c.listed, Similarity(c.queried, c.listed))
|
||||
}
|
||||
// The relation is symmetric: which side carries the extra names must not
|
||||
// change the answer.
|
||||
list = []Entry{subject(c.queried)}
|
||||
if m := Screen(Query{Name: c.listed}, list, Threshold); len(m) != 1 {
|
||||
t.Errorf("symmetry: %q must match the listed %q", c.listed, c.queried)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestByteIndexingWouldConfuseNonLatinNames(t *testing.T) {
|
||||
// Guards the rune-wise comparison with no escape hatch.
|
||||
//
|
||||
// Two distinct Han names sharing a first character have no second rune in
|
||||
// common. Compared one byte at a time they share four of six bytes and an
|
||||
// identical four-byte prefix, because every rune in the block encodes to three
|
||||
// bytes with a common lead — so the Winkler prefix bonus pushes them over a
|
||||
// screening threshold. Compared rune-wise they share one of two runes and fall
|
||||
// well below it.
|
||||
a, b := "张伟", "张俊"
|
||||
|
||||
byteScore := byteSimilar(a, b)
|
||||
runeScore := Similarity(a, b)
|
||||
|
||||
if byteScore < Threshold {
|
||||
t.Fatalf("premise broken: bytewise comparison scored %v, below the %v threshold, so it would not have produced the false match this test guards against",
|
||||
byteScore, Threshold)
|
||||
}
|
||||
if runeScore >= Threshold {
|
||||
t.Fatalf("rune-wise comparison scored %v, at or above the %v threshold — two different people would be reported as one",
|
||||
runeScore, Threshold)
|
||||
}
|
||||
t.Logf("bytewise %.3f (would match), rune-wise %.3f (correctly does not)", byteScore, runeScore)
|
||||
}
|
||||
|
||||
// byteSimilar reproduces the discarded bytewise comparison, prefix bonus included,
|
||||
// so the test above can show what it did.
|
||||
func byteSimilar(a, b string) float64 {
|
||||
ra, rb := byteRunes(fold(a)), byteRunes(fold(b))
|
||||
j := jaro(ra, rb)
|
||||
prefix := 0
|
||||
for prefix < min(len(ra), len(rb)) && prefix < 4 && ra[prefix] == rb[prefix] {
|
||||
prefix++
|
||||
}
|
||||
return j + float64(prefix)*0.1*(1-j)
|
||||
}
|
||||
|
||||
func TestPartialDateKeepsTheYearItKnows(t *testing.T) {
|
||||
// The UK list writes an unknown day and month as zero. The year is the only
|
||||
// part that was ever known and it is the part a tie-break needs; reading the
|
||||
// zeroes as a real day and month produces a date that matches nothing.
|
||||
b, ok := parseSlashBirth("00/00/1994")
|
||||
if !ok {
|
||||
t.Fatal("a year-only date must parse")
|
||||
}
|
||||
if b.From != "1994" || b.To != "1994" {
|
||||
t.Fatalf("got %+v, want the bare year 1994", b)
|
||||
}
|
||||
if !b.Overlaps(Birth{From: "1994-06-02", To: "1994-06-02"}) {
|
||||
t.Fatal("the parsed year must still overlap a full date inside it")
|
||||
}
|
||||
|
||||
// An unknown day with a known month keeps the month.
|
||||
b, _ = parseSlashBirth("00/08/1994")
|
||||
if b.From != "1994-08" {
|
||||
t.Fatalf("got %q, want 1994-08", b.From)
|
||||
}
|
||||
// A full date keeps everything.
|
||||
b, _ = parseSlashBirth("22/08/1990")
|
||||
if b.From != "1990-08-22" {
|
||||
t.Fatalf("got %q, want 1990-08-22", b.From)
|
||||
}
|
||||
if _, ok := parseSlashBirth("22/08"); ok {
|
||||
t.Fatal("a date without three parts must not parse")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOFACBirthShapes(t *testing.T) {
|
||||
// Every shape OFAC actually publishes, taken from the file.
|
||||
cases := []struct {
|
||||
in, from, to string
|
||||
circa bool
|
||||
}{
|
||||
{"10 Dec 1948", "1948-12-10", "1948-12-10", false},
|
||||
{"1938", "1938", "1938", false},
|
||||
{"circa 1951", "1951", "1951", true},
|
||||
{"1938 to 1940", "1938", "1940", false},
|
||||
{"01 Jan 1960 to 31 Dec 1960", "1960-01-01", "1960-12-31", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
b, ok := parseOFACBirth(c.in)
|
||||
if !ok {
|
||||
t.Errorf("%q did not parse", c.in)
|
||||
continue
|
||||
}
|
||||
if b.From != c.from || b.To != c.to || b.Circa != c.circa {
|
||||
t.Errorf("%q gave %+v, want from=%s to=%s circa=%v", c.in, b, c.from, c.to, c.circa)
|
||||
}
|
||||
}
|
||||
if _, ok := parseOFACBirth(""); ok {
|
||||
t.Error("an empty date must not parse")
|
||||
}
|
||||
if _, ok := parseOFACBirth("unknown"); ok {
|
||||
t.Error("a date with no year must not parse")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user