mirror of
https://github.com/luxfi/aml.git
synced 2026-09-24 00:25:20 +00:00
zeekayandHanzo Dev
e6f58397ab
Each parser was written against the published file, not against a description of it. Verified counts: OFAC 19,175 (matching the Record_Count the file declares), UN 1,011, EU 6,017, OFSI 5,135. What the previous single parser lost, and why each matters: OFAC publishes 963 digital currency addresses across 19 networks in idList. Dropping idList meant a payment to a designated blockchain address could not be detected by any means. OFAC marks 4,383 of its 24,546 aliases weak and the UN and OFSI mark theirs Low; flattening them to plain strings matched fragments at full confidence. OFAC flags which of several dates of birth is the main one, and for 37 subjects it is not the first — taking the first dropped the tie-break for exactly the subjects that need one. UN, EU and OFSI were parsed with the OFAC parser on the stated grounds that the formats are similar. They are not: the roots are CONSOLIDATED_LIST, export and a CSV. All three failed every night, logged a line, and continued. OFSI publishes one row per name and joins them by group; 19,761 rows are 5,135 subjects. It also puts the header on line two behind a preamble, orders the name columns Name 6 first, and writes partial dates as 00/00/1994. A parse that yields fewer records than the publisher declares is refused rather than loaded, because a partial sanctions list clears designated subjects. Co-authored-by: Hanzo Dev <[email protected]>
83 lines
2.1 KiB
Go
83 lines
2.1 KiB
Go
package sanctions
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
// TestPublishedLists parses the real published files.
|
|
//
|
|
// It runs only when AML_LISTS points at a directory holding sdn.xml, un.xml,
|
|
// eu.xml and ofsi.csv as downloaded from the publishers. A fixture proves the
|
|
// parser handles the shapes we thought to write down; only the published file
|
|
// proves it handles the shapes the publisher actually emits, and every defect
|
|
// this work fixed in the previous parser was of the second kind.
|
|
func TestPublishedLists(t *testing.T) {
|
|
dir := os.Getenv("AML_LISTS")
|
|
if dir == "" {
|
|
t.Skip("set AML_LISTS to a directory of published list files to run")
|
|
}
|
|
|
|
cases := []struct {
|
|
file string
|
|
parse func([]byte) ([]Entry, error)
|
|
least int
|
|
}{
|
|
{"sdn.xml", ParseOFAC, 19000},
|
|
{"un.xml", ParseUN, 900},
|
|
{"eu.xml", ParseEU, 5000},
|
|
{"ofsi.csv", ParseOFSI, 4000},
|
|
}
|
|
|
|
for _, c := range cases {
|
|
t.Run(c.file, func(t *testing.T) {
|
|
data, err := os.ReadFile(filepath.Join(dir, c.file))
|
|
if err != nil {
|
|
t.Skipf("%v", err)
|
|
}
|
|
entries, err := c.parse(data)
|
|
if err != nil {
|
|
t.Fatalf("parse: %v", err)
|
|
}
|
|
if len(entries) < c.least {
|
|
t.Fatalf("parsed %d entries, expected at least %d", len(entries), c.least)
|
|
}
|
|
|
|
var noName, withBirth, withDoc, withProgram, weak, addrs int
|
|
kinds := map[string]int{}
|
|
for _, e := range entries {
|
|
if e.PrimaryName() == "" {
|
|
noName++
|
|
}
|
|
if len(e.Births) > 0 {
|
|
withBirth++
|
|
}
|
|
if len(e.Documents) > 0 {
|
|
withDoc++
|
|
}
|
|
if len(e.Programs) > 0 {
|
|
withProgram++
|
|
}
|
|
for _, n := range e.Names {
|
|
if !n.Strong {
|
|
weak++
|
|
}
|
|
}
|
|
addrs += len(e.Chains())
|
|
kinds[e.Kind]++
|
|
}
|
|
t.Logf("%s: %d entries, kinds=%v, births=%d, documents=%d, programmes=%d, weak names=%d, chain addresses=%d",
|
|
c.file, len(entries), kinds, withBirth, withDoc, withProgram, weak, addrs)
|
|
|
|
// A subject with no name at all cannot be screened. One is a parser bug.
|
|
if noName > 0 {
|
|
t.Errorf("%d entries parsed with no name — those subjects are unscreenable", noName)
|
|
}
|
|
if len(kinds) == 0 {
|
|
t.Error("no subject kinds recorded")
|
|
}
|
|
})
|
|
}
|
|
}
|