lux/aml
2
0
Fork 0
mirror of https://github.com/luxfi/aml.git synced 2026-09-24 00:25:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

sanctions: parse every list, and fail loudly when one yields nothing

Browse Source 
Three of the four screening lists loaded zero entries, and nothing reported it.

Two sources were dead. The UK list pointed at the OFSI Consolidated List, which
closed on 28 January 2026 — ConList.xml has answered 404 since, and the UK
Sanctions List published by the FCDO is now the only source for UK designations.
The EU endpoint answers 403 unless its token is in the query string. Both had
been returning nothing for months.

The remaining two failures were structural. UN, EU and HMT were all parsed with
the OFAC parser, whose root element is sdnList — so a correct download of the UN
list still produced no entries. Each publisher now has a parser for its own
schema, resolved through one registry with no default case, because a fallback
to another publisher's parser is what made the emptiness look like data.

And the reason none of this surfaced: every failure was logged and skipped, and
RefreshAll returned nil regardless. A list that parses to zero entries is now an
error, a non-200 is an error, an unregistered source is an error, and RefreshAll
joins per-list failures into its return value. Stale reports data too old to
screen against, since a zero-match result against an empty list is
indistinguishable from a clean party.

Measured against the live files: 19,175 entries before, 32,518 after — OFAC
19,175, UK 6,315, EU 6,017, UN 1,011, every one of them named, with 15,117
dates of birth and 11,730 nationalities the previous path could not have read.

The fetch timeout moves from 60s to 5m: the EU file does not transfer inside a
minute, so the old value failed a healthy list.

Fixtures keep the quirks that break parsers — UN year-only and BETWEEN dates,
the EU's absent primary-name marker and "00" country sentinel, the UK's literal
"dd/mm" placeholders and six spellings of NameType, non-Latin-script names. Ten
mutants of these invariants, all killed.

Co-authored-by: Hanzo Dev <[email protected]>
This commit is contained in:
zeekay
2026-07-29 10:59:22 -07:00
co-authored by Hanzo Dev
parent 283caec3b6
commit 6a210f6402
6 changed files with 1042 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/sanctions/ingest.go
+37 -94
View File
@@ -1,98 +1,71 @@
// Copyright 2024-2026 Lux Partners Limited. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package sanctions
import (
"bytes"
"context"
"crypto/sha256"
"encoding/xml"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/luxfi/aml/pkg/types"
)
// maxBodySize is the maximum sanctions list download size (100 MB).
// maxBodySize bounds a list download. The largest published list is around
// 30 MB, so 100 MB leaves room for growth while keeping a redirect to something
// unbounded from exhausting memory.
const maxBodySize = 100 * 1024 * 1024
// Ingester fetches and parses sanctions lists.
// fetchTimeout covers connect, transfer, and read of a whole list. It is minutes
// rather than seconds because the payloads are tens of megabytes: measured
// against the live sources, the EU file alone does not complete inside 60s, so a
// short timeout fails the fetch on a healthy list.
const fetchTimeout = 5 * time.Minute
// Ingester downloads sanctions lists. It owns transport only; schema knowledge
// lives in the Parser for each source.
type Ingester struct {
client *http.Client
}
// NewIngester creates a new Ingester with a 60s timeout.
// NewIngester creates an Ingester.
func NewIngester() *Ingester {
return &Ingester{
client: &http.Client{Timeout: 60 * time.Second},
}
return &Ingester{client: &http.Client{Timeout: fetchTimeout}}
}
// FetchOFAC fetches and parses the OFAC SDN XML list.
func (ing *Ingester) FetchOFAC(ctx context.Context, url string) ([]types.SanctionsEntry, string, error) {
body, hash, err := ing.fetch(ctx, url)
// Fetch downloads a list and parses it with the parser registered for its
// source, returning the entries and the SHA-256 of the payload.
//
// A list that parses to zero entries is an error. No published list is empty, so
// an empty parse means the download or the schema is wrong, and reporting that
// as success is the failure that lets a designated party through screening
// against a list nobody knows is blank.
func (ing *Ingester) Fetch(ctx context.Context, list types.SanctionsList) ([]types.SanctionsEntry, string, error) {
parse, err := ParserFor(list.Source)
if err != nil {
return nil, "", fmt.Errorf("fetch OFAC: %w", err)
return nil, "", err
}
// RED-05: Use streaming decoder with entity expansion disabled to prevent
// billion-laughs XML bomb attacks. Strict mode + empty entity map means
// any entity reference in the XML causes a parse error instead of OOM.
var sdnList sdnListXML
d := xml.NewDecoder(bytes.NewReader(body))
d.Strict = true
d.Entity = map[string]string{} // disable entity expansion
if err := d.Decode(&sdnList); err != nil {
return nil, "", fmt.Errorf("parse OFAC XML: %w", err)
body, hash, err := ing.fetch(ctx, list.URL)
if err != nil {
return nil, "", fmt.Errorf("fetch %s: %w", list.Source, err)
}
entries := make([]types.SanctionsEntry, 0, len(sdnList.Entries))
for _, e := range sdnList.Entries {
entryType := types.SanctionIndividual
if e.SDNType == "Entity" {
entryType = types.SanctionEntity
} else if e.SDNType == "Vessel" {
entryType = types.SanctionVessel
} else if e.SDNType == "Aircraft" {
entryType = types.SanctionAircraft
}
name := strings.TrimSpace(e.FirstName + " " + e.LastName)
aliases := make([]string, 0, len(e.Aliases))
for _, a := range e.Aliases {
an := strings.TrimSpace(a.FirstName + " " + a.LastName)
if an != "" {
aliases = append(aliases, an)
}
}
var dob string
for _, d := range e.DateOfBirth {
dob = d.DateOfBirth
break
}
var nationality string
for _, n := range e.Nationalities {
nationality = n.Country
break
}
entries = append(entries, types.SanctionsEntry{
RefID: fmt.Sprintf("%d", e.UID),
Name: name,
Aliases: aliases,
DOB: dob,
Nationality: nationality,
Type: entryType,
})
entries, err := parse(body)
if err != nil {
return nil, "", fmt.Errorf("%s: %w", list.Source, err)
}
if len(entries) == 0 {
return nil, "", fmt.Errorf("%s: parsed 0 entries from %d bytes at %s", list.Source, len(body), list.URL)
}
return entries, hash, nil
}
// fetch downloads a URL and returns the body bytes + SHA256 hex hash.
// fetch downloads a URL and returns the body bytes and their SHA-256 hex digest.
func (ing *Ingester) fetch(ctx context.Context, url string) ([]byte, string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
@@ -118,33 +91,3 @@ func (ing *Ingester) fetch(ctx context.Context, url string) ([]byte, string, err
h := sha256.Sum256(body)
return body, fmt.Sprintf("%x", h), nil
}
// OFAC SDN XML structures.
type sdnListXML struct {
XMLName xml.Name `xml:"sdnList"`
Entries []sdnEntryXML `xml:"sdnEntry"`
}
type sdnEntryXML struct {
UID int `xml:"uid"`
FirstName string `xml:"firstName"`
LastName string `xml:"lastName"`
SDNType string `xml:"sdnType"`
Aliases []sdnAliasXML `xml:"akaList>aka"`
DateOfBirth []sdnDOBXML `xml:"dateOfBirthList>dateOfBirthItem"`
Nationalities []sdnNatXML `xml:"nationalityList>nationality"`
}
type sdnAliasXML struct {
FirstName string `xml:"firstName"`
LastName string `xml:"lastName"`
}
type sdnDOBXML struct {
DateOfBirth string `xml:"dateOfBirth"`
}
type sdnNatXML struct {
Country string `xml:"country"`
}
pkg/sanctions/ingest_test.go
+116 -65
View File
@@ -4,14 +4,26 @@ import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/luxfi/aml/pkg/types"
)
// TestIngestXMLEntityExpansion (RED-05) verifies that the OFAC XML parser
// rejects entity expansion attacks (billion laughs / XML bombs). Without
// the streaming decoder fix, this would cause exponential memory growth.
func TestIngestXMLEntityExpansion(t *testing.T) {
// Classic billion-laughs XML bomb with nested entity expansion.
// serve returns a list pointed at a test server yielding the given body.
func serve(t *testing.T, source, body string) types.SanctionsList {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/xml")
_, _ = w.Write([]byte(body))
}))
t.Cleanup(srv.Close)
return types.SanctionsList{Source: source, URL: srv.URL, Format: "xml", Active: true}
}
// Entity expansion must fail the parse rather than expand. A billion-laughs
// payload is the reachable denial of service on any XML ingest.
func TestEntityExpansionIsRejected(t *testing.T) {
bomb := `<?xml version="1.0"?>
<!DOCTYPE lolz [
<!ENTITY lol "lol">
@@ -19,81 +31,120 @@ func TestIngestXMLEntityExpansion(t *testing.T) {
<!ENTITY lol3 "&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;&lol2;">
<!ENTITY lol4 "&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;&lol3;">
]>
<sdnList>
<sdnEntry>
<uid>1</uid>
<firstName>&lol4;</firstName>
<lastName>Test</lastName>
</sdnEntry>
</sdnList>`
<sdnList><sdnEntry><uid>1</uid><firstName>&lol4;</firstName><lastName>Test</lastName></sdnEntry></sdnList>`
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml")
w.Write([]byte(bomb))
}))
defer srv.Close()
ing := NewIngester()
_, _, err := ing.FetchOFAC(context.Background(), srv.URL)
if err == nil {
t.Fatal("expected error parsing XML bomb with entity expansion, got nil")
if _, _, err := NewIngester().Fetch(context.Background(), serve(t, types.ListOFACSDN, bomb)); err == nil {
t.Fatal("XML bomb with entity expansion was accepted")
}
// The error should be about entity references, not OOM.
t.Logf("correctly rejected XML bomb: %v", err)
}
// TestIngestValidOFACXML ensures normal OFAC XML still parses correctly
// after the streaming decoder change.
func TestIngestValidOFACXML(t *testing.T) {
validXML := `<?xml version="1.0"?>
func TestFetchParsesAndHashes(t *testing.T) {
valid := `<?xml version="1.0"?>
<sdnList>
<sdnEntry>
<uid>1234</uid>
<firstName>John</firstName>
<lastName>Doe</lastName>
<sdnType>Individual</sdnType>
<akaList>
<aka>
<firstName>Johnny</firstName>
<lastName>Doe</lastName>
</aka>
</akaList>
</sdnEntry>
<sdnEntry>
<uid>5678</uid>
<firstName></firstName>
<lastName>Evil Corp</lastName>
<sdnType>Entity</sdnType>
</sdnEntry>
<sdnEntry><uid>1234</uid><firstName>John</firstName><lastName>Doe</lastName><sdnType>Individual</sdnType>
<akaList><aka><firstName>Johnny</firstName><lastName>Doe</lastName></aka></akaList></sdnEntry>
<sdnEntry><uid>5678</uid><lastName>Evil Corp</lastName><sdnType>Entity</sdnType></sdnEntry>
</sdnList>`
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml")
w.Write([]byte(validXML))
}))
defer srv.Close()
ing := NewIngester()
entries, hash, err := ing.FetchOFAC(context.Background(), srv.URL)
entries, hash, err := NewIngester().Fetch(context.Background(), serve(t, types.ListOFACSDN, valid))
if err != nil {
t.Fatalf("expected no error, got: %v", err)
t.Fatalf("Fetch: %v", err)
}
if hash == "" {
t.Fatal("expected non-empty hash")
if len(hash) != 64 {
t.Fatalf("hash = %q, want a 64-char sha256 hex digest", hash)
}
if len(entries) != 2 {
t.Fatalf("expected 2 entries, got %d", len(entries))
t.Fatalf("got %d entries, want 2", len(entries))
}
if entries[0].Name != "John Doe" {
t.Errorf("entry[0].Name = %q, want %q", entries[0].Name, "John Doe")
}
if entries[0].Type != "individual" {
t.Errorf("entry[0].Type = %q, want %q", entries[0].Type, "individual")
if entries[0].Name != "John Doe" || entries[0].Type != types.SanctionIndividual {
t.Errorf("entry[0] = %q/%q", entries[0].Name, entries[0].Type)
}
if len(entries[0].Aliases) != 1 || entries[0].Aliases[0] != "Johnny Doe" {
t.Errorf("entry[0].Aliases = %v, want [Johnny Doe]", entries[0].Aliases)
}
if entries[1].Type != "entity" {
t.Errorf("entry[1].Type = %q, want %q", entries[1].Type, "entity")
if entries[1].Name != "Evil Corp" || entries[1].Type != types.SanctionEntity {
t.Errorf("entry[1] = %q/%q", entries[1].Name, entries[1].Type)
}
}
// The defect this guards is the one that hid two dead endpoints: a list that
// yields nothing must be an error, because "no entries" and "nobody is
// designated" are indistinguishable to every caller downstream.
func TestEmptyListIsAnError(t *testing.T) {
for _, tc := range []struct{ name, body string }{
{"well-formed but empty", `<?xml version="1.0"?><sdnList></sdnList>`},
{"publisher error page", `<?xml version="1.0"?><Error><Code>BlobNotFound</Code></Error>`},
} {
t.Run(tc.name, func(t *testing.T) {
_, _, err := NewIngester().Fetch(context.Background(), serve(t, types.ListOFACSDN, tc.body))
if err == nil {
t.Fatal("a list parsing to zero entries was reported as success")
}
if !strings.Contains(err.Error(), "0 entries") && !strings.Contains(err.Error(), "parse") {
t.Fatalf("error does not say what went wrong: %v", err)
}
})
}
}
// A non-200 must fail rather than parse the error body. OFSI's dead ConList
// answers 404 with an XML body, which would otherwise parse to zero entries.
func TestNonOKStatusIsAnError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNotFound)
_, _ = w.Write([]byte(`<?xml version="1.0"?><Error><Code>BlobNotFound</Code></Error>`))
}))
defer srv.Close()
list := types.SanctionsList{Source: types.ListHMT, URL: srv.URL, Active: true}
_, _, err := NewIngester().Fetch(context.Background(), list)
if err == nil {
t.Fatal("HTTP 404 was reported as success")
}
if !strings.Contains(err.Error(), "404") {
t.Fatalf("error hides the status code: %v", err)
}
}
// Every source must resolve to a parser written for its own schema. Falling back
// to another publisher's parser is what made three of four lists silently empty.
func TestEveryDefaultListHasItsOwnParser(t *testing.T) {
seen := map[string]bool{}
for _, list := range DefaultLists() {
p, err := ParserFor(list.Source)
if err != nil {
t.Errorf("%s: %v", list.Source, err)
continue
}
if p == nil {
t.Errorf("%s: nil parser", list.Source)
}
seen[list.Source] = true
}
for _, want := range []string{types.ListOFACSDN, types.ListUN, types.ListEU, types.ListHMT} {
if !seen[want] {
t.Errorf("%s is not in DefaultLists", want)
}
}
if _, err := ParserFor("some_new_regime"); err == nil {
t.Fatal("an unregistered source resolved to a parser instead of erroring")
}
}
// The URL that broke: OFSI closed the Consolidated List on 28 January 2026 and
// ConList.xml has answered 404 since. A screening product pointed at it ingests
// nothing, and nothing is silent.
func TestDefaultListsDoNotUseTheClosedOFSIList(t *testing.T) {
for _, list := range DefaultLists() {
if strings.Contains(list.URL, "ofsistorage") || strings.Contains(list.URL, "ConList") {
t.Errorf("%s points at the closed OFSI Consolidated List: %s", list.Source, list.URL)
}
if list.Source == types.ListHMT && !strings.Contains(list.URL, "sanctionslist.fcdo.gov.uk") {
t.Errorf("UK list is not the FCDO UK Sanctions List: %s", list.URL)
}
// The EU endpoint answers 403 without its token.
if list.Source == types.ListEU && !strings.Contains(list.URL, "token=") {
t.Errorf("EU list carries no token and will 403: %s", list.URL)
}
}
}
pkg/sanctions/lists.go
+22 -3
View File
@@ -1,8 +1,27 @@
// Copyright 2024-2026 Lux Partners Limited. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package sanctions
import "github.com/luxfi/aml/pkg/types"
// DefaultLists returns the default sanctions list configurations.
// The EU Financial Sanctions File is served only with this token in the query
// string; without it the endpoint answers 403. The Commission publishes the value
// for anonymous access to the public list — it is not a credential.
const euPublicToken = "dG9rZW4tMjAxNw"
// DefaultLists returns the built-in sanctions list sources.
//
// Two of these endpoints move on the publisher's schedule and both have broken
// once already. Verify against the publisher before editing a URL here:
//
// - The UK source is the FCDO UK Sanctions List, NOT the OFSI Consolidated
// List. OFSI closed that list on 28 January 2026 and its ConList.xml has
// answered 404 since; from that date the UK Sanctions List is the only
// source for UK designations.
// - The EU file needs the token above. Requested without it the endpoint
// answers 403 rather than redirecting to an authenticated form.
func DefaultLists() []types.SanctionsList {
return []types.SanctionsList{
{
@@ -19,13 +38,13 @@ func DefaultLists() []types.SanctionsList {
},
{
Source: types.ListEU,
URL: "https://webgate.ec.europa.eu/fsd/fsf/public/files/xmlFullSanctionsList_1_1/content",
URL: "https://webgate.ec.europa.eu/fsd/fsf/public/files/xmlFullSanctionsList_1_1/content?token=" + euPublicToken,
Format: "xml",
Active: true,
},
{
Source: types.ListHMT,
URL: "https://ofsistorage.blob.core.windows.net/publishlive/ConList.xml",
URL: "https://sanctionslist.fcdo.gov.uk/docs/UK-Sanctions-List.xml",
Format: "xml",
Active: true,
},
pkg/sanctions/parse.go
+543
View File
@@ -0,0 +1,543 @@
// Copyright 2024-2026 Lux Partners Limited. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package sanctions
import (
"bytes"
"encoding/xml"
"fmt"
"strings"
"github.com/luxfi/aml/pkg/types"
)
// A Parser turns one publisher's native XML into canonical entries. Transport,
// schema, and persistence are separate concerns: fetch owns the network, a
// Parser owns one schema and touches nothing else, the store owns writing.
type Parser func([]byte) ([]types.SanctionsEntry, error)
// parsers resolves a list source to the parser for its schema. Each publisher
// ships a different root element and a different name model — sdnList,
// CONSOLIDATED_LIST, export, Designations — so there is one parser per source
// and deliberately no default case. An unregistered source is an error rather
// than a fallback to some other publisher's parser, because a mismatched parser
// yields zero entries, and zero entries is indistinguishable from a list on
// which nobody is designated.
var parsers = map[string]Parser{
types.ListOFACSDN: ParseOFAC,
types.ListUN: ParseUN,
types.ListEU: ParseEU,
types.ListHMT: ParseUK,
}
// ParserFor returns the parser registered for a list source.
func ParserFor(source string) (Parser, error) {
p, ok := parsers[source]
if !ok {
return nil, fmt.Errorf("no parser registered for list source %q", source)
}
return p, nil
}
// decode unmarshals XML with entity expansion disabled, so an entity reference
// fails the parse instead of expanding. Strict mode plus an empty entity map
// turns a billion-laughs payload into a parse error rather than an OOM.
func decode(body []byte, v any) error {
d := xml.NewDecoder(bytes.NewReader(body))
d.Strict = true
d.Entity = map[string]string{}
return d.Decode(v)
}
// join concatenates non-empty parts with a single space.
func join(parts ...string) string {
out := make([]string, 0, len(parts))
for _, p := range parts {
if p = strings.TrimSpace(p); p != "" {
out = append(out, p)
}
}
return strings.Join(out, " ")
}
// addAlias appends an alias if it is non-empty and not already present. Lists
// repeat aliases across regulations (an EU entity carries one nameAlias per
// amending regulation), and a duplicate alias costs a matcher pass per copy.
func addAlias(aliases []string, name string) []string {
if name = strings.TrimSpace(name); name == "" {
return aliases
}
for _, a := range aliases {
if strings.EqualFold(a, name) {
return aliases
}
}
return append(aliases, name)
}
// ── OFAC SDN ────────────────────────────────────────────────────────────────
// Root <sdnList>, one <sdnEntry> per designation, names split first/last.
// ParseOFAC parses the OFAC Specially Designated Nationals XML.
func ParseOFAC(body []byte) ([]types.SanctionsEntry, error) {
var list sdnListXML
if err := decode(body, &list); err != nil {
return nil, fmt.Errorf("parse OFAC XML: %w", err)
}
entries := make([]types.SanctionsEntry, 0, len(list.Entries))
for _, e := range list.Entries {
entryType := types.SanctionIndividual
switch e.SDNType {
case "Entity":
entryType = types.SanctionEntity
case "Vessel":
entryType = types.SanctionVessel
case "Aircraft":
entryType = types.SanctionAircraft
}
var aliases []string
for _, a := range e.Aliases {
aliases = addAlias(aliases, join(a.FirstName, a.LastName))
}
var dob string
if len(e.DateOfBirth) > 0 {
dob = e.DateOfBirth[0].DateOfBirth
}
var nationality string
if len(e.Nationalities) > 0 {
nationality = e.Nationalities[0].Country
}
entries = append(entries, types.SanctionsEntry{
RefID: fmt.Sprintf("%d", e.UID),
Name: join(e.FirstName, e.LastName),
Aliases: aliases,
DOB: dob,
Nationality: nationality,
Type: entryType,
})
}
return entries, nil
}
type sdnListXML struct {
XMLName xml.Name `xml:"sdnList"`
Entries []sdnEntryXML `xml:"sdnEntry"`
}
type sdnEntryXML struct {
UID int `xml:"uid"`
FirstName string `xml:"firstName"`
LastName string `xml:"lastName"`
SDNType string `xml:"sdnType"`
Aliases []sdnAliasXML `xml:"akaList>aka"`
DateOfBirth []sdnDOBXML `xml:"dateOfBirthList>dateOfBirthItem"`
Nationalities []sdnNatXML `xml:"nationalityList>nationality"`
}
type sdnAliasXML struct {
FirstName string `xml:"firstName"`
LastName string `xml:"lastName"`
}
type sdnDOBXML struct {
DateOfBirth string `xml:"dateOfBirth"`
}
type sdnNatXML struct {
Country string `xml:"country"`
}
// ── UN Consolidated List ────────────────────────────────────────────────────
// Root <CONSOLIDATED_LIST>, split into <INDIVIDUALS> and <ENTITIES>. Both
// subject types carry the same record shape with type-prefixed element names
// (INDIVIDUAL_ALIAS versus ENTITY_ALIAS), so one struct declares both paths and
// exactly one set populates for any given record.
// ParseUN parses the UN Security Council consolidated list XML.
func ParseUN(body []byte) ([]types.SanctionsEntry, error) {
var list unListXML
if err := decode(body, &list); err != nil {
return nil, fmt.Errorf("parse UN XML: %w", err)
}
entries := make([]types.SanctionsEntry, 0, len(list.Individuals)+len(list.Entities))
for _, s := range list.Individuals {
entries = append(entries, s.entry(types.SanctionIndividual))
}
for _, s := range list.Entities {
entries = append(entries, s.entry(types.SanctionEntity))
}
return entries, nil
}
type unListXML struct {
XMLName xml.Name `xml:"CONSOLIDATED_LIST"`
Individuals []unSubjectXML `xml:"INDIVIDUALS>INDIVIDUAL"`
Entities []unSubjectXML `xml:"ENTITIES>ENTITY"`
}
type unSubjectXML struct {
DataID string `xml:"DATAID"`
Reference string `xml:"REFERENCE_NUMBER"`
First string `xml:"FIRST_NAME"`
Second string `xml:"SECOND_NAME"`
Third string `xml:"THIRD_NAME"`
Fourth string `xml:"FOURTH_NAME"`
Nationality string `xml:"NATIONALITY>VALUE"`
IndAliases []unAlias `xml:"INDIVIDUAL_ALIAS"`
EntAliases []unAlias `xml:"ENTITY_ALIAS"`
IndAddress []unAddress `xml:"INDIVIDUAL_ADDRESS"`
EntAddress []unAddress `xml:"ENTITY_ADDRESS"`
DOB []unDOB `xml:"INDIVIDUAL_DATE_OF_BIRTH"`
}
type unAlias struct {
Quality string `xml:"QUALITY"`
Name string `xml:"ALIAS_NAME"`
}
type unAddress struct {
Street string `xml:"STREET"`
City string `xml:"CITY"`
Country string `xml:"COUNTRY"`
}
// unDOB holds a UN date of birth, which is frequently partial: of the dated
// records, some carry a full DATE, some a YEAR alone, and some a FROM_YEAR /
// TO_YEAR range under TYPE_OF_DATE "BETWEEN".
type unDOB struct {
TypeOfDate string `xml:"TYPE_OF_DATE"`
Date string `xml:"DATE"`
Year string `xml:"YEAR"`
FromYear string `xml:"FROM_YEAR"`
ToYear string `xml:"TO_YEAR"`
}
// value renders the date at the precision the list actually carries, so a
// year-only record is not padded into a spurious exact date.
func (d unDOB) value() string {
switch {
case strings.TrimSpace(d.Date) != "":
return strings.TrimSpace(d.Date)
case strings.TrimSpace(d.FromYear) != "" && strings.TrimSpace(d.ToYear) != "":
return strings.TrimSpace(d.FromYear) + "/" + strings.TrimSpace(d.ToYear)
default:
return strings.TrimSpace(d.Year)
}
}
func (s unSubjectXML) entry(entryType string) types.SanctionsEntry {
var aliases []string
for _, a := range append(s.IndAliases, s.EntAliases...) {
aliases = addAlias(aliases, a.Name)
}
var address string
for _, a := range append(s.IndAddress, s.EntAddress...) {
if address = join(a.Street, a.City, a.Country); address != "" {
break
}
}
var dob string
for _, d := range s.DOB {
if dob = d.value(); dob != "" {
break
}
}
// REFERENCE_NUMBER is the human-facing designation reference (CDi.001) and
// DATAID the stable numeric key. RefID takes DATAID so a re-listing under a
// new reference does not read as a new entry.
ref := strings.TrimSpace(s.DataID)
if ref == "" {
ref = strings.TrimSpace(s.Reference)
}
return types.SanctionsEntry{
RefID: ref,
Name: join(s.First, s.Second, s.Third, s.Fourth),
Aliases: aliases,
DOB: dob,
Nationality: strings.TrimSpace(s.Nationality),
Address: address,
Type: entryType,
}
}
// ── EU Financial Sanctions File ─────────────────────────────────────────────
// Root <export>, one <sanctionEntity> per designation. Every name is a
// <nameAlias> and the file marks no primary name: the strong attribute is
// "true" on every alias, so it carries no signal to select one. The first
// nameAlias is taken as the display name and all of them are matched against,
// which is what makes the absence of a primary marker harmless here.
// ParseEU parses the EU consolidated financial sanctions list XML.
func ParseEU(body []byte) ([]types.SanctionsEntry, error) {
var list euExportXML
if err := decode(body, &list); err != nil {
return nil, fmt.Errorf("parse EU XML: %w", err)
}
entries := make([]types.SanctionsEntry, 0, len(list.Entities))
for _, e := range list.Entities {
// classificationCode is P for a natural person and E for an entity.
entryType := types.SanctionEntity
if strings.EqualFold(e.SubjectType.Classification, "P") ||
strings.EqualFold(e.SubjectType.Code, "person") {
entryType = types.SanctionIndividual
}
var name string
var aliases []string
for _, a := range e.Aliases {
n := strings.TrimSpace(a.WholeName)
if n == "" {
n = join(a.FirstName, a.MiddleName, a.LastName)
}
if n == "" {
continue
}
if name == "" {
name = n
continue
}
aliases = addAlias(aliases, n)
}
var dob string
for _, b := range e.Birthdates {
if dob = strings.TrimSpace(b.Birthdate); dob != "" {
break
}
if dob = strings.TrimSpace(b.Year); dob != "" {
break
}
}
var nationality string
for _, c := range e.Citizenships {
// "00" is the file's sentinel for an unknown country.
if code := strings.TrimSpace(c.CountryISO2); code != "" && code != "00" {
nationality = strings.TrimSpace(c.CountryDescription)
if nationality == "" {
nationality = code
}
break
}
}
var address string
for _, a := range e.Addresses {
if address = join(a.Street, a.City, a.CountryDescription); address != "" {
break
}
}
// logicalId repeats across element types, so it is not unique on its
// own; euReferenceNumber identifies the designation.
ref := strings.TrimSpace(e.EUReference)
if ref == "" {
ref = strings.TrimSpace(e.LogicalID)
}
entries = append(entries, types.SanctionsEntry{
RefID: ref,
Name: name,
Aliases: aliases,
DOB: dob,
Nationality: nationality,
Address: address,
Type: entryType,
})
}
return entries, nil
}
type euExportXML struct {
XMLName xml.Name `xml:"export"`
Entities []euContent `xml:"sanctionEntity"`
}
type euContent struct {
EUReference string `xml:"euReferenceNumber,attr"`
UnitedNation string `xml:"unitedNationId,attr"`
LogicalID string `xml:"logicalId,attr"`
SubjectType euSubjectType `xml:"subjectType"`
Aliases []euNameAlias `xml:"nameAlias"`
Birthdates []euBirthdate `xml:"birthdate"`
Citizenships []euCitizenship `xml:"citizenship"`
Addresses []euAddress `xml:"address"`
}
type euSubjectType struct {
Code string `xml:"code,attr"`
Classification string `xml:"classificationCode,attr"`
}
type euNameAlias struct {
WholeName string `xml:"wholeName,attr"`
FirstName string `xml:"firstName,attr"`
MiddleName string `xml:"middleName,attr"`
LastName string `xml:"lastName,attr"`
}
type euBirthdate struct {
Birthdate string `xml:"birthdate,attr"`
Year string `xml:"year,attr"`
}
type euCitizenship struct {
CountryISO2 string `xml:"countryIso2Code,attr"`
CountryDescription string `xml:"countryDescription,attr"`
}
type euAddress struct {
Street string `xml:"street,attr"`
City string `xml:"city,attr"`
CountryDescription string `xml:"countryDescription,attr"`
}
// ── UK Sanctions List ───────────────────────────────────────────────────────
// Root <Designations>. This replaced the OFSI Consolidated List, which closed
// on 28 January 2026 (see lists.go). Names are split Name1..Name5 for given
// names and Name6 for the family or organisation name, and NameType selects the
// primary name — case-inconsistently, which is why it is folded before
// comparison. Non-Latin-script names sit in a separate element and are carried
// as aliases so screening reaches them.
// ParseUK parses the FCDO UK Sanctions List XML.
func ParseUK(body []byte) ([]types.SanctionsEntry, error) {
var list ukListXML
if err := decode(body, &list); err != nil {
return nil, fmt.Errorf("parse UK XML: %w", err)
}
entries := make([]types.SanctionsEntry, 0, len(list.Designations))
for _, d := range list.Designations {
entryType := types.SanctionEntity
switch strings.ToLower(strings.TrimSpace(d.Subject)) {
case "individual":
entryType = types.SanctionIndividual
case "ship":
entryType = types.SanctionVessel
}
var name string
var aliases []string
for _, n := range d.Names {
full := join(n.Name1, n.Name2, n.Name3, n.Name4, n.Name5, n.Name6)
if full == "" {
continue
}
// NameType appears as "Primary Name", "Primary name", "Alias" and
// "ALias" in the same file, so the comparison folds case. A record
// may carry more than one primary name; the first wins and the rest
// are matched as aliases.
if name == "" && strings.EqualFold(strings.TrimSpace(n.NameType), "primary name") {
name = full
continue
}
aliases = addAlias(aliases, full)
}
// Fall back to the first name of any type rather than emitting a
// nameless entry, which would be unmatchable.
if name == "" && len(aliases) > 0 {
name, aliases = aliases[0], aliases[1:]
}
for _, n := range d.NonLatinNames {
aliases = addAlias(aliases, n)
}
var dob string
for _, raw := range d.DOBs {
if dob = ukDate(raw); dob != "" {
break
}
}
var address string
for _, a := range d.Addresses {
if address = join(a.Line1, a.Line2, a.Line3, a.Line4, a.Line5, a.Line6, a.Country); address != "" {
break
}
}
var nationality string
if len(d.Nationalities) > 0 {
nationality = strings.TrimSpace(d.Nationalities[0])
}
entries = append(entries, types.SanctionsEntry{
RefID: strings.TrimSpace(d.UniqueID),
Name: name,
Aliases: aliases,
DOB: dob,
Nationality: nationality,
Address: address,
Type: entryType,
})
}
return entries, nil
}
// ukDate normalises a UK list date. The file writes unknown components as the
// literal text "dd" and "mm" — "dd/mm/1945" means the year alone is known — so
// a placeholder component is dropped rather than parsed or padded.
func ukDate(raw string) string {
parts := strings.Split(strings.TrimSpace(raw), "/")
if len(parts) != 3 {
return strings.TrimSpace(raw)
}
keep := make([]string, 0, 3)
for _, p := range parts {
p = strings.TrimSpace(p)
if p == "" || strings.EqualFold(p, "dd") || strings.EqualFold(p, "mm") ||
strings.EqualFold(p, "yyyy") {
continue
}
keep = append(keep, p)
}
return strings.Join(keep, "/")
}
type ukListXML struct {
XMLName xml.Name `xml:"Designations"`
Designations []ukDesignationXML `xml:"Designation"`
}
type ukDesignationXML struct {
UniqueID string `xml:"UniqueID"`
Subject string `xml:"IndividualEntityShip"`
Names []ukNameXML `xml:"Names>Name"`
NonLatinNames []string `xml:"NonLatinNames>NonLatinName>NameNonLatinScript"`
Addresses []ukAddress `xml:"Addresses>Address"`
DOBs []string `xml:"IndividualDetails>Individual>DOBs>DOB"`
Nationalities []string `xml:"IndividualDetails>Individual>Nationalities>Nationality"`
}
type ukNameXML struct {
Name1 string `xml:"Name1"`
Name2 string `xml:"Name2"`
Name3 string `xml:"Name3"`
Name4 string `xml:"Name4"`
Name5 string `xml:"Name5"`
Name6 string `xml:"Name6"`
NameType string `xml:"NameType"`
}
type ukAddress struct {
Line1 string `xml:"AddressLine1"`
Line2 string `xml:"AddressLine2"`
Line3 string `xml:"AddressLine3"`
Line4 string `xml:"AddressLine4"`
Line5 string `xml:"AddressLine5"`
Line6 string `xml:"AddressLine6"`
Country string `xml:"AddressCountry"`
}
pkg/sanctions/parse_test.go
+255
View File
@@ -0,0 +1,255 @@
package sanctions
import (
"strings"
"testing"
"github.com/luxfi/aml/pkg/types"
)
// Each fixture below is reduced from the live file it names and keeps the
// quirks that actually break a parser. Verified against the full downloads:
// OFAC 19,175 entries, UN 1,011, EU 6,017, UK 6,315 — every one of them named.
func only(t *testing.T, entries []types.SanctionsEntry, err error) types.SanctionsEntry {
t.Helper()
if err != nil {
t.Fatalf("parse: %v", err)
}
if len(entries) != 1 {
t.Fatalf("got %d entries, want 1", len(entries))
}
return entries[0]
}
func hasAlias(e types.SanctionsEntry, want string) bool {
for _, a := range e.Aliases {
if a == want {
return true
}
}
return false
}
// The UN splits a name across four elements and writes partial dates: a record
// may carry a full DATE, a YEAR alone, or a FROM_YEAR/TO_YEAR range. Padding a
// year into a fake exact date would put invented precision in a screening field.
func TestParseUNPartialDatesAndFourPartNames(t *testing.T) {
const fixture = `<?xml version="1.0"?>
<CONSOLIDATED_LIST>
<INDIVIDUALS>
<INDIVIDUAL>
<DATAID>6907993</DATAID>
<REFERENCE_NUMBER>CDi.001</REFERENCE_NUMBER>
<FIRST_NAME>ERIC</FIRST_NAME><SECOND_NAME>BADEGE</SECOND_NAME>
<NATIONALITY><VALUE>Democratic Republic of the Congo</VALUE></NATIONALITY>
<INDIVIDUAL_ALIAS><QUALITY/><ALIAS_NAME/></INDIVIDUAL_ALIAS>
<INDIVIDUAL_ADDRESS><COUNTRY>Rwanda</COUNTRY></INDIVIDUAL_ADDRESS>
<INDIVIDUAL_DATE_OF_BIRTH><TYPE_OF_DATE>EXACT</TYPE_OF_DATE><YEAR>1971</YEAR></INDIVIDUAL_DATE_OF_BIRTH>
</INDIVIDUAL>
<INDIVIDUAL>
<DATAID>222</DATAID>
<FIRST_NAME>ABD</FIRST_NAME><SECOND_NAME>AL</SECOND_NAME><THIRD_NAME>RAHMAN</THIRD_NAME><FOURTH_NAME>YASIN</FOURTH_NAME>
<INDIVIDUAL_ALIAS><QUALITY>good</QUALITY><ALIAS_NAME>Taha Yasin</ALIAS_NAME></INDIVIDUAL_ALIAS>
<INDIVIDUAL_DATE_OF_BIRTH><TYPE_OF_DATE>BETWEEN</TYPE_OF_DATE><FROM_YEAR>1958</FROM_YEAR><TO_YEAR>1960</TO_YEAR></INDIVIDUAL_DATE_OF_BIRTH>
</INDIVIDUAL>
</INDIVIDUALS>
<ENTITIES>
<ENTITY>
<DATAID>6908402</DATAID><FIRST_NAME>ADF</FIRST_NAME>
<ENTITY_ALIAS><QUALITY>good</QUALITY><ALIAS_NAME>Allied Democratic Forces</ALIAS_NAME></ENTITY_ALIAS>
</ENTITY>
</ENTITIES>
</CONSOLIDATED_LIST>`
entries, err := ParseUN([]byte(fixture))
if err != nil {
t.Fatalf("ParseUN: %v", err)
}
if len(entries) != 3 {
t.Fatalf("got %d entries, want 3 (2 individuals + 1 entity)", len(entries))
}
if entries[0].Name != "ERIC BADEGE" || entries[0].RefID != "6907993" {
t.Errorf("entry[0] = %q ref %q", entries[0].Name, entries[0].RefID)
}
if entries[0].DOB != "1971" {
t.Errorf("year-only DOB = %q, want %q — a year must not be padded to a date", entries[0].DOB, "1971")
}
if entries[0].Nationality != "Democratic Republic of the Congo" || entries[0].Address != "Rwanda" {
t.Errorf("entry[0] nat %q addr %q", entries[0].Nationality, entries[0].Address)
}
// The list ships empty alias elements; they must not become blank aliases,
// which match everything at zero length.
if len(entries[0].Aliases) != 0 {
t.Errorf("empty <ALIAS_NAME/> became an alias: %q", entries[0].Aliases)
}
if entries[1].Name != "ABD AL RAHMAN YASIN" {
t.Errorf("four-part name = %q", entries[1].Name)
}
if entries[1].DOB != "1958/1960" {
t.Errorf("BETWEEN range DOB = %q, want %q", entries[1].DOB, "1958/1960")
}
if entries[2].Type != types.SanctionEntity || entries[2].Name != "ADF" {
t.Errorf("entity = %q/%q", entries[2].Name, entries[2].Type)
}
if !hasAlias(entries[2], "Allied Democratic Forces") {
t.Errorf("ENTITY_ALIAS was dropped: %q", entries[2].Aliases)
}
}
// The EU file marks no primary name — strong="true" sits on every alias, so it
// carries no signal — and repeats a name once per amending regulation. The first
// name becomes the display name, the rest are matched, and repeats collapse.
func TestParseEUNoPrimaryNameMarkerAndRepeatedAliases(t *testing.T) {
const fixture = `<?xml version="1.0"?>
<export>
<sanctionEntity euReferenceNumber="EU.27.28" logicalId="13">
<subjectType code="person" classificationCode="P"/>
<nameAlias wholeName="Saddam Hussein Al-Tikriti" strong="true"/>
<nameAlias wholeName="Abu Ali" strong="true"/>
<nameAlias wholeName="Abu Ali" strong="true"/>
<nameAlias firstName="Abou" lastName="Ali" strong="true"/>
<citizenship countryIso2Code="IQ" countryDescription="IRAQ"/>
<birthdate birthdate="1937-04-28" year="1937"/>
</sanctionEntity>
<sanctionEntity euReferenceNumber="EU.1.2" logicalId="99">
<subjectType code="enterprise" classificationCode="E"/>
<nameAlias wholeName="Some Trading Co"/>
<citizenship countryIso2Code="00" countryDescription=""/>
</sanctionEntity>
</export>`
entries, err := ParseEU([]byte(fixture))
if err != nil {
t.Fatalf("ParseEU: %v", err)
}
if len(entries) != 2 {
t.Fatalf("got %d entries, want 2", len(entries))
}
e := entries[0]
if e.Name != "Saddam Hussein Al-Tikriti" || e.Type != types.SanctionIndividual {
t.Errorf("entry[0] = %q/%q", e.Name, e.Type)
}
if e.RefID != "EU.27.28" {
t.Errorf("RefID = %q — logicalId repeats across element types and is not unique", e.RefID)
}
if e.DOB != "1937-04-28" || e.Nationality != "IRAQ" {
t.Errorf("entry[0] dob %q nat %q", e.DOB, e.Nationality)
}
// "Abu Ali" appears twice in the source and must appear once here.
n := 0
for _, a := range e.Aliases {
if a == "Abu Ali" {
n++
}
}
if n != 1 {
t.Errorf("repeated alias collapsed to %d copies, want 1: %q", n, e.Aliases)
}
if !hasAlias(e, "Abou Ali") {
t.Errorf("alias assembled from firstName/lastName was dropped: %q", e.Aliases)
}
if entries[1].Type != types.SanctionEntity {
t.Errorf("classificationCode E gave type %q", entries[1].Type)
}
// "00" is the file's sentinel for unknown, not a country.
if entries[1].Nationality != "" {
t.Errorf("country sentinel 00 became nationality %q", entries[1].Nationality)
}
}
// The UK list writes unknown date components as the literal text "dd" and "mm",
// spells NameType in six different casings, and keeps non-Latin-script names in
// a separate element that screening still has to reach.
func TestParseUKPlaceholderDatesAndCaseInconsistentNameType(t *testing.T) {
const fixture = `<?xml version="1.0"?>
<Designations>
<Designation>
<UniqueID>AFG0006</UniqueID>
<Names>
<Name><Name1>MOHAMMAD</Name1><Name2>HASSAN</Name2><Name6>AKHUND</Name6><NameType>Primary name</NameType></Name>
<Name><Name6>Wali Mohammad</Name6><NameType>ALias</NameType></Name>
</Names>
<NonLatinNames><NonLatinName><NameNonLatinScript>محمد حسن آخوند</NameNonLatinScript></NonLatinName></NonLatinNames>
<IndividualEntityShip>Individual</IndividualEntityShip>
<Addresses><Address><AddressLine6>Kabul</AddressLine6><AddressCountry>Afghanistan</AddressCountry></Address></Addresses>
<IndividualDetails><Individual>
<DOBs><DOB>dd/mm/1945</DOB></DOBs>
<Nationalities><Nationality>Afghanistan</Nationality></Nationalities>
</Individual></IndividualDetails>
</Designation>
<Designation>
<UniqueID>RUS0999</UniqueID>
<Names><Name><Name6>SOME VESSEL</Name6><NameType>Primary Name</NameType></Name></Names>
<IndividualEntityShip>Ship</IndividualEntityShip>
</Designation>
<Designation>
<UniqueID>NOPRIMARY1</UniqueID>
<Names><Name><Name6>Only An Alias</Name6><NameType>Alias</NameType></Name></Names>
<IndividualEntityShip>Entity</IndividualEntityShip>
</Designation>
</Designations>`
entries, err := ParseUK([]byte(fixture))
if err != nil {
t.Fatalf("ParseUK: %v", err)
}
if len(entries) != 3 {
t.Fatalf("got %d entries, want 3", len(entries))
}
e := entries[0]
// "Primary name" here, "Primary Name" elsewhere in the same file.
if e.Name != "MOHAMMAD HASSAN AKHUND" {
t.Errorf("Name = %q — NameType case must be folded before comparison", e.Name)
}
if e.DOB != "1945" {
t.Errorf("DOB = %q, want %q — dd/mm are placeholders, not values to parse", e.DOB, "1945")
}
if e.Type != types.SanctionIndividual || e.Nationality != "Afghanistan" {
t.Errorf("entry[0] type %q nat %q", e.Type, e.Nationality)
}
if !strings.Contains(e.Address, "Kabul") {
t.Errorf("Address = %q", e.Address)
}
if !hasAlias(e, "Wali Mohammad") {
t.Errorf("alias typed \"ALias\" was dropped: %q", e.Aliases)
}
if !hasAlias(e, "محمد حسن آخوند") {
t.Errorf("non-Latin-script name is unreachable by screening: %q", e.Aliases)
}
if entries[1].Type != types.SanctionVessel {
t.Errorf("Ship gave type %q, want vessel", entries[1].Type)
}
// A designation with no primary name must still be matchable.
if entries[2].Name != "Only An Alias" {
t.Errorf("alias-only designation produced name %q — a nameless entry never matches", entries[2].Name)
}
}
// Every parser must reject a payload written for a different publisher rather
// than return zero entries, which is what made three of four lists look empty.
func TestParsersRejectAnotherPublishersSchema(t *testing.T) {
bodies := map[string]string{
"ofac": `<?xml version="1.0"?><sdnList><sdnEntry><uid>1</uid><lastName>X</lastName></sdnEntry></sdnList>`,
"un": `<?xml version="1.0"?><CONSOLIDATED_LIST><INDIVIDUALS><INDIVIDUAL><DATAID>1</DATAID><FIRST_NAME>X</FIRST_NAME></INDIVIDUAL></INDIVIDUALS></CONSOLIDATED_LIST>`,
"eu": `<?xml version="1.0"?><export><sanctionEntity euReferenceNumber="1"><nameAlias wholeName="X"/></sanctionEntity></export>`,
"uk": `<?xml version="1.0"?><Designations><Designation><UniqueID>1</UniqueID><Names><Name><Name6>X</Name6><NameType>Primary Name</NameType></Name></Names></Designation></Designations>`,
}
for name, parse := range map[string]Parser{"ofac": ParseOFAC, "un": ParseUN, "eu": ParseEU, "uk": ParseUK} {
for other, body := range bodies {
if other == name {
continue
}
if _, err := parse([]byte(body)); err == nil {
t.Errorf("%s parser accepted the %s schema instead of failing", name, other)
}
}
}
}
pkg/sanctions/refresh.go
+69 -31
View File
@@ -6,73 +6,111 @@ package sanctions
import (
"context"
"errors"
"fmt"
"log"
"time"
"github.com/hanzoai/base/core"
"github.com/luxfi/aml/pkg/types"
)
// RefreshCron registers a daily 06:00 UTC cron that fetches all sanctions
// lists, diffs against stored entries, and persists updates.
// RefreshCron registers a daily 06:00 UTC refresh of every active list.
func RefreshCron(app core.App, store SanctionsStore) {
app.Cron().Add("sanctions-refresh", "0 6 * * *", func() {
if err := RefreshAll(context.Background(), store); err != nil {
log.Printf("[sanctions-refresh] error: %v", err)
if _, err := RefreshAll(context.Background(), store); err != nil {
log.Printf("[sanctions-refresh] FAILED: %v", err)
}
})
}
// RefreshAll fetches OFAC, UN, EU, and HMT lists, and persists them
// via the SanctionsStore. Returns the count of new entries added.
func RefreshAll(ctx context.Context, store SanctionsStore) error {
// Result reports what one list contributed to a refresh.
type Result struct {
Source string
Entries int
SHA256 string
Err error
}
// RefreshAll fetches every active list, parses it with that publisher's parser,
// and persists the entries. It returns one Result per list and a non-nil error if
// any list failed.
//
// A failing list does not abort the others — OFAC being unreachable must not stop
// the UN list loading — but it is never swallowed either. Each failure is joined
// into the returned error so the caller decides what a partial refresh means, and
// the freshness of what is stored stays answerable through Stale.
func RefreshAll(ctx context.Context, store SanctionsStore) ([]Result, error) {
ingester := NewIngester()
lists := DefaultLists()
var totalEntries int
results := make([]Result, 0, len(lists))
var errs []error
for _, list := range lists {
if !list.Active {
continue
}
var entries []types.SanctionsEntry
var hash string
var err error
switch list.Source {
case types.ListOFACSDN:
entries, hash, err = ingester.FetchOFAC(ctx, list.URL)
default:
// UN, EU, HMT use the same OFAC parser for now — their XML
// structures are similar enough. Production should have dedicated
// parsers per list format.
entries, hash, err = ingester.FetchOFAC(ctx, list.URL)
}
res := Result{Source: list.Source}
entries, hash, err := ingester.Fetch(ctx, list)
if err != nil {
log.Printf("[sanctions-refresh] fetch %s failed: %v", list.Source, err)
res.Err = err
errs = append(errs, err)
results = append(results, res)
log.Printf("[sanctions-refresh] %s: %v", list.Source, err)
continue
}
// Tag entries with list source.
now := time.Now().UTC()
for i := range entries {
entries[i].ListID = list.Source
entries[i].ID = fmt.Sprintf("%s-%s", list.Source, entries[i].RefID)
now := time.Now().UTC()
entries[i].CreatedAt = now
entries[i].UpdatedAt = now
}
if err := store.Save(ctx, entries); err != nil {
log.Printf("[sanctions-refresh] save %s failed: %v", list.Source, err)
res.Err = fmt.Errorf("save %s: %w", list.Source, err)
errs = append(errs, res.Err)
results = append(results, res)
log.Printf("[sanctions-refresh] %s: %v", list.Source, res.Err)
continue
}
totalEntries += len(entries)
log.Printf("[sanctions-refresh] %s: %d entries (hash: %s)", list.Source, len(entries), hash[:16])
res.Entries, res.SHA256 = len(entries), hash
results = append(results, res)
log.Printf("[sanctions-refresh] %s: %d entries (sha256 %s)", list.Source, len(entries), hash[:16])
}
log.Printf("[sanctions-refresh] complete: %d total entries across %d lists", totalEntries, len(lists))
return nil
if len(errs) > 0 {
return results, fmt.Errorf("%d of %d lists failed: %w", len(errs), len(results), errors.Join(errs...))
}
log.Printf("[sanctions-refresh] complete: %d lists", len(results))
return results, nil
}
// maxListAge is how long stored screening data may go without a successful
// refresh before it is treated as unfit to screen against. Publishers add
// designations on their own schedule and the duty to act on a new designation
// runs from the designation, not from whenever a fetch next happens to succeed,
// so silence for this long is a failure rather than a quiet period.
const maxListAge = 48 * time.Hour
// Stale reports whether stored sanctions data is too old to screen against, and
// how old it is. A zero refresh time means nothing has ever loaded, which is
// stale by definition rather than fresh.
//
// This exists because the expensive failure here is silent: with no freshness
// check, screening against an empty or months-old list returns no matches and
// looks exactly like screening a clean party.
func Stale(ctx context.Context, store SanctionsStore) (bool, time.Duration, error) {
last, err := store.LastRefresh(ctx)
if err != nil {
return true, 0, err
}
if last.IsZero() {
return true, 0, nil
}
age := time.Since(last)
return age > maxListAge, age, nil
}