Merge branch 'blue/scoring' into blue/scoring-land

Co-authored-by: Hanzo Dev <[email protected]>

pkg/calibrate/calibrate.go

@@ -0,0 +1,654 @@
+// Package calibrate turns a score into a probability.
+//
+// THE PROBLEM IT SOLVES. A half-space tree's score is a density: how isolated a
+// point is among the mass this tenant's recent traffic laid down. It ranks well
+// and it means nothing on its own. 0.8 is not "80% likely to be fraud" — it is
+// not a likelihood at all, and two tenants' 0.8s are not comparable because the
+// mass under them is different. Every threshold set on such a score is a number
+// someone liked, every cost calculation over it is arithmetic on the wrong
+// units, and an adverse decision defended with it cannot answer "how likely was
+// this, actually?".
+//
+// A calibration is the missing map. Fitted on a tenant's OWN judged decisions,
+// it answers: among the decisions that scored near here, what share turned out
+// to be productive? That number IS a probability, it is comparable across
+// tenants and across model versions, and it is the only thing a cost-weighted
+// policy can be stated over honestly.
+//
+// TWO METHODS, AND WHEN EACH IS RIGHT.
+//
+//	Isotonic  the non-parametric one. Assumes only that a higher score is never
+//	          less likely to be productive — the one assumption the detector
+//	          actually guarantees. Fits the exact least-squares monotone step
+//	          function by pool-adjacent-violators, then interpolates linearly
+//	          between block centroids so a policy threshold landing inside a
+//	          plateau still means something. Needs data: with a few dozen
+//	          positives it will fit the noise.
+//	Platt     the parametric one: one logistic through the scores. Two
+//	          parameters, so it survives a small sample, at the cost of assuming
+//	          a shape. Fitted with Platt's own label smoothing, which is not
+//	          optional on imbalanced data — without it the fit runs off to
+//	          certainty on whichever class is scarce.
+//
+// WHAT IT REFUSES. A calibration fitted on one class is a constant, and a
+// constant reported as a probability is a lie with a decimal point on it. Fit
+// refuses below a floor of judged rows and below a floor of each class, and
+// names which. An absent calibration is a fact a caller can act on; an invented
+// one is not.
+//
+// THE SHAPE GATE — this is the training–serving skew control. A Map records the
+// digest of the scorer it was fitted under (anomaly.Store.Digest(), which covers
+// the feature inventory in order, the neutral values and the detector geometry).
+// A Map cannot be read at all: Under(shape) is the only way to obtain a Reader,
+// and it refuses for a scorer whose shape differs. So a calibration cannot
+// outlive the model it describes: change a feature, and every map fitted before
+// the change stops answering instead of silently mapping coordinates that no
+// longer mean what they meant.
+//
+// The gate is at the BIND and not at the read, because a gate every read has to
+// pass is a gate every read can be made to pass — the shape argument was
+// available from the map itself, so passing m.Shape cleared the control and
+// looked exactly like clearing it honestly. Reading is total once the bind has
+// happened, so there is one place to get it right and no way downstream to get
+// it wrong.
+package calibrate
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"math"
+	"sort"
+	"strconv"
+)
+
+// Methods.
+const (
+	// Isotonic is the non-parametric monotone fit.
+	Isotonic = "isotonic"
+	// Platt is the two-parameter logistic fit.
+	Platt = "platt"
+)
+
+// Floors. Below these a fit is refused rather than produced.
+//
+// Rows is a small number on purpose: the point is to refuse the degenerate case,
+// not to impose a statistical opinion about sufficiency. The caller sees Rows
+// and Positive on the Map and decides for itself whether to trust the fit; a
+// package that silently declined to answer until it judged the sample large
+// enough would be making that decision in the wrong place.
+const (
+	// MinRows is the fewest judged observations a fit will look at.
+	MinRows = 20
+	// MinClass is the fewest of EACH class. One of each is arithmetically
+	// sufficient and statistically meaningless; five is the point at which the
+	// isotonic blocks stop being individual observations.
+	MinClass = 5
+)
+
+// Errors. Each names the thing that was missing, because "calibration failed" is
+// not something an operator can act on.
+var (
+	ErrThin       = errors.New("calibrate: too few judged observations to fit a probability")
+	ErrOneClass   = errors.New("calibrate: the judged observations are all one class, so any fit would be a constant")
+	ErrMethod     = errors.New("calibrate: unknown method")
+	ErrShape      = errors.New("calibrate: this map was fitted under a different model shape")
+	ErrNoShape    = errors.New("calibrate: a map must record the model shape it was fitted under")
+	ErrNotFitted  = errors.New("calibrate: this map holds no fit")
+	ErrConverging = errors.New("calibrate: the logistic fit did not converge")
+)
+
+// Sample is one judged observation: what the model said, and what turned out to
+// be true.
+//
+// Productive is the engine's own vocabulary (replay.Disposition), reduced to the
+// one bit a fit reads. Unjudged observations are not Samples — a caller filters
+// them out, because a fit over "unknown treated as negative" is a fit against
+// the incumbent policy rather than against the world.
+type Sample struct {
+	// Score is the raw model score in [0,1].
+	Score float64
+	// Productive is what a human or a processor concluded: true when the
+	// decision was right to be worried.
+	Productive bool
+	// Weight lets a caller down-weight a low-confidence label. Zero or negative
+	// means one, so the zero value of Sample is an ordinary unweighted
+	// observation.
+	Weight float64
+}
+
+func (s Sample) weight() float64 {
+	if s.Weight <= 0 {
+		return 1
+	}
+	return s.Weight
+}
+
+// Knot is one point of the fitted map: at this score, this probability. Between
+// two knots P interpolates linearly; outside the outermost pair it clamps.
+type Knot struct {
+	Score float64 `json:"score"`
+	P     float64 `json:"p"`
+}
+
+// Map is a fitted calibration. It is a VALUE: immutable once returned, safe to
+// share across goroutines, and cheap enough to hold one per tenant in memory and
+// to write as a row.
+type Map struct {
+	// Method is isotonic or platt.
+	Method string `json:"method"`
+	// Knots is the isotonic fit, ascending by score. Empty for platt.
+	Knots []Knot `json:"knots,omitempty"`
+	// A and B are the platt fit: p = 1/(1+exp(A*score+B)). Zero for isotonic.
+	A float64 `json:"a,omitempty"`
+	B float64 `json:"b,omitempty"`
+	// Shape is the digest of the scorer this was fitted under. P refuses under
+	// any other shape, which is what stops a calibration from outliving the
+	// coordinates it describes.
+	Shape string `json:"shape"`
+	// Rows, Positive and Negative are the sample it was fitted on, so a caller
+	// can see how much evidence is behind the number before acting on it.
+	Rows     int `json:"rows"`
+	Positive int `json:"positive"`
+	Negative int `json:"negative"`
+	// Prevalence is the productive share of the fitting sample. A probability
+	// read without it is uninterpretable: 0.4 is alarming at a base rate of
+	// 1-in-1000 and unremarkable at 1-in-3.
+	Prevalence float64 `json:"prevalence"`
+	// Brier is the mean squared error of this map on its own fitting sample. It
+	// is the in-sample number and is therefore optimistic; the honest one comes
+	// from evaluating the map on data it was not fitted on, which is what the
+	// learning curve does.
+	Brier float64 `json:"brier"`
+	// Digest identifies this fit exactly. A decision records it, so an auditor
+	// can pin which map turned which score into which probability.
+	Digest string `json:"digest"`
+}
+
+// Fitted reports whether this map can answer.
+func (m Map) Fitted() bool {
+	return m.Shape != "" && (len(m.Knots) > 0 || m.Method == Platt)
+}
+
+// Under binds this map to the scoring shape now in force, refusing when they
+// differ. It is the ONLY way to read a probability out of a Map.
+//
+// A GATE A CALLER CAN SATISFY IS NOT A GATE. The earlier read was
+// P(score, shape), and every internal call site passed m.Shape — the value being
+// compared came from the value being checked, so the training–serving skew
+// control was trivially true everywhere except the one place a live shape
+// happened to be threaded in. Nothing about that was visible at the call site;
+// each one looked like it was passing the gate.
+//
+// Binding separates the two concerns. ONE place decides whether this map applies
+// to these coordinates, at the boundary where the current shape is a fact the
+// server observed rather than a value a caller supplied. Everything downstream
+// takes a Reader, for which the question is already answered and cannot be
+// re-asked wrongly — a Reader has no shape to pass and no Map to read one off.
+func (m Map) Under(shape string) (Reader, error) {
+	if !m.Fitted() {
+		return Reader{}, ErrNotFitted
+	}
+	if shape == "" {
+		return Reader{}, ErrNoShape
+	}
+	if shape != m.Shape {
+		return Reader{}, fmt.Errorf("%w: fitted under %s, asked under %s", ErrShape, short(m.Shape), short(shape))
+	}
+	if m.Method != Isotonic && m.Method != Platt {
+		return Reader{}, fmt.Errorf("%w: %q", ErrMethod, m.Method)
+	}
+	return Reader{m: m, ok: true}, nil
+}
+
+// Reader is a calibration that has already been checked against the coordinates
+// it is about to be read in: a map bound to one scoring shape.
+//
+// It has no exported fields and no constructor other than Map.Under, so a
+// function taking a Reader is a function whose caller has been through the gate,
+// and that is a property the compiler checks rather than a reviewer.
+//
+// The ZERO Reader is the honest "no probability here": it is what a caller holds
+// when this organisation has no calibration at all AND what it holds when the
+// shape has moved, which are the two facts every consumer downstream has to
+// treat the same way — silently, by not stating a probability.
+type Reader struct {
+	m  Map
+	ok bool
+}
+
+// Fitted reports whether this reader can answer.
+func (r Reader) Fitted() bool { return r.ok }
+
+// Digest names the fit behind this reader, so a record can pin which map turned
+// which score into which probability. Empty when it cannot answer.
+func (r Reader) Digest() string {
+	if !r.ok {
+		return ""
+	}
+	return r.m.Digest
+}
+
+// P maps one score to a probability. It is TOTAL: the only thing that could have
+// failed is the shape, and holding a Reader is proof it did not.
+func (r Reader) P(score float64) float64 {
+	if !r.ok {
+		return 0
+	}
+	if r.m.Method == Platt {
+		return clamp01(1 / (1 + math.Exp(r.m.A*score+r.m.B)))
+	}
+	return clamp01(interpolate(r.m.Knots, score))
+}
+
+// interpolate reads the piecewise-linear map at score, clamping outside the
+// fitted range.
+//
+// Linear between knots rather than a step, deliberately. The exact
+// pool-adjacent-violators solution is a step function, and a step means every
+// score inside one block gets one probability — so a policy threshold that lands
+// inside a block is arbitrary within it, and moving the threshold by a hair
+// either changes nothing or changes everything. Interpolating keeps the fit
+// monotone (both endpoints are, and a line between two non-decreasing values is)
+// while making the map strictly informative about where in the block a score
+// sat.
+func interpolate(k []Knot, score float64) float64 {
+	if len(k) == 0 {
+		return 0
+	}
+	if score <= k[0].Score {
+		return k[0].P
+	}
+	last := len(k) - 1
+	if score >= k[last].Score {
+		return k[last].P
+	}
+	i := sort.Search(len(k), func(i int) bool { return k[i].Score >= score })
+	lo, hi := k[i-1], k[i]
+	if hi.Score == lo.Score {
+		return hi.P
+	}
+	t := (score - lo.Score) / (hi.Score - lo.Score)
+	return lo.P + t*(hi.P-lo.P)
+}
+
+// Fit produces a calibration from judged observations.
+//
+// shape is the scorer's own digest and is required: a map that does not know
+// what it was fitted under cannot refuse to be misapplied, and refusing to be
+// misapplied is most of what a calibration is for.
+//
+// The fit is DETERMINISTIC. Samples are sorted by score with ties broken by
+// class, so two calls on the same multiset — in any order — return byte-identical
+// knots and the same digest. That is what lets a fit be a record: an auditor
+// re-running it gets the same map or knows the inputs changed.
+func Fit(samples []Sample, method, shape string) (Map, error) {
+	if shape == "" {
+		return Map{}, ErrNoShape
+	}
+	switch method {
+	case Isotonic, Platt:
+	default:
+		return Map{}, fmt.Errorf("%w: %q", ErrMethod, method)
+	}
+
+	rows, pos, neg := 0, 0, 0
+	for _, s := range samples {
+		if math.IsNaN(s.Score) || math.IsInf(s.Score, 0) {
+			continue
+		}
+		rows++
+		if s.Productive {
+			pos++
+		} else {
+			neg++
+		}
+	}
+	if rows < MinRows {
+		return Map{}, fmt.Errorf("%w: %d judged, %d needed", ErrThin, rows, MinRows)
+	}
+	if pos < MinClass || neg < MinClass {
+		return Map{}, fmt.Errorf("%w: %d productive, %d unproductive, %d of each needed", ErrOneClass, pos, neg, MinClass)
+	}
+
+	clean := make([]Sample, 0, rows)
+	for _, s := range samples {
+		if math.IsNaN(s.Score) || math.IsInf(s.Score, 0) {
+			continue
+		}
+		clean = append(clean, s)
+	}
+	// Score ascending, so the fit does not depend on the caller's order. The
+	// tie-break by class is cosmetic and stays only because a stable, stated order
+	// is easier to read in a dump: pava pools every sample at one score into one
+	// block before it starts, so the order WITHIN a tie cannot reach the answer.
+	// Relying on that order instead of pooling is exactly the bug pooling fixes.
+	sort.SliceStable(clean, func(i, j int) bool {
+		if clean[i].Score != clean[j].Score {
+			return clean[i].Score < clean[j].Score
+		}
+		return !clean[i].Productive && clean[j].Productive
+	})
+
+	m := Map{
+		Method: method, Shape: shape,
+		Rows: rows, Positive: pos, Negative: neg,
+		Prevalence: float64(pos) / float64(rows),
+	}
+	var err error
+	switch method {
+	case Isotonic:
+		m.Knots = pava(clean)
+	case Platt:
+		m.A, m.B, err = platt(clean, pos, neg)
+		if err != nil {
+			return Map{}, err
+		}
+	}
+	// The in-sample Brier reads the map through the shape it was just fitted
+	// under, which is the one place that identity is a fact rather than an
+	// assumption — the shape was handed in and stamped on the same value two lines
+	// above. Everywhere else the shape comes from the live scorer.
+	r, err := m.Under(shape)
+	if err != nil {
+		return Map{}, err
+	}
+	m.Brier = brier(clean, r)
+	m.Digest = digest(m)
+	return m, nil
+}
+
+// pava is pool-adjacent-violators: the exact least-squares fit of a
+// non-decreasing function to the labels, in one pass over the sorted samples.
+//
+// TIES ARE POOLED BEFORE THE ALGORITHM STARTS, and that is not a refinement —
+// it is what makes the answer isotonic regression at all. A monotone function
+// has ONE value at one score, so every sample sharing a score is one block by
+// definition. Fed individual samples instead, a run of unproductive-then-
+// productive at a single score is already non-decreasing, so the merge condition
+// never fires, every sample stays its own block, and the score ends up mapped to
+// whichever label the sort happened to put last: 1 wherever any positive sits at
+// the top of a tie, 0 wherever none does.
+//
+// That is the ordinary case here, not an edge. A decision score is
+// 1 - prod(1-weight) over a FIXED set of rule weights, rounded to four places, so
+// the score alphabet is a handful of atoms and the modal atom is exactly zero.
+// Un-pooled, the whole map collapses to a step function pinned at 0 and 1 — every
+// plateau reported as certainty, at the top of the range where declines happen.
+//
+// Each block then holds a weighted mean; whenever a block's mean is not greater
+// than its predecessor's the two are merged, which is the only operation the
+// algorithm has. The result is the unique monotone least-squares solution — not
+// an approximation and not an iterative one — which is why this is 60 lines and
+// needs no numerical library.
+//
+// The knots are the block CENTROIDS (the weighted mean score inside the block)
+// rather than the block edges. An edge is where two blocks meet and belongs to
+// neither; the centroid is where the block's evidence actually sits, so
+// interpolating between centroids puts the map's steepest movement where the
+// data is densest.
+func pava(sorted []Sample) []Knot {
+	type block struct{ sum, weight, score float64 }
+	blocks := make([]block, 0, len(sorted))
+	push := func(b block) {
+		blocks = append(blocks, b)
+		for len(blocks) > 1 {
+			last := len(blocks) - 1
+			if blocks[last-1].sum/blocks[last-1].weight <= blocks[last].sum/blocks[last].weight {
+				break
+			}
+			blocks[last-1].sum += blocks[last].sum
+			blocks[last-1].weight += blocks[last].weight
+			blocks[last-1].score += blocks[last].score
+			blocks = blocks[:last]
+		}
+	}
+	// One block per DISTINCT score. weight() is never zero, so every block the
+	// loop pushes has mass and the means below are total.
+	for i := 0; i < len(sorted); {
+		var b block
+		j := i
+		for ; j < len(sorted) && sorted[j].Score == sorted[i].Score; j++ {
+			w := sorted[j].weight()
+			if sorted[j].Productive {
+				b.sum += w
+			}
+			b.weight += w
+			b.score += sorted[j].Score * w
+		}
+		push(b)
+		i = j
+	}
+
+	// A block's centroid lies inside its own score range, and the ranges are
+	// disjoint and ordered, so the centroids ascend strictly — until round6 puts
+	// two of them on one score. Those two are POOLED rather than one of them
+	// discarded: either endpoint alone states a probability the pooled evidence
+	// does not support, and the pooled mean sits between the neighbours it
+	// replaced, so the map stays monotone and interpolate stays total.
+	knots := make([]Knot, 0, len(blocks))
+	for i := 0; i < len(blocks); {
+		at := round6(blocks[i].score / blocks[i].weight)
+		var sum, weight float64
+		j := i
+		for ; j < len(blocks) && round6(blocks[j].score/blocks[j].weight) == at; j++ {
+			sum += blocks[j].sum
+			weight += blocks[j].weight
+		}
+		knots = append(knots, Knot{Score: at, P: round6(sum / weight)})
+		i = j
+	}
+	return knots
+}
+
+// platt fits p = 1/(1+exp(A*score+B)) by Newton's method on the regularised
+// log-likelihood.
+//
+// The targets are SMOOTHED — t+ = (N+ + 1)/(N+ + 2) and t- = 1/(N- + 2) — which
+// is Platt's own correction and is not decoration. On a fraud stream the classes
+// are wildly unbalanced, and against hard 0/1 targets the maximum-likelihood fit
+// drives the scarce class to a probability of exactly one, which is both false
+// and unusable in a cost calculation. The smoothed targets are the posterior
+// under a uniform prior over the observed counts, so the fit stays inside the
+// open unit interval whatever the imbalance.
+func platt(sorted []Sample, pos, neg int) (float64, float64, error) {
+	hi := (float64(pos) + 1) / (float64(pos) + 2)
+	lo := 1 / (float64(neg) + 2)
+
+	a, b := 0.0, math.Log((float64(neg)+1)/(float64(pos)+1))
+	const iterations = 128
+	const tolerance = 1e-10
+	for range iterations {
+		var g0, g1, h00, h01, h11 float64
+		for _, s := range sorted {
+			t := lo
+			if s.Productive {
+				t = hi
+			}
+			w := s.weight()
+			z := a*s.Score + b
+			// p is written in the numerically stable branch for each sign of z;
+			// exp of a large positive z overflows, and a probability that came
+			// back as NaN would poison the whole Hessian.
+			var p float64
+			if z >= 0 {
+				p = math.Exp(-z) / (1 + math.Exp(-z))
+			} else {
+				p = 1 / (1 + math.Exp(z))
+			}
+			// The negative log-likelihood differentiates to (t - p) with respect
+			// to z = a*score + b, because p is written as 1/(1+exp(z)) and so
+			// dp/dz is -p(1-p) rather than +p(1-p). Getting that sign wrong turns
+			// Newton's method into gradient ascent, which converges — to
+			// certainty, in the wrong direction, silently.
+			d := w * (t - p)
+			g0 += d * s.Score
+			g1 += d
+			v := w * p * (1 - p)
+			h00 += v * s.Score * s.Score
+			h01 += v * s.Score
+			h11 += v
+		}
+		// A small ridge keeps the Hessian invertible when every score is equal,
+		// which is exactly the degenerate input a warming model produces.
+		h00 += 1e-12
+		h11 += 1e-12
+		det := h00*h11 - h01*h01
+		if det == 0 || math.IsNaN(det) {
+			return 0, 0, ErrConverging
+		}
+		da := (h11*g0 - h01*g1) / det
+		db := (h00*g1 - h01*g0) / det
+		a -= da
+		b -= db
+		if math.Abs(da) < tolerance && math.Abs(db) < tolerance {
+			if math.IsNaN(a) || math.IsNaN(b) {
+				return 0, 0, ErrConverging
+			}
+			return round6(a), round6(b), nil
+		}
+	}
+	if math.IsNaN(a) || math.IsNaN(b) {
+		return 0, 0, ErrConverging
+	}
+	return round6(a), round6(b), nil
+}
+
+// brier is the mean squared error of a map against the labels it was fitted on.
+// Lower is better; 0.25 is what a constant at the base rate of a balanced set
+// scores, so a number above that on a balanced set means the map is worse than
+// saying nothing.
+func brier(samples []Sample, r Reader) float64 {
+	if len(samples) == 0 || !r.Fitted() {
+		return 0
+	}
+	var sum, w float64
+	for _, s := range samples {
+		p := r.P(s.Score)
+		y := 0.0
+		if s.Productive {
+			y = 1
+		}
+		sum += s.weight() * (p - y) * (p - y)
+		w += s.weight()
+	}
+	if w == 0 {
+		return 0
+	}
+	return round6(sum / w)
+}
+
+// Bin is one bucket of a reliability report: what the map predicted here, and
+// what actually happened.
+type Bin struct {
+	// From and To bound the predicted probability, half-open.
+	From float64 `json:"from"`
+	To   float64 `json:"to"`
+	// Rows is how many observations fell in the bin, and Predicted and Observed
+	// are the mean predicted probability and the realised productive share. A
+	// calibrated map has them equal; the gap between them is the whole report.
+	Rows      int      `json:"rows"`
+	Predicted *float64 `json:"predicted,omitempty"`
+	Observed  *float64 `json:"observed,omitempty"`
+}
+
+// Reliability is the calibration report a console renders as a diagonal: bins of
+// predicted probability against the share that turned out productive.
+//
+// Predicted and Observed are POINTERS. An empty bin has no mean and no realised
+// share, and reporting either as 0.0 would draw a point on the chart at perfect
+// confidence in innocence — which is the most misleading thing this whole
+// package could do.
+//
+// A reader that cannot answer gets NO CHART, not an empty one. A calibration
+// whose shape has moved refuses to state a probability; a reliability report is
+// that same map applied to every row, so shipping one beside the refusal would
+// draw the chart the prose just said does not exist.
+func Reliability(samples []Sample, r Reader, bins int) []Bin {
+	if !r.Fitted() {
+		return nil
+	}
+	if bins <= 0 {
+		bins = 10
+	}
+	out := make([]Bin, bins)
+	sumP := make([]float64, bins)
+	sumY := make([]float64, bins)
+	w := make([]float64, bins)
+	for i := range out {
+		out[i] = Bin{From: float64(i) / float64(bins), To: float64(i+1) / float64(bins)}
+	}
+	for _, s := range samples {
+		p := r.P(s.Score)
+		i := int(p * float64(bins))
+		if i >= bins {
+			i = bins - 1
+		}
+		if i < 0 {
+			i = 0
+		}
+		y := 0.0
+		if s.Productive {
+			y = 1
+		}
+		out[i].Rows++
+		sumP[i] += s.weight() * p
+		sumY[i] += s.weight() * y
+		w[i] += s.weight()
+	}
+	for i := range out {
+		if w[i] == 0 {
+			continue
+		}
+		p := round6(sumP[i] / w[i])
+		y := round6(sumY[i] / w[i])
+		out[i].Predicted, out[i].Observed = &p, &y
+	}
+	return out
+}
+
+// digest identifies a fit by its parameters, so two maps that answer identically
+// have one identity and a map that has moved has another. It covers the shape it
+// was fitted under, because the same knots under a different model are a
+// different map.
+func digest(m Map) string {
+	h := sha256.New()
+	fmt.Fprintf(h, "v1|%s|%s|%d|%d|", m.Method, m.Shape, m.Positive, m.Negative)
+	fmt.Fprintf(h, "%s|%s|", strconv.FormatFloat(m.A, 'g', 17, 64), strconv.FormatFloat(m.B, 'g', 17, 64))
+	for _, k := range m.Knots {
+		fmt.Fprintf(h, "%s:%s|", strconv.FormatFloat(k.Score, 'g', 17, 64), strconv.FormatFloat(k.P, 'g', 17, 64))
+	}
+	return hex.EncodeToString(h.Sum(nil))
+}
+
+// short renders a digest for a message. A whole SHA-256 in an error is noise a
+// reader skips; twelve hex characters is enough to tell two apart.
+func short(s string) string {
+	if len(s) <= 12 {
+		return s
+	}
+	return s[:12]
+}
+
+func clamp01(v float64) float64 {
+	switch {
+	case math.IsNaN(v):
+		return 0
+	case v < 0:
+		return 0
+	case v > 1:
+		return 1
+	}
+	return v
+}
+
+// round6 trims to six places. A probability is reported to a person and stored
+// as a record; seventeen digits of float noise makes two identical fits look
+// different and invites a caller to compare the fifteenth decimal.
+func round6(v float64) float64 {
+	if math.IsNaN(v) || math.IsInf(v, 0) {
+		return 0
+	}
+	return math.Round(v*1e6) / 1e6
+}

pkg/calibrate/calibrate_test.go

@@ -0,0 +1,425 @@
+package calibrate
+
+import (
+	"errors"
+	"math"
+	"math/rand"
+	"testing"
+)
+
+const shape = "sha256:model-shape-under-test"
+
+// stream builds a sample whose true probability of being productive is a known
+// function of the score, so the fit can be checked against the truth rather than
+// against itself.
+func stream(n int, seed int64, truth func(float64) float64) []Sample {
+	rng := rand.New(rand.NewSource(seed))
+	out := make([]Sample, 0, n)
+	for range n {
+		s := rng.Float64()
+		out = append(out, Sample{Score: s, Productive: rng.Float64() < truth(s)})
+	}
+	return out
+}
+
+// THE calibration test. A calibration exists to make a score MEAN a probability,
+// so the property to prove is that the fitted probability tracks the true one —
+// not that the function returns a number.
+func TestIsotonicRecoversTheTrueProbability(t *testing.T) {
+	truth := func(s float64) float64 { return s * s } // convex: a raw score badly overstates risk at the low end
+	m, err := Fit(stream(20000, 7, truth), Isotonic, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r, err := m.Under(shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var worst float64
+	for _, s := range []float64{0.05, 0.15, 0.3, 0.5, 0.7, 0.85, 0.95} {
+		if d := math.Abs(r.P(s) - truth(s)); d > worst {
+			worst = d
+		}
+	}
+	if worst > 0.05 {
+		t.Errorf("the fitted probability is off the truth by %.3f at worst; the map does not mean what it says", worst)
+	}
+	// The raw score is what a policy would have used without this package. It is
+	// wrong by a wide margin at the low end, which is the whole reason the
+	// calibration is not optional.
+	if d := math.Abs(0.3 - truth(0.3)); d < 0.15 {
+		t.Fatal("the test's own premise is broken: the raw score was already calibrated")
+	}
+}
+
+// A calibration must never reorder. If it could, ROC and PR would move under it
+// and every metric measured before a fit would be incomparable with one measured
+// after.
+func TestFitIsMonotone(t *testing.T) {
+	for _, method := range []string{Isotonic, Platt} {
+		m, err := Fit(stream(4000, 11, func(s float64) float64 { return 0.02 + 0.9*s*s*s }), method, shape)
+		if err != nil {
+			t.Fatalf("%s: %v", method, err)
+		}
+		r, err := m.Under(shape)
+		if err != nil {
+			t.Fatalf("%s: %v", method, err)
+		}
+		prev := -1.0
+		for i := range 1001 {
+			s := float64(i) / 1000
+			p := r.P(s)
+			if p < prev-1e-12 {
+				t.Fatalf("%s: not monotone — p(%g)=%g after %g", method, s, p, prev)
+			}
+			if p < 0 || p > 1 {
+				t.Fatalf("%s: p(%g)=%g is not a probability", method, s, p)
+			}
+			prev = p
+		}
+	}
+}
+
+// The fit is a record: an auditor re-running it must get the same map. Order of
+// the input must not matter, because the caller's order is an accident of how
+// rows came back from a store.
+func TestFitIsDeterministicUnderReordering(t *testing.T) {
+	base := stream(3000, 23, func(s float64) float64 { return s })
+	a, err := Fit(base, Isotonic, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	shuffled := append([]Sample(nil), base...)
+	rng := rand.New(rand.NewSource(99))
+	rng.Shuffle(len(shuffled), func(i, j int) { shuffled[i], shuffled[j] = shuffled[j], shuffled[i] })
+	b, err := Fit(shuffled, Isotonic, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if a.Digest != b.Digest {
+		t.Fatalf("the same multiset in a different order produced two maps: %s vs %s", a.Digest[:12], b.Digest[:12])
+	}
+	if len(a.Knots) != len(b.Knots) {
+		t.Fatalf("knot counts differ: %d vs %d", len(a.Knots), len(b.Knots))
+	}
+}
+
+// THE training-serving skew gate. A calibration fitted under one feature space
+// must refuse to answer under another. Silent misapplication is the failure mode
+// this whole design exists to prevent, and it is invisible without this check —
+// every number still looks like a probability.
+func TestMapRefusesAnotherModelShape(t *testing.T) {
+	m, err := Fit(stream(1000, 3, func(s float64) float64 { return s }), Isotonic, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := m.Under(shape); err != nil {
+		t.Fatalf("refused its own shape: %v", err)
+	}
+	if _, err := m.Under("sha256:a-tenth-feature-was-added"); !errors.Is(err, ErrShape) {
+		t.Fatalf("bound under a different shape (err=%v); a moved feature space would go unnoticed", err)
+	}
+	if _, err := m.Under(""); !errors.Is(err, ErrNoShape) {
+		t.Fatalf("bound with no shape stated (err=%v)", err)
+	}
+	if _, err := (Map{}).Under(shape); !errors.Is(err, ErrNotFitted) {
+		t.Fatalf("bound an unfitted map (err=%v)", err)
+	}
+	if _, err := Fit(stream(1000, 3, func(s float64) float64 { return s }), Isotonic, ""); !errors.Is(err, ErrNoShape) {
+		t.Fatal("fitted a map that cannot refuse to be misapplied")
+	}
+}
+
+// A fit on one class is a constant. Reporting a constant as a probability is the
+// single most damaging thing this package could do, so it is refused by name.
+func TestFitRefusesTheDegenerateSample(t *testing.T) {
+	var oneClass []Sample
+	for i := range 200 {
+		oneClass = append(oneClass, Sample{Score: float64(i) / 200})
+	}
+	if _, err := Fit(oneClass, Isotonic, shape); !errors.Is(err, ErrOneClass) {
+		t.Fatalf("fitted on one class: %v", err)
+	}
+
+	thin := []Sample{{Score: 0.1}, {Score: 0.9, Productive: true}}
+	if _, err := Fit(thin, Isotonic, shape); !errors.Is(err, ErrThin) {
+		t.Fatalf("fitted on two rows: %v", err)
+	}
+
+	// Enough rows, not enough of the scarce class.
+	var scarce []Sample
+	for i := range 100 {
+		scarce = append(scarce, Sample{Score: float64(i) / 100, Productive: i > 97})
+	}
+	if _, err := Fit(scarce, Isotonic, shape); !errors.Is(err, ErrOneClass) {
+		t.Fatalf("fitted on %d positives: %v", 2, err)
+	}
+}
+
+// Platt's label smoothing is not decoration. Without it the maximum-likelihood
+// fit on an imbalanced sample runs to a probability of exactly one, which is
+// false and unusable in a cost calculation.
+func TestPlattStaysInsideTheOpenInterval(t *testing.T) {
+	// 2% base rate with near-perfect separation — the shape that makes an
+	// unsmoothed fit diverge.
+	var s []Sample
+	for i := range 2000 {
+		score := float64(i) / 2000
+		s = append(s, Sample{Score: score, Productive: score > 0.98})
+	}
+	m, err := Fit(s, Platt, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r, err := m.Under(shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	hi, lo := r.P(1), r.P(0)
+	if hi >= 1 || lo <= 0 {
+		t.Fatalf("the fit reached certainty: p(0)=%g p(1)=%g", lo, hi)
+	}
+	if hi <= lo {
+		t.Fatalf("the fit is not increasing: p(0)=%g p(1)=%g", lo, hi)
+	}
+}
+
+// Reliability is what a console draws as the diagonal. An empty bin must report
+// absence, not a point at zero — a point at zero on that chart claims perfect
+// confidence in innocence.
+func TestReliabilityReportsAbsenceNotZero(t *testing.T) {
+	m, err := Fit(stream(3000, 5, func(s float64) float64 { return 0.5 * s }), Isotonic, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r, err := m.Under(shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	bins := Reliability(stream(3000, 6, func(s float64) float64 { return 0.5 * s }), r, 10)
+	if len(bins) != 10 {
+		t.Fatalf("%d bins, want 10", len(bins))
+	}
+	var empty, filled int
+	for _, b := range bins {
+		if b.Rows == 0 {
+			empty++
+			if b.Predicted != nil || b.Observed != nil {
+				t.Errorf("empty bin [%g,%g) reports numbers", b.From, b.To)
+			}
+			continue
+		}
+		filled++
+		if b.Predicted == nil || b.Observed == nil {
+			t.Errorf("bin [%g,%g) has %d rows and no numbers", b.From, b.To, b.Rows)
+		}
+	}
+	if filled == 0 {
+		t.Fatal("no bin was filled; the test proves nothing")
+	}
+	// The truth caps at 0.5, so the upper bins must be empty — which is exactly
+	// the case the absence rule exists for.
+	if empty == 0 {
+		t.Fatal("no bin was empty; the absence rule is untested")
+	}
+}
+
+// A calibration should reduce the Brier score against the raw score, which is
+// the whole claim being made when one is fitted.
+func TestCalibrationImprovesBrier(t *testing.T) {
+	truth := func(s float64) float64 { return s * s * s }
+	train := stream(20000, 31, truth)
+	test := stream(20000, 32, truth)
+
+	m, err := Fit(train, Isotonic, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r, err := m.Under(shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var fitted, raw float64
+	for _, s := range test {
+		y := 0.0
+		if s.Productive {
+			y = 1
+		}
+		p := r.P(s.Score)
+		fitted += (p - y) * (p - y)
+		raw += (s.Score - y) * (s.Score - y)
+	}
+	fitted /= float64(len(test))
+	raw /= float64(len(test))
+	if fitted >= raw {
+		t.Fatalf("calibrated Brier %.4f is no better than the raw score's %.4f", fitted, raw)
+	}
+}
+
+// THE TIE TEST, and the one that matters most in production. The scores this
+// package is asked to calibrate are not a continuum: a decision score is
+// 1 - prod(1-weight) over a FIXED set of rule weights, rounded to four places,
+// so thousands of decisions land on a handful of atoms and the modal atom is
+// exactly zero.
+//
+// Isotonic regression assigns ONE value per score, so every sample sharing a
+// score is one block before the algorithm starts. Fed sample by sample instead,
+// the run of unproductive-then-productive at a tied score is already
+// non-decreasing, nothing merges, and each plateau ends up reported as whichever
+// label sorted last — 1 at every plateau that holds a positive. The map then
+// says CERTAINTY at the top of the range, which is where declines happen.
+func TestIsotonicPoolsTiedScores(t *testing.T) {
+	atoms := []struct {
+		score        float64
+		clean, fraud int
+	}{
+		{0.00, 900, 2},  // 0.002217
+		{0.35, 190, 10}, // 0.05
+		{0.50, 85, 15},  // 0.15
+		{0.70, 45, 55},  // 0.55
+		{0.90, 10, 40},  // 0.80
+	}
+	var s []Sample
+	for _, a := range atoms {
+		for range a.clean {
+			s = append(s, Sample{Score: a.score})
+		}
+		for range a.fraud {
+			s = append(s, Sample{Score: a.score, Productive: true})
+		}
+	}
+	m, err := Fit(s, Isotonic, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r, err := m.Under(shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// The observed rates are already non-decreasing, so pooling alone is the
+	// whole fit and the map must reproduce them exactly. Nothing here is a
+	// tolerance on statistical noise: this is arithmetic.
+	for _, a := range atoms {
+		want := round6(float64(a.fraud) / float64(a.clean+a.fraud))
+		if got := r.P(a.score); math.Abs(got-want) > 1e-6 {
+			t.Errorf("score %.2f -> %.6f, want %.6f (%d productive of %d)",
+				a.score, got, want, a.fraud, a.clean+a.fraud)
+		}
+	}
+	// One knot per distinct score, no more: a plateau is one place on the map.
+	if len(m.Knots) != len(atoms) {
+		t.Errorf("%d knots over %d distinct scores; ties were not pooled into blocks",
+			len(m.Knots), len(atoms))
+	}
+}
+
+// A tie whose samples arrive in the other order is the same evidence and must
+// produce the same map. Pooling makes that true by construction; ordering the
+// sort by class makes it look true without being true.
+func TestTiedFitDoesNotDependOnLabelOrder(t *testing.T) {
+	build := func(positiveFirst bool) []Sample {
+		var s []Sample
+		for _, sc := range []float64{0.1, 0.4, 0.8} {
+			pos, neg := 20, 30
+			if sc == 0.8 {
+				pos, neg = 40, 10
+			}
+			if positiveFirst {
+				for range pos {
+					s = append(s, Sample{Score: sc, Productive: true})
+				}
+				for range neg {
+					s = append(s, Sample{Score: sc})
+				}
+				continue
+			}
+			for range neg {
+				s = append(s, Sample{Score: sc})
+			}
+			for range pos {
+				s = append(s, Sample{Score: sc, Productive: true})
+			}
+		}
+		return s
+	}
+	a, err := Fit(build(false), Isotonic, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, err := Fit(build(true), Isotonic, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if a.Digest != b.Digest {
+		t.Fatalf("the same tied evidence in two orders produced two maps: %.12s vs %.12s", a.Digest, b.Digest)
+	}
+	ra, _ := a.Under(shape)
+	if got := ra.P(0.1); math.Abs(got-0.4) > 1e-6 {
+		t.Errorf("p(0.1) = %.6f, want 0.4 (20 productive of 50)", got)
+	}
+}
+
+// A reader that cannot answer draws no chart. The reliability report IS the map
+// applied to every row, so shipping one beside a refusal states in a picture
+// exactly what the prose has just refused to state.
+func TestReliabilityRefusesWhenTheMapDoes(t *testing.T) {
+	m, err := Fit(stream(2000, 13, func(s float64) float64 { return s }), Isotonic, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rows := stream(2000, 14, func(s float64) float64 { return s })
+	if _, err := m.Under("sha256:a-rule-was-written"); !errors.Is(err, ErrShape) {
+		t.Fatalf("bound under a moved shape: %v", err)
+	}
+	if bins := Reliability(rows, Reader{}, 10); len(bins) != 0 {
+		t.Errorf("a map that refuses to answer shipped %d reliability bins", len(bins))
+	}
+	good, err := m.Under(shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bins := Reliability(rows, good, 10); len(bins) != 10 {
+		t.Fatalf("the bound reader drew %d bins, want 10", len(bins))
+	}
+}
+
+// TestACentroidCollisionPoolsRatherThanPicks pins the second half of the tie
+// defect, and it is a different failure from the first.
+//
+// Two blocks whose centroids are distinct floats but round to one place on the
+// map cannot both be a knot — the map has one value at one score. Keeping either
+// one states a probability the pooled evidence does not support: here the lower
+// block is 2-in-10 and the upper is 8-in-10, so a map that keeps the max says
+// 0.8 where the evidence says 0.5, at a score that is one place on the map and
+// carries both blocks' rows. Pooling is the only answer that is a mean of what
+// was actually observed there.
+//
+// The collision is engineered rather than waited for, because that is what makes
+// this a test: round6 collapses anything inside 5e-7, and a decision score is
+// round4'd before it ever gets here.
+func TestACentroidCollisionPoolsRatherThanPicks(t *testing.T) {
+	var s []Sample
+	// Two distinct scores 1e-9 apart: two blocks, one centroid after round6.
+	for i := range 10 {
+		s = append(s, Sample{Score: 0.5, Productive: i < 2}) // 0.2
+	}
+	for i := range 10 {
+		s = append(s, Sample{Score: 0.5 + 1e-9, Productive: i < 8}) // 0.8
+	}
+	m, err := Fit(s, Isotonic, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(m.Knots) != 1 {
+		t.Fatalf("%d knots at one rounded score %v — a map has one value at one score", len(m.Knots), m.Knots)
+	}
+	r, err := m.Under(shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// 10 of 20 productive across the two pooled blocks.
+	if got := r.P(0.5); math.Abs(got-0.5) > 1e-6 {
+		t.Errorf("P(0.5) = %.6f, want 0.500000 — the colliding blocks were picked between rather than pooled "+
+			"(0.8 is the upper block alone, 0.2 the lower)", got)
+	}
+}

pkg/evaluate/evaluate.go

@@ -0,0 +1,574 @@
+// Package evaluate measures a decision rule against what actually happened, and
+// replays a candidate one over recorded history to see what it would have done.
+//
+// WHY NOT ACCURACY. Fraud is rare. A stream at one productive event in a
+// thousand is 99.9% accurate if the model says "fine" to everything, and that
+// model is worth nothing. Accuracy on an imbalanced stream is not a weak metric,
+// it is an actively misleading one, so this package does not compute it. What it
+// computes instead:
+//
+//	ROC-AUC     the probability that a productive event outranks an unproductive
+//	            one. Insensitive to the base rate, which makes it comparable
+//	            across tenants — and, for the same reason, optimistic-looking on
+//	            a rare-event stream, because the enormous negative class makes
+//	            false positives cheap in its arithmetic.
+//	PR-AUC      average precision: the area under precision-recall, computed as
+//	            the sum of precision at each recall step rather than by
+//	            trapezoid, because interpolating between operating points on a PR
+//	            curve credits a model with points it cannot actually reach. This
+//	            is the number that moves when a rare-event model gets better, and
+//	            it is the one to argue about.
+//	Prevalence  reported beside both, always. An AUC without the base rate is
+//	            uninterpretable, and a PR-AUC is bounded below by the prevalence,
+//	            so 0.1 is excellent at 1-in-1000 and worthless at 1-in-5.
+//	Cost        the only metric with a unit anybody outside the team cares
+//	            about. A miss and a false alarm have different prices; the
+//	            cost-minimising threshold is where the marginal one of each
+//	            balances, and it is almost never where a round number sits.
+//
+// EVERY RATE IS A POINTER. An unmeasured proportion reported as 0.0 reads as a
+// perfect model, and on a plane where most events are unjudged that is the
+// default state. Absent means absent.
+//
+// THE COST ARITHMETIC IS EXACT. Prices are int64 nano-units multiplied by
+// counts. No float touches money here: a cost curve compared across thresholds
+// in floating point can order two thresholds by rounding noise, and the answer
+// is which one to put in production.
+package evaluate
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"math"
+	"sort"
+	"strconv"
+	"time"
+
+	"github.com/luxfi/aml/pkg/calibrate"
+	"github.com/luxfi/aml/pkg/policy"
+	"github.com/luxfi/aml/pkg/replay"
+)
+
+// Observation is one recorded decision: what the model said, what the system
+// did, and what turned out to be true.
+//
+// Disposition is replay.Disposition and not a second vocabulary. The engine's
+// exhaustive search already refuses to name a winner without judged events, and
+// a parallel set of label words would leave it permanently unable to read the
+// labels this plane collects.
+type Observation struct {
+	// ID identifies the decision, so a report can point at the rows behind a
+	// number instead of only stating it.
+	ID string
+	// At is when it was decided. The order of a slice of these is the order they
+	// happened, and every temporal split in this package relies on that.
+	At time.Time
+	// Score is the raw model score in [0,1] AS RECORDED AT THE TIME. It is not
+	// recomputed: the rings have moved on, and re-scoring history through a model
+	// that has since learned from that same history is how a backtest comes to
+	// know the future.
+	Score float64
+	// Action is what the live system actually did.
+	Action string
+	// Disposition is what a human or a processor concluded, when anyone has.
+	Disposition replay.Disposition
+	// Weight down-weights a low-confidence label. Zero means one.
+	Weight float64
+}
+
+func (o Observation) weight() float64 {
+	if o.Weight <= 0 {
+		return 1
+	}
+	return o.Weight
+}
+
+func (o Observation) judged() bool {
+	return o.Disposition == replay.Productive || o.Disposition == replay.Unproductive
+}
+
+func (o Observation) productive() bool { return o.Disposition == replay.Productive }
+
+// Confusion is the four counts at one operating point, weighted.
+type Confusion struct {
+	// TP is productive and flagged; FN is productive and let through — the
+	// misses, which is what Cost.Miss prices.
+	TP float64 `json:"tp"`
+	FN float64 `json:"fn"`
+	// FP is unproductive and flagged — the false alarms, which Cost.Alarm
+	// prices; TN is unproductive and let through.
+	FP float64 `json:"fp"`
+	TN float64 `json:"tn"`
+}
+
+// Metrics is the report. Every proportion is absent rather than zero when
+// nothing supports it.
+type Metrics struct {
+	// Rows is every observation looked at; Judged is how many carry a
+	// disposition. The gap between them is the honest limit on everything below,
+	// and it is reported first for that reason.
+	Rows   int `json:"rows"`
+	Judged int `json:"judged"`
+	// Productive and Unproductive are the two judged classes, so the imbalance is
+	// visible before anyone reads a rate computed over it.
+	Productive   int `json:"productive"`
+	Unproductive int `json:"unproductive"`
+	// Prevalence is the productive share of the judged set.
+	Prevalence *float64 `json:"prevalence,omitempty"`
+	// ROC is the area under the receiver-operating curve, with midranks for
+	// ties. Absent without both classes.
+	ROC *float64 `json:"roc,omitempty"`
+	// PR is average precision — the area under precision-recall by the step
+	// definition. This is the imbalanced-data number.
+	PR *float64 `json:"pr,omitempty"`
+	// Threshold is the operating point everything below is measured at.
+	Threshold *float64 `json:"threshold,omitempty"`
+	// Precision, Recall and F1 at that point.
+	Precision *float64 `json:"precision,omitempty"`
+	Recall    *float64 `json:"recall,omitempty"`
+	F1        *float64 `json:"f1,omitempty"`
+	// Alarm is the share of the judged stream flagged at that point — the
+	// realised review rate, which is the number an operations team is actually
+	// resourced against.
+	Alarm *float64 `json:"alarm,omitempty"`
+	// Lift is precision divided by prevalence: how much better than chance the
+	// flagged set is. One means the model is choosing at random.
+	Lift *float64 `json:"lift,omitempty"`
+	// Confusion is the four counts at the operating point.
+	Confusion *Confusion `json:"confusion,omitempty"`
+	// CostNano is what being wrong cost at that point, in nano-units, and
+	// BestNano is the lowest it reaches anywhere, at BestThreshold. Absent when
+	// the tenant has not stated a price, because a cost computed from unstated
+	// prices is a number with no meaning and a decimal point on it.
+	CostNano      *int64   `json:"cost_nano,omitempty"`
+	BestNano      *int64   `json:"best_nano,omitempty"`
+	BestThreshold *float64 `json:"best_threshold,omitempty"`
+	// Brier is the calibration error of the supplied map on this data. A
+	// well-ranked, badly-calibrated model makes every rung of the policy mean
+	// something other than what it says, and ROC and PR cannot see that at all —
+	// both are rank statistics, and a monotone calibration cannot change either.
+	Brier *float64 `json:"brier,omitempty"`
+	// Refusal names why a report is thin when it is. Silence and a clean result
+	// are the same bytes otherwise.
+	Refusal string `json:"refusal,omitempty"`
+}
+
+// Point is one operating point of a curve.
+type Point struct {
+	// Threshold is the score at or above which an observation is flagged.
+	Threshold float64 `json:"threshold"`
+	// TruePositive and FalsePositive are the ROC axes (recall and fall-out).
+	TruePositive  float64 `json:"tpr"`
+	FalsePositive float64 `json:"fpr"`
+	// Precision is the PR axis against Recall, which equals TruePositive.
+	Precision float64 `json:"precision"`
+	// CostNano is what this point costs under the stated prices. Zero when no
+	// price is stated; Metrics is where absence is expressed, and a curve is a
+	// dense array a console plots.
+	CostNano int64 `json:"cost_nano"`
+}
+
+// Measure computes the report at one operating point.
+//
+// threshold is on the SCORE, not the probability, because the score is what was
+// recorded and a calibration can be refitted afterwards. A caller holding a
+// policy over probabilities converts once, at the boundary, rather than making
+// every metric here depend on a map that may not exist.
+//
+// cal may be the zero Reader — no calibration, or one whose shape has moved.
+// Brier is then absent and everything else is unaffected, since every other
+// metric here is a rank statistic. It is a READER and not a Map because the
+// question "does this map apply to these coordinates" is answered once, at the
+// boundary that knows the live shape, and never re-asked here.
+func Measure(obs []Observation, threshold float64, cost policy.Cost, cal calibrate.Reader) Metrics {
+	m := Metrics{Rows: len(obs)}
+	judged := make([]Observation, 0, len(obs))
+	for _, o := range obs {
+		if !o.judged() || math.IsNaN(o.Score) || math.IsInf(o.Score, 0) {
+			continue
+		}
+		judged = append(judged, o)
+		if o.productive() {
+			m.Productive++
+		} else {
+			m.Unproductive++
+		}
+	}
+	m.Judged = len(judged)
+	if m.Judged == 0 {
+		m.Refusal = "nothing in this window has been judged, so no rate can be measured"
+		return m
+	}
+	prev := float64(m.Productive) / float64(m.Judged)
+	m.Prevalence = ptr(round6(prev))
+
+	if m.Productive == 0 || m.Unproductive == 0 {
+		m.Refusal = "the judged observations are all one class, so separation cannot be measured"
+	} else {
+		m.ROC = ptr(round6(roc(judged)))
+		m.PR = ptr(round6(averagePrecision(judged)))
+	}
+
+	c := confusionAt(judged, threshold)
+	m.Threshold = ptr(round6(threshold))
+	m.Confusion = &c
+	if flagged := c.TP + c.FP; flagged > 0 {
+		p := c.TP / flagged
+		m.Precision = ptr(round6(p))
+		if prev > 0 {
+			m.Lift = ptr(round6(p / prev))
+		}
+	}
+	if actual := c.TP + c.FN; actual > 0 {
+		m.Recall = ptr(round6(c.TP / actual))
+	}
+	if total := c.TP + c.FP + c.TN + c.FN; total > 0 {
+		m.Alarm = ptr(round6((c.TP + c.FP) / total))
+	}
+	if m.Precision != nil && m.Recall != nil && (*m.Precision+*m.Recall) > 0 {
+		m.F1 = ptr(round6(2 * *m.Precision * *m.Recall / (*m.Precision + *m.Recall)))
+	}
+	if cost.Stated() {
+		v, ok := nano(c, cost)
+		best, at, okBest := minimum(judged, cost)
+		if ok && okBest {
+			m.CostNano = ptrInt(v)
+			m.BestNano, m.BestThreshold = ptrInt(best), ptr(round6(at))
+		} else if m.Refusal == "" {
+			m.Refusal = "the stated prices cannot be multiplied by this many observations without overflowing, so no cost is reported"
+		}
+	}
+	if b, ok := brier(judged, cal); ok {
+		m.Brier = ptr(round6(b))
+	}
+	return m
+}
+
+// Curve is the whole sweep, one point per distinct score, descending. It is what
+// a console renders as ROC, as precision-recall and as a cost curve — three
+// charts from one pass, because they are three projections of the same sweep.
+//
+// One point per DISTINCT score, not per observation: two observations at the
+// same score cannot be separated by any threshold, so a curve with a point
+// between them describes an operating point that does not exist.
+func Curve(obs []Observation, cost policy.Cost) []Point {
+	judged := make([]Observation, 0, len(obs))
+	var pos, neg float64
+	for _, o := range obs {
+		if !o.judged() || math.IsNaN(o.Score) || math.IsInf(o.Score, 0) {
+			continue
+		}
+		judged = append(judged, o)
+		if o.productive() {
+			pos += o.weight()
+		} else {
+			neg += o.weight()
+		}
+	}
+	if len(judged) == 0 || pos == 0 || neg == 0 {
+		return nil
+	}
+	sort.SliceStable(judged, func(i, j int) bool { return judged[i].Score > judged[j].Score })
+	priced := cost.Stated()
+
+	out := make([]Point, 0, len(judged))
+	var tp, fp float64
+	for i := 0; i < len(judged); {
+		j := i
+		for j < len(judged) && judged[j].Score == judged[i].Score {
+			if judged[j].productive() {
+				tp += judged[j].weight()
+			} else {
+				fp += judged[j].weight()
+			}
+			j++
+		}
+		p := Point{
+			Threshold:     round6(judged[i].Score),
+			TruePositive:  round6(tp / pos),
+			FalsePositive: round6(fp / neg),
+			Precision:     round6(tp / (tp + fp)),
+		}
+		if priced {
+			v, ok := nano(Confusion{TP: tp, FP: fp, FN: pos - tp, TN: neg - fp}, cost)
+			if !ok {
+				// One unrepresentable point makes the whole cost dimension
+				// unreadable: a curve with a hole in it plots as a cliff. Drop it
+				// for the sweep and let Metrics carry the refusal.
+				priced = false
+				for k := range out {
+					out[k].CostNano = 0
+				}
+			}
+			p.CostNano = v
+		}
+		out = append(out, p)
+		i = j
+	}
+	return sample(out)
+}
+
+// sample bounds what a curve RETURNS without moving where its points sit.
+//
+// The sweep above is over every distinct score and stays that way: Metrics and
+// the cost minimum are exact whatever this does. A curve is a CHART, and the
+// argument that bounds Moved applies unchanged — a person reading one does not
+// need the ten-thousandth point, and a response carrying fifty thousand of them
+// is a megabyte nobody renders.
+//
+// It takes an evenly spaced subsequence and always keeps both ends, so the
+// shape, the extremes and the monotonicity survive. Truncating instead would cut
+// the curve off at one end and describe a model that stops.
+func sample(points []Point) []Point {
+	if len(points) <= Plotted {
+		return points
+	}
+	out := make([]Point, 0, Plotted)
+	last := len(points) - 1
+	for i := range Plotted - 1 {
+		out = append(out, points[i*last/(Plotted-1)])
+	}
+	return append(out, points[last])
+}
+
+// confusionAt counts the four cells at one threshold. Flagged is score >= t, so
+// a threshold of zero flags everything and a threshold above one flags nothing —
+// both ends reachable, which is what makes the sweep total.
+func confusionAt(judged []Observation, t float64) Confusion {
+	var c Confusion
+	for _, o := range judged {
+		w := o.weight()
+		switch {
+		case o.Score >= t && o.productive():
+			c.TP += w
+		case o.Score >= t:
+			c.FP += w
+		case o.productive():
+			c.FN += w
+		default:
+			c.TN += w
+		}
+	}
+	return c
+}
+
+// nano prices a confusion, and says whether the price is representable.
+//
+// Counts are weighted and therefore float; the MULTIPLICATION is done once, per
+// cell, and rounded to the nearest nano before summing, so the result is an
+// exact integer number of nano-units and two calls on the same counts cannot
+// differ.
+//
+// It returns false rather than a wrapped number. policy.MaxPrice already keeps a
+// sealed ladder's prices inside the range where this cannot happen, but a
+// wrapped cost is the one arithmetic failure here that does not look like a
+// failure — it looks like a recommendation — so the arithmetic refuses on its
+// own account and does not rely on having been called correctly. A policy
+// recorded before that bound existed reads back through this same path.
+func nano(c Confusion, cost policy.Cost) (int64, bool) {
+	miss, ok := mul(int64(math.Round(c.FN)), cost.Miss)
+	if !ok {
+		return 0, false
+	}
+	alarm, ok := mul(int64(math.Round(c.FP)), cost.Alarm)
+	if !ok {
+		return 0, false
+	}
+	sum := miss + alarm
+	if sum < miss {
+		return 0, false
+	}
+	return sum, true
+}
+
+// mul multiplies a count by a price, reporting whether the product fits. Both
+// are non-negative here, so the check is the single division that inverts the
+// multiplication exactly.
+func mul(count, price int64) (int64, bool) {
+	if count <= 0 || price <= 0 {
+		return 0, true
+	}
+	if count > math.MaxInt64/price {
+		return 0, false
+	}
+	return count * price, true
+}
+
+// Plotted bounds how many points a curve returns. A thousand is denser than any
+// screen, and the metrics behind it are computed over every distinct score
+// regardless — this bounds the rendering, never the arithmetic.
+const Plotted = 1000
+
+// minimum sweeps every reachable threshold and returns the lowest cost and where
+// it is reached.
+//
+// Ties go to the HIGHER threshold — the one that flags less. Two thresholds that
+// cost the same are not equivalent to the business: the one that stops fewer
+// customers is the one to choose, and leaving the tie to iteration order would
+// make the recommendation move between runs on identical data.
+func minimum(judged []Observation, cost policy.Cost) (int64, float64, bool) {
+	sorted := append([]Observation(nil), judged...)
+	sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Score > sorted[j].Score })
+
+	var pos, neg float64
+	for _, o := range sorted {
+		if o.productive() {
+			pos += o.weight()
+		} else {
+			neg += o.weight()
+		}
+	}
+	// The empty flagged set is a real operating point — flag nothing, pay for
+	// every miss — and it is the right answer when alarms cost more than misses.
+	best, ok := nano(Confusion{FN: pos, TN: neg}, cost)
+	if !ok {
+		return 0, 0, false
+	}
+	at := math.Nextafter(1, 2)
+	var tp, fp float64
+	for i := 0; i < len(sorted); {
+		j := i
+		for j < len(sorted) && sorted[j].Score == sorted[i].Score {
+			if sorted[j].productive() {
+				tp += sorted[j].weight()
+			} else {
+				fp += sorted[j].weight()
+			}
+			j++
+		}
+		v, ok := nano(Confusion{TP: tp, FP: fp, FN: pos - tp, TN: neg - fp}, cost)
+		if !ok {
+			return 0, 0, false
+		}
+		if v < best {
+			best, at = v, sorted[i].Score
+		}
+		i = j
+	}
+	return best, at, true
+}
+
+// roc is the area under the receiver-operating curve, by the Mann-Whitney
+// identity: the probability that a random productive observation outranks a
+// random unproductive one.
+//
+// Computed from midranks, so ties count as half a win rather than a whole one.
+// Without that a model that gives every observation the same score — which is
+// exactly what a warming detector does — scores 1.0 instead of 0.5.
+func roc(judged []Observation) float64 {
+	sorted := append([]Observation(nil), judged...)
+	sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Score < sorted[j].Score })
+
+	var sumRanks, pos, neg float64
+	for i := 0; i < len(sorted); {
+		j := i
+		for j < len(sorted) && sorted[j].Score == sorted[i].Score {
+			j++
+		}
+		// Ranks are one-based; the midrank of the block [i,j) is the mean of the
+		// ranks it spans.
+		mid := (float64(i+1) + float64(j)) / 2
+		for k := i; k < j; k++ {
+			w := sorted[k].weight()
+			if sorted[k].productive() {
+				sumRanks += mid * w
+				pos += w
+			} else {
+				neg += w
+			}
+		}
+		i = j
+	}
+	if pos == 0 || neg == 0 {
+		return 0
+	}
+	return (sumRanks - pos*(pos+1)/2) / (pos * neg)
+}
+
+// averagePrecision is the area under precision-recall by the STEP definition:
+// the sum over operating points of (recall gained) x (precision there).
+//
+// Not the trapezoid. Interpolating between two PR operating points draws a line
+// through points the classifier cannot actually reach, and it flatters exactly
+// the rare-event case this metric exists for. Ties are handled as one block, so
+// two observations at one score contribute one step and the answer does not
+// depend on which of them the sort happened to put first.
+func averagePrecision(judged []Observation) float64 {
+	sorted := append([]Observation(nil), judged...)
+	sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Score > sorted[j].Score })
+
+	var pos float64
+	for _, o := range sorted {
+		if o.productive() {
+			pos += o.weight()
+		}
+	}
+	if pos == 0 {
+		return 0
+	}
+	var tp, seen, ap, prevRecall float64
+	for i := 0; i < len(sorted); {
+		j := i
+		for j < len(sorted) && sorted[j].Score == sorted[i].Score {
+			w := sorted[j].weight()
+			seen += w
+			if sorted[j].productive() {
+				tp += w
+			}
+			j++
+		}
+		recall := tp / pos
+		precision := tp / seen
+		ap += (recall - prevRecall) * precision
+		prevRecall = recall
+		i = j
+	}
+	return ap
+}
+
+// brier is the mean squared error of the calibration on these observations. It
+// is the out-of-sample number when the caller passes data the map was not fitted
+// on, which is the only version of it worth reporting.
+func brier(judged []Observation, cal calibrate.Reader) (float64, bool) {
+	if !cal.Fitted() {
+		return 0, false
+	}
+	var sum, w float64
+	for _, o := range judged {
+		p := cal.P(o.Score)
+		y := 0.0
+		if o.productive() {
+			y = 1
+		}
+		sum += o.weight() * (p - y) * (p - y)
+		w += o.weight()
+	}
+	if w == 0 {
+		return 0, false
+	}
+	return sum / w, true
+}
+
+func ptr(v float64) *float64 { return &v }
+func ptrInt(v int64) *int64  { return &v }
+
+func round6(v float64) float64 {
+	if math.IsNaN(v) || math.IsInf(v, 0) {
+		return 0
+	}
+	return math.Round(v*1e6) / 1e6
+}
+
+// canonical renders a float for a digest: fixed precision, no exponent, so two
+// runs that agree to six places agree byte for byte.
+func canonical(v float64) string { return strconv.FormatFloat(round6(v), 'f', 6, 64) }
+
+// fingerprint folds a sequence of strings into one identity.
+func fingerprint(parts ...string) string {
+	h := sha256.New()
+	for _, p := range parts {
+		fmt.Fprintf(h, "%d:%s|", len(p), p)
+	}
+	return hex.EncodeToString(h.Sum(nil))
+}

pkg/evaluate/evaluate_test.go

@@ -0,0 +1,290 @@
+package evaluate
+
+import (
+	"math"
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/luxfi/aml/pkg/calibrate"
+	"github.com/luxfi/aml/pkg/policy"
+	"github.com/luxfi/aml/pkg/replay"
+)
+
+const shape = "sha256:model-shape-under-test"
+
+func obs(score float64, productive bool) Observation {
+	d := replay.Unproductive
+	if productive {
+		d = replay.Productive
+	}
+	return Observation{Score: score, Disposition: d}
+}
+
+// A perfect ranker scores 1.0 and a coin scores 0.5. Both ends pinned, because
+// an AUC implementation that is wrong is wrong in a way that still looks like a
+// number between zero and one.
+func TestROCAtBothEnds(t *testing.T) {
+	perfect := []Observation{obs(0.1, false), obs(0.2, false), obs(0.8, true), obs(0.9, true)}
+	if got := roc(perfect); got != 1 {
+		t.Errorf("a perfect ranker scored %g", got)
+	}
+	inverted := []Observation{obs(0.1, true), obs(0.2, true), obs(0.8, false), obs(0.9, false)}
+	if got := roc(inverted); got != 0 {
+		t.Errorf("an exactly wrong ranker scored %g", got)
+	}
+	// Every score identical — which is exactly what a warming detector produces.
+	// Midranks make this 0.5; counting ties as wins would make it 1.0 and report
+	// a model that says nothing as flawless.
+	flat := []Observation{obs(0.5, true), obs(0.5, false), obs(0.5, true), obs(0.5, false)}
+	if got := roc(flat); got != 0.5 {
+		t.Errorf("a model with no opinion scored %g, want 0.5 — ties are not wins", got)
+	}
+}
+
+// Average precision must use the step definition. The trapezoid credits a
+// classifier with operating points between two it actually has, which flatters
+// exactly the rare-event case the metric exists for.
+func TestAveragePrecisionIsTheStepArea(t *testing.T) {
+	// Ranked: + - + -. Precision at the two recall steps is 1/1 and 2/3.
+	// Step AP = 0.5*1 + 0.5*(2/3) = 0.8333...
+	got := averagePrecision([]Observation{
+		obs(0.9, true), obs(0.8, false), obs(0.7, true), obs(0.6, false),
+	})
+	want := 0.5*1 + 0.5*(2.0/3.0)
+	if math.Abs(got-want) > 1e-12 {
+		t.Errorf("AP = %.6f, want %.6f", got, want)
+	}
+
+	// A ranker with no signal converges on the prevalence.
+	rng := rand.New(rand.NewSource(4))
+	var noise []Observation
+	for range 20000 {
+		noise = append(noise, obs(rng.Float64(), rng.Float64() < 0.05))
+	}
+	if ap := averagePrecision(noise); math.Abs(ap-0.05) > 0.02 {
+		t.Errorf("a signal-free ranker scored AP %.4f at a 5%% base rate, want ~0.05", ap)
+	}
+}
+
+// Ties must not let the caller's ordering change the answer.
+func TestMetricsAreOrderInvariant(t *testing.T) {
+	base := []Observation{
+		obs(0.5, true), obs(0.5, false), obs(0.5, false), obs(0.5, true),
+		obs(0.9, true), obs(0.1, false), obs(0.9, false),
+	}
+	a, b := roc(base), averagePrecision(base)
+	rng := rand.New(rand.NewSource(17))
+	for range 50 {
+		s := append([]Observation(nil), base...)
+		rng.Shuffle(len(s), func(i, j int) { s[i], s[j] = s[j], s[i] })
+		if roc(s) != a || averagePrecision(s) != b {
+			t.Fatalf("a reshuffle moved the metrics: roc %g->%g ap %g->%g", a, roc(s), b, averagePrecision(s))
+		}
+	}
+}
+
+// Absence is not zero. On a plane where most events are unjudged this is the
+// default state, and a rate reported as 0.0 reads as a perfect model.
+func TestUnjudgedReportsAbsenceNotZero(t *testing.T) {
+	m := Measure([]Observation{{Score: 0.9}, {Score: 0.1}}, 0.5, policy.Cost{}, calibrate.Reader{})
+	if m.Rows != 2 || m.Judged != 0 {
+		t.Fatalf("counted %d rows / %d judged", m.Rows, m.Judged)
+	}
+	if m.ROC != nil || m.PR != nil || m.Precision != nil || m.Recall != nil || m.Prevalence != nil {
+		t.Errorf("reported rates over nothing: %+v", m)
+	}
+	if m.Refusal == "" {
+		t.Error("an unmeasurable report did not say so")
+	}
+
+	// One class only: prevalence is real, separation is not.
+	one := []Observation{obs(0.9, true), obs(0.8, true), obs(0.7, true)}
+	m = Measure(one, 0.5, policy.Cost{}, calibrate.Reader{})
+	if m.Prevalence == nil || *m.Prevalence != 1 {
+		t.Errorf("prevalence over one class is %v, want 1", m.Prevalence)
+	}
+	if m.ROC != nil || m.PR != nil {
+		t.Error("separation was reported over a single class")
+	}
+}
+
+// THE cost-weighted test. A miss and a false alarm have different prices, and
+// the cost-optimal threshold moves with the ratio — which is the whole reason
+// the metric exists and the reason it must be exact.
+func TestCostWeightedOptimumMovesWithThePrices(t *testing.T) {
+	// 100 unproductive spread over [0,0.6), 10 productive over [0.4,1.0).
+	var o []Observation
+	for i := range 100 {
+		o = append(o, obs(0.6*float64(i)/100, false))
+	}
+	for i := range 10 {
+		o = append(o, obs(0.4+0.6*float64(i)/10, true))
+	}
+
+	// A miss priced far above an alarm should push the threshold DOWN — flag
+	// more, catch more.
+	cheap := Measure(o, 0.5, policy.Cost{Miss: 1_000_000_000_000, Alarm: 1_000_000}, calibrate.Reader{})
+	// An alarm priced above a miss should push it UP — flag almost nothing.
+	dear := Measure(o, 0.5, policy.Cost{Miss: 1_000_000, Alarm: 1_000_000_000_000}, calibrate.Reader{})
+
+	if cheap.BestThreshold == nil || dear.BestThreshold == nil {
+		t.Fatal("no cost-optimal threshold was found under stated prices")
+	}
+	if *cheap.BestThreshold >= *dear.BestThreshold {
+		t.Errorf("cheap-alarm optimum %g is not below dear-alarm optimum %g",
+			*cheap.BestThreshold, *dear.BestThreshold)
+	}
+	if cheap.BestNano == nil || *cheap.BestNano > *cheap.CostNano {
+		t.Errorf("the reported optimum (%v) is worse than the operating point (%v)", cheap.BestNano, cheap.CostNano)
+	}
+	// Exactness: the cost at the operating point is counts times prices, in
+	// integers, with nothing rounded away.
+	c := cheap.Confusion
+	want := int64(math.Round(c.FN))*1_000_000_000_000 + int64(math.Round(c.FP))*1_000_000
+	if *cheap.CostNano != want {
+		t.Errorf("cost is %d, want %d — the money arithmetic is not exact", *cheap.CostNano, want)
+	}
+}
+
+// An unstated price must produce no cost at all, not a cost of zero.
+func TestNoStatedPriceMeansNoCost(t *testing.T) {
+	m := Measure([]Observation{obs(0.9, true), obs(0.1, false)}, 0.5, policy.Cost{}, calibrate.Reader{})
+	if m.CostNano != nil || m.BestNano != nil || m.BestThreshold != nil {
+		t.Errorf("priced a mistake nobody put a price on: %+v", m)
+	}
+}
+
+// A monotone calibration cannot reorder, so it cannot move a rank statistic.
+// This is worth pinning because it is the property that makes ROC and PR
+// comparable across calibration versions — and the property a reader most often
+// assumes the other way round.
+func TestCalibrationMovesBrierAndNotSeparation(t *testing.T) {
+	rng := rand.New(rand.NewSource(8))
+	var o []Observation
+	var samples []calibrate.Sample
+	for range 4000 {
+		s := rng.Float64()
+		p := s * s
+		productive := rng.Float64() < p
+		o = append(o, obs(s, productive))
+		samples = append(samples, calibrate.Sample{Score: s, Productive: productive})
+	}
+	m, err := calibrate.Fit(samples, calibrate.Isotonic, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cal, err := m.Under(shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw := Measure(o, 0.5, policy.Cost{}, calibrate.Reader{})
+	fitted := Measure(o, 0.5, policy.Cost{}, cal)
+
+	if *raw.ROC != *fitted.ROC || *raw.PR != *fitted.PR {
+		t.Errorf("a monotone calibration moved a rank statistic: roc %g->%g pr %g->%g",
+			*raw.ROC, *fitted.ROC, *raw.PR, *fitted.PR)
+	}
+	if raw.Brier != nil {
+		t.Error("a Brier score was reported with no calibration to score")
+	}
+	if fitted.Brier == nil {
+		t.Fatal("no Brier score was reported with a calibration in hand")
+	}
+}
+
+// The curve is one point per REACHABLE operating point. A point between two
+// observations at one score describes a threshold that separates nothing.
+func TestCurveHasOnePointPerDistinctScore(t *testing.T) {
+	o := []Observation{obs(0.9, true), obs(0.9, false), obs(0.5, true), obs(0.1, false)}
+	c := Curve(o, policy.Cost{Miss: 1, Alarm: 1})
+	if len(c) != 3 {
+		t.Fatalf("%d points over 3 distinct scores", len(c))
+	}
+	for i := 1; i < len(c); i++ {
+		if c[i].Threshold >= c[i-1].Threshold {
+			t.Errorf("the curve is not descending: %v", c)
+		}
+		if c[i].TruePositive < c[i-1].TruePositive || c[i].FalsePositive < c[i-1].FalsePositive {
+			t.Errorf("recall or fall-out went backwards as the threshold fell: %v", c)
+		}
+	}
+	last := c[len(c)-1]
+	if last.TruePositive != 1 || last.FalsePositive != 1 {
+		t.Errorf("the lowest threshold does not flag everything: %+v", last)
+	}
+}
+
+func TestCurveIsEmptyWithoutBothClasses(t *testing.T) {
+	if got := Curve([]Observation{obs(0.9, true), obs(0.8, true)}, policy.Cost{}); got != nil {
+		t.Errorf("drew a curve over one class: %v", got)
+	}
+	if got := Curve(nil, policy.Cost{}); got != nil {
+		t.Errorf("drew a curve over nothing: %v", got)
+	}
+}
+
+// A price that cannot be multiplied by the count without wrapping produces NO
+// cost and says why. The wrapped number would be negative, would be the minimum
+// of the sweep, and would therefore be returned as the threshold to adopt — an
+// arithmetic failure wearing the shape of a recommendation.
+func TestAnUnrepresentablePriceIsAbsentAndNotWrapped(t *testing.T) {
+	var obsv []Observation
+	for i := range 200 {
+		obsv = append(obsv, obs(float64(i)/200, i%3 == 0))
+	}
+	huge := policy.Cost{Miss: math.MaxInt64 / 4, Alarm: math.MaxInt64 / 4}
+	m := Measure(obsv, 0.5, huge, calibrate.Reader{})
+	if m.CostNano != nil {
+		t.Errorf("a cost of %d was reported from a product that does not fit in an int64", *m.CostNano)
+	}
+	if m.BestNano != nil || m.BestThreshold != nil {
+		t.Error("an operating point was recommended from arithmetic that overflowed")
+	}
+	if m.Refusal == "" {
+		t.Error("the cost was dropped in silence, which reads as an unpriced policy")
+	}
+	// The rank statistics are unaffected: they never touch the prices.
+	if m.ROC == nil || m.PR == nil {
+		t.Error("separation went missing because a price did not fit")
+	}
+	// And nowhere on the curve is a wrapped number plotted.
+	for i, p := range Curve(obsv, huge) {
+		if p.CostNano != 0 {
+			t.Fatalf("curve point %d carries %d, a price the arithmetic could not represent", i, p.CostNano)
+		}
+	}
+}
+
+// A curve is a chart and the response carrying it is read by a browser. The
+// metrics behind it stay exact over every distinct score; what is RETURNED is
+// bounded, keeps both ends, and stays descending — a truncated curve would
+// describe a model that stops.
+func TestTheCurveIsBoundedAndKeepsItsEnds(t *testing.T) {
+	n := Plotted * 7
+	var obsv []Observation
+	for i := range n {
+		obsv = append(obsv, obs(float64(i)/float64(n), i%5 == 0))
+	}
+	full := Curve(obsv, policy.Cost{})
+	if len(full) > Plotted {
+		t.Fatalf("the curve returned %d points, above the bound of %d", len(full), Plotted)
+	}
+	if len(full) < Plotted {
+		t.Fatalf("the curve returned %d points from %d distinct scores; it should fill the bound", len(full), n)
+	}
+	last := full[len(full)-1]
+	if last.TruePositive != 1 || last.FalsePositive != 1 {
+		t.Errorf("the sampled curve lost its far end: %+v", last)
+	}
+	if full[0].Threshold <= full[len(full)-1].Threshold {
+		t.Error("the sampled curve is not descending")
+	}
+	for i := 1; i < len(full); i++ {
+		if full[i].TruePositive < full[i-1].TruePositive || full[i].FalsePositive < full[i-1].FalsePositive {
+			t.Fatalf("sampling reordered the curve at %d", i)
+		}
+	}
+}
+
+func at(i int) time.Time { return time.Unix(int64(1_700_000_000+i*60), 0).UTC() }

pkg/evaluate/history.go

@@ -0,0 +1,327 @@
+package evaluate
+
+// history.go replays a candidate calibration and policy over recorded history:
+// what WOULD this have decided, on the events that already happened.
+//
+// WHAT IS REPLAYED, AND WHAT DELIBERATELY IS NOT. The recorded SCORE is
+// replayed. The model is not re-run. Two reasons, and both are the difference
+// between a backtest that means something and one that does not:
+//
+//	The rings have moved on. A score is a statement about how unusual an event
+//	was against the tenant's traffic AT THAT MOMENT. Recomputing it today against
+//	today's aggregates answers a different question and calls it the same one.
+//
+//	The model has since learned from these very events. Re-scoring history
+//	through a detector that folded that history into its own reference is the
+//	purest form of a backtest knowing the future — and it produces a beautiful
+//	number.
+//
+// So the score is a recorded fact and the candidate is a pure function of it.
+// That is also what makes the replay DETERMINISTIC: no clock, no random source,
+// no model state, no map iteration in anything that reaches the output. Two runs
+// over the same rows return the same digest, which is what lets a replay be
+// evidence rather than an anecdote.
+//
+// WHAT IT CANNOT TELL YOU, STATED RATHER THAN HIDDEN. A replay cannot know what
+// would have happened to an event the live system blocked, because a blocked
+// event has no outcome — the counterfactual is unobserved and no arithmetic
+// recovers it. That is the feedback loop every scoring plane has, and the only
+// honest treatment is to report how much of the judged set came from the
+// below-the-line sample arm, which is the part of the stream the incumbent
+// policy did NOT choose. Report.Explore is that number, and a replay whose
+// evidence is entirely the incumbent's own alerts is measuring agreement with
+// the incumbent.
+
+import (
+	"sort"
+	"time"
+
+	"github.com/luxfi/aml/pkg/calibrate"
+	"github.com/luxfi/aml/pkg/policy"
+	"github.com/luxfi/aml/pkg/replay"
+)
+
+// Recorded extends an Observation with where its judgement came from, which is
+// what makes the exploration share computable.
+type Recorded struct {
+	Observation
+	// Source is how this observation came to be judged: a dispute, a case
+	// outcome, an analyst review, or the below-the-line sample. Free-form here;
+	// Sample is the one value this package reads.
+	Source string
+}
+
+// Sample is the source name for a judgement that came from the below-the-line
+// reproducible sample — the arm that is chosen by hash rather than by the
+// incumbent policy, and therefore the only evidence about what the policy is
+// missing.
+const Sample = "sample"
+
+// Tally is one action and how often it was reached. A slice of these rather than
+// a map: a map's iteration order is randomised per process, so a report built
+// from one cannot have a stable digest and cannot be compared byte-for-byte
+// between two runs.
+type Tally struct {
+	Action string `json:"action"`
+	Count  int    `json:"count"`
+}
+
+// Move is one observation the candidate would have decided differently.
+type Move struct {
+	// ID is the decision.
+	ID string `json:"id"`
+	// At is when it happened.
+	At time.Time `json:"at,omitzero"`
+	// Was is what the live system did; Would is what the candidate would do.
+	Was   string `json:"was"`
+	Would string `json:"would"`
+	// Score is what the model said, and Probability what the candidate
+	// calibration makes of it.
+	Score       float64  `json:"score"`
+	Probability *float64 `json:"probability,omitempty"`
+	// Disposition is what turned out to be true, when anyone judged it. It is
+	// what turns a list of differences into a list of improvements or of
+	// regressions.
+	Disposition replay.Disposition `json:"disposition,omitempty"`
+}
+
+// Report is the answer.
+type Report struct {
+	// Rows is how many observations were replayed, and From/To the period.
+	Rows int       `json:"rows"`
+	From time.Time `json:"from,omitzero"`
+	To   time.Time `json:"to,omitzero"`
+
+	// Was and Would are the action distributions, ascending by action name.
+	Was   []Tally `json:"was"`
+	Would []Tally `json:"would"`
+
+	// Moved is every observation that would be decided differently, in the order
+	// they happened, cut at Listed. Changed is the exact count and is never cut.
+	Changed int    `json:"changed"`
+	Moved   []Move `json:"moved,omitempty"`
+
+	// Tightened and Loosened split Changed by direction against the caller's own
+	// action ranking: how many the candidate would act harder on, and how many it
+	// would let through that the incumbent did not.
+	Tightened int `json:"tightened"`
+	Loosened  int `json:"loosened"`
+
+	// Metrics is the candidate measured against the judged subset, at the
+	// probability the policy's FIRST rung sits at — the point where the candidate
+	// stops doing nothing, which is the operating point a policy actually
+	// implies.
+	Metrics Metrics `json:"metrics"`
+
+	// Explore is the share of the judged observations whose judgement came from
+	// the below-the-line sample rather than from something the incumbent policy
+	// chose to look at. Near zero means this report is measuring agreement with
+	// the incumbent, and that is a property of the evidence, not of the
+	// candidate.
+	Explore float64 `json:"explore"`
+
+	// Policy and Calibration are the digests of what was replayed, so a report
+	// names exactly the candidate it describes.
+	Policy      string `json:"policy"`
+	Calibration string `json:"calibration,omitempty"`
+
+	// Digest is the identity of this report. Two replays of one candidate over
+	// one history agree on it; if they do not, the inputs moved.
+	Digest string `json:"digest"`
+
+	// Refusal names why a report is empty when it is. An empty history and a
+	// candidate that changes nothing are opposite facts that render identically.
+	Refusal string `json:"refusal,omitempty"`
+}
+
+// Listed bounds the Moved list. The counts are exact whatever this is; the list
+// is evidence for a person, and a person reading a decision does not need the
+// ten-thousandth row.
+const Listed = 500
+
+// Replay runs a candidate calibration and policy over recorded history.
+//
+// rank orders the caller's own action vocabulary so Tightened and Loosened can
+// be counted without this package knowing what the actions mean — the same
+// separation policy.Policy makes, for the same reason. A nil rank counts every
+// difference as changed and neither tightened nor loosened, which is honest for
+// a caller that has no ordering.
+//
+// The history is taken in the order given, which is the order it happened. It is
+// not sorted here: sorting would silently repair a caller that handed rows in
+// the wrong order, and a replay over shuffled history is a different question
+// that would then be indistinguishable from this one.
+func Replay(history []Recorded, cal calibrate.Reader, pol policy.Policy, rank func(string) int) Report {
+	r := Report{Rows: len(history), Policy: pol.Digest, Calibration: cal.Digest()}
+	if len(history) == 0 {
+		r.Refusal = "no history to replay, so the result would be indistinguishable from a candidate that decides nothing"
+		r.Digest = fingerprint("evaluate.replay.v1", r.Refusal)
+		return r
+	}
+	if err := pol.Validate(); err != nil {
+		r.Refusal = "the candidate policy is not a ladder: " + err.Error()
+		r.Digest = fingerprint("evaluate.replay.v1", r.Refusal)
+		return r
+	}
+
+	was := map[string]int{}
+	would := map[string]int{}
+	obs := make([]Observation, 0, len(history))
+	var judged, explored int
+
+	for _, h := range history {
+		if r.From.IsZero() || h.At.Before(r.From) {
+			r.From = h.At
+		}
+		if h.At.After(r.To) {
+			r.To = h.At
+		}
+		obs = append(obs, h.Observation)
+		if h.judged() {
+			judged++
+			if h.Source == Sample {
+				explored++
+			}
+		}
+
+		var prob *float64
+		action := pol.Floor
+		if cal.Fitted() {
+			p := cal.P(h.Score)
+			v := round6(p)
+			prob = &v
+			action = pol.Action(p)
+		}
+		// An uncalibrated candidate cannot reach any rung: the ladder is over
+		// probabilities and there is no probability. The floor is the honest
+		// answer, and Refusal below says why every row landed on it.
+
+		was[h.Action]++
+		would[action]++
+		if action == h.Action {
+			continue
+		}
+		r.Changed++
+		if rank != nil {
+			switch {
+			case rank(action) > rank(h.Action):
+				r.Tightened++
+			case rank(action) < rank(h.Action):
+				r.Loosened++
+			}
+		}
+		if len(r.Moved) < Listed {
+			r.Moved = append(r.Moved, Move{
+				ID: h.ID, At: h.At, Was: h.Action, Would: action,
+				Score: round6(h.Score), Probability: prob, Disposition: h.Disposition,
+			})
+		}
+	}
+
+	if !cal.Fitted() {
+		r.Refusal = "no calibration, so no probability, so no rung of the ladder is reachable and every row takes the floor"
+	}
+	r.Was, r.Would = tally(was), tally(would)
+	if judged > 0 {
+		r.Explore = round6(float64(explored) / float64(judged))
+	}
+
+	// The operating point a policy IMPLIES is its first rung: below it the policy
+	// does nothing, so that is where it starts to act and the only point at which
+	// its precision and recall are its own. Measured on the score, by inverting
+	// the calibration through a sweep rather than analytically — the map is
+	// piecewise linear and monotone, so the first score whose probability reaches
+	// the rung is exactly the boundary.
+	r.Metrics = Measure(obs, scoreAt(cal, pol.Rungs[0].At), pol.Cost, cal)
+
+	r.Digest = digestOf(r)
+	return r
+}
+
+// scoreAt inverts a monotone calibration: the lowest score whose probability
+// reaches p. A binary search over the unit interval rather than over the knots,
+// so it works identically for isotonic and for platt and there is one
+// implementation instead of two.
+//
+// Returns a threshold above one when no score reaches p, which flags nothing —
+// the correct reading of a rung the candidate can never reach.
+func scoreAt(cal calibrate.Reader, p float64) float64 {
+	if !cal.Fitted() {
+		// Without a calibration the sweep has no probability to invert; a
+		// threshold above every score flags nothing, which is what a policy with
+		// no reachable rung does.
+		return 1.0000001
+	}
+	lo, hi := 0.0, 1.0
+	if cal.P(hi) < p {
+		return 1.0000001
+	}
+	// 40 halvings resolves the unit interval to about 1e-12, far finer than the
+	// six places a score is recorded at, so the answer is exact at the recorded
+	// precision and the loop is bounded.
+	for range 40 {
+		mid := (lo + hi) / 2
+		if cal.P(mid) >= p {
+			hi = mid
+		} else {
+			lo = mid
+		}
+	}
+	return round6(hi)
+}
+
+// tally renders a count map as an ascending slice. This is the ONE place a map
+// becomes output in this package, and it sorts — which is the whole reason the
+// digest is stable.
+func tally(m map[string]int) []Tally {
+	out := make([]Tally, 0, len(m))
+	for k, v := range m {
+		out = append(out, Tally{Action: k, Count: v})
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Action < out[j].Action })
+	return out
+}
+
+// digestOf is the determinism receipt: everything the report asserts, rendered
+// in a fixed order at a fixed precision.
+//
+// It covers the counts, the distributions, the metric values and the candidate's
+// identity — and not the wall-clock time, which is not an input. The Moved list
+// is folded in by id and direction so a report that lists different rows has a
+// different identity even when the totals happen to agree.
+func digestOf(r Report) string {
+	parts := []string{
+		"evaluate.replay.v1",
+		strconvInt(r.Rows), strconvInt(r.Changed), strconvInt(r.Tightened), strconvInt(r.Loosened),
+		r.Policy, r.Calibration, canonical(r.Explore),
+	}
+	for _, t := range r.Was {
+		parts = append(parts, "was:"+t.Action+"="+strconvInt(t.Count))
+	}
+	for _, t := range r.Would {
+		parts = append(parts, "would:"+t.Action+"="+strconvInt(t.Count))
+	}
+	for _, m := range r.Moved {
+		parts = append(parts, "moved:"+m.ID+":"+m.Was+">"+m.Would)
+	}
+	parts = append(parts,
+		strconvInt(r.Metrics.Judged), strconvInt(r.Metrics.Productive), strconvInt(r.Metrics.Unproductive),
+		opt(r.Metrics.ROC), opt(r.Metrics.PR), opt(r.Metrics.Precision), opt(r.Metrics.Recall),
+		opt(r.Metrics.Brier), optInt(r.Metrics.CostNano), optInt(r.Metrics.BestNano), opt(r.Metrics.BestThreshold),
+	)
+	return fingerprint(parts...)
+}
+
+func opt(v *float64) string {
+	if v == nil {
+		return "-"
+	}
+	return canonical(*v)
+}
+
+func optInt(v *int64) string {
+	if v == nil {
+		return "-"
+	}
+	return strconvInt64(*v)
+}

pkg/evaluate/history_test.go

@@ -0,0 +1,248 @@
+package evaluate
+
+import (
+	"math/rand"
+	"testing"
+
+	"github.com/luxfi/aml/pkg/calibrate"
+	"github.com/luxfi/aml/pkg/policy"
+	"github.com/luxfi/aml/pkg/replay"
+)
+
+func rank(a string) int {
+	switch a {
+	case "allow":
+		return 1
+	case "challenge":
+		return 2
+	case "review":
+		return 3
+	case "block":
+		return 4
+	}
+	return 0
+}
+
+func candidate(t *testing.T) policy.Policy {
+	t.Helper()
+	p, err := policy.Seal(policy.Policy{
+		Stage: "payment", Floor: "allow",
+		Rungs: []policy.Rung{
+			{At: 0.20, Action: "challenge"},
+			{At: 0.60, Action: "review"},
+			{At: 0.90, Action: "block"},
+		},
+		Cost: policy.Cost{Miss: 40_000_000_000, Alarm: 2_000_000_000},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return p
+}
+
+// history builds a recorded stream whose truth is a known function of the score,
+// with the live system's own action recorded beside it.
+func history(n int, seed int64) ([]Recorded, calibrate.Reader, error) {
+	rng := rand.New(rand.NewSource(seed))
+	var h []Recorded
+	var samples []calibrate.Sample
+	for i := range n {
+		s := rng.Float64()
+		productive := rng.Float64() < s*s
+		d := replay.Unproductive
+		if productive {
+			d = replay.Productive
+		}
+		action := "allow"
+		if s > 0.8 {
+			action = "review"
+		}
+		source := "review"
+		if i%7 == 0 {
+			source = Sample
+		}
+		h = append(h, Recorded{
+			Observation: Observation{
+				ID: "d_" + strconvInt(i), At: at(i), Score: s, Action: action, Disposition: d,
+			},
+			Source: source,
+		})
+		samples = append(samples, calibrate.Sample{Score: s, Productive: productive})
+	}
+	m, err := calibrate.Fit(samples, calibrate.Isotonic, shape)
+	if err != nil {
+		return h, calibrate.Reader{}, err
+	}
+	cal, err := m.Under(shape)
+	return h, cal, err
+}
+
+// THE backtest determinism test. A replay is evidence only if re-running it
+// reproduces it exactly — including under Go's randomised map iteration, which
+// is why nothing that reaches the output may be a map.
+func TestReplayIsDeterministic(t *testing.T) {
+	h, cal, err := history(2000, 41)
+	if err != nil {
+		t.Fatal(err)
+	}
+	pol := candidate(t)
+
+	first := Replay(h, cal, pol, rank)
+	if first.Refusal != "" {
+		t.Fatalf("refused: %s", first.Refusal)
+	}
+	if first.Digest == "" {
+		t.Fatal("a report with no identity cannot be evidence")
+	}
+	for i := range 25 {
+		again := Replay(h, cal, pol, rank)
+		if again.Digest != first.Digest {
+			t.Fatalf("run %d produced a different report: %s vs %s", i, again.Digest[:12], first.Digest[:12])
+		}
+		if len(again.Was) != len(first.Was) || len(again.Would) != len(first.Would) {
+			t.Fatalf("run %d produced different distributions", i)
+		}
+		for j := range again.Was {
+			if again.Was[j] != first.Was[j] || again.Would[j] != first.Would[j] {
+				t.Fatalf("run %d moved a tally: %+v vs %+v", i, again.Was[j], first.Was[j])
+			}
+		}
+	}
+}
+
+// The digest must actually discriminate, or determinism is trivially satisfied
+// by a constant.
+func TestReplayDigestMovesWithTheCandidate(t *testing.T) {
+	h, cal, err := history(1500, 42)
+	if err != nil {
+		t.Fatal(err)
+	}
+	base := Replay(h, cal, candidate(t), rank)
+
+	tighter := candidate(t)
+	tighter.Rungs[0].At = 0.10
+	tighter, err = policy.Seal(tighter)
+	if err != nil {
+		t.Fatal(err)
+	}
+	moved := Replay(h, cal, tighter, rank)
+	if moved.Digest == base.Digest {
+		t.Fatal("moving the first rung left the report identical")
+	}
+	if moved.Would[0].Count == base.Would[0].Count && moved.Changed == base.Changed {
+		t.Fatal("a lower first rung changed nothing")
+	}
+	// A lower first rung can only act harder.
+	if moved.Tightened < base.Tightened {
+		t.Errorf("lowering the challenge rung reduced the tightened count: %d -> %d", base.Tightened, moved.Tightened)
+	}
+}
+
+// A tally is ordered, not a map. This is the property the determinism rests on
+// and it is worth asserting directly.
+func TestTalliesAreOrdered(t *testing.T) {
+	h, cal, err := history(500, 43)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := Replay(h, cal, candidate(t), rank)
+	for _, set := range [][]Tally{r.Was, r.Would} {
+		for i := 1; i < len(set); i++ {
+			if set[i-1].Action >= set[i].Action {
+				t.Fatalf("tallies are not ascending: %+v", set)
+			}
+		}
+	}
+}
+
+// An empty history and a candidate that changes nothing render identically. The
+// difference is the whole reason a sandbox exists, so it must be stated.
+func TestReplayRefusesAnEmptyHistory(t *testing.T) {
+	r := Replay(nil, calibrate.Reader{}, candidate(t), rank)
+	if r.Refusal == "" {
+		t.Fatal("an empty replay reported as if it had run")
+	}
+	if r.Changed != 0 || r.Rows != 0 {
+		t.Fatalf("counted something over nothing: %+v", r)
+	}
+}
+
+// A candidate with no calibration cannot reach any rung, because the ladder is
+// over probabilities. Every row taking the floor is the correct answer and the
+// report must say why rather than presenting it as a policy that allows
+// everything.
+func TestReplayWithoutCalibrationSaysWhyNothingActs(t *testing.T) {
+	h, _, err := history(400, 44)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := Replay(h, calibrate.Reader{}, candidate(t), rank)
+	if r.Refusal == "" {
+		t.Fatal("every row took the floor and the report did not say why")
+	}
+	if len(r.Would) != 1 || r.Would[0].Action != "allow" {
+		t.Fatalf("something acted with no probability to act on: %+v", r.Would)
+	}
+}
+
+// A ladder that does not validate must be refused rather than replayed, or the
+// report describes a policy that could never be put in force.
+func TestReplayRefusesABrokenLadder(t *testing.T) {
+	h, cal, err := history(400, 45)
+	if err != nil {
+		t.Fatal(err)
+	}
+	broken := candidate(t)
+	broken.Rungs[1].At = 0.05 // descending
+	r := Replay(h, cal, broken, rank)
+	if r.Refusal == "" {
+		t.Fatal("replayed a ladder that cannot be put in force")
+	}
+	if r.Changed != 0 {
+		t.Fatalf("counted changes under a refused ladder: %d", r.Changed)
+	}
+}
+
+// The exploration share is the honest limit on every replay: evidence that came
+// only from what the incumbent chose to look at measures agreement with the
+// incumbent.
+func TestExploreIsMeasuredNotAssumed(t *testing.T) {
+	h, cal, err := history(700, 46) // every seventh judgement comes from the sample arm
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := Replay(h, cal, candidate(t), rank)
+	if r.Explore <= 0.1 || r.Explore >= 0.2 {
+		t.Errorf("explore share is %g, want about 1/7", r.Explore)
+	}
+
+	for i := range h {
+		h[i].Source = "review"
+	}
+	r = Replay(h, cal, candidate(t), rank)
+	if r.Explore != 0 {
+		t.Errorf("explore share is %g with no sample arm at all", r.Explore)
+	}
+}
+
+// The Moved list is evidence for a person and is cut; the count is exact and
+// never is.
+func TestMovedIsCutAndChangedIsNot(t *testing.T) {
+	h, cal, err := history(4000, 47)
+	if err != nil {
+		t.Fatal(err)
+	}
+	all := candidate(t)
+	all.Rungs = []policy.Rung{{At: 0, Action: "block"}}
+	all, err = policy.Seal(all)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := Replay(h, cal, all, rank)
+	if r.Changed <= Listed {
+		t.Fatalf("only %d changed; the cut is untested", r.Changed)
+	}
+	if len(r.Moved) != Listed {
+		t.Fatalf("listed %d rows, want the bound of %d", len(r.Moved), Listed)
+	}
+}

pkg/evaluate/learning.go

@@ -0,0 +1,167 @@
+package evaluate
+
+// learning.go answers the question an operator asks before spending money on
+// more labels: is this getting better, and would more data help?
+//
+// A learning curve is the metric as a function of how much history the fit had.
+// The one built here has TWO arms, and the pair is the point:
+//
+//	train  the metric on the same rows the calibration was fitted on. Optimistic
+//	       by construction.
+//	ahead  the metric on the rows that came AFTER — data the fit had not seen,
+//	       in the direction time actually runs.
+//
+// A curve where both arms are still falling means more labels would help. A
+// curve where train keeps falling and ahead has stopped means the fit has
+// started memorising the sample, and the next thing to change is the method, not
+// the volume. A curve where ahead is flat and equal to train means the fit is as
+// good as this feature space allows. Those are three different decisions and one
+// arm cannot distinguish them.
+//
+// THE SPLIT IS TEMPORAL AND ONLY TEMPORAL. Never random. Two reasons, and the
+// second is the one that actually bites:
+//
+//	A random split lets a fit see the future. Fraud arrives in campaigns; a
+//	random holdout puts half of one campaign in training and half in test, and
+//	the fit scores brilliantly on a pattern it has already been shown.
+//
+//	A random split lets a fit see the same ENTITY on both sides. One device, one
+//	card, one account appearing in train and in test means the model is being
+//	rewarded for memorising an identifier rather than for recognising behaviour,
+//	and that is invisible in every metric. Ordering by time and cutting once is
+//	the only split that cannot do either.
+//
+// SEPARATION IS RANK-INVARIANT AND CALIBRATION IS NOT — which is why both are
+// reported and why they say different things. A calibration is monotone, so it
+// cannot reorder anything: ROC and PR are IDENTICAL before and after fitting
+// one. Brier is not. So the ROC/PR arm of the curve measures the SCORER getting
+// better as the tenant's model learns, and the Brier arm measures the
+// CALIBRATION getting better as labels accrue. Reading either as the other is
+// the most common way a learning curve is misused.
+
+import (
+	"strconv"
+
+	"github.com/luxfi/aml/pkg/calibrate"
+	"github.com/luxfi/aml/pkg/policy"
+)
+
+// Step is one point of a learning curve.
+type Step struct {
+	// Rows is how many observations the fit was given, and Judged how many of
+	// those carried a disposition — the number that actually constrains the fit.
+	Rows   int `json:"rows"`
+	Judged int `json:"judged"`
+	// Held is the size of the fixed later window every step is measured against.
+	// It is the same at every step, which is what makes two steps comparable.
+	Held int `json:"held"`
+	// Productive is how many of the judged were positive. A step with two
+	// positives produces a number; it does not produce evidence, and this column
+	// is how a reader sees which is which.
+	Productive int `json:"productive"`
+	// Train is the report on the rows the fit saw.
+	Train Metrics `json:"train"`
+	// Ahead is the report on the fixed later window — the same rows at every
+	// step, all of them after every row the fit saw.
+	Ahead Metrics `json:"ahead"`
+	// Refusal names why a step produced no fit — almost always that too few of
+	// one class had accrued by that point. It is stated rather than skipped, so
+	// the curve shows where the plane became able to answer at all.
+	Refusal string `json:"refusal,omitempty"`
+}
+
+// Learning walks a growing temporal prefix of the history, fitting a calibration
+// on each prefix and measuring it on the prefix and on ONE FIXED window of
+// later data.
+//
+// THE AHEAD WINDOW IS THE SAME AT EVERY STEP, and that is the difference between
+// a curve and a set of unrelated numbers. The obvious construction — train on
+// the first n, test on everything after n — changes the test set at every step:
+// the last point is measured on a handful of rows and the first on almost all of
+// them, so the curve's own noise grows along the x-axis and a rise at the right
+// end says nothing about the fit. Holding the window fixed makes two steps
+// comparable, which is the only reason to draw them on one chart.
+//
+// The window is the final 1/(steps+1) of the history and the training pool is
+// everything before it, so nothing in the pool is later than anything in the
+// window and no step can see the future.
+//
+// steps is how many points the curve has. Ten is the default and is deliberate:
+// enough to see a shape, few enough that the answer is a chart rather than a
+// data set, and the same ten the topology search already uses so two curves in
+// one console share an x-axis.
+//
+// history must be in time order. It is not sorted here, for the reason stated in
+// Replay: silently repairing a caller's ordering would make a leaked split
+// indistinguishable from a clean one.
+func Learning(history []Observation, method, shape string, steps int, cost policy.Cost) []Step {
+	if steps <= 0 {
+		steps = 10
+	}
+	if len(history) < 2 {
+		return nil
+	}
+	cut := len(history) - len(history)/(steps+1)
+	if cut >= len(history) {
+		cut = len(history) - 1
+	}
+	pool, ahead := history[:cut], history[cut:]
+
+	out := make([]Step, 0, steps)
+	for i := 1; i <= steps; i++ {
+		n := len(pool) * i / steps
+		if n == 0 {
+			continue
+		}
+		train := pool[:n]
+		s := Step{Rows: n, Held: len(ahead)}
+		samples := make([]calibrate.Sample, 0, n)
+		for _, o := range train {
+			if !o.judged() {
+				continue
+			}
+			s.Judged++
+			if o.productive() {
+				s.Productive++
+			}
+			samples = append(samples, calibrate.Sample{
+				Score: o.Score, Productive: o.productive(), Weight: o.Weight,
+			})
+		}
+		cal, err := calibrate.Fit(samples, method, shape)
+		if err != nil {
+			s.Refusal = err.Error()
+			// The separation arm needs no calibration — it is a rank statistic —
+			// so it is still measured and reported. A curve that went blank
+			// wherever the calibration refused would hide the model improving
+			// underneath it.
+			s.Train = Measure(train, 0, cost, calibrate.Reader{})
+			s.Ahead = Measure(ahead, 0, cost, calibrate.Reader{})
+			out = append(out, s)
+			continue
+		}
+		// The step's own fit is read under the shape it was just fitted under —
+		// the one place that identity is a fact rather than an assumption. A bind
+		// that refuses here is a fit that cannot describe its own sample, which is
+		// a refusal and not a step.
+		read, err := cal.Under(shape)
+		if err != nil {
+			s.Refusal = err.Error()
+			s.Train = Measure(train, 0, cost, calibrate.Reader{})
+			s.Ahead = Measure(ahead, 0, cost, calibrate.Reader{})
+			out = append(out, s)
+			continue
+		}
+		// The operating point is the score at the median fitted probability, so
+		// every step is measured at a comparable place on its own map rather than
+		// at a constant score that means something different at each step.
+		at := scoreAt(read, 0.5)
+		s.Train = Measure(train, at, cost, read)
+		s.Ahead = Measure(ahead, at, cost, read)
+		out = append(out, s)
+	}
+	return out
+}
+
+func strconvInt(v int) string     { return strconv.Itoa(v) }
+func strconvInt64(v int64) string { return strconv.FormatInt(v, 10) }

pkg/evaluate/learning_test.go

@@ -0,0 +1,109 @@
+package evaluate
+
+import (
+	"math/rand"
+	"testing"
+
+	"github.com/luxfi/aml/pkg/calibrate"
+	"github.com/luxfi/aml/pkg/policy"
+	"github.com/luxfi/aml/pkg/replay"
+)
+
+// curveData builds a stream in time order whose truth is stationary, so the only
+// thing that changes along the curve is how much of it the fit has seen.
+func curveData(n int, seed int64) []Observation {
+	rng := rand.New(rand.NewSource(seed))
+	out := make([]Observation, 0, n)
+	for i := range n {
+		s := rng.Float64()
+		d := replay.Unproductive
+		if rng.Float64() < s*s {
+			d = replay.Productive
+		}
+		out = append(out, Observation{ID: "d_" + strconvInt(i), At: at(i), Score: s, Disposition: d})
+	}
+	return out
+}
+
+// A learning curve with one arm cannot distinguish "more data would help" from
+// "the fit has started memorising". Both arms, the split is temporal, and the
+// ahead window is the SAME at every step — otherwise two points are measured on
+// different data and the curve is a set of unrelated numbers.
+func TestLearningHasBothArmsAndImprovesAhead(t *testing.T) {
+	data := curveData(6000, 51)
+	steps := Learning(data, calibrate.Isotonic, shape, 10, policy.Cost{})
+	if len(steps) != 10 {
+		t.Fatalf("%d steps, want 10", len(steps))
+	}
+	pool := len(data) - len(data)/11
+	for i, s := range steps {
+		if want := pool * (i + 1) / 10; s.Rows != want {
+			t.Errorf("step %d saw %d rows, want %d — the split is not a growing temporal prefix", i, s.Rows, want)
+		}
+		if s.Held != len(data)-pool {
+			t.Errorf("step %d is measured against %d held rows, want the fixed %d", i, s.Held, len(data)-pool)
+		}
+		if s.Ahead.Rows != s.Held {
+			t.Errorf("step %d: the ahead report covers %d rows, the window is %d", i, s.Ahead.Rows, s.Held)
+		}
+	}
+
+	// The whole point of the fixed window: out-of-sample calibration error is
+	// comparable end to end, and it should fall as labels accrue.
+	early, late := steps[0].Ahead.Brier, steps[len(steps)-1].Ahead.Brier
+	if early == nil || late == nil {
+		t.Fatal("no out-of-sample Brier was measured at both ends of the curve")
+	}
+	if *late > *early {
+		t.Errorf("out-of-sample calibration error rose with more data: %.4f -> %.4f", *early, *late)
+	}
+}
+
+// The split must never let a fit see the future: every held row lies strictly
+// after every row any step trained on.
+func TestLearningSplitIsTemporalAndDisjoint(t *testing.T) {
+	data := curveData(1000, 52)
+	steps := Learning(data, calibrate.Isotonic, shape, 5, policy.Cost{})
+	if len(steps) == 0 {
+		t.Fatal("no steps")
+	}
+	held := steps[0].Held
+	cut := len(data) - held
+	for i, s := range steps {
+		if s.Held != held {
+			t.Fatalf("step %d moved the held window: %d vs %d", i, s.Held, held)
+		}
+		if s.Rows > cut {
+			t.Fatalf("step %d trained on %d rows, past the cut at %d — the fit saw the future", i, s.Rows, cut)
+		}
+		if !data[s.Rows-1].At.Before(data[cut].At) {
+			t.Fatalf("step %d: the held window does not lie strictly after the training prefix", i)
+		}
+	}
+}
+
+// A step whose sample is too thin to fit must SAY so and still report the
+// separation, which needs no calibration. A curve that went blank would hide the
+// model improving underneath it.
+func TestLearningStatesWhyAStepCouldNotFit(t *testing.T) {
+	// 40 rows over 10 steps: the first steps have four rows each.
+	steps := Learning(curveData(40, 53), calibrate.Isotonic, shape, 10, policy.Cost{})
+	var refused int
+	for _, s := range steps {
+		if s.Refusal != "" {
+			refused++
+			if s.Train.Rows == 0 && s.Train.Judged == 0 {
+				t.Errorf("a refused step reported nothing at all: %+v", s)
+			}
+		}
+	}
+	if refused == 0 {
+		t.Fatal("no step refused on a sample this thin; the refusal path is untested")
+	}
+}
+
+func TestLearningOverNothing(t *testing.T) {
+	if got := Learning(nil, calibrate.Isotonic, shape, 10, policy.Cost{}); got != nil {
+		t.Errorf("drew a curve over no history: %v", got)
+	}
+}

pkg/policy/policy.go

@@ -0,0 +1,254 @@
+// Package policy is the ladder from a probability to an action, as a value that
+// can be recorded, versioned and replayed.
+//
+// WHY IT IS A VALUE AND NOT A CONSTANT. Where the line sits between letting a
+// customer through and stopping them is not a technical fact — it is a business
+// decision with a price on either side, and it belongs to the organisation
+// making it. A threshold compiled into the scoring path is a decision nobody
+// signed, cannot be varied per tenant, cannot be reviewed, and cannot be shown
+// to have been in force on the day a particular customer was declined. So a
+// policy here is an ordinary immutable value with a version, an author, a stated
+// reason and a digest — a record, in the same sense a decision is a record.
+//
+// WHY A LADDER RATHER THAN FOUR NAMED THRESHOLDS. Named fields (challenge,
+// review, restrict, block) hard-code one product's action vocabulary into the
+// arithmetic, and then a tenant that wants three rungs has to leave one field at
+// a sentinel value that every reader has to remember to check. A ladder is the
+// general shape: an ordered list of (probability, action) steps, the last step
+// at or below the probability wins, and below the first step the floor applies.
+// Adding a rung is adding an element.
+//
+// ACTION NAMES ARE OPAQUE HERE, DELIBERATELY. This package validates the
+// arithmetic — thresholds inside the unit interval, non-decreasing, no
+// duplicates, no empty action — and knows nothing about what the actions mean.
+// The closed vocabulary belongs to the surface that receives the policy off the
+// wire, which is where a caller-supplied string has to be validated anyway.
+// Restating the vocabulary here would put it in two places and let them drift.
+//
+// COST IS PART OF THE POLICY, AND IN INTEGERS. A threshold is only defensible
+// against a stated price for being wrong in each direction, and those two prices
+// are almost never equal: a missed fraud costs the disputed amount plus the
+// network fee, a false decline costs a customer. They are int64 nano-units, not
+// floats, because they are money and they are multiplied by counts.
+package policy
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"math"
+	"strconv"
+	"time"
+)
+
+// Errors. Each names the property that failed, because a policy is refused at
+// the moment somebody is trying to change one and "invalid" is not actionable.
+var (
+	ErrNoStage    = errors.New("policy: a policy applies to one lifecycle stage and this one names none")
+	ErrNoRungs    = errors.New("policy: a ladder with no rungs decides nothing")
+	ErrNoFloor    = errors.New("policy: the action below the first rung must be named")
+	ErrRange      = errors.New("policy: a rung sits outside the unit interval, so no probability can reach it")
+	ErrOrder      = errors.New("policy: the rungs are not ascending, so a higher probability would ask for less")
+	ErrDuplicate  = errors.New("policy: two rungs at one probability, so which one applies is undecided")
+	ErrNoAction   = errors.New("policy: a rung with no action")
+	ErrSameAction = errors.New("policy: a rung asks for the action already in force below it")
+	ErrCost       = errors.New("policy: a cost cannot be negative")
+	ErrPrice      = errors.New("policy: a price above the bound cannot be multiplied by a count without overflowing, and a wrapped cost reads as a recommendation")
+)
+
+// Rung is one step of the ladder: at or above this probability, this action.
+type Rung struct {
+	// At is the probability boundary, in [0,1]. It is a PROBABILITY and not a
+	// raw score: a ladder over an uncalibrated score is a ladder over units that
+	// mean nothing, and the cost arithmetic below would be meaningless too.
+	At float64 `json:"at"`
+	// Action is what the organisation wants done at or above At. Opaque here;
+	// the surface that accepts it validates it against its own closed set.
+	Action string `json:"action"`
+}
+
+// Cost is what being wrong is worth, in nano-units of the tenant's accounting
+// currency. Integers, because this is money and it is multiplied by counts.
+//
+// Miss is the price of letting through something that should have been stopped —
+// on a payment lane, typically the disputed amount plus the network's dispute
+// fee. Alarm is the price of stopping something that was fine — a customer's
+// abandoned purchase, a support contact, and whatever share of those customers
+// do not come back. Neither is knowable to three significant figures, and that
+// is not an argument for leaving them out: stating them badly makes the
+// trade-off arguable, and leaving them out makes it invisible.
+type Cost struct {
+	Miss  int64 `json:"miss"`
+	Alarm int64 `json:"alarm"`
+}
+
+// Volume and MaxPrice are the bound that keeps the cost arithmetic exact.
+//
+// A price is multiplied by a COUNT of observations and the products are summed
+// in int64. An int64 that overflows does not fail — it wraps, silently, and the
+// cost-minimising threshold then becomes a number chosen by the wrap rather than
+// by the evidence. That is the worst failure this package could have: it does
+// not look like an error, it looks like a recommendation.
+//
+// Volume is the largest number of observations any one measurement will price.
+// It is set well above the bounded read every measurement is taken under, so
+// raising that read does not quietly invalidate the invariant. MaxPrice is
+// int64's ceiling divided by it, which comes to roughly 92 thousand units of the
+// tenant's currency for the price of ONE mistake — far above any average a
+// lifecycle stage would state, and below the point where the arithmetic stops
+// being exact.
+const (
+	Volume   = 100_000
+	MaxPrice = math.MaxInt64 / Volume
+)
+
+// Stated reports whether this cost says anything. A zero cost is not a free
+// mistake, it is an unstated price, and every metric derived from it is absent
+// rather than zero.
+func (c Cost) Stated() bool { return c.Miss > 0 || c.Alarm > 0 }
+
+// Policy is one tenant's ladder for one stage, as a record.
+type Policy struct {
+	// Stage is the lifecycle moment this governs — signup, payment, session,
+	// payout. One ladder per stage: the price of a false decline at signup and at
+	// payout are different numbers about different things.
+	Stage string `json:"stage"`
+	// Version is monotone within (tenant, stage). A change mints the next one;
+	// nothing is ever edited, so what was in force on a given day is a lookup and
+	// not a reconstruction.
+	Version int `json:"version"`
+	// At and By are when it took effect and who decided. A control that changed
+	// and cannot say who changed it is a control nobody owns.
+	At time.Time `json:"at"`
+	By string    `json:"by"`
+	// Reason is why. It is required of the surface, not of this package, for the
+	// same reason the action vocabulary is: the wire is where a caller-supplied
+	// string is checked.
+	Reason string `json:"reason,omitempty"`
+	// Floor is the action below the first rung.
+	Floor string `json:"floor"`
+	// Rungs is the ladder, ascending.
+	Rungs []Rung `json:"rungs"`
+	// Shadow observes without acting: every threshold is computed and recorded
+	// and the action returned is always the floor. It is the second gate in
+	// series with the engine's own Shadow, so a deployment-wide flag flipped by
+	// mistake still cannot make one tenant act.
+	Shadow bool `json:"shadow"`
+	// Cost is what this tenant says being wrong is worth.
+	Cost Cost `json:"cost"`
+	// Digest identifies this policy exactly, so a decision can cite the ladder
+	// that produced it and an auditor can pin it.
+	Digest string `json:"digest"`
+}
+
+// Action is the ladder read at one probability: the highest rung at or below it,
+// or the floor.
+//
+// In shadow it is always the floor, and that is the whole of the shadow
+// mechanism — the caller still computes everything and still records what would
+// have happened, because a shadow that skipped the computation would prove
+// nothing about the live path.
+func (p Policy) Action(prob float64) string {
+	if p.Shadow {
+		return p.Floor
+	}
+	action := p.Floor
+	for _, r := range p.Rungs {
+		if prob >= r.At {
+			action = r.Action
+			continue
+		}
+		break
+	}
+	return action
+}
+
+// Reached is the rung that decided, for a decision that wants to cite one.
+// Absent when the floor applied, which is a real answer: nothing was reached.
+func (p Policy) Reached(prob float64) *Rung {
+	if p.Shadow {
+		return nil
+	}
+	var hit *Rung
+	for i := range p.Rungs {
+		if prob >= p.Rungs[i].At {
+			hit = &p.Rungs[i]
+			continue
+		}
+		break
+	}
+	return hit
+}
+
+// Validate checks everything about a ladder that can be checked without knowing
+// what the actions mean.
+//
+// Every one of these refusals is a real failure mode. A rung above one and no
+// probability reaches it, so the action is unreachable and the operator believes
+// a control is in place that is not. Rungs out of order and a higher probability
+// asks for LESS, which is the opposite of a risk policy. Two rungs at one
+// probability and which applies depends on iteration order. A rung repeating the
+// action below it is dead: it changes nothing and hides that the ladder has
+// fewer real steps than it appears to.
+func (p Policy) Validate() error {
+	if p.Stage == "" {
+		return ErrNoStage
+	}
+	if p.Floor == "" {
+		return ErrNoFloor
+	}
+	if len(p.Rungs) == 0 {
+		return ErrNoRungs
+	}
+	if p.Cost.Miss < 0 || p.Cost.Alarm < 0 {
+		return ErrCost
+	}
+	if p.Cost.Miss > MaxPrice || p.Cost.Alarm > MaxPrice {
+		return fmt.Errorf("%w: %d and %d against a bound of %d", ErrPrice, p.Cost.Miss, p.Cost.Alarm, MaxPrice)
+	}
+	prev, prevAction := -1.0, p.Floor
+	for i, r := range p.Rungs {
+		switch {
+		case r.Action == "":
+			return fmt.Errorf("%w: rung %d", ErrNoAction, i)
+		case math.IsNaN(r.At), r.At < 0, r.At > 1:
+			return fmt.Errorf("%w: rung %d at %g", ErrRange, i, r.At)
+		case r.At == prev:
+			return fmt.Errorf("%w: rungs %d and %d at %g", ErrDuplicate, i-1, i, r.At)
+		case r.At < prev:
+			return fmt.Errorf("%w: rung %d at %g follows %g", ErrOrder, i, r.At, prev)
+		case r.Action == prevAction:
+			return fmt.Errorf("%w: rung %d repeats %q", ErrSameAction, i, r.Action)
+		}
+		prev, prevAction = r.At, r.Action
+	}
+	return nil
+}
+
+// Seal validates a ladder and returns it with its digest set. It is the ONE way
+// to obtain a policy that carries an identity, so an unvalidated ladder cannot
+// be recorded as if it had been agreed.
+func Seal(p Policy) (Policy, error) {
+	if err := p.Validate(); err != nil {
+		return Policy{}, err
+	}
+	p.Digest = Digest(p)
+	return p, nil
+}
+
+// Digest identifies a ladder by what it DECIDES — stage, floor, rungs, shadow
+// and cost — and deliberately not by who wrote it or when.
+//
+// Two policies with the same digest decide identically, so a backtest of one is
+// a backtest of the other and a version bump that changed nothing is visible as
+// such. Folding the author and the timestamp in would make every re-save a
+// different policy and destroy exactly that property.
+func Digest(p Policy) string {
+	h := sha256.New()
+	fmt.Fprintf(h, "v1|%s|%s|%t|%d|%d|", p.Stage, p.Floor, p.Shadow, p.Cost.Miss, p.Cost.Alarm)
+	for _, r := range p.Rungs {
+		fmt.Fprintf(h, "%s:%s|", strconv.FormatFloat(r.At, 'g', 17, 64), r.Action)
+	}
+	return hex.EncodeToString(h.Sum(nil))
+}

pkg/policy/policy_test.go

@@ -0,0 +1,184 @@
+package policy
+
+import (
+	"errors"
+	"math"
+	"testing"
+	"time"
+)
+
+func ladder() Policy {
+	return Policy{
+		Stage: "payment", Version: 3, At: time.Unix(1, 0).UTC(), By: "ops@acme",
+		Reason: "the dispute rate on new cards doubled",
+		Floor:  "allow",
+		Rungs: []Rung{
+			{At: 0.20, Action: "challenge"},
+			{At: 0.55, Action: "review"},
+			{At: 0.80, Action: "restrict"},
+			{At: 0.95, Action: "block"},
+		},
+		Cost: Cost{Miss: 42_000_000_000, Alarm: 3_000_000_000},
+	}
+}
+
+func TestActionReadsTheLadder(t *testing.T) {
+	p, err := Seal(ladder())
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range []struct {
+		prob float64
+		want string
+	}{
+		{0, "allow"}, {0.1999, "allow"},
+		{0.20, "challenge"}, {0.5499, "challenge"},
+		{0.55, "review"}, {0.7999, "review"},
+		{0.80, "restrict"}, {0.9499, "restrict"},
+		{0.95, "block"}, {1, "block"},
+	} {
+		if got := p.Action(c.prob); got != c.want {
+			t.Errorf("p=%g decided %q, want %q", c.prob, got, c.want)
+		}
+	}
+}
+
+// The rung a decision cites has to be the one that actually decided, or an
+// adverse-action notice names a threshold the customer did not cross.
+func TestReachedNamesTheDecidingRung(t *testing.T) {
+	p, _ := Seal(ladder())
+	if r := p.Reached(0.1); r != nil {
+		t.Errorf("the floor applied and a rung was cited: %+v", r)
+	}
+	r := p.Reached(0.9)
+	if r == nil || r.Action != "restrict" || r.At != 0.80 {
+		t.Fatalf("cited %+v, want the 0.80 restrict rung", r)
+	}
+}
+
+// Shadow must compute everything and act on nothing. A shadow that short-circuits
+// would prove nothing about the live path.
+func TestShadowAlwaysTakesTheFloor(t *testing.T) {
+	l := ladder()
+	l.Shadow = true
+	p, _ := Seal(l)
+	for _, prob := range []float64{0, 0.5, 0.99, 1} {
+		if got := p.Action(prob); got != "allow" {
+			t.Errorf("in shadow p=%g acted %q", prob, got)
+		}
+		if r := p.Reached(prob); r != nil {
+			t.Errorf("in shadow p=%g cited a rung", prob)
+		}
+	}
+}
+
+// Each of these is a real way a ladder silently stops being a risk policy.
+func TestValidateRefusesEveryBrokenLadder(t *testing.T) {
+	for _, c := range []struct {
+		name string
+		mut  func(*Policy)
+		want error
+	}{
+		{"no stage", func(p *Policy) { p.Stage = "" }, ErrNoStage},
+		{"no floor", func(p *Policy) { p.Floor = "" }, ErrNoFloor},
+		{"no rungs", func(p *Policy) { p.Rungs = nil }, ErrNoRungs},
+		{"rung above one", func(p *Policy) { p.Rungs[3].At = 1.5 }, ErrRange},
+		{"rung below zero", func(p *Policy) { p.Rungs[0].At = -0.1 }, ErrRange},
+		{"descending", func(p *Policy) { p.Rungs[2].At = 0.3 }, ErrOrder},
+		{"duplicate", func(p *Policy) { p.Rungs[2].At = 0.55 }, ErrDuplicate},
+		{"unnamed action", func(p *Policy) { p.Rungs[1].Action = "" }, ErrNoAction},
+		{"dead rung", func(p *Policy) { p.Rungs[1].Action = "challenge" }, ErrSameAction},
+		{"negative cost", func(p *Policy) { p.Cost.Miss = -1 }, ErrCost},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			p := ladder()
+			c.mut(&p)
+			if err := p.Validate(); !errors.Is(err, c.want) {
+				t.Fatalf("accepted (err=%v), want %v", err, c.want)
+			}
+			if _, err := Seal(p); err == nil {
+				t.Fatal("Seal minted a digest for a ladder that does not validate")
+			}
+		})
+	}
+}
+
+// The digest identifies what a policy DECIDES. Two policies that decide
+// identically must share it, or a backtest of one is not a backtest of the
+// other; two that decide differently must not.
+func TestDigestIsOverTheDecisionAndNothingElse(t *testing.T) {
+	a, _ := Seal(ladder())
+
+	l := ladder()
+	l.Version = 99
+	l.At = time.Now()
+	l.By = "someone-else@acme"
+	l.Reason = "a different sentence"
+	b, _ := Seal(l)
+	if a.Digest != b.Digest {
+		t.Error("re-saving an unchanged ladder minted a new identity, so a no-op edit looks like a policy change")
+	}
+
+	l = ladder()
+	l.Rungs[0].At = 0.21
+	c, _ := Seal(l)
+	if a.Digest == c.Digest {
+		t.Error("moving a threshold left the identity unchanged")
+	}
+
+	l = ladder()
+	l.Cost.Miss = 1
+	d, _ := Seal(l)
+	if a.Digest == d.Digest {
+		t.Error("changing the stated cost left the identity unchanged; the cost is part of what the policy decides")
+	}
+
+	l = ladder()
+	l.Shadow = true
+	e, _ := Seal(l)
+	if a.Digest == e.Digest {
+		t.Error("turning shadow on left the identity unchanged")
+	}
+}
+
+// An unstated price is not a free mistake. Everything derived from it must be
+// absent, and Stated is the one predicate that decides.
+func TestUnstatedCostIsNotZeroCost(t *testing.T) {
+	if (Cost{}).Stated() {
+		t.Error("an empty cost claims to state a price")
+	}
+	if !(Cost{Alarm: 1}).Stated() {
+		t.Error("a stated alarm price reads as unstated")
+	}
+}
+
+// A price is multiplied by a count and summed in int64. Above the bound that
+// product wraps, and a wrapped cost does not read as an error — it reads as a
+// recommendation about where to put a threshold. The ladder is refused at the
+// door, which is the one place a caller-supplied number arrives.
+func TestAPriceAboveTheBoundIsRefused(t *testing.T) {
+	l := ladder()
+	l.Cost = Cost{Miss: MaxPrice + 1, Alarm: 1}
+	if _, err := Seal(l); !errors.Is(err, ErrPrice) {
+		t.Errorf("a price of %d was accepted: %v", l.Cost.Miss, err)
+	}
+	l.Cost = Cost{Miss: 1, Alarm: MaxPrice + 1}
+	if _, err := Seal(l); !errors.Is(err, ErrPrice) {
+		t.Errorf("an alarm price of %d was accepted: %v", l.Cost.Alarm, err)
+	}
+	// And the bound itself is reachable: refusing at the boundary would make the
+	// stated maximum a number nobody can use.
+	l.Cost = Cost{Miss: MaxPrice, Alarm: MaxPrice}
+	if _, err := Seal(l); err != nil {
+		t.Errorf("the stated maximum was refused: %v", err)
+	}
+	// The bound has to be worth something. A price that cannot express an
+	// ordinary large loss would push a tenant into understating one.
+	if MaxPrice < 10_000*1_000_000_000 {
+		t.Errorf("MaxPrice is %d nano — under ten thousand units, which is too small to state a real loss", MaxPrice)
+	}
+	// And it has to actually prevent the wrap it exists for.
+	if MaxPrice > math.MaxInt64/Volume {
+		t.Errorf("MaxPrice %d times Volume %d overflows int64", MaxPrice, Volume)
+	}
+}

pkg/reason/reason.go

@@ -0,0 +1,270 @@
+// Package reason turns a model's attribution into the artefact an adverse
+// decision is defended with: a small closed set of codes, ranked by how much
+// each one moved the decision, each carrying a sentence a person can be shown.
+//
+// WHY A CLOSED VOCABULARY. A free-text reason cannot be counted, cannot be
+// tested, and cannot be compared between two decisions or two tenants. Every
+// regime that lets a person question a decision — Reg B §1002.9(b)(2) for
+// credit, GDPR Art. 22(3) and Art. 13(2)(f) for automated processing, and the
+// supervisory expectation behind every alert an investigator writes up — asks
+// for the PRINCIPAL reasons, specifically and accurately. That is a ranked list
+// over a fixed vocabulary, not a paragraph.
+//
+// WHY THE VOCABULARY IS DERIVED AND NOT DECLARED. The codes come from
+// anomaly.Inventory() — the same nine features the model actually reads, in the
+// same order, with the same neutral values. A second hand-written table of codes
+// would be a second description of the model, and the two would drift the first
+// time a feature was added. Codes() is a pure function of the inventory, so the
+// vocabulary cannot describe a model that is not there and cannot omit one that
+// is.
+//
+// WHY THE WEIGHTS ARE THE MODEL'S OWN. A Reason's weight is types.Cause.Share,
+// which anomaly computes by moving one coordinate to its neutral value and
+// rescoring on the same trees — an exact counterfactual on the model that
+// decided, not a surrogate fitted afterwards. That is the whole reason the
+// detector is a half-space tree and not a net: attributability was the
+// selection criterion.
+package reason
+
+import (
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/luxfi/aml/pkg/anomaly"
+	"github.com/luxfi/aml/pkg/types"
+)
+
+// Direction says which way a feature moved away from unremarkable. It is part of
+// the code because "much larger than this customer's norm" and "much smaller
+// than this customer's norm" are different findings that would otherwise share
+// one code and be uncountable apart.
+const (
+	// Above is a coordinate higher than the feature's neutral value.
+	Above = "above"
+	// Below is a coordinate lower than it. Only reachable for the features whose
+	// neutral sits inside the unit interval; a feature neutral at zero has no
+	// below.
+	Below = "below"
+)
+
+// Rule is the ONE source that is not the model: one of the tenant's own written
+// rules fired.
+//
+// It is a single code and the specific rule is named in the reason's Source,
+// rather than minting a code per rule — otherwise the vocabulary grows by one
+// entry every time a tenant writes a rule and stops being countable, which is
+// the property that makes it defensible.
+//
+// A rule that reads a list is still a rule. The list is data the rule consults,
+// not a separate authority, and citing "list" beside "rule" for one firing would
+// double-count one piece of evidence.
+//
+// There is deliberately no code for anything this plane cannot emit. A published
+// code nothing can reach is a branch a consumer writes and never runs, which is
+// the same defect as an unreachable direction and is refused for the same
+// reason.
+const Rule = "rule"
+
+// Code is one entry of the vocabulary.
+type Code struct {
+	// Code is the identifier a decision cites and a report counts. Stable: it is
+	// part of the published contract, so renaming one invalidates every stored
+	// decision that cites it.
+	Code string `json:"code"`
+	// Source is where a reason with this code comes from: "model" for a feature
+	// the detector reads, or Rule for a rule the tenant wrote.
+	Source string `json:"source"`
+	// Says is the sentence template a person is shown, with no numbers in it. A
+	// Reason fills the numbers in.
+	Says string `json:"says"`
+	// Typology, Indicator and Citation are the model features' own provenance,
+	// carried through unchanged from anomaly.Inventory so the published reason
+	// and the model's own inventory cannot tell two different stories. Empty for
+	// the rule code, which cites the tenant's own detection instead.
+	Typology  string `json:"typology,omitempty"`
+	Indicator string `json:"indicator,omitempty"`
+	Citation  string `json:"citation,omitempty"`
+	// Severity is the grading a reason with this code carries.
+	Severity string `json:"severity"`
+}
+
+// Reason is one code as it applies to one decision.
+type Reason struct {
+	// Code is from the vocabulary Codes() publishes. A code not in that set is a
+	// bug, and Rank cannot produce one: every code it emits is built by the same
+	// two functions that build the vocabulary.
+	Code string `json:"code"`
+	// Weight is this reason's part of the decision, in [0,1]. For a model reason
+	// it is the exact counterfactual share; for a rule it is the weight that rule
+	// carried into the combination.
+	Weight float64 `json:"weight"`
+	// Says is the sentence shown to an investigator or to a declined customer,
+	// with this decision's own numbers in it.
+	Says string `json:"says"`
+	// Source names the specific rule, which the code deliberately does not.
+	// Empty for a model feature, whose code already names it exactly.
+	Source string `json:"source,omitempty"`
+	// Severity is the grading.
+	Severity string `json:"severity,omitempty"`
+}
+
+// codeOf renders the identifier for one feature and direction. It is the ONE
+// place the code's spelling is decided, so Codes and Rank cannot disagree about
+// what a code looks like.
+func codeOf(feature, direction string) string { return feature + "." + direction }
+
+// Codes is the whole vocabulary, in a stable order: every reachable model code
+// first, in the model's own coordinate order, then the one rule code.
+//
+// Reachability is read off the feature's neutral value rather than listed:
+// a feature neutral at zero is a share or a count that cannot go below it, so
+// there is no "below" finding to report and publishing one would be a code
+// nothing can ever cite. Deriving it means adding a tenth feature adds its codes
+// automatically and correctly.
+func Codes() []Code {
+	inv := anomaly.Inventory()
+	out := make([]Code, 0, 2*anomaly.Dims+1)
+	for _, f := range inv {
+		out = append(out, feature(f, Above))
+		if f.Neutral > 0 {
+			out = append(out, feature(f, Below))
+		}
+	}
+	return append(out, Code{
+		Code: Rule, Source: Rule, Severity: types.SeverityMedium,
+		Says: "one of this organisation's own rules, named in the reason's source",
+	})
+}
+
+// feature renders one model code from one inventory entry.
+func feature(f anomaly.Feature, direction string) Code {
+	return Code{
+		Code:      codeOf(f.Name, direction),
+		Source:    "model",
+		Says:      says(f, direction),
+		Typology:  f.Typology,
+		Indicator: f.Indicator,
+		Citation:  f.Citation,
+		Severity:  f.Severity,
+	}
+}
+
+// says renders the sentence for one feature and direction, with no numbers.
+func says(f anomaly.Feature, direction string) string {
+	word := "higher than"
+	if direction == Below {
+		word = "lower than"
+	}
+	return f.Indicator + " — " + word + " this subject's own pattern, measured in " + f.Unit
+}
+
+// Known reports whether a code is in the vocabulary. It is what a caller that
+// stored a code and read it back later checks before showing it, and what a test
+// asserts over every reason a decision produced.
+func Known(code string) bool {
+	for _, c := range Codes() {
+		if c.Code == code {
+			return true
+		}
+	}
+	return false
+}
+
+// Rank turns a model's attribution into the reasons a decision cites, strongest
+// first.
+//
+// The direction is read from the coordinate the model actually used, not from
+// the raw observation: a coordinate is where the model was, and the raw number
+// is what it was computed from. Cause carries Observed and Baseline, and the
+// direction is decided by comparing them the way the feature's own projection
+// does — a ratio feature is above when the observation exceeds the baseline, and
+// a share feature (neutral zero) can only ever be above.
+//
+// top bounds how many reasons come back. Reg B asks for the principal reasons
+// and expressly does not ask for all of them; a list of nine is not an
+// explanation. Zero means the caller wants them all.
+func Rank(causes []types.Cause, top int) []Reason {
+	inv := anomaly.Inventory()
+	byName := map[string]anomaly.Feature{}
+	for _, f := range inv {
+		byName[f.Name] = f
+	}
+
+	out := make([]Reason, 0, len(causes))
+	for _, c := range causes {
+		f, ok := byName[c.Feature]
+		if !ok {
+			// A cause naming a feature the model does not have cannot be turned
+			// into a code in the vocabulary, and inventing one would publish a
+			// reason nothing can count. Dropped, and the drop is visible as a
+			// reason list shorter than the cause list.
+			continue
+		}
+		dir := Above
+		if f.Neutral > 0 && c.Observed < c.Baseline {
+			dir = Below
+		}
+		out = append(out, Reason{
+			Code:     codeOf(f.Name, dir),
+			Weight:   c.Share,
+			Says:     sentence(f, dir, c),
+			Severity: f.Severity,
+		})
+	}
+
+	// Stable by weight so that two reasons contributing equally keep the order
+	// the model gave them — which is its own ordering by distance from neutral
+	// when no single feature accounts for the score.
+	sort.SliceStable(out, func(i, j int) bool { return out[i].Weight > out[j].Weight })
+	if top > 0 && len(out) > top {
+		out = out[:top]
+	}
+	return out
+}
+
+// sentence renders one reason with this decision's own numbers.
+func sentence(f anomaly.Feature, direction string, c types.Cause) string {
+	var b strings.Builder
+	b.WriteString(f.Indicator)
+	b.WriteString(" — ")
+	if direction == Below {
+		b.WriteString("lower")
+	} else {
+		b.WriteString("higher")
+	}
+	b.WriteString(" than this subject's own pattern (observed ")
+	b.WriteString(number(c.Observed))
+	if c.Baseline != 0 {
+		b.WriteString(" against ")
+		b.WriteString(number(c.Baseline))
+	}
+	b.WriteString(" ")
+	b.WriteString(f.Unit)
+	b.WriteString(")")
+	return b.String()
+}
+
+// number renders a quantity the way a person reads one: no exponent, no
+// seventeen digits of float noise, and no trailing zeros.
+func number(v float64) string {
+	s := strconv.FormatFloat(v, 'f', 4, 64)
+	s = strings.TrimRight(s, "0")
+	return strings.TrimSuffix(s, ".")
+}
+
+// OfRule builds the reason for one rule firing. The rule is named in Source and
+// the code stays Rule, so a report can count how often rules decided against how
+// often the model did, and can still say WHICH rule for any one decision.
+//
+// says is the rule's own name — the sentence the tenant wrote when it declared
+// the detection, which is the only description of it anyone has agreed to.
+func OfRule(id, says, severity string, weight float64) Reason {
+	if severity == "" {
+		severity = types.SeverityMedium
+	}
+	if says == "" {
+		says = id
+	}
+	return Reason{Code: Rule, Weight: weight, Says: says, Source: id, Severity: severity}
+}

pkg/reason/reason_test.go

@@ -0,0 +1,181 @@
+package reason
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/luxfi/aml/pkg/anomaly"
+	"github.com/luxfi/aml/pkg/types"
+)
+
+// The vocabulary is derived from the model, so it must cover exactly the model
+// and nothing else. A code for a feature that is not there is a reason nothing
+// can cite; a feature with no code is an alert nobody can be told the reason for.
+func TestVocabularyCoversExactlyTheModel(t *testing.T) {
+	inv := anomaly.Inventory()
+	seen := map[string]int{}
+	for _, c := range Codes() {
+		if c.Source != "model" {
+			continue
+		}
+		feature, _, ok := strings.Cut(c.Code, ".")
+		if !ok {
+			t.Fatalf("model code %q is not <feature>.<direction>", c.Code)
+		}
+		seen[feature]++
+	}
+	if len(seen) != anomaly.Dims {
+		t.Fatalf("vocabulary names %d features, the model reads %d", len(seen), anomaly.Dims)
+	}
+	for _, f := range inv {
+		want := 1
+		if f.Neutral > 0 {
+			want = 2
+		}
+		if seen[f.Name] != want {
+			t.Errorf("feature %q (neutral %g): %d codes, want %d", f.Name, f.Neutral, seen[f.Name], want)
+		}
+	}
+}
+
+// A one-sided feature cannot go below its neutral of zero, so publishing a
+// "below" code for it would be a code nothing can ever reach.
+func TestNoUnreachableCode(t *testing.T) {
+	neutral := map[string]float64{}
+	for _, f := range anomaly.Inventory() {
+		neutral[f.Name] = f.Neutral
+	}
+	for _, c := range Codes() {
+		if c.Source != "model" {
+			continue
+		}
+		feature, direction, _ := strings.Cut(c.Code, ".")
+		if direction == Below && neutral[feature] == 0 {
+			t.Errorf("%q is unreachable: %q is neutral at zero", c.Code, feature)
+		}
+	}
+}
+
+// Every code carries the provenance a reviewer asks for. A reason with no
+// citation is an assertion.
+func TestEveryModelCodeCarriesProvenance(t *testing.T) {
+	for _, c := range Codes() {
+		if c.Source != "model" {
+			continue
+		}
+		if c.Typology == "" || c.Indicator == "" || c.Citation == "" || c.Severity == "" {
+			t.Errorf("%q is missing provenance: %+v", c.Code, c)
+		}
+		if c.Says == "" {
+			t.Errorf("%q has no sentence to show anyone", c.Code)
+		}
+	}
+}
+
+// THE reason-code test. Rank must emit only codes the vocabulary publishes, must
+// order them by contribution, and must read the direction off the arithmetic the
+// model actually did.
+func TestRankEmitsOnlyPublishedCodesStrongestFirst(t *testing.T) {
+	causes := []types.Cause{
+		{Feature: "novelty", Observed: 0, Baseline: 0, Share: 0.10},
+		{Feature: "amount", Observed: 9000, Baseline: 300, Share: 0.55},
+		{Feature: "count", Observed: 1, Baseline: 12, Share: 0.35},
+		{Feature: "nonesuch", Observed: 1, Baseline: 0, Share: 0.99},
+	}
+	got := Rank(causes, 0)
+
+	if len(got) != 3 {
+		t.Fatalf("ranked %d reasons, want 3 (the unknown feature must be dropped, not invented)", len(got))
+	}
+	for _, r := range got {
+		if !Known(r.Code) {
+			t.Errorf("%q is not in the vocabulary", r.Code)
+		}
+		if r.Says == "" {
+			t.Errorf("%q has no sentence", r.Code)
+		}
+	}
+	if got[0].Code != "amount.above" {
+		t.Errorf("strongest is %q, want amount.above", got[0].Code)
+	}
+	// count is a ratio feature (neutral 0.5) and the observation is BELOW the
+	// baseline, so the finding is "lower than this subject's own pattern".
+	if got[1].Code != "count.below" {
+		t.Errorf("second is %q, want count.below — observed %g against baseline %g", got[1].Code, 1.0, 12.0)
+	}
+	if got[2].Code != "novelty.above" {
+		t.Errorf("third is %q, want novelty.above", got[2].Code)
+	}
+	for i := 1; i < len(got); i++ {
+		if got[i-1].Weight < got[i].Weight {
+			t.Errorf("reasons are not strongest first: %v", got)
+		}
+	}
+	if !strings.Contains(got[0].Says, "9000") {
+		t.Errorf("the sentence does not carry this decision's own numbers: %q", got[0].Says)
+	}
+}
+
+// Reg B asks for the PRINCIPAL reasons and expressly does not ask for all of
+// them. A cap that did not cap would put nine reasons in an adverse-action
+// notice, which is not an explanation.
+func TestRankCaps(t *testing.T) {
+	var causes []types.Cause
+	for i, f := range anomaly.Inventory() {
+		causes = append(causes, types.Cause{Feature: f.Name, Observed: float64(i + 2), Baseline: 1, Share: float64(9-i) / 45})
+	}
+	if got := Rank(causes, 4); len(got) != 4 {
+		t.Fatalf("cap of 4 returned %d", len(got))
+	}
+	if got := Rank(causes, 0); len(got) != anomaly.Dims {
+		t.Fatalf("no cap returned %d, want %d", len(got), anomaly.Dims)
+	}
+}
+
+// A reason minted for every rule a tenant writes would make the vocabulary
+// uncountable, which is the property that makes it defensible. The rule goes in
+// Source; the code stays one.
+func TestOfRuleKeepsTheVocabularyClosed(t *testing.T) {
+	a := OfRule("rule_7f3", "velocity over 24h on this card", types.SeverityHigh, 0.4)
+	b := OfRule("rule_ab1", "a denied address", types.SeverityCritical, 0.9)
+	if a.Code != Rule || b.Code != Rule {
+		t.Errorf("two rules minted two codes: %q and %q", a.Code, b.Code)
+	}
+	if a.Source != "rule_7f3" || b.Source != "rule_ab1" {
+		t.Errorf("the rule id did not reach Source: %q %q", a.Source, b.Source)
+	}
+	if !Known(a.Code) {
+		t.Errorf("%q is not in the vocabulary", a.Code)
+	}
+	if a.Severity != types.SeverityHigh {
+		t.Errorf("severity is %q, want the rule's own", a.Severity)
+	}
+	if got := OfRule("rule_x", "", "", 0.1); got.Says != "rule_x" || got.Severity == "" {
+		t.Errorf("a nameless rule produced an unshowable reason: %+v", got)
+	}
+}
+
+// The vocabulary must publish nothing this plane cannot emit. Two sources: the
+// model's own features, and a rule firing.
+func TestVocabularyPublishesOnlyEmittableSources(t *testing.T) {
+	sources := map[string]int{}
+	for _, c := range Codes() {
+		sources[c.Source]++
+	}
+	if len(sources) != 2 || sources["model"] == 0 || sources[Rule] != 1 {
+		t.Fatalf("sources are %v, want exactly {model: many, rule: 1}", sources)
+	}
+}
+
+// Every code the vocabulary publishes must be recognised by Known — otherwise a
+// stored decision could cite a code the reader rejects.
+func TestKnownAgreesWithCodes(t *testing.T) {
+	for _, c := range Codes() {
+		if !Known(c.Code) {
+			t.Errorf("Codes publishes %q and Known rejects it", c.Code)
+		}
+	}
+	if Known("amount.sideways") {
+		t.Error("Known accepted a code Codes does not publish")
+	}
+}