calibrate: pool tied scores, and bind a map to a shape before it can be read

Two defects in the same value.

TIES. A decision score is 1 - prod(1-weight) over a fixed set of rule
weights, rounded to four places, so the score alphabet is a handful of
atoms and the modal atom is exactly zero. pava ran over individual
samples, and a run of unproductive-then-productive at one score is
already non-decreasing — so nothing merged, every sample stayed its own
block, and the dedup that followed kept the MAX of two blocks sharing a
centroid. Every plateau holding a positive was reported as certainty, at
the top of the range where declines happen. Ties are now pooled into one
block per distinct score before the algorithm starts, which is what makes
the answer isotonic regression; a rounding collision between two
centroids pools rather than picks.

THE SHAPE GATE. P(score, shape) let the caller supply the value being
compared, and every internal call site supplied m.Shape — so the
training-serving skew control was trivially satisfied everywhere except
the one place a live shape was threaded in. Reading is now impossible
without Under(shape), which returns a Reader: the gate is at the bind,
where the current shape is a fact the server observed, and every
consumer downstream holds a value for which the question is already
answered and has no shape to pass. Reliability draws no chart for a
reader that cannot answer.

Co-authored-by: Hanzo Dev <[email protected]>

pkg/calibrate/calibrate.go

@@ -39,10 +39,18 @@
 // THE SHAPE GATE — this is the training–serving skew control. A Map records the
 // digest of the scorer it was fitted under (anomaly.Store.Digest(), which covers
 // the feature inventory in order, the neutral values and the detector geometry).
-// P refuses to answer for a scorer whose shape differs. So a calibration cannot
+// A Map cannot be read at all: Under(shape) is the only way to obtain a Reader,
+// and it refuses for a scorer whose shape differs. So a calibration cannot
 // outlive the model it describes: change a feature, and every map fitted before
 // the change stops answering instead of silently mapping coordinates that no
 // longer mean what they meant.
+//
+// The gate is at the BIND and not at the read, because a gate every read has to
+// pass is a gate every read can be made to pass — the shape argument was
+// available from the map itself, so passing m.Shape cleared the control and
+// looked exactly like clearing it honestly. Reading is total once the bind has
+// happened, so there is one place to get it right and no way downstream to get
+// it wrong.
 package calibrate
 
 import (
@@ -163,32 +171,75 @@ func (m Map) Fitted() bool {
 	return m.Shape != "" && (len(m.Knots) > 0 || m.Method == Platt)
 }
 
-// P maps one score to a probability under this fit, refusing when the scorer's
-// shape is not the one it was fitted under.
+// Under binds this map to the scoring shape now in force, refusing when they
+// differ. It is the ONLY way to read a probability out of a Map.
 //
-// The shape argument is not optional and there is no variant without it. A
-// calibration applied to coordinates that have moved is the classic
-// training–serving skew failure, and it fails silently: every number still looks
-// like a probability. Making the shape a required argument turns that into a
-// refusal at the one place every read has to pass.
-func (m Map) P(score float64, shape string) (float64, error) {
+// A GATE A CALLER CAN SATISFY IS NOT A GATE. The earlier read was
+// P(score, shape), and every internal call site passed m.Shape — the value being
+// compared came from the value being checked, so the training–serving skew
+// control was trivially true everywhere except the one place a live shape
+// happened to be threaded in. Nothing about that was visible at the call site;
+// each one looked like it was passing the gate.
+//
+// Binding separates the two concerns. ONE place decides whether this map applies
+// to these coordinates, at the boundary where the current shape is a fact the
+// server observed rather than a value a caller supplied. Everything downstream
+// takes a Reader, for which the question is already answered and cannot be
+// re-asked wrongly — a Reader has no shape to pass and no Map to read one off.
+func (m Map) Under(shape string) (Reader, error) {
 	if !m.Fitted() {
-		return 0, ErrNotFitted
+		return Reader{}, ErrNotFitted
 	}
 	if shape == "" {
-		return 0, ErrNoShape
+		return Reader{}, ErrNoShape
 	}
 	if shape != m.Shape {
-		return 0, fmt.Errorf("%w: fitted under %s, asked under %s", ErrShape, short(m.Shape), short(shape))
+		return Reader{}, fmt.Errorf("%w: fitted under %s, asked under %s", ErrShape, short(m.Shape), short(shape))
 	}
-	switch m.Method {
-	case Platt:
-		return clamp01(1 / (1 + math.Exp(m.A*score+m.B))), nil
-	case Isotonic:
-		return clamp01(interpolate(m.Knots, score)), nil
-	default:
-		return 0, ErrMethod
+	if m.Method != Isotonic && m.Method != Platt {
+		return Reader{}, fmt.Errorf("%w: %q", ErrMethod, m.Method)
 	}
+	return Reader{m: m, ok: true}, nil
+}
+
+// Reader is a calibration that has already been checked against the coordinates
+// it is about to be read in: a map bound to one scoring shape.
+//
+// It has no exported fields and no constructor other than Map.Under, so a
+// function taking a Reader is a function whose caller has been through the gate,
+// and that is a property the compiler checks rather than a reviewer.
+//
+// The ZERO Reader is the honest "no probability here": it is what a caller holds
+// when this organisation has no calibration at all AND what it holds when the
+// shape has moved, which are the two facts every consumer downstream has to
+// treat the same way — silently, by not stating a probability.
+type Reader struct {
+	m  Map
+	ok bool
+}
+
+// Fitted reports whether this reader can answer.
+func (r Reader) Fitted() bool { return r.ok }
+
+// Digest names the fit behind this reader, so a record can pin which map turned
+// which score into which probability. Empty when it cannot answer.
+func (r Reader) Digest() string {
+	if !r.ok {
+		return ""
+	}
+	return r.m.Digest
+}
+
+// P maps one score to a probability. It is TOTAL: the only thing that could have
+// failed is the shape, and holding a Reader is proof it did not.
+func (r Reader) P(score float64) float64 {
+	if !r.ok {
+		return 0
+	}
+	if r.m.Method == Platt {
+		return clamp01(1 / (1 + math.Exp(r.m.A*score+r.m.B)))
+	}
+	return clamp01(interpolate(r.m.Knots, score))
 }
 
 // interpolate reads the piecewise-linear map at score, clamping outside the
@@ -268,9 +319,11 @@ func Fit(samples []Sample, method, shape string) (Map, error) {
 		}
 		clean = append(clean, s)
 	}
-	// Total order, so the fit does not depend on the caller's order. Score
-	// ascending; unproductive before productive at equal score, which is the
-	// order pool-adjacent-violators would reach anyway and makes it explicit.
+	// Score ascending, so the fit does not depend on the caller's order. The
+	// tie-break by class is cosmetic and stays only because a stable, stated order
+	// is easier to read in a dump: pava pools every sample at one score into one
+	// block before it starts, so the order WITHIN a tie cannot reach the answer.
+	// Relying on that order instead of pooling is exactly the bug pooling fixes.
 	sort.SliceStable(clean, func(i, j int) bool {
 		if clean[i].Score != clean[j].Score {
 			return clean[i].Score < clean[j].Score
@@ -293,7 +346,15 @@ func Fit(samples []Sample, method, shape string) (Map, error) {
 			return Map{}, err
 		}
 	}
-	m.Brier = brier(clean, m)
+	// The in-sample Brier reads the map through the shape it was just fitted
+	// under, which is the one place that identity is a fact rather than an
+	// assumption — the shape was handed in and stamped on the same value two lines
+	// above. Everywhere else the shape comes from the live scorer.
+	r, err := m.Under(shape)
+	if err != nil {
+		return Map{}, err
+	}
+	m.Brier = brier(clean, r)
 	m.Digest = digest(m)
 	return m, nil
 }
@@ -301,10 +362,25 @@ func Fit(samples []Sample, method, shape string) (Map, error) {
 // pava is pool-adjacent-violators: the exact least-squares fit of a
 // non-decreasing function to the labels, in one pass over the sorted samples.
 //
-// Each block holds a weighted mean; whenever a block's mean is not greater than
-// its predecessor's the two are merged, which is the only operation the
+// TIES ARE POOLED BEFORE THE ALGORITHM STARTS, and that is not a refinement —
+// it is what makes the answer isotonic regression at all. A monotone function
+// has ONE value at one score, so every sample sharing a score is one block by
+// definition. Fed individual samples instead, a run of unproductive-then-
+// productive at a single score is already non-decreasing, so the merge condition
+// never fires, every sample stays its own block, and the score ends up mapped to
+// whichever label the sort happened to put last: 1 wherever any positive sits at
+// the top of a tie, 0 wherever none does.
+//
+// That is the ordinary case here, not an edge. A decision score is
+// 1 - prod(1-weight) over a FIXED set of rule weights, rounded to four places, so
+// the score alphabet is a handful of atoms and the modal atom is exactly zero.
+// Un-pooled, the whole map collapses to a step function pinned at 0 and 1 — every
+// plateau reported as certainty, at the top of the range where declines happen.
+//
+// Each block then holds a weighted mean; whenever a block's mean is not greater
+// than its predecessor's the two are merged, which is the only operation the
 // algorithm has. The result is the unique monotone least-squares solution — not
-// an approximation and not an iterative one — which is why this is 40 lines and
+// an approximation and not an iterative one — which is why this is 60 lines and
 // needs no numerical library.
 //
 // The knots are the block CENTROIDS (the weighted mean score inside the block)
@@ -315,13 +391,7 @@ func Fit(samples []Sample, method, shape string) (Map, error) {
 func pava(sorted []Sample) []Knot {
 	type block struct{ sum, weight, score float64 }
 	blocks := make([]block, 0, len(sorted))
-	for _, s := range sorted {
-		w := s.weight()
-		y := 0.0
-		if s.Productive {
-			y = 1
-		}
-		b := block{sum: y * w, weight: w, score: s.Score * w}
+	push := func(b block) {
 		blocks = append(blocks, b)
 		for len(blocks) > 1 {
 			last := len(blocks) - 1
@@ -334,24 +404,42 @@ func pava(sorted []Sample) []Knot {
 			blocks = blocks[:last]
 		}
 	}
-	knots := make([]Knot, 0, len(blocks))
-	for _, b := range blocks {
-		knots = append(knots, Knot{
-			Score: round6(b.score / b.weight),
-			P:     round6(b.sum / b.weight),
-		})
-	}
-	// Two blocks can share a centroid when every sample in both carried the same
-	// score. Keep the last, which holds the merged mean, so the knots are
-	// strictly ascending and interpolate is total.
-	out := knots[:0]
-	for i, k := range knots {
-		if i+1 < len(knots) && knots[i+1].Score == k.Score {
-			continue
+	// One block per DISTINCT score. weight() is never zero, so every block the
+	// loop pushes has mass and the means below are total.
+	for i := 0; i < len(sorted); {
+		var b block
+		j := i
+		for ; j < len(sorted) && sorted[j].Score == sorted[i].Score; j++ {
+			w := sorted[j].weight()
+			if sorted[j].Productive {
+				b.sum += w
+			}
+			b.weight += w
+			b.score += sorted[j].Score * w
 		}
-		out = append(out, k)
+		push(b)
+		i = j
 	}
-	return out
+
+	// A block's centroid lies inside its own score range, and the ranges are
+	// disjoint and ordered, so the centroids ascend strictly — until round6 puts
+	// two of them on one score. Those two are POOLED rather than one of them
+	// discarded: either endpoint alone states a probability the pooled evidence
+	// does not support, and the pooled mean sits between the neighbours it
+	// replaced, so the map stays monotone and interpolate stays total.
+	knots := make([]Knot, 0, len(blocks))
+	for i := 0; i < len(blocks); {
+		at := round6(blocks[i].score / blocks[i].weight)
+		var sum, weight float64
+		j := i
+		for ; j < len(blocks) && round6(blocks[j].score/blocks[j].weight) == at; j++ {
+			sum += blocks[j].sum
+			weight += blocks[j].weight
+		}
+		knots = append(knots, Knot{Score: at, P: round6(sum / weight)})
+		i = j
+	}
+	return knots
 }
 
 // platt fits p = 1/(1+exp(A*score+B)) by Newton's method on the regularised
@@ -431,16 +519,13 @@ func platt(sorted []Sample, pos, neg int) (float64, float64, error) {
 // Lower is better; 0.25 is what a constant at the base rate of a balanced set
 // scores, so a number above that on a balanced set means the map is worse than
 // saying nothing.
-func brier(samples []Sample, m Map) float64 {
-	if len(samples) == 0 {
+func brier(samples []Sample, r Reader) float64 {
+	if len(samples) == 0 || !r.Fitted() {
 		return 0
 	}
 	var sum, w float64
 	for _, s := range samples {
-		p, err := m.P(s.Score, m.Shape)
-		if err != nil {
-			return 0
-		}
+		p := r.P(s.Score)
 		y := 0.0
 		if s.Productive {
 			y = 1
@@ -475,7 +560,15 @@ type Bin struct {
 // share, and reporting either as 0.0 would draw a point on the chart at perfect
 // confidence in innocence — which is the most misleading thing this whole
 // package could do.
-func Reliability(samples []Sample, m Map, bins int) []Bin {
+//
+// A reader that cannot answer gets NO CHART, not an empty one. A calibration
+// whose shape has moved refuses to state a probability; a reliability report is
+// that same map applied to every row, so shipping one beside the refusal would
+// draw the chart the prose just said does not exist.
+func Reliability(samples []Sample, r Reader, bins int) []Bin {
+	if !r.Fitted() {
+		return nil
+	}
 	if bins <= 0 {
 		bins = 10
 	}
@@ -487,10 +580,7 @@ func Reliability(samples []Sample, m Map, bins int) []Bin {
 		out[i] = Bin{From: float64(i) / float64(bins), To: float64(i+1) / float64(bins)}
 	}
 	for _, s := range samples {
-		p, err := m.P(s.Score, m.Shape)
-		if err != nil {
-			continue
-		}
+		p := r.P(s.Score)
 		i := int(p * float64(bins))
 		if i >= bins {
 			i = bins - 1

pkg/calibrate/calibrate_test.go

@@ -31,13 +31,13 @@ func TestIsotonicRecoversTheTrueProbability(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	r, err := m.Under(shape)
+	if err != nil {
+		t.Fatal(err)
+	}
 	var worst float64
 	for _, s := range []float64{0.05, 0.15, 0.3, 0.5, 0.7, 0.85, 0.95} {
-		got, err := m.P(s, shape)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if d := math.Abs(got - truth(s)); d > worst {
+		if d := math.Abs(r.P(s) - truth(s)); d > worst {
 			worst = d
 		}
 	}
@@ -61,13 +61,14 @@ func TestFitIsMonotone(t *testing.T) {
 		if err != nil {
 			t.Fatalf("%s: %v", method, err)
 		}
+		r, err := m.Under(shape)
+		if err != nil {
+			t.Fatalf("%s: %v", method, err)
+		}
 		prev := -1.0
 		for i := range 1001 {
 			s := float64(i) / 1000
-			p, err := m.P(s, shape)
-			if err != nil {
-				t.Fatalf("%s: %v", method, err)
-			}
+			p := r.P(s)
 			if p < prev-1e-12 {
 				t.Fatalf("%s: not monotone — p(%g)=%g after %g", method, s, p, prev)
 			}
@@ -112,14 +113,17 @@ func TestMapRefusesAnotherModelShape(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if _, err := m.P(0.5, shape); err != nil {
+	if _, err := m.Under(shape); err != nil {
 		t.Fatalf("refused its own shape: %v", err)
 	}
-	if _, err := m.P(0.5, "sha256:a-tenth-feature-was-added"); !errors.Is(err, ErrShape) {
-		t.Fatalf("answered under a different shape (err=%v); a moved feature space would go unnoticed", err)
+	if _, err := m.Under("sha256:a-tenth-feature-was-added"); !errors.Is(err, ErrShape) {
+		t.Fatalf("bound under a different shape (err=%v); a moved feature space would go unnoticed", err)
 	}
-	if _, err := m.P(0.5, ""); !errors.Is(err, ErrNoShape) {
-		t.Fatalf("answered with no shape stated (err=%v)", err)
+	if _, err := m.Under(""); !errors.Is(err, ErrNoShape) {
+		t.Fatalf("bound with no shape stated (err=%v)", err)
+	}
+	if _, err := (Map{}).Under(shape); !errors.Is(err, ErrNotFitted) {
+		t.Fatalf("bound an unfitted map (err=%v)", err)
 	}
 	if _, err := Fit(stream(1000, 3, func(s float64) float64 { return s }), Isotonic, ""); !errors.Is(err, ErrNoShape) {
 		t.Fatal("fitted a map that cannot refuse to be misapplied")
@@ -167,14 +171,11 @@ func TestPlattStaysInsideTheOpenInterval(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	hi, err := m.P(1, shape)
-	if err != nil {
-		t.Fatal(err)
-	}
-	lo, err := m.P(0, shape)
+	r, err := m.Under(shape)
 	if err != nil {
 		t.Fatal(err)
 	}
+	hi, lo := r.P(1), r.P(0)
 	if hi >= 1 || lo <= 0 {
 		t.Fatalf("the fit reached certainty: p(0)=%g p(1)=%g", lo, hi)
 	}
@@ -191,7 +192,11 @@ func TestReliabilityReportsAbsenceNotZero(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	bins := Reliability(stream(3000, 6, func(s float64) float64 { return 0.5 * s }), m, 10)
+	r, err := m.Under(shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	bins := Reliability(stream(3000, 6, func(s float64) float64 { return 0.5 * s }), r, 10)
 	if len(bins) != 10 {
 		t.Fatalf("%d bins, want 10", len(bins))
 	}
@@ -230,16 +235,17 @@ func TestCalibrationImprovesBrier(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	r, err := m.Under(shape)
+	if err != nil {
+		t.Fatal(err)
+	}
 	var fitted, raw float64
 	for _, s := range test {
 		y := 0.0
 		if s.Productive {
 			y = 1
 		}
-		p, err := m.P(s.Score, shape)
-		if err != nil {
-			t.Fatal(err)
-		}
+		p := r.P(s.Score)
 		fitted += (p - y) * (p - y)
 		raw += (s.Score - y) * (s.Score - y)
 	}
@@ -249,3 +255,130 @@ func TestCalibrationImprovesBrier(t *testing.T) {
 		t.Fatalf("calibrated Brier %.4f is no better than the raw score's %.4f", fitted, raw)
 	}
 }
+
+// THE TIE TEST, and the one that matters most in production. The scores this
+// package is asked to calibrate are not a continuum: a decision score is
+// 1 - prod(1-weight) over a FIXED set of rule weights, rounded to four places,
+// so thousands of decisions land on a handful of atoms and the modal atom is
+// exactly zero.
+//
+// Isotonic regression assigns ONE value per score, so every sample sharing a
+// score is one block before the algorithm starts. Fed sample by sample instead,
+// the run of unproductive-then-productive at a tied score is already
+// non-decreasing, nothing merges, and each plateau ends up reported as whichever
+// label sorted last — 1 at every plateau that holds a positive. The map then
+// says CERTAINTY at the top of the range, which is where declines happen.
+func TestIsotonicPoolsTiedScores(t *testing.T) {
+	atoms := []struct {
+		score        float64
+		clean, fraud int
+	}{
+		{0.00, 900, 2},  // 0.002217
+		{0.35, 190, 10}, // 0.05
+		{0.50, 85, 15},  // 0.15
+		{0.70, 45, 55},  // 0.55
+		{0.90, 10, 40},  // 0.80
+	}
+	var s []Sample
+	for _, a := range atoms {
+		for range a.clean {
+			s = append(s, Sample{Score: a.score})
+		}
+		for range a.fraud {
+			s = append(s, Sample{Score: a.score, Productive: true})
+		}
+	}
+	m, err := Fit(s, Isotonic, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r, err := m.Under(shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// The observed rates are already non-decreasing, so pooling alone is the
+	// whole fit and the map must reproduce them exactly. Nothing here is a
+	// tolerance on statistical noise: this is arithmetic.
+	for _, a := range atoms {
+		want := round6(float64(a.fraud) / float64(a.clean+a.fraud))
+		if got := r.P(a.score); math.Abs(got-want) > 1e-6 {
+			t.Errorf("score %.2f -> %.6f, want %.6f (%d productive of %d)",
+				a.score, got, want, a.fraud, a.clean+a.fraud)
+		}
+	}
+	// One knot per distinct score, no more: a plateau is one place on the map.
+	if len(m.Knots) != len(atoms) {
+		t.Errorf("%d knots over %d distinct scores; ties were not pooled into blocks",
+			len(m.Knots), len(atoms))
+	}
+}
+
+// A tie whose samples arrive in the other order is the same evidence and must
+// produce the same map. Pooling makes that true by construction; ordering the
+// sort by class makes it look true without being true.
+func TestTiedFitDoesNotDependOnLabelOrder(t *testing.T) {
+	build := func(positiveFirst bool) []Sample {
+		var s []Sample
+		for _, sc := range []float64{0.1, 0.4, 0.8} {
+			pos, neg := 20, 30
+			if sc == 0.8 {
+				pos, neg = 40, 10
+			}
+			if positiveFirst {
+				for range pos {
+					s = append(s, Sample{Score: sc, Productive: true})
+				}
+				for range neg {
+					s = append(s, Sample{Score: sc})
+				}
+				continue
+			}
+			for range neg {
+				s = append(s, Sample{Score: sc})
+			}
+			for range pos {
+				s = append(s, Sample{Score: sc, Productive: true})
+			}
+		}
+		return s
+	}
+	a, err := Fit(build(false), Isotonic, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, err := Fit(build(true), Isotonic, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if a.Digest != b.Digest {
+		t.Fatalf("the same tied evidence in two orders produced two maps: %.12s vs %.12s", a.Digest, b.Digest)
+	}
+	ra, _ := a.Under(shape)
+	if got := ra.P(0.1); math.Abs(got-0.4) > 1e-6 {
+		t.Errorf("p(0.1) = %.6f, want 0.4 (20 productive of 50)", got)
+	}
+}
+
+// A reader that cannot answer draws no chart. The reliability report IS the map
+// applied to every row, so shipping one beside a refusal states in a picture
+// exactly what the prose has just refused to state.
+func TestReliabilityRefusesWhenTheMapDoes(t *testing.T) {
+	m, err := Fit(stream(2000, 13, func(s float64) float64 { return s }), Isotonic, shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rows := stream(2000, 14, func(s float64) float64 { return s })
+	if _, err := m.Under("sha256:a-rule-was-written"); !errors.Is(err, ErrShape) {
+		t.Fatalf("bound under a moved shape: %v", err)
+	}
+	if bins := Reliability(rows, Reader{}, 10); len(bins) != 0 {
+		t.Errorf("a map that refuses to answer shipped %d reliability bins", len(bins))
+	}
+	good, err := m.Under(shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bins := Reliability(rows, good, 10); len(bins) != 10 {
+		t.Fatalf("the bound reader drew %d bins, want 10", len(bins))
+	}
+}

pkg/evaluate/evaluate.go

@@ -176,9 +176,12 @@ type Point struct {
 // policy over probabilities converts once, at the boundary, rather than making
 // every metric here depend on a map that may not exist.
 //
-// cal may be an unfitted zero Map: Brier is then absent and everything else is
-// unaffected, since every other metric here is a rank statistic.
-func Measure(obs []Observation, threshold float64, cost policy.Cost, cal calibrate.Map) Metrics {
+// cal may be the zero Reader — no calibration, or one whose shape has moved.
+// Brier is then absent and everything else is unaffected, since every other
+// metric here is a rank statistic. It is a READER and not a Map because the
+// question "does this map apply to these coordinates" is answered once, at the
+// boundary that knows the live shape, and never re-asked here.
+func Measure(obs []Observation, threshold float64, cost policy.Cost, cal calibrate.Reader) Metrics {
 	m := Metrics{Rows: len(obs)}
 	judged := make([]Observation, 0, len(obs))
 	for _, o := range obs {
@@ -236,10 +239,8 @@ func Measure(obs []Observation, threshold float64, cost policy.Cost, cal calibra
 			m.Refusal = "the stated prices cannot be multiplied by this many observations without overflowing, so no cost is reported"
 		}
 	}
-	if cal.Fitted() {
-		if b, ok := brier(judged, cal); ok {
-			m.Brier = ptr(round6(b))
-		}
+	if b, ok := brier(judged, cal); ok {
+		m.Brier = ptr(round6(b))
 	}
 	return m
 }
@@ -529,13 +530,13 @@ func averagePrecision(judged []Observation) float64 {
 // brier is the mean squared error of the calibration on these observations. It
 // is the out-of-sample number when the caller passes data the map was not fitted
 // on, which is the only version of it worth reporting.
-func brier(judged []Observation, cal calibrate.Map) (float64, bool) {
+func brier(judged []Observation, cal calibrate.Reader) (float64, bool) {
+	if !cal.Fitted() {
+		return 0, false
+	}
 	var sum, w float64
 	for _, o := range judged {
-		p, err := cal.P(o.Score, cal.Shape)
-		if err != nil {
-			return 0, false
-		}
+		p := cal.P(o.Score)
 		y := 0.0
 		if o.productive() {
 			y = 1

pkg/evaluate/evaluate_test.go

@@ -87,7 +87,7 @@ func TestMetricsAreOrderInvariant(t *testing.T) {
 // Absence is not zero. On a plane where most events are unjudged this is the
 // default state, and a rate reported as 0.0 reads as a perfect model.
 func TestUnjudgedReportsAbsenceNotZero(t *testing.T) {
-	m := Measure([]Observation{{Score: 0.9}, {Score: 0.1}}, 0.5, policy.Cost{}, calibrate.Map{})
+	m := Measure([]Observation{{Score: 0.9}, {Score: 0.1}}, 0.5, policy.Cost{}, calibrate.Reader{})
 	if m.Rows != 2 || m.Judged != 0 {
 		t.Fatalf("counted %d rows / %d judged", m.Rows, m.Judged)
 	}
@@ -100,7 +100,7 @@ func TestUnjudgedReportsAbsenceNotZero(t *testing.T) {
 
 	// One class only: prevalence is real, separation is not.
 	one := []Observation{obs(0.9, true), obs(0.8, true), obs(0.7, true)}
-	m = Measure(one, 0.5, policy.Cost{}, calibrate.Map{})
+	m = Measure(one, 0.5, policy.Cost{}, calibrate.Reader{})
 	if m.Prevalence == nil || *m.Prevalence != 1 {
 		t.Errorf("prevalence over one class is %v, want 1", m.Prevalence)
 	}
@@ -124,9 +124,9 @@ func TestCostWeightedOptimumMovesWithThePrices(t *testing.T) {
 
 	// A miss priced far above an alarm should push the threshold DOWN — flag
 	// more, catch more.
-	cheap := Measure(o, 0.5, policy.Cost{Miss: 1_000_000_000_000, Alarm: 1_000_000}, calibrate.Map{})
+	cheap := Measure(o, 0.5, policy.Cost{Miss: 1_000_000_000_000, Alarm: 1_000_000}, calibrate.Reader{})
 	// An alarm priced above a miss should push it UP — flag almost nothing.
-	dear := Measure(o, 0.5, policy.Cost{Miss: 1_000_000, Alarm: 1_000_000_000_000}, calibrate.Map{})
+	dear := Measure(o, 0.5, policy.Cost{Miss: 1_000_000, Alarm: 1_000_000_000_000}, calibrate.Reader{})
 
 	if cheap.BestThreshold == nil || dear.BestThreshold == nil {
 		t.Fatal("no cost-optimal threshold was found under stated prices")
@@ -149,7 +149,7 @@ func TestCostWeightedOptimumMovesWithThePrices(t *testing.T) {
 
 // An unstated price must produce no cost at all, not a cost of zero.
 func TestNoStatedPriceMeansNoCost(t *testing.T) {
-	m := Measure([]Observation{obs(0.9, true), obs(0.1, false)}, 0.5, policy.Cost{}, calibrate.Map{})
+	m := Measure([]Observation{obs(0.9, true), obs(0.1, false)}, 0.5, policy.Cost{}, calibrate.Reader{})
 	if m.CostNano != nil || m.BestNano != nil || m.BestThreshold != nil {
 		t.Errorf("priced a mistake nobody put a price on: %+v", m)
 	}
@@ -170,11 +170,15 @@ func TestCalibrationMovesBrierAndNotSeparation(t *testing.T) {
 		o = append(o, obs(s, productive))
 		samples = append(samples, calibrate.Sample{Score: s, Productive: productive})
 	}
-	cal, err := calibrate.Fit(samples, calibrate.Isotonic, shape)
+	m, err := calibrate.Fit(samples, calibrate.Isotonic, shape)
 	if err != nil {
 		t.Fatal(err)
 	}
-	raw := Measure(o, 0.5, policy.Cost{}, calibrate.Map{})
+	cal, err := m.Under(shape)
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw := Measure(o, 0.5, policy.Cost{}, calibrate.Reader{})
 	fitted := Measure(o, 0.5, policy.Cost{}, cal)
 
 	if *raw.ROC != *fitted.ROC || *raw.PR != *fitted.PR {
@@ -230,7 +234,7 @@ func TestAnUnrepresentablePriceIsAbsentAndNotWrapped(t *testing.T) {
 		obsv = append(obsv, obs(float64(i)/200, i%3 == 0))
 	}
 	huge := policy.Cost{Miss: math.MaxInt64 / 4, Alarm: math.MaxInt64 / 4}
-	m := Measure(obsv, 0.5, huge, calibrate.Map{})
+	m := Measure(obsv, 0.5, huge, calibrate.Reader{})
 	if m.CostNano != nil {
 		t.Errorf("a cost of %d was reported from a product that does not fit in an int64", *m.CostNano)
 	}

pkg/evaluate/history.go

@@ -151,11 +151,8 @@ const Listed = 500
 // not sorted here: sorting would silently repair a caller that handed rows in
 // the wrong order, and a replay over shuffled history is a different question
 // that would then be indistinguishable from this one.
-func Replay(history []Recorded, cal calibrate.Map, pol policy.Policy, rank func(string) int) Report {
-	r := Report{Rows: len(history), Policy: pol.Digest}
-	if cal.Fitted() {
-		r.Calibration = cal.Digest
-	}
+func Replay(history []Recorded, cal calibrate.Reader, pol policy.Policy, rank func(string) int) Report {
+	r := Report{Rows: len(history), Policy: pol.Digest, Calibration: cal.Digest()}
 	if len(history) == 0 {
 		r.Refusal = "no history to replay, so the result would be indistinguishable from a candidate that decides nothing"
 		r.Digest = fingerprint("evaluate.replay.v1", r.Refusal)
@@ -189,7 +186,8 @@ func Replay(history []Recorded, cal calibrate.Map, pol policy.Policy, rank func(
 
 		var prob *float64
 		action := pol.Floor
-		if p, err := cal.P(h.Score, cal.Shape); err == nil {
+		if cal.Fitted() {
+			p := cal.P(h.Score)
 			v := round6(p)
 			prob = &v
 			action = pol.Action(p)
@@ -247,7 +245,7 @@ func Replay(history []Recorded, cal calibrate.Map, pol policy.Policy, rank func(
 //
 // Returns a threshold above one when no score reaches p, which flags nothing —
 // the correct reading of a rung the candidate can never reach.
-func scoreAt(cal calibrate.Map, p float64) float64 {
+func scoreAt(cal calibrate.Reader, p float64) float64 {
 	if !cal.Fitted() {
 		// Without a calibration the sweep has no probability to invert; a
 		// threshold above every score flags nothing, which is what a policy with
@@ -255,7 +253,7 @@ func scoreAt(cal calibrate.Map, p float64) float64 {
 		return 1.0000001
 	}
 	lo, hi := 0.0, 1.0
-	if v, err := cal.P(hi, cal.Shape); err != nil || v < p {
+	if cal.P(hi) < p {
 		return 1.0000001
 	}
 	// 40 halvings resolves the unit interval to about 1e-12, far finer than the
@@ -263,11 +261,7 @@ func scoreAt(cal calibrate.Map, p float64) float64 {
 	// precision and the loop is bounded.
 	for range 40 {
 		mid := (lo + hi) / 2
-		v, err := cal.P(mid, cal.Shape)
-		if err != nil {
-			return 1.0000001
-		}
-		if v >= p {
+		if cal.P(mid) >= p {
 			hi = mid
 		} else {
 			lo = mid

pkg/evaluate/history_test.go

@@ -42,7 +42,7 @@ func candidate(t *testing.T) policy.Policy {
 
 // history builds a recorded stream whose truth is a known function of the score,
 // with the live system's own action recorded beside it.
-func history(n int, seed int64) ([]Recorded, calibrate.Map, error) {
+func history(n int, seed int64) ([]Recorded, calibrate.Reader, error) {
 	rng := rand.New(rand.NewSource(seed))
 	var h []Recorded
 	var samples []calibrate.Sample
@@ -69,7 +69,11 @@ func history(n int, seed int64) ([]Recorded, calibrate.Map, error) {
 		})
 		samples = append(samples, calibrate.Sample{Score: s, Productive: productive})
 	}
-	cal, err := calibrate.Fit(samples, calibrate.Isotonic, shape)
+	m, err := calibrate.Fit(samples, calibrate.Isotonic, shape)
+	if err != nil {
+		return h, calibrate.Reader{}, err
+	}
+	cal, err := m.Under(shape)
 	return h, cal, err
 }
 
@@ -154,7 +158,7 @@ func TestTalliesAreOrdered(t *testing.T) {
 // An empty history and a candidate that changes nothing render identically. The
 // difference is the whole reason a sandbox exists, so it must be stated.
 func TestReplayRefusesAnEmptyHistory(t *testing.T) {
-	r := Replay(nil, calibrate.Map{}, candidate(t), rank)
+	r := Replay(nil, calibrate.Reader{}, candidate(t), rank)
 	if r.Refusal == "" {
 		t.Fatal("an empty replay reported as if it had run")
 	}
@@ -172,7 +176,7 @@ func TestReplayWithoutCalibrationSaysWhyNothingActs(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r := Replay(h, calibrate.Map{}, candidate(t), rank)
+	r := Replay(h, calibrate.Reader{}, candidate(t), rank)
 	if r.Refusal == "" {
 		t.Fatal("every row took the floor and the report did not say why")
 	}

pkg/evaluate/learning.go

@@ -135,17 +135,29 @@ func Learning(history []Observation, method, shape string, steps int, cost polic
 			// so it is still measured and reported. A curve that went blank
 			// wherever the calibration refused would hide the model improving
 			// underneath it.
-			s.Train = Measure(train, 0, cost, calibrate.Map{})
-			s.Ahead = Measure(ahead, 0, cost, calibrate.Map{})
+			s.Train = Measure(train, 0, cost, calibrate.Reader{})
+			s.Ahead = Measure(ahead, 0, cost, calibrate.Reader{})
+			out = append(out, s)
+			continue
+		}
+		// The step's own fit is read under the shape it was just fitted under —
+		// the one place that identity is a fact rather than an assumption. A bind
+		// that refuses here is a fit that cannot describe its own sample, which is
+		// a refusal and not a step.
+		read, err := cal.Under(shape)
+		if err != nil {
+			s.Refusal = err.Error()
+			s.Train = Measure(train, 0, cost, calibrate.Reader{})
+			s.Ahead = Measure(ahead, 0, cost, calibrate.Reader{})
 			out = append(out, s)
 			continue
 		}
 		// The operating point is the score at the median fitted probability, so
 		// every step is measured at a comparable place on its own map rather than
 		// at a constant score that means something different at each step.
-		at := scoreAt(cal, 0.5)
-		s.Train = Measure(train, at, cost, cal)
-		s.Ahead = Measure(ahead, at, cost, cal)
+		at := scoreAt(read, 0.5)
+		s.Train = Measure(train, at, cost, read)
+		s.Ahead = Measure(ahead, at, cost, read)
 		out = append(out, s)
 	}
 	return out