Files
laya/tests/test_batch.py
50df9bb3da feat(agent): report the calibrated confidence alongside the existing one (#126)
`confidence` carries two definitions in one response payload:

    choice / score  ->  1 - H(p) / log(k)          (normalized entropy)
    noul            ->  max(p_true, 1 - p_true)    == max(p)

They are not on the same scale. The same two-option distribution comes back as
0.90 from a `noul` and 0.53 from an equivalent two-option `choice`. Every shipped
preset mixes the types in a single call -- `moderation_questions()` is four `noul`
and one `score` -- and the README's "Automated Confidence Gating" section gates all
of them on one threshold, so at P(true)=0.90 the same evidence is automated as a
noul and escalated as a choice.

Only max(p) has the property that section relies on. Both benchmark harnesses take
`conf = max(probs)` before calling `ece_score` (research/scripts/bench_local.py,
build_benchmark_nb.py), so every ECE figure in the README describes max(p), and
temperature scaling fits the same quantity. Normalized entropy carries no such
guarantee: on perfectly calibrated synthetic data, ECE on max(p) is 0.0111 while
ECE on normalized entropy is 0.3682.

This is additive. `confidence` is untouched on every question type, so nothing a
caller gates on today moves. A new `answer_confidence` reports max(p) on all three
types, so a caller can gate across types on one number now, and `answer_confidence`
is exported next to `confidence_from_probs` with each docstring saying what it does
and does not promise.

It also leaves a clean path for changing `confidence` itself in a deliberate
release: callers migrate to `answer_confidence` first, and the later switch is then
a no-op for them.

tests/test_confidence.py covers it without loading weights: max(p) behaviour, the
new field agreeing with the shipped noul value across the range, the threshold
disagreement between the two scales, and an end-to-end calibration check showing
ECE near zero on max(p) and far from it on entropy.

Co-authored-by: NandhaKishorM <[email protected]>
2026-09-24 08:47:36 +05:30

279 lines
12 KiB
Python

"""Batch inference: predict_batch packs many states into shared forward passes.
These tests need no model weights. They cover:
1. Contract — predict_batch exists and system_one is defined in terms of it, so the single-state
and batched paths can never numerically drift apart.
2. Orchestration — empty input, bad input, per-state row mapping, and batch_size chunking, all
exercised against a fake whose forward is stubbed. The numerical equivalence of the real
model path (predict_batch(states) == [system_one(s) for s in states]) is checked with weights
in tests/test_local_e2e.py.
3. Decoding — calibrated answer values and confidence, including mixed question types.
"""
import inspect
import os
import sys
from unittest.mock import patch
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import numpy as np # noqa: E402
from laya import agent as _agent # noqa: E402
from laya.agent import Agent # noqa: E402
PASS, FAIL = [], []
def check(name, got, want):
if got == want:
PASS.append(name)
else:
FAIL.append("%s:\n got %r\n want %r" % (name, got, want))
def check_true(name, cond, detail=""):
if cond:
PASS.append(name)
else:
FAIL.append("%s %s" % (name, detail))
def check_raises(name, exc, fn):
try:
fn()
except exc:
PASS.append(name)
except Exception as e: # noqa: BLE001
FAIL.append("%s: raised %r, expected %s" % (name, e, exc.__name__))
else:
FAIL.append("%s: did not raise %s" % (name, exc.__name__))
# --------------------------------------------------------------- contract (source inspection)
check_true("contract/predict_batch is defined", callable(getattr(Agent, "predict_batch", None)))
_so = inspect.getsource(Agent.system_one)
check_true("contract/system_one delegates to predict_batch", "self.predict_batch(" in _so)
check_true("contract/predict alias preserved", Agent.predict is Agent.system_one)
_sig = inspect.signature(Agent.predict_batch)
check_true("contract/predict_batch has batch_size kwarg", "batch_size" in _sig.parameters)
# --------------------------------------------------------------- orchestration (weight-free fake)
# We drive the real predict_batch method but stub the three helpers it composes, so no encoder,
# tokenizer vocabulary, or weights are needed. collate_items runs for real on the fake items.
NQ = 2
QUESTIONS = {"a": {"type": "noul", "instructions": "?"}, "b": {"type": "noul", "instructions": "?"}}
def make_fake():
fake = _agent.Agent.__new__(_agent.Agent)
fake.tok = type("Tok", (), {"pad_token_id": 0})()
fake._to_internal = staticmethod(Agent._to_internal).__func__ # reuse the real normalizer
fake._forward_calls = []
def _encode_state(state, ids, internal):
# one 3-token, 2-marker item per question; content is irrelevant to the mapping test
return [{"ids": [1, 2, 3], "markers": [0, 1], "qtype": 2} for _ in ids]
def _forward(b):
n = b["input_ids"].shape[0]
fake._forward_calls.append(n)
# deterministic logits so decode is stable; act pre-softmaxed as _forward would return it
return np.zeros((n, 2), dtype=np.float32), np.full((n, 2), 0.5, dtype=np.float32)
def _decode_answers(logits, act, items, ids, internal, offset):
# record the row offset this state was decoded from, to verify per-state alignment
return {"_offset": offset}
fake._encode_state = _encode_state
fake._forward = _forward
fake._decode_answers = _decode_answers
return fake
# --------------------------------------------------------------- length grouping and restoration
def make_length_fake():
fake = make_fake()
fake.encoded = []
fake.batch_rows = []
fake.slots = 0
fake.encoded_at_forward = []
def encode(state, ids, internal, **overrides):
fake.encoded.append(state)
limit = overrides.get("max_len", 100)
# Question rows have different lengths; sorting must use the longest.
return [{"ids": [state["id"] + 1] * min(limit, size), "markers": [0, 1], "qtype": 2}
for size in (3, state["length"])]
def forward(batch):
fake.slots += batch["input_ids"].numel()
fake.batch_rows.append(batch["input_ids"][:, 0].tolist())
fake.encoded_at_forward.append(len(fake.encoded))
return batch["input_ids"][:, :1].numpy(), None
def decode(logits, act, items, ids, internal, offset):
return {qid: int(logits[offset + i, 0]) - 1 for i, qid in enumerate(ids)}
fake._encode_state = encode
fake._forward = forward
fake._decode_answers = decode
return fake
states = [{"id": i, "length": [40, 5, 25, 8][i % 4]} for i in range(37)]
plain, grouped = make_length_fake(), make_length_fake()
expected = plain.predict_batch(states, QUESTIONS, batch_size=2)
events = []
actual = grouped.predict_batch(states, QUESTIONS, batch_size=2, sort_by_length=True,
on_predict_end=lambda ctx: events.append((ctx.results, ctx.usage)))
check("length/answers and usage retain input order across windows", actual, expected)
check("length/each state encoded once in input order", grouped.encoded, states)
check_true("length/grouping reduces padded slots", grouped.slots < plain.slots)
check("length/same number of forward passes", len(grouped.batch_rows), len(plain.batch_rows))
check_true("length/batch size caps all forward passes", all(len(rows) <= 4 for rows in grouped.batch_rows))
check("length/partial final batch", len(grouped.batch_rows[-1]), 2)
check("length/first forward buffers at most eight batches", grouped.encoded_at_forward[0], 16)
check("length/end hook sees restored results", events[0][0], expected)
check("length/end hook usage excludes padding", events[0][1]["input_tokens"],
sum(3 + st["length"] for st in states))
check("length/does not mutate caller states", [st["id"] for st in states], list(range(37)))
# A hook may replace both inputs and the length budget before grouping occurs.
def rewrite_length_inputs(ctx):
ctx.states = states
ctx.max_len = 10
rewritten = make_length_fake().predict_batch(["unused"], QUESTIONS, batch_size=2,
sort_by_length=True, on_predict_start=rewrite_length_inputs)
reference = make_length_fake().predict_batch(states, QUESTIONS, batch_size=2, max_len=10)
check("length/hook rewrites and truncation budgets precede sorting", rewritten, reference)
for batch_size in (None, 0, -1, 1, 100):
plain, grouped = make_length_fake(), make_length_fake()
check("length/no-op results for batch_size=%r" % batch_size,
grouped.predict_batch(states, QUESTIONS, batch_size=batch_size, sort_by_length=True),
plain.predict_batch(states, QUESTIONS, batch_size=batch_size))
check("length/no-op grouping for batch_size=%r" % batch_size, grouped.batch_rows, plain.batch_rows)
f = make_length_fake()
check("length/empty inputs", f.predict_batch([], QUESTIONS, sort_by_length=True), [])
check("length/empty questions", f.predict_batch(states[:1], {}, sort_by_length=True)[0]["answers"], {})
check("length/empty cases do not encode", f.encoded, [])
f.predict_batch(states, QUESTIONS, sort_by_length=True, on_predict_start=lambda ctx: ctx.skip([]))
check("length/cache skip does not encode", f.encoded, [])
plain, grouped = make_length_fake(), make_length_fake()
equal_lengths = [{"id": i, "length": 10} for i in range(20)]
plain.predict_batch(equal_lengths, QUESTIONS, batch_size=3)
grouped.predict_batch(equal_lengths, QUESTIONS, batch_size=3, sort_by_length=True)
check("length/equal length ties preserve row order", grouped.batch_rows, plain.batch_rows)
errors = []
class RecordFailure:
def on_error(self, ctx):
errors.append(ctx.error)
def fail_forward(batch):
raise RuntimeError("forward failed")
f = make_length_fake()
f._forward = fail_forward
check_raises("length/forward error propagates", RuntimeError,
lambda: f.predict_batch(states, QUESTIONS, batch_size=2,
sort_by_length=True, hooks=[RecordFailure()]))
check("length/error hook receives failure once", [str(e) for e in errors], ["forward failed"])
f = make_fake()
check("orchestration/empty states -> empty list", f.predict_batch([], QUESTIONS), [])
f = make_fake()
res = f.predict_batch(["s0", "s1", "s2"], QUESTIONS)
check("orchestration/one result per state", len(res), 3)
check("orchestration/state 0 decoded from row 0", res[0]["answers"]["_offset"], 0)
check("orchestration/state 1 decoded from row NQ", res[1]["answers"]["_offset"], NQ)
check("orchestration/state 2 decoded from row 2*NQ", res[2]["answers"]["_offset"], 2 * NQ)
check_true("orchestration/result shape matches system_one",
all(set(r) == {"model", "answers", "usage"} for r in res))
check("orchestration/single forward pass by default", f._forward_calls, [6]) # 3 states x 2 questions
f = make_fake()
f.predict_batch(["s0", "s1", "s2"], QUESTIONS, batch_size=1)
check("orchestration/batch_size=1 chunks into three passes", f._forward_calls, [2, 2, 2])
f = make_fake()
f.predict_batch(["s0", "s1", "s2", "s3", "s4"], QUESTIONS, batch_size=2)
check("orchestration/batch_size=2 chunks 5 states as 2+2+1", f._forward_calls, [4, 4, 2])
f = make_fake()
check_raises("orchestration/bare string rejected", TypeError, lambda: f.predict_batch("just a string", QUESTIONS))
f = make_fake()
check_raises("orchestration/bare dict rejected", TypeError, lambda: f.predict_batch({"body": "x"}, QUESTIONS))
# --------------------------------------------------------------- real answer decoding (no weights)
# Exercise the real decoder with a row offset and distinct per-type/bucket temperatures.
# Noul uses top probability for confidence; its unused entropy calculation can be skipped.
decoder = Agent.__new__(Agent)
decoder.temperature = {0: 1.0, 1: 2.0, 2: 3.0}
decoder.temperature_by_options = {"choice:2": 0.5}
decode_ids = ["pick", "level", "flag"]
decode_internal = {
"pick": {"t": "choice", "crit": {"left": "left", "right": "right"}},
"level": {"t": "score", "crit": ["low", "high"]},
"flag": {"t": "noul", "crit": None},
}
decode_items = [{"markers": [0, 1]} for _ in decode_ids]
decode_logits = np.array([
[99.0, -99.0], # preceding state: must not be decoded
np.log([0.25, 0.75]) * 0.5,
np.log([0.25, 0.75]) * 2.0,
np.log([0.8, 0.2]) * 3.0,
])
decode_act = np.array([[0.0, 1.0], [0.125, 0.875], [0.25, 0.75], [0.75, 0.25]])
with patch.object(_agent, "confidence_from_probs", wraps=_agent.confidence_from_probs) as entropy:
decoded = decoder._decode_answers(
decode_logits, decode_act, decode_items, decode_ids, decode_internal, 1
)
check("decode/mixed answers retain calibrated values", decoded, {
"pick": {"type": "choice", "choice": "right",
"probabilities": {"left": 0.25, "right": 0.75}, "confidence": 0.1887,
"answer_confidence": 0.75,
"action": {"act_probability": 0.125}},
"level": {"type": "score", "score": 0.75, "legend": {"0": "low", "1": "high"},
"probabilities": {"0": 0.25, "1": 0.75}, "confidence": 0.1887,
"answer_confidence": 0.75,
"action": {"act_probability": 0.25}},
"flag": {"type": "noul", "noul": 0.2, "confidence": 0.8,
# max(p) over two options is max(p_true, 1 - p_true): the same number
"answer_confidence": 0.8,
"action": {"act_probability": 0.75}},
})
check("decode/entropy only computed for choice and score", entropy.call_count, 2)
entropy.reset_mock()
for true_probability in (0.1, 0.5, 0.9):
logits = np.log([[1.0 - true_probability, true_probability]]) * 3.0
result = decoder._decode_answers(
logits, decode_act[-1:], decode_items[-1:], ["flag"], decode_internal, 0
)
check("decode/noul confidence at p=%s" % true_probability, result["flag"], {
"type": "noul", "noul": true_probability,
"confidence": max(true_probability, 1.0 - true_probability),
"answer_confidence": max(true_probability, 1.0 - true_probability),
"action": {"act_probability": 0.75},
})
check("decode/noul-only answers skip entropy", entropy.call_count, 0)
# --------------------------------------------------------------------------- report
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f_ in FAIL:
print(" FAIL " + f_)
sys.exit(1 if FAIL else 0)