Files
laya/tests/test_training.py
5877415e3c fix: reject targets longer than their own marker count in mixed batches (#311) (#312)
collate_items checked len(target) against the batch-wide kmax, so a target
longer than its own item's options slipped through whenever a sibling row
was wider; the extra entries landed under masked options. Compare against
the item's own marker count and report it in the error. The TS collateItems
port had no guard at all and now raises the same message.

Co-authored-by: NandhaKishorM <[email protected]>
2026-09-24 08:46:04 +05:30

316 lines
15 KiB
Python

"""Training-side helpers: the scoring rules Laya is trained against, and the metrics around them.
`proper_reward` is the reward the model is optimised for, so its *properness* is a correctness
property, not a detail: if the reward were maximised by something other than the true
distribution, the calibration claims in the README would not hold. `td_lambda_targets` builds
bootstrapped targets for multi-turn episodes, and `ece_score` / `confidence_from_probs` are the
metrics used to report calibration.
No model weights are loaded, so this runs anywhere torch does.
"""
import math
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import numpy as np # noqa: E402
import torch # noqa: E402
from laya.common import ( # noqa: E402
QTYPES,
QTYPE_NAMES,
amp_dtype,
build_sequence,
collate_items,
confidence_from_probs,
ece_score,
proper_reward,
temp_bucket,
td_lambda_targets,
)
PASS, FAIL = [], []
def check(name, got, want):
if got == want:
PASS.append(name)
else:
FAIL.append("%s:\n got %r\n want %r" % (name, got, want))
def check_true(name, cond, detail=""):
if cond:
PASS.append(name)
else:
FAIL.append("%s %s" % (name, detail))
def close(name, got, want, tol=1e-6):
check_true(name, abs(got - want) <= tol, "got %r want %r" % (got, want))
def one_hot(i, k):
row = [0.0] * k
row[i] = 1.0
return row
# =============================================================== proper_reward
# The reward is a weighted sum of a log score and a spherical score, minus a ranked probability
# score for ordinal (`score`) questions only.
Q = torch.tensor([[0.7, 0.3]])
T = torch.tensor([[1.0, 0.0]])
M = torch.tensor([[1.0, 1.0]])
QT = torch.tensor([QTYPES["choice"]])
r_perfect = proper_reward(torch.tensor([[0.7, 0.3]]), T, QT, M).item()
r_wrong = proper_reward(torch.tensor([[0.3, 0.7]]), T, QT, M).item()
check_true("reward/perfect beats wrong", r_perfect > r_wrong, "%.4f vs %.4f" % (r_perfect, r_wrong))
# Expected reward under the target must be maximised at q == target (strict properness).
# E[r(q, y)] = sum_i p_i * r(q, one_hot(i)) -- the property that makes the reported
# probabilities meaningful.
def expected_reward(q_vec, target, qtype=QTYPES["choice"], w_sph=0.5, w_rps=1.0):
q = torch.tensor([q_vec], dtype=torch.float32)
qt = torch.tensor([qtype])
mask = torch.ones_like(q)
total = 0.0
for i, p in enumerate(target):
if p == 0:
continue
tgt = torch.tensor([one_hot(i, len(target))], dtype=torch.float32)
total += p * proper_reward(q, tgt, qt, mask, w_sph=w_sph, w_rps=w_rps).item()
return total
target = [0.7, 0.3]
grid = [0.5, 0.6, 0.7, 0.8, 0.9]
rewards = [(q, expected_reward([q, 1 - q], target)) for q in grid]
best = max(rewards, key=lambda kv: kv[1])[0]
close("reward/strictly proper: argmax at q == target (choice)", best, 0.7)
# same property for the ordinal (score) rule, where the RPS term also applies
rewards_score = [(q, expected_reward([q, 1 - q], target, qtype=QTYPES["score"])) for q in grid]
close("reward/strictly proper: argmax at q == target (score)",
max(rewards_score, key=lambda kv: kv[1])[0], 0.7)
# the RPS term is score-only: the same inputs must score differently for the two types
r_choice = proper_reward(Q, T, torch.tensor([QTYPES["choice"]]), M).item()
r_score = proper_reward(Q, T, torch.tensor([QTYPES["score"]]), M).item()
check_true("reward/rps applies to score but not choice", r_score < r_choice,
"score %.4f vs choice %.4f" % (r_score, r_choice))
# a masked-out option cannot influence the reward
r_masked_a = proper_reward(torch.tensor([[0.9, 0.1]]), T, QT, torch.tensor([[1.0, 0.0]])).item()
r_masked_b = proper_reward(torch.tensor([[0.9, 0.9]]), T, QT, torch.tensor([[1.0, 0.0]])).item()
close("reward/masked option is ignored", r_masked_a, r_masked_b)
# zero probability is clamped by log_floor rather than producing -inf
r_zero = proper_reward(torch.tensor([[1.0, 0.0]]), T, QT, M).item()
check_true("reward/zero probability is finite", math.isfinite(r_zero), "%r" % r_zero)
# permuting options consistently leaves the reward unchanged
perm = proper_reward(torch.tensor([[0.3, 0.7]]), torch.tensor([[0.0, 1.0]]), QT, M).item()
close("reward/invariant to consistent permutation", perm, proper_reward(Q, T, QT, M).item())
# the spherical weight is a real knob
hi = proper_reward(Q, T, QT, M, w_sph=2.0).item()
lo = proper_reward(Q, T, QT, M, w_sph=0.0).item()
check_true("reward/w_sph changes the reward", hi > lo, "%.4f vs %.4f" % (hi, lo))
# multi-row batches are handled independently
batch_r = proper_reward(torch.tensor([[0.7, 0.3], [0.1, 0.9]]),
torch.tensor([[1.0, 0.0], [0.0, 1.0]]),
torch.tensor([QTYPES["choice"], QTYPES["choice"]]),
torch.ones(2, 2))
check("reward/batch shape", tuple(batch_r.shape), (2,))
# =============================================================== td_lambda_targets
p_true = torch.tensor([0.1, 0.4, 0.9])
# without episode grouping the targets are returned untouched
batch = {"target": torch.tensor([[1.0, 0.0], [1.0, 0.0], [1.0, 0.0]])}
out = td_lambda_targets(p_true, batch, lam=1.0)
check_true("td_lambda/no groups -> unchanged targets",
torch.equal(out, batch["target"]))
# grouped: rows are ordered by ep_step, and the terminal row's truth bootstraps backwards
batch = {"target": torch.tensor([[1.0, 0.0], [1.0, 0.0], [1.0, 0.0]]),
"ep_group": torch.tensor([0, 0, 0]),
"ep_step": torch.tensor([0, 1, 2])}
lam1 = td_lambda_targets(p_true, {"target": batch["target"].clone(),
"ep_group": batch["ep_group"],
"ep_step": batch["ep_step"]}, lam=1.0)
close("td_lambda/lam=1 propagates terminal truth to every step", float(lam1[0, 1]), 0.0)
close("td_lambda/lam=1 terminal step unchanged", float(lam1[2, 1]), 0.0)
check_true("td_lambda/lam=1 all steps equal the terminal target",
torch.allclose(lam1[:, 0], 1 - lam1[:, 1]))
lam0 = td_lambda_targets(p_true, {"target": batch["target"].clone(),
"ep_group": batch["ep_group"],
"ep_step": batch["ep_step"]}, lam=0.0)
close("td_lambda/lam=0 step0 uses the next step's p_true", float(lam0[0, 1]), 0.4)
close("td_lambda/lam=0 step1 uses the next step's p_true", float(lam0[1, 1]), 0.9)
close("td_lambda/lam=0 terminal keeps the terminal target", float(lam0[2, 1]), 0.0)
# row order must not matter: steps are sorted by ep_step inside each group, and `p_true` is
# indexed by row, so the same logical episode written out of step order must agree per step.
shuffled = td_lambda_targets(torch.tensor([0.9, 0.1, 0.4]), # rows are [step2, step0, step1]
{"target": torch.tensor([[1.0, 0.0]] * 3),
"ep_group": torch.tensor([0, 0, 0]),
"ep_step": torch.tensor([2, 0, 1])}, lam=0.0)
ordered = td_lambda_targets(torch.tensor([0.1, 0.4, 0.9]), # the same episode in step order
{"target": torch.tensor([[1.0, 0.0]] * 3),
"ep_group": torch.tensor([0, 0, 0]),
"ep_step": torch.tensor([0, 1, 2])}, lam=0.0)
check_true("td_lambda/rows are processed in ep_step order",
float(ordered[0, 1]) == float(shuffled[1, 1]) and
float(ordered[1, 1]) == float(shuffled[2, 1]) and
float(ordered[2, 1]) == float(shuffled[0, 1]),
"ordered=%s shuffled=%s" % (ordered[:, 1].tolist(), shuffled[:, 1].tolist()))
# groups are independent: each episode bootstraps from its own terminal row
two = td_lambda_targets(torch.tensor([0.1, 0.2, 0.8, 0.9]),
{"target": torch.tensor([[1.0, 0.0], [0.2, 0.8], [1.0, 0.0], [0.6, 0.4]]),
"ep_group": torch.tensor([0, 0, 1, 1]),
"ep_step": torch.tensor([0, 1, 0, 1])}, lam=1.0)
check("td_lambda/group 0 uses its own terminal", [round(float(v), 4) for v in two[0:2, 1]], [0.8, 0.8])
check("td_lambda/group 1 uses its own terminal", [round(float(v), 4) for v in two[2:4, 1]], [0.4, 0.4])
# =============================================================== ece_score
# perfectly calibrated: 90% confident, right 90% of the time
conf = np.full(100, 0.9)
correct = np.array([1] * 90 + [0] * 10)
close("ece/perfectly calibrated is ~0", ece_score(conf, correct), 0.0, tol=1e-9)
# maximally overconfident: certain and always wrong
close("ece/certain and wrong is 1.0", ece_score(np.ones(10), np.zeros(10, dtype=int)), 1.0, tol=1e-9)
# empty input is nan rather than an exception
check_true("ece/empty is nan", math.isnan(ece_score(np.array([]), np.array([]))))
# overconfidence costs: same accuracy, higher confidence -> higher ECE
base_correct = np.array([1] * 80 + [0] * 20)
conf_a = np.full(100, 0.8) # calibrated
conf_b = np.full(100, 0.99) # overconfident
check_true("ece/overconfidence raises ECE", ece_score(conf_b, base_correct) > ece_score(conf_a, base_correct),
"%.4f vs %.4f" % (ece_score(conf_b, base_correct), ece_score(conf_a, base_correct)))
# bins argument changes granularity, not correctness of the perfect case
close("ece/bins=5 perfectly calibrated", ece_score(conf, correct, bins=5), 0.0, tol=1e-9)
# =============================================================== confidence_from_probs
close("confidence/uniform over 2 is 0", confidence_from_probs(np.array([0.5, 0.5]), 2), 0.0)
close("confidence/uniform over 5 is 0", confidence_from_probs(np.full(5, 0.2), 5), 0.0)
close("confidence/one-hot is 1", confidence_from_probs(np.array([1.0, 0.0]), 2), 1.0)
close("confidence/single option is 1 by definition", confidence_from_probs(np.array([1.0]), 1), 1.0)
spread = confidence_from_probs(np.array([0.6, 0.3, 0.1]), 3)
peaked = confidence_from_probs(np.array([0.9, 0.05, 0.05]), 3)
check_true("confidence/decreases with entropy", peaked > spread, "%.4f vs %.4f" % (peaked, spread))
close("confidence/only the first k entries count", confidence_from_probs(np.array([0.5, 0.5, 0.0]), 2), 0.0)
# =============================================================== small helpers
check("qtypes/choice", QTYPES["choice"], 0)
check("qtypes/score", QTYPES["score"], 1)
check("qtypes/noul", QTYPES["noul"], 2)
check("qtypes/names round-trip", QTYPE_NAMES[QTYPES["noul"]], "noul")
check("temp_bucket/2 options is noul:2", temp_bucket(QTYPES["noul"], 2), "noul:2")
check("temp_bucket/3-5", temp_bucket(QTYPES["choice"], 4), "choice:3-5")
check("temp_bucket/6-10", temp_bucket(QTYPES["choice"], 10), "choice:6-10")
check("temp_bucket/11+", temp_bucket(QTYPES["choice"], 77), "choice:11+")
check("temp_bucket/score bucket", temp_bucket(QTYPES["score"], 3), "score:3-5")
check("amp_dtype/bf16", amp_dtype("bf16"), torch.bfloat16)
check("amp_dtype/anything else is fp16", amp_dtype(None), torch.float16)
# =============================================================== collate_items
items = [{"ids": [1, 2, 3], "markers": [1, 2], "qtype": 0, "label": 1},
{"ids": [4, 5], "markers": [1], "qtype": 2, "label": -1}]
b = collate_items([items], pad_id=0)
check("collate/batch size", tuple(b["input_ids"].shape), (2, 3))
check("collate/marker columns = longest marker list", tuple(b["marker_pos"].shape), (2, 2))
check("collate/padding filled with pad_id", b["input_ids"][1].tolist(), [4, 5, 0])
check("collate/attention mask marks real tokens", b["attention_mask"][1].tolist(), [1, 1, 0])
check("collate/marker mask marks real markers", b["marker_mask"][1].tolist(), [True, False])
check("collate/qtype carried", b["qtype"].tolist(), [0, 2])
check("collate/no target unless requested", "target" in b, False)
b2 = collate_items([[{"ids": [1], "markers": [0, 1], "qtype": 1, "target": [0.0, 1.0]}]], pad_id=0)
check("collate/target included when present", tuple(b2["target"].shape), (1, 2))
check("collate/empty batch returns None", collate_items([], pad_id=0), None)
# a target with more entries than the item has options is a data bug; it must say so, not
# surface as a tensor shape error from inside the assignment
try:
collate_items([[{"ids": [1], "markers": [0], "qtype": 1, "target": [0.0, 1.0]}]], pad_id=0)
FAIL.append("collate/oversized target raises ValueError (nothing raised)")
except ValueError as e:
check_true("collate/oversized target raises a clear ValueError",
"one entry per option" in str(e), str(e)[:90])
except Exception as e: # noqa: BLE001
FAIL.append("collate/oversized target raised %s instead of ValueError: %s" % (type(e).__name__, e))
# #311: in a mixed-width batch the limit is each item's own marker count, not the batch-wide max
try:
collate_items([[
{"ids": [1], "markers": [0, 1], "qtype": 1, "target": [0.2, 0.3, 0.5]},
{"ids": [2], "markers": [0, 1, 2, 3], "qtype": 1, "target": [0.25, 0.25, 0.25, 0.25]},
]], pad_id=0)
FAIL.append("collate/mixed-width oversized target raises ValueError (nothing raised)")
except ValueError as e:
check_true("collate/mixed-width oversized target raises a clear ValueError",
"one entry per option" in str(e), str(e)[:90])
except Exception as e: # noqa: BLE001
FAIL.append("collate/mixed-width oversized target raised %s instead of ValueError: %s"
% (type(e).__name__, e))
# a target shorter than the item's own markers is still padded with zeros, not rejected
b3 = collate_items([[{"ids": [1], "markers": [0, 1, 2], "qtype": 1, "target": [1.0]}]], pad_id=0)
check("collate/short target padded with zeros", b3["target"][0].tolist(), [1.0, 0.0, 0.0])
# =============================================================== build_sequence
class _Tok:
"""Minimal stand-in for a fast tokenizer: one id per character."""
cls_token_id, sep_token_id, mask_token_id, pad_token_id = 1, 2, 3, 0
mask_token = "[M]"
def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):
ids = [10 + (ord(c) % 50) for c in text]
if truncation and max_length:
ids = ids[:max_length]
return {"input_ids": ids}
tok = _Tok()
q = {"t": "choice", "ins": "Which team?", "crit": {"a": "first", "b": "second", "c": "third"}}
seq, markers = build_sequence(tok, "some state text", q, max_len=128, head_max_len=64)
check("build_sequence/one marker per option", len(markers), 3)
check_true("build_sequence/markers point inside the sequence", all(0 <= m < len(seq) for m in markers))
check_true("build_sequence/starts with CLS", seq[0] == tok.cls_token_id)
check_true("build_sequence/ends with SEP", seq[-1] == tok.sep_token_id)
check_true("build_sequence/respects max_len", len(seq) <= 128)
long_seq, long_markers = build_sequence(tok, "x" * 5000, q, max_len=64, head_max_len=32)
check_true("build_sequence/truncates long state to max_len", len(long_seq) <= 64, "len=%d" % len(long_seq))
noul_seq, noul_markers = build_sequence(tok, "state", {"t": "noul", "ins": "Is it?", "crit": None}, 128, 64)
check("build_sequence/noul always offers two options", len(noul_markers), 2)
score_seq, score_markers = build_sequence(tok, "state", {"t": "score", "ins": "How bad?",
"crit": ["low", "mid", "high"]}, 128, 64)
check("build_sequence/score has one marker per level", len(score_markers), 3)
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)
sys.exit(1 if FAIL else 0)