mirror of
https://github.com/hanzoai/laya.git
synced 2026-10-30 19:33:31 +00:00
* research: re-run the 51-language sweep in both temperature regimes (the #208 ask) The committed `cpu_51_language_sweep.json` stores `ece` and `mean_confidence` from before #42 clamped temperatures to `[0.5, 5]`, so its calibration columns no longer reproduce. This adds the re-run #208 asked for, with the old file left in place so the before/after is visible. `research/results/cpu_51_language_sweep_clamped.json` holds the committed columns, an `--unclamped` re-run and a default re-run side by side, per language, from the same 51 languages and 5,100 cases. committed unclamped re-run clamped re-run 0.2269 0.2269 0.2269 macro accuracy 0.7331 0.7331 0.5709 macro ECE 0.2053 0.2053 0.2053 macro F1 Macro accuracy reproduces at 0.2269 and the raw-temperature re-run reproduces the committed macro ECE at 0.7331, so the clamp is the only variable left. Per language the unclamped run agrees with the committed file on `accuracy` and `macro_f1` in 51/51, on `ece` in 48/51 and on `mean_confidence` in 49/51; the three that differ do so by 0.0001, the last stored digit, because the committed run used torch 2.8.0 and this one 2.14.0. The clamped run differs on `ece` and `mean_confidence` in 51/51, every one lower. `choice:11+` is the only bucket the clamp moves and every case here is a 20-option question, so it applies to all 5,100. `accuracy` and `macro_f1` cannot move with T at all: a temperature-scaled softmax has the same argmax at every positive temperature. Worth recording because it cuts against the reading that the clamp only distorts the published number: `acc_at_50_coverage` is the one rank-quality column that consumes the confidence values, and it goes up, macro 0.3004 -> 0.3020 and `en` 0.94 -> 0.98. The new file is guarded by `research/eval/test_laya_eval.py`, which fails if the committed columns stopped matching the unclamped re-run or if the clamped re-run started matching them. Deliberate choices tested: - `bench_local.py` was not modified. It produced the committed file, and rewriting it to reproduce its own pre-clamp output would make the old and new runs come from different scripts. `laya_eval.py` already has `--unclamped` for this, and the three columns it does not compute (`brier`, `nll`, `seconds`, `dropped`) are recorded in the new file rather than quietly dropped. - The clamped re-run is committed alongside rather than replacing anything, so `cpu_51_language_sweep.json` keeps the accuracy columns other tables cite. - `ci.yml` is untouched. #184 is adding the check that every model-free suite is wired into both lanes, and this suite is currently in neither; adding it here would collide with that PR and with the verbatim `test` job in #212. Known limitation: only the `english` checkpoint was re-run. The committed file's `part_b` covers the English checkpoint only, and the calibration columns #208 tracks are in `part_a`, so the multilingual half is untouched and unmeasured here. Verified: `python research/eval/test_laya_eval.py` -> 62 passed, 0 failed. * research: record the multilingual eliminations for the committed-row gap Three more candidates for the unreconciled multilingual row are now ruled out on re-measurement rather than by inspection: the shipped head_max_len (256, read from the checkpoint's own config and recorded in the run), which of the two multilingual copies was measured (bundled subfolder and standalone repo both give 0.4008 / 6-of-51), and a checkpoint change since the committed sweep (weights and config are identical by size and hash at every revision in that window). Also corrects the test count in the same file, which the new #208 guard changes. --------- Co-authored-by: PerryLink <[email protected]> Co-authored-by: NandhaKishorM <[email protected]>
251 lines
11 KiB
Python
251 lines
11 KiB
Python
"""Offline tests for the layered evaluation harness.
|
|
|
|
No checkpoint and no network: every function under test is pure, so this runs in
|
|
CI next to the other suites. The model-dependent paths (`score_cases`,
|
|
`run_language`) are exercised by `tests/test_local_e2e.py` when a checkpoint is
|
|
present.
|
|
|
|
Run: python research/eval/test_laya_eval.py
|
|
"""
|
|
import os
|
|
import random
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
|
|
|
from research.eval.laya_eval import ( # noqa: E402
|
|
ECE_BINS, INSTRUCTIONS, N_OPTS, SEED, build_case, build_suite, ece, macro_f1,
|
|
render_label, summarise, temperature_for,
|
|
)
|
|
|
|
PASS, FAIL = [], []
|
|
|
|
|
|
def check(name, got, want):
|
|
if got == want:
|
|
PASS.append(name)
|
|
else:
|
|
FAIL.append("%s:\n got %r\n want %r" % (name, got, want))
|
|
|
|
|
|
def check_true(name, cond, detail=""):
|
|
if cond:
|
|
PASS.append(name)
|
|
else:
|
|
FAIL.append("%s %s" % (name, detail))
|
|
|
|
|
|
# --------------------------------------------------------------- label rendering
|
|
check("render/underscores become spaces", render_label("alarm_set"), "alarm set")
|
|
check("render/dots become colon-space", render_label("news.»"), "news: »".replace("»", "»"))
|
|
check("render/query_definition", render_label("qa_definition"), "qa definition")
|
|
check("render/no underscores left", "_" in render_label("a_b_c"), False)
|
|
# must match bench_local.py:176 exactly
|
|
check("render/upstream parity", render_label("datetime_query"), "datetime query")
|
|
|
|
|
|
# ------------------------------------------------------------------ one case
|
|
_rng = random.Random(SEED)
|
|
_state, _q, _gold_idx = build_case("wake me up at five am", "alarm_set",
|
|
["a", "b", "c"], _rng, 4)
|
|
check("case/state key is utterance", list(_state), ["utterance"])
|
|
check("case/single question id", list(_q), ["intent"])
|
|
check("case/type is choice", _q["intent"]["type"], "choice")
|
|
check("case/instructions fixed", _q["intent"]["instructions"], INSTRUCTIONS)
|
|
check("case/option count", len(_q["intent"]["criteria"]), 4)
|
|
check("case/gold is present", "alarm_set" in _q["intent"]["criteria"], True)
|
|
check("case/gold index points at gold",
|
|
list(_q["intent"]["criteria"])[_gold_idx], "alarm_set")
|
|
|
|
|
|
# ------------------------------------------------- determinism and seed parity
|
|
ROWS = [{"text": "t%d" % i, "label_text": "lab%d" % (i % 7)} for i in range(30)]
|
|
LABELS = sorted({r["label_text"] for r in ROWS})
|
|
|
|
a, b, c = build_suite(ROWS, LABELS, 10, 5, SEED)
|
|
d, e, f = build_suite(ROWS, LABELS, 10, 5, SEED)
|
|
check("suite/same seed same cases", a == d, True)
|
|
check("suite/same seed same gold", b == e, True)
|
|
check("suite/same seed same options", c == f, True)
|
|
|
|
g, h, i = build_suite(ROWS, LABELS, 10, 5, SEED + 1)
|
|
check_true("suite/different seed different options", c != i)
|
|
|
|
# a fresh Random(SEED) per language is what upstream does; two languages with the
|
|
# same row content must therefore produce identical option sets
|
|
ROWS_B = [dict(r) for r in ROWS]
|
|
_b1 = build_suite(ROWS, LABELS, 10, 5, SEED)[2]
|
|
_b2 = build_suite(ROWS_B, LABELS, 10, 5, SEED)[2]
|
|
check("suite/rng reset per language", _b1, _b2)
|
|
|
|
check("suite/per_lang caps cases", len(a), 10)
|
|
check("suite/gold within options",
|
|
all(0 <= gi < len(opts) for gi, opts in zip(b, c)), True)
|
|
check("suite/every gold is in its option set",
|
|
all(opts[gi] == ROWS[i]["label_text"] for i, (gi, opts) in enumerate(zip(b, c))), True)
|
|
check("suite/options are distinct",
|
|
all(len(set(o)) == len(o) for o in c), True)
|
|
check("suite/n_opts respected", all(len(o) == 5 for o in c), True)
|
|
|
|
# more options requested than labels available: must not crash or duplicate
|
|
tiny = [{"text": "x", "label_text": "only"}]
|
|
_tc, _tg, _to = build_suite(tiny, ["only"], 1, 20, SEED)
|
|
check("suite/fewer labels than n_opts", len(_to[0]), 1)
|
|
|
|
|
|
# --------------------------------------------------------------------- metrics
|
|
import math # noqa: E402
|
|
|
|
check_true("ece/empty is nan", math.isnan(ece([], [])))
|
|
check("ece/perfect on one bin", round(ece([0.95] * 50, [1.0] * 50), 4), 0.05)
|
|
check("ece/all wrong and confident", round(ece([0.95] * 50, [0.0] * 50), 4), 0.95)
|
|
check("ece/normalised by count", round(ece([0.9] * 100, [1.0] * 100), 4),
|
|
round(ece([0.9] * 1000, [1.0] * 1000), 4))
|
|
check("ece/one sample", round(ece([1.0], [1.0]), 4), 0.0)
|
|
|
|
# Bin boundary: the first bin is closed at the bottom, so conf == 0.0 IS counted. This
|
|
# harness tested `conf > lo` for every bin until the divergence was found, which made it
|
|
# the only one of the four implementations that binned differently --
|
|
# `laya.common.ece_score`, `research/scripts/bench_local.py` and
|
|
# `research/scripts/build_benchmark_nb.py` all settled on this boundary in #39.
|
|
check("ece/conf==0.0 is binned (matches the other three)",
|
|
round(ece([0.0, 0.0], [1.0, 1.0]), 4), 1.0)
|
|
check("ece/conf==0.0 carries its bin weight",
|
|
round(ece([0.0, 1.0], [1.0, 1.0]), 4), 0.5)
|
|
check("ece/conf==0.0 and correct costs nothing",
|
|
round(ece([0.0, 0.0], [0.0, 0.0]), 4), 0.0)
|
|
check_true("ece/conf slightly above 0 IS binned",
|
|
ece([1e-9, 1e-9], [1.0, 1.0]) > 0.0)
|
|
|
|
check_true("f1/empty is nan", math.isnan(macro_f1([], [])))
|
|
check("f1/perfect", macro_f1([0, 1, 2], [0, 1, 2]), 1.0)
|
|
check("f1/all wrong", round(macro_f1([0, 1], [1, 0]), 4), 0.0)
|
|
check("f1/disjoint labels", round(macro_f1([0], [1]), 4), 0.0)
|
|
|
|
_s = summarise([0.9, 0.8, 0.7, 0.6], [1.0, 1.0, 0.0, 0.0], [0, 1, 2, 3], [0, 1, 3, 2])
|
|
check("summary/n", _s["n"], 4)
|
|
check("summary/accuracy", _s["accuracy"], 0.5)
|
|
check("summary/mean_confidence", _s["mean_confidence"], 0.75)
|
|
check("summary/acc_at_50_coverage takes the confident half",
|
|
_s["acc_at_50_coverage"], 1.0)
|
|
check("summary/empty", summarise([], [], [], []), {"n": 0})
|
|
|
|
|
|
# ------------------------------------------------------- temperature selection
|
|
class _FakeAgent:
|
|
temperature_by_options = {"choice:11+": 0.5}
|
|
temperature = [1.0, 1.0, 1.0]
|
|
temperature_by_options_raw = {"choice:11+": 0.10058280825614929}
|
|
temperature_raw = [1.0, 1.0, 1.0]
|
|
|
|
|
|
_fa = _FakeAgent()
|
|
from laya.common import QTYPES # noqa: E402
|
|
|
|
check("temp/clamped path is used by default",
|
|
temperature_for(_fa, QTYPES["choice"], 20), 0.5)
|
|
check("temp/raw path under unclamped",
|
|
round(temperature_for(_fa, QTYPES["choice"], 20, unclamped=True), 6), 0.100583)
|
|
check("temp/falls back to the per-type list",
|
|
temperature_for(_fa, QTYPES["choice"], 3), 1.0)
|
|
check("temp/raw falls back too",
|
|
temperature_for(_fa, QTYPES["choice"], 3, unclamped=True), 1.0)
|
|
|
|
|
|
# ------------------------------------------------------------------- constants
|
|
check("const/seed matches upstream", SEED, 13)
|
|
check("const/n_opts matches upstream", N_OPTS, 20)
|
|
check("const/bins", ECE_BINS, 15)
|
|
check("const/instructions match bench_local.py",
|
|
INSTRUCTIONS, "What is the user asking for in `utterance`?")
|
|
|
|
|
|
# ------------------------------------------- the #208 before/after re-run file
|
|
# research/results/cpu_51_language_sweep_clamped.json records the committed sweep,
|
|
# the pre-clamp re-run and the served-temperature re-run side by side. It is only
|
|
# useful if it still agrees with the committed file, so that agreement is a test.
|
|
import json # noqa: E402
|
|
|
|
_RESULTS = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
|
|
"research", "results")
|
|
|
|
|
|
def _load(name):
|
|
with open(os.path.join(_RESULTS, name), encoding="utf-8") as fh:
|
|
return json.load(fh)
|
|
|
|
|
|
_rerun = _load("cpu_51_language_sweep_clamped.json")
|
|
_sweep = _load("cpu_51_language_sweep.json")["part_a"]["by_model"]["english"]
|
|
_langs = _sweep["per_language"]
|
|
|
|
# Both files round each per-language figure to 4 decimals (bench_local.py:138-144), so
|
|
# agreement has to be judged at that resolution: two files can disagree by 1 in the
|
|
# last stored digit for reasons that have nothing to do with the temperatures, and
|
|
# three languages do (am, el, ro) because the committed run used torch 2.8.0 and this
|
|
# one 2.14.0. The macro figures match exactly, which is why that is the headline.
|
|
_QUANT = 1.5e-4
|
|
|
|
|
|
def _close(a, b, tol=_QUANT):
|
|
return abs(a - b) <= tol
|
|
|
|
check("208/one entry per language", len(_rerun["per_language"]), len(_langs))
|
|
check("208/case count is languages x per_lang",
|
|
_rerun["config"]["n_cases"],
|
|
_rerun["config"]["languages"] * _rerun["config"]["per_lang"])
|
|
check("208/macro accuracy copied from the committed sweep",
|
|
_rerun["macro"]["committed"]["accuracy"], _sweep["macro_accuracy"])
|
|
check("208/macro ece copied from the committed sweep",
|
|
_rerun["macro"]["committed"]["ece"], _sweep["macro_ece"])
|
|
|
|
# accuracy is argmax of a temperature-scaled softmax, so it cannot move with T
|
|
check_true("208/accuracy identical in all three regimes",
|
|
all(len({_rerun["per_language"][lg][r]["accuracy"] for r in
|
|
("committed", "unclamped_rerun", "clamped_rerun")}) == 1
|
|
for lg in _langs))
|
|
check_true("208/macro_f1 identical in all three regimes",
|
|
all(len({_rerun["per_language"][lg][r]["macro_f1"] for r in
|
|
("committed", "unclamped_rerun", "clamped_rerun")}) == 1
|
|
for lg in _langs))
|
|
|
|
# the committed calibration columns must match the unclamped re-run, and only the
|
|
# clamped re-run may differ -- that is the whole claim
|
|
check_true("208/ece matches the committed file in the unclamped re-run",
|
|
all(_close(_rerun["per_language"][lg]["unclamped_rerun"]["ece"],
|
|
_langs[lg]["ece"]) for lg in _langs))
|
|
check_true("208/ece differs from the committed file in the clamped re-run",
|
|
all(not _close(_rerun["per_language"][lg]["clamped_rerun"]["ece"],
|
|
_langs[lg]["ece"]) for lg in _langs))
|
|
check_true("208/mean_confidence matches in the unclamped re-run",
|
|
all(_close(_rerun["per_language"][lg]["unclamped_rerun"]["mean_confidence"],
|
|
_langs[lg]["mean_confidence"]) for lg in _langs))
|
|
check_true("208/mean_confidence differs in the clamped re-run",
|
|
all(not _close(_rerun["per_language"][lg]["clamped_rerun"]["mean_confidence"],
|
|
_langs[lg]["mean_confidence"]) for lg in _langs))
|
|
|
|
# the unclamped re-run is a reproduction, not an approximation: name the tolerance
|
|
# so a future change that widens it has to say so
|
|
check_true("208/unclamped reproduction agrees in at least 48 of 51 languages",
|
|
sum(1 for lg in _langs
|
|
if _close(_rerun["per_language"][lg]["unclamped_rerun"]["ece"], _langs[lg]["ece"])) >= 48)
|
|
|
|
# the clamp can lower ECE everywhere without lowering rank quality, so this is a
|
|
# guard against the misleading "the clamp makes it worse" reading
|
|
check_true("208/the clamp lowers macro ece",
|
|
_rerun["macro"]["clamped_rerun"]["macro_ece"]
|
|
< _rerun["macro"]["unclamped_rerun"]["macro_ece"])
|
|
check_true("208/every per-language delta is reported",
|
|
all("delta_ece" in v and "delta_mean_confidence" in v
|
|
for v in _rerun["per_language"].values()))
|
|
check("208/only choice:11+ is the clamped bucket",
|
|
_rerun["temperature_choice_11_plus"]["unclamped_rerun"], 0.10058280825614929)
|
|
check("208/the served bucket is 0.5",
|
|
_rerun["temperature_choice_11_plus"]["clamped_rerun"], 0.5)
|
|
|
|
|
|
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
|
|
for f in FAIL:
|
|
print(" FAIL " + f)
|
|
sys.exit(1 if FAIL else 0)
|