Files
laya/tests/test_lang_guess.py
RishitandGitHub 1bee2d0732 fix(lang): identify plain-ASCII German instead of labelling it English (#130)
* fix(lang): identify plain-ASCII German instead of labelling it English

Short German requests without umlauts were labelled English and routed to
the English checkpoint (#54). The `de` stopword list held only 23 words and
missed most of the function words assistant traffic is made of (ich, wie,
mir, bitte, meinen, im, heute), while `in` and `was` counted for English
alone. German needs a two-hit margin over English, so a sentence like
"wie lautet die temperatur in fulda in hessen" scored de=1 against en=2 and
fell through to "en".

Add 38 high-frequency German function words that no other list claims, and
list `in` and `was` for German too so they no longer count as evidence for
English only.

test_lang_guess pinned "Mein Konto wurde zweimal belastet, bitte erstatten
Sie" to the English checkpoint; it now routes to multilingual.

MASSIVE 1.1 test split, share routed to the English checkpoint:
  de-DE 0.507 -> 0.247 (accents stripped: 0.753 -> 0.364)
  en-US 1.000 -> 1.000; 0 of 16,521 en-US utterances change label or route
  every other locale unchanged, except af-ZA (-0.013) and one stripped
  vi-VN state, which move to multilingual

German with no identified language and no umlauts still takes the router's
`default` (English unless configured); that case is a separate follow-up
agreed on #54.

* test(lang): keep es and du out of the German stopwords

`es` and `du` are German function words, but Spanish and French claim them
too, and a word two lists share names neither language. With them in the
German list, "que hora es en australia" and "baisse le volume du
haut-parleur" are left undecided and routed to the English checkpoint, and
the MASSIVE es-ES share routed to multilingual falls from 0.711 to 0.692
(0.488 to 0.447 with accents stripped; fr-FR 0.822 to 0.816).

Pin both so a later German addition cannot bring them back.

* test(lang): pin English that shares words with the German list

`in` and `was` now count for German as well as English, and `im` and `den`
are English tokens too. Pin two English states that must keep routing to
English: "turn off smart lamp in den", the en-US MASSIVE utterance nearest
to flipping (de=2 against en=1, three needed), and a chat line with `im`
that flips if any one of `am`, `an` or `so` joins the German list. Making
`in`/`was` German-only flips both.
2026-09-23 23:17:58 +05:30

183 lines
8.7 KiB
Python

"""Regression tests for the caller-supplied language hint (`lang_guess`).
A caller who already runs a language-identification model can hand routing the answer instead
of being silently misrouted by the built-in stopword heuristic:
analyse("Care este ora in Tokyo?")
# {'script': 'latin', 'language': 'en', 'is_english': True} -> routes to ENGLISH
The hint answers one question -- can the English checkpoint read this state -- so it is checked
after an explicit `lang` and before detection, and a hint that resolves to nothing falls
through to detection so a LID model can abstain.
Run: python tests/test_lang_guess.py
"""
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from laya.router import Router, _english_from_code # noqa: E402
PASS, FAIL = [], []
def check(name, got, want):
if got == want:
PASS.append(name)
else:
FAIL.append("%s:\n got %r\n want %r" % (name, got, want))
def check_true(name, cond, detail=""):
if cond:
PASS.append(name)
else:
FAIL.append("%s %s" % (name, detail))
# The state the maintainer used on #35: a short Romanian request the heuristic cannot place.
ROMANIAN = "Care este ora in Tokyo?"
GENERIC = {"intent": {"type": "choice", "instructions": "x", "criteria": ["a", "b"]}}
# ------------------------------------------------------------------ the baseline
r0 = Router()
check("baseline/Romanian is not identified by the heuristic",
r0.route(ROMANIAN, GENERIC)["detection"]["language"], "en")
# pinned, not "either checkpoint": this is the defect the hint exists to fix, so the
# assertion has to distinguish english from the correct answer to mean anything.
check("baseline/Romanian therefore reaches english",
r0.route(ROMANIAN, GENERIC)["model"], "english")
# ------------------------------------------------------------------ codes
check("code/Romanian routes multilingual", r0.route(ROMANIAN, GENERIC, lang_guess="ro")["model"],
"multilingual")
check("code/English routes english", r0.route(ROMANIAN, GENERIC, lang_guess="en")["model"], "english")
check("code/POSIX underscore is read", r0.route(ROMANIAN, GENERIC, lang_guess="en_US")["model"], "english")
check("code/POSIX with encoding is read",
r0.route(ROMANIAN, GENERIC, lang_guess="en_US.UTF-8")["model"], "english")
check("code/hyphen subtag is read", r0.route(ROMANIAN, GENERIC, lang_guess="de-DE")["model"], "multilingual")
check("code/case is ignored", r0.route(ROMANIAN, GENERIC, lang_guess="RO")["model"], "multilingual")
check("code/whitespace is ignored", r0.route(ROMANIAN, GENERIC, lang_guess=" en ")["model"], "english")
check("code/unknown code still means non-English",
r0.route(ROMANIAN, GENERIC, lang_guess="qq")["model"], "multilingual")
# an abstaining hint must fall through to detection, not force a checkpoint
for empty in (None, "", " "):
d = r0.route(ROMANIAN, GENERIC, lang_guess=empty)
check("abstain/%r falls through to detection" % (empty,), d["detection"] is not None, True)
# ------------------------------------------------------------------ callables
check("callable/code is used",
r0.route(ROMANIAN, GENERIC, lang_guess=lambda s: "ro")["model"], "multilingual")
check("callable/receives the state",
r0.route(ROMANIAN, GENERIC, lang_guess=lambda s: "en" if "Tokyo" in str(s) else "ro")["model"],
"english")
check("callable/None falls through to detection",
r0.route(ROMANIAN, GENERIC, lang_guess=lambda s: None)["detection"] is not None, True)
check("callable/empty string falls through",
r0.route(ROMANIAN, GENERIC, lang_guess=lambda s: "")["detection"] is not None, True)
check("callable/Romanian model that returns None does not change the default route",
r0.route(ROMANIAN, GENERIC, lang_guess=lambda s: None)["model"],
r0.route(ROMANIAN, GENERIC)["model"])
# ------------------------------------------------------------------ installed on the Router
r_inst = Router(lang_guess="ro")
check("installed/applies without a per-call hint", r_inst.route(ROMANIAN, GENERIC)["model"], "multilingual")
check("installed/per-call overrides the installed one",
r_inst.route(ROMANIAN, GENERIC, lang_guess="en")["model"], "english")
r_fn = Router(lang_guess=lambda s: "ro")
check("installed/callable works too", r_fn.route(ROMANIAN, GENERIC)["model"], "multilingual")
check("installed/absent by default", r0.lang_guess, None)
check("installed/a hint that abstains leaves detection intact",
Router(lang_guess=lambda s: None).route(ROMANIAN, GENERIC)["detection"] is not None, True)
# ------------------------------------------------------------------ precedence
check("precedence/explicit model beats the hint",
r0.route(ROMANIAN, GENERIC, model="english", lang_guess="ro")["model"], "english")
check("precedence/explicit task beats the hint",
r0.route(ROMANIAN, GENERIC, task="typed_decisions", lang_guess="ro")["model"], "typed-decisions")
check("precedence/explicit lang beats the hint",
r0.route(ROMANIAN, GENERIC, lang="en", lang_guess="ro")["model"], "english")
check_true("precedence/an explicit lang is still reported as explicit",
"explicit lang" in r0.route(ROMANIAN, GENERIC, lang="en", lang_guess="ro")["reason"])
# ------------------------------------------------------------------ the decision payload
d = r0.route(ROMANIAN, GENERIC, lang_guess="ro")
check("payload/model", d["model"], "multilingual")
check("payload/repo is a string", isinstance(d["repo"], str), True)
check("payload/reason records the caller hint", "lang_guess" in d["reason"], True)
check_true("payload/detection is None when the hint decided it", d["detection"] is None)
check("payload/installed hint names its source",
"Router(lang_guess=...)" in Router(lang_guess="ro").route(ROMANIAN, GENERIC)["reason"], True)
check_true("payload/repo points at the bundle", "convaiinnovations/laya" in d["repo"])
# ------------------------------------------------------------------ predict forwards it
class _FakeAgent:
"""Stands in for a loaded checkpoint so this suite stays offline and fast."""
def __init__(self):
self.calls = []
def system_one(self, state, questions):
self.calls.append((state, questions))
return {"answers": {}, "usage": {"input_tokens": 0, "output_tokens": 0}}
def _router_with_stub(**kwargs):
"""A Router whose `load` is stubbed, so `predict` exercises routing without weights."""
r = Router(**kwargs)
stub = _FakeAgent()
def _load(name):
r._agents[name] = stub
return stub
r.load = _load
return r, stub
r_p, stub = _router_with_stub(lang_guess="ro")
out = r_p.predict(ROMANIAN, GENERIC)
check("predict/uses the hint", out["routing"]["model"], "multilingual")
check("predict/forwards a per-call hint",
r_p.predict(ROMANIAN, GENERIC, lang_guess="en")["routing"]["model"], "english")
check("predict/reaches the model", len(stub.calls), 2)
check_true("predict/passes the state through unchanged", stub.calls[0][0] == ROMANIAN)
check_true("predict/keeps the routing block",
set(out["routing"]) >= {"model", "repo", "reason", "detection", "workflow"})
# ------------------------------------------------------------------ the helper itself
check("helper/None", _english_from_code(None), None)
check("helper/empty", _english_from_code(""), None)
check("helper/spaces", _english_from_code(" "), None)
check("helper/en", _english_from_code("en"), True)
check("helper/en_US", _english_from_code("en_US"), True)
check("helper/zh_CN", _english_from_code("zh_CN"), False)
check("helper/strips after the dot", _english_from_code("en.UTF-8"), True)
check("helper/only a dot", _english_from_code("."), None)
# the standalone-repo mapping is untouched
r_alone = Router(standalone_repos=True, lang_guess="ro")
check("standalone/hint still uses the standalone repo",
r_alone.route(ROMANIAN, GENERIC)["repo"], "convaiinnovations/laya-multilingual")
# nothing without a hint moves
# The expected model is pinned per case. Comparing `r0.route(...)` against a fresh
# `Router().route(...)` cannot fail, because both sides are the same pure call on an
# equally configured router -- a regression would move both together.
BEFORE = [("plain english", "I was charged twice and want a refund", "english"),
("German with umlauts", "Mein Konto wurde zweimal belastet, bitte erstatten Sie", "multilingual"),
("Hindi", "यह एक हिंदी वाक्य है", "multilingual"),
("empty", "", "english"),
("digits", "12345", "english")]
for label, s, want in BEFORE:
check("unchanged/" + label, r0.route(s, GENERIC)["model"], want)
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)
sys.exit(1 if FAIL else 0)