mirror of
https://github.com/hanzoai/laya.git
synced 2026-10-30 19:33:31 +00:00
`Router.predict` forwards the language of the request to the agent so that per-language
temperatures can be applied; the batched path forwarded only the token budgets. The same request
therefore scored differently depending on which entry point served it, while being routed
identically — the routed checkpoint was never the difference, which is what made it hard to see.
```python
# predict, router.py:532-534 and :551
effective_lang = lang
if effective_lang is None and decision.get("detection") and decision["detection"].get("language"):
effective_lang = decision["detection"]["language"]
...
result = agent.system_one(ctx.states[0], ctx.questions, lang=effective_lang, **overrides)
# predict_batch, before
batch_results = agent.predict_batch([...], group["questions"], batch_size=batch_size,
**group["overrides"]) # no lang
```
Measured on an agent carrying `lang_temperatures={"de": ...}`, one identical request:
```
Router.predict : noul 0.572
Router.predict_batch : noul 0.608
same request, same route, answers equal: False
Agent.system_one(DE, Q, lang='de') -> 0.572 # the language temperature does work
Agent.system_one(DE, Q) -> 0.608
```
`lang` is consumed only by the `lang_temperatures` lookup when decoding answers, so nothing else
about the batch changes. It contradicts `predict_batch`'s own docstring ("Requests may
independently specify `model`, `task`, `lang` or `lang_guess`") and
`docs/hooks/lifecycle.md` ("Each result is what `predict` returns for that request"), and it is
only reachable through `attach()` — building the Agent yourself is the only way to put one with
per-language temperatures behind a Router, which is exactly the case `#258` added.
Deliberate choices tested:
- **Gate the whole thing on `agent.lang_temperatures`.** This is the part that matters. Adding `lang` to the group key unconditionally would split a group that shares one forward pass today, changing batching for every caller that never uses the feature. Restoring the key unconditionally fails the existing `test_mixed_groups_keep_order_and_lru` cases; with the gate, a three-request batch on an agent without the attribute is still one call of three, asserted by its own test.
- **Group by language as well as schema and budget.** The language decision is per request, so requests that disagree about it cannot share a batched call that takes one `lang` — the group key has to carry it or the fix would apply one request's language to another's scoring.
- **Same derivation as `predict`.** The request's own `lang` wins, and the routed detection's `language` is the fallback, so a request that never named a language still gets the detected one, exactly as the single-request path does.
- **The same `TypeError` tolerance `predict` has.** An Agent-like object whose `predict_batch` predates the `lang` argument is retried without it, matching `router.py:552-556`, so the fix cannot break a third-party agent that already works.
- **No dependency, no default change.** For every agent without `lang_temperatures` the grouping, the number of forward passes and every argument passed are byte-identical.
The new checks in `tests/test_router_batch.py` use a fake agent that records the `lang` it
received, and fail without the forwarding:
```
# with the lang kwarg removed from the batched call
2 failed, 19 passed
FAILED test_router_batch.py::test_lang_reaches_the_agent_when_it_carries_lang_temperatures
FAILED test_router_batch.py::test_predict_and_predict_batch_pass_the_same_lang
# restored
21 passed
```
They cover the three properties separately, because each can regress on its own: the language
reaching the agent and grouping by it, the group key staying untouched without the attribute,
and the two entry points agreeing on the value they pass.
```
$ python -m pytest tests/test_router_batch.py -q 21 passed
$ python -m pytest tests/test_predict_batch.py tests/test_system_one_lang.py \
tests/test_truncation_direction.py -q 13 passed
$ python tests/test_router.py all routing tests passed
$ python tests/test_router_memory.py OK
$ python tests/test_batch.py 49 passed, 0 failed
$ python tests/test_lang_guess.py 52 passed, 0 failed
$ python tests/test_hooks.py all hook tests passed
$ python -m ruff check laya/ --select=E9,F63,F7,F82,F401,F811 --line-length=120
All checks passed!
$ python -m compileall -q laya/ tests/
```
Known limitation: only `Router.predict` and `Router.predict_batch` are covered. `Agent.predict_batch`
already accepts `lang` through its own overrides, so an agent used directly is unaffected, and
this PR does not touch that path. `Router.route_batch` is untouched too — it decides checkpoints,
not scores, and the language there is already used for routing.
295 lines
12 KiB
Python
295 lines
12 KiB
Python
"""Deterministic regression coverage for heterogeneous Router batching."""
|
|
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from threading import Barrier, Lock
|
|
from time import sleep
|
|
|
|
import pytest
|
|
|
|
from laya.router import Router
|
|
|
|
|
|
Q = {"intent": {"type": "noul", "instructions": "Relevant?"}}
|
|
|
|
|
|
def request(state, **overrides):
|
|
return {"state": state, "questions": Q, **overrides}
|
|
|
|
|
|
@pytest.fixture
|
|
def fake_agent(monkeypatch):
|
|
import laya.agent
|
|
|
|
built = []
|
|
calls = []
|
|
|
|
class Agent:
|
|
def __init__(self, repo, *, device, token, subfolder):
|
|
self.checkpoint = subfolder or "english"
|
|
built.append(self.checkpoint)
|
|
|
|
def predict_batch(self, states, questions, batch_size=None):
|
|
calls.append((self.checkpoint, list(states), questions, batch_size))
|
|
if "raise" in states:
|
|
raise RuntimeError("inference failed")
|
|
return [
|
|
{"model": "laya-rl-agent", "answers": {"seen": state}, "usage": {}}
|
|
for state in states
|
|
]
|
|
|
|
def system_one(self, state, questions):
|
|
return self.predict_batch([state], questions)[0]
|
|
|
|
monkeypatch.setattr(laya.agent, "Agent", Agent)
|
|
return built, calls
|
|
|
|
|
|
@pytest.mark.parametrize("capacity,expected", [
|
|
(1, (["english", "multilingual", "typed-decisions"], ["typed-decisions"])),
|
|
(2, (["english", "multilingual", "typed-decisions"], ["multilingual", "typed-decisions"])),
|
|
(3, (["english", "multilingual", "typed-decisions"],
|
|
["english", "multilingual", "typed-decisions"])),
|
|
])
|
|
def test_mixed_groups_keep_order_and_lru(fake_agent, capacity, expected):
|
|
built, calls = fake_agent
|
|
router = Router(max_loaded=capacity, auto_task_detection=True)
|
|
typed = {key: {"type": "noul", "instructions": "?"} for key in
|
|
("action", "needs_review", "outcome", "risk", "urgency")}
|
|
items = [request("English text one"), request("مرحبا"),
|
|
request("English text two"), {"state": "decision", "questions": typed},
|
|
request("forced", model="ml"), request("forced english", lang="en"),
|
|
request("explicit task", task="typed_decisions")]
|
|
decisions = router.route_batch(items)
|
|
assert router.loaded == []
|
|
results = router.predict_batch(items)
|
|
assert [r["answers"]["seen"] for r in results] == [r["state"] for r in items]
|
|
assert [r["routing"] for r in results] == list(map(dict, decisions))
|
|
assert built == expected[0]
|
|
assert router.loaded == expected[1]
|
|
assert [(c[0], c[1]) for c in calls] == [
|
|
("english", ["English text one", "English text two", "forced english"]),
|
|
("multilingual", ["مرحبا", "forced"]),
|
|
("typed-decisions", ["decision"]),
|
|
("typed-decisions", ["explicit task"]),
|
|
]
|
|
assert router.predict_many([]) == []
|
|
assert router.route_batch(()) == []
|
|
|
|
|
|
@pytest.mark.parametrize("items,error,fragment", [
|
|
(None, TypeError, "requests must be a sequence"),
|
|
({}, TypeError, "requests must be a sequence"),
|
|
("text", TypeError, "requests must be a sequence"),
|
|
([None], TypeError, "request 0"),
|
|
([{"questions": Q}], ValueError, "request 0 is missing required key 'state'"),
|
|
([{"state": "x"}], ValueError, "request 0 is missing required key 'questions'"),
|
|
([request("x"), {"state": "y", "questions": None}], TypeError, "request 1 'questions'"),
|
|
([request("x"), request("y", model="invalid")], ValueError, "unknown model"),
|
|
])
|
|
def test_invalid_batch_fails_before_loading(fake_agent, items, error, fragment):
|
|
built, _ = fake_agent
|
|
router = Router()
|
|
with pytest.raises(error, match=fragment):
|
|
router.predict_batch(items)
|
|
assert built == []
|
|
assert router.loaded == []
|
|
|
|
|
|
def test_inference_exception_propagates_and_cache_remains_consistent(fake_agent):
|
|
built, calls = fake_agent
|
|
router = Router(max_loaded=1)
|
|
with pytest.raises(RuntimeError, match="inference failed"):
|
|
router.predict_batch([request("first"), request("raise", lang="ar"),
|
|
request("unreached", lang="ar")])
|
|
assert built == ["english", "multilingual"]
|
|
assert [(c[0], c[1]) for c in calls] == [
|
|
("english", ["first"]),
|
|
("multilingual", ["raise", "unreached"]),
|
|
]
|
|
assert router.loaded == ["multilingual"]
|
|
assert list(router._agents) == router.loaded
|
|
assert router.predict("after failure", Q, lang="ar")["answers"]["seen"] == "after failure"
|
|
|
|
|
|
def test_warm_cache_and_repeated_batches(fake_agent):
|
|
built, _ = fake_agent
|
|
router = Router(max_loaded=2)
|
|
router.load("multilingual")
|
|
result = router.predict_batch([request("en", model="english"),
|
|
request("ar", model="multilingual"),
|
|
request("en again", model="english")])
|
|
assert [r["answers"]["seen"] for r in result] == ["en", "ar", "en again"]
|
|
assert built == ["multilingual", "english"]
|
|
assert router.loaded == ["english", "multilingual"]
|
|
router.predict_batch([request("ar", model="multilingual"), request("en", model="english")])
|
|
assert built == ["multilingual", "english"]
|
|
|
|
|
|
|
|
def test_same_checkpoint_same_questions_uses_one_agent_batch(fake_agent):
|
|
_, calls = fake_agent
|
|
router = Router(max_loaded=2)
|
|
items = [
|
|
request("one", model="english"),
|
|
request("two", model="english"),
|
|
request("three", model="english"),
|
|
]
|
|
|
|
results = router.predict_batch(items, batch_size=2)
|
|
|
|
assert [r["answers"]["seen"] for r in results] == ["one", "two", "three"]
|
|
assert len(calls) == 1
|
|
assert calls[0][0] == "english"
|
|
assert calls[0][1] == ["one", "two", "three"]
|
|
assert calls[0][2] == Q
|
|
assert calls[0][3] == 2
|
|
|
|
|
|
def test_same_checkpoint_different_questions_split_agent_batches(fake_agent):
|
|
_, calls = fake_agent
|
|
router = Router(max_loaded=2)
|
|
q2 = {"risk": {"type": "noul", "instructions": "Risky?"}}
|
|
items = [
|
|
{"state": "one", "questions": Q, "model": "english"},
|
|
{"state": "two", "questions": q2, "model": "english"},
|
|
{"state": "three", "questions": Q, "model": "english"},
|
|
]
|
|
|
|
results = router.predict_batch(items)
|
|
|
|
assert [r["answers"]["seen"] for r in results] == ["one", "two", "three"]
|
|
assert len(calls) == 2
|
|
assert calls[0][0] == "english"
|
|
assert calls[0][1] == ["one", "three"]
|
|
assert calls[0][2] == Q
|
|
assert calls[1][0] == "english"
|
|
assert calls[1][1] == ["two"]
|
|
assert calls[1][2] == q2
|
|
|
|
|
|
def test_route_batch_forwards_lang_guess(fake_agent):
|
|
built, _ = fake_agent
|
|
router = Router()
|
|
|
|
decisions = router.route_batch([
|
|
request("hola", lang_guess="es"),
|
|
request("hello", lang_guess="en-US"),
|
|
])
|
|
|
|
assert [decision.model for decision in decisions] == ["multilingual", "english"]
|
|
assert built == []
|
|
|
|
def test_concurrent_batch_load_deduplicates(monkeypatch):
|
|
import laya.agent
|
|
|
|
built = []
|
|
guard = Lock()
|
|
|
|
class SlowAgent:
|
|
def __init__(self, repo, *, device, token, subfolder):
|
|
sleep(0.01)
|
|
with guard:
|
|
built.append(subfolder or "english")
|
|
|
|
def predict_batch(self, states, questions, batch_size=None):
|
|
return [
|
|
{"model": "stub", "answers": {"seen": state}, "usage": {}}
|
|
for state in states
|
|
]
|
|
|
|
def system_one(self, state, questions):
|
|
return self.predict_batch([state], questions)[0]
|
|
|
|
monkeypatch.setattr(laya.agent, "Agent", SlowAgent)
|
|
router = Router(max_loaded=2)
|
|
start = Barrier(8)
|
|
|
|
def worker(_):
|
|
start.wait()
|
|
return router.predict_batch([request("hello", model="english"),
|
|
request("مرحبا", model="multilingual")])
|
|
|
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
results = list(pool.map(worker, range(8)))
|
|
assert built == ["english", "multilingual"]
|
|
assert router.loaded == ["english", "multilingual"]
|
|
assert all([r["answers"]["seen"] for r in result] == ["hello", "مرحبا"]
|
|
for result in results)
|
|
|
|
|
|
def test_equal_questions_with_different_option_order_score_separately(fake_agent):
|
|
"""A question schema that arrives with a different key order must be scored with
|
|
its own order. Dict equality ignores insertion order, but options are positional
|
|
in the rendered sequence, so grouping reordered-but-equal schemas would make the
|
|
second request's batched answers differ from its single-request answers."""
|
|
built, calls = fake_agent
|
|
ordered = {"intent": {"type": "choice", "instructions": "Pick one",
|
|
"criteria": {"zulu": "last", "alpha": "first"}}}
|
|
reordered = {"intent": {"type": "choice", "instructions": "Pick one",
|
|
"criteria": {"alpha": "first", "zulu": "last"}}}
|
|
requests = [{"state": "one", "questions": ordered},
|
|
{"state": "two", "questions": reordered}]
|
|
results = Router(max_loaded=1, default="english").predict_batch(requests)
|
|
assert len(results) == 2
|
|
# separate agent calls, each carrying its own caller's option order
|
|
orders = [list(call[2]["intent"]["criteria"]) for call in calls]
|
|
assert orders == [["zulu", "alpha"], ["alpha", "zulu"]]
|
|
|
|
|
|
def _lang_recording_router(monkeypatch, lang_temperatures):
|
|
"""A Router over a fake agent that records the `lang` each batch call received."""
|
|
import laya.agent
|
|
|
|
calls = []
|
|
|
|
class Agent:
|
|
def __init__(self, repo, *, device, token, subfolder):
|
|
self.checkpoint = subfolder or "english"
|
|
if lang_temperatures is not None:
|
|
self.lang_temperatures = lang_temperatures
|
|
|
|
def predict_batch(self, states, questions, batch_size=None, **overrides):
|
|
calls.append({"n": len(states), "lang": overrides.get("lang")})
|
|
return [{"model": "fake", "answers": {}} for _ in states]
|
|
|
|
def system_one(self, state, questions, **overrides):
|
|
calls.append({"n": 1, "lang": overrides.get("lang")})
|
|
return {"model": "fake", "answers": {}}
|
|
|
|
monkeypatch.setattr(laya.agent, "Agent", Agent)
|
|
return Router(max_loaded=1, default="english"), calls
|
|
|
|
|
|
def test_lang_reaches_the_agent_when_it_carries_lang_temperatures(monkeypatch):
|
|
"""`predict` forwards the request's language so per-language temperatures apply; the batch
|
|
path forwarded only the token budgets, so the same request scored differently depending on
|
|
which entry point served it. The request is routed as German either way, which is what made
|
|
the difference hard to see."""
|
|
router, calls = _lang_recording_router(monkeypatch, {"de": [1.5, 1.5, 1.5]})
|
|
router.predict_batch([request("a", lang="de"), request("b", lang="de"), request("c", lang="fr")])
|
|
assert sorted(c["lang"] for c in calls) == ["de", "fr"]
|
|
# requests sharing a language still share one forward pass
|
|
assert {"n": 2, "lang": "de"} in calls
|
|
assert {"n": 1, "lang": "fr"} in calls
|
|
|
|
|
|
def test_lang_is_not_added_to_the_group_key_without_lang_temperatures(monkeypatch):
|
|
"""An agent with no per-language temperatures does not use `lang`, so naming it must not
|
|
split a group that shares one forward pass today. The explicit `model` keeps every request
|
|
on one checkpoint, so the only thing that could split the group is the lang key."""
|
|
router, calls = _lang_recording_router(monkeypatch, None)
|
|
router.predict_batch([request("a", model="english", lang="de"),
|
|
request("b", model="english", lang="fr"),
|
|
request("c", model="english")])
|
|
assert calls == [{"n": 3, "lang": None}]
|
|
|
|
|
|
def test_predict_and_predict_batch_pass_the_same_lang(monkeypatch):
|
|
"""The invariant that was violated: one request, either entry point, same language."""
|
|
router, calls = _lang_recording_router(monkeypatch, {"de": [1.5, 1.5, 1.5]})
|
|
router.predict("a", Q, model="english", lang="de")
|
|
via_predict = calls[-1]["lang"]
|
|
calls.clear()
|
|
router.predict_batch([request("a", model="english", lang="de")])
|
|
assert calls[-1]["lang"] == via_predict == "de"
|