Files
laya/tests/test_truncation_direction.py
34d27dfa8e perf(agent): reuse the tokenized state across questions; autocast on MPS (#109)
* perf(agent): reuse the tokenized state across questions; autocast on MPS

- system_one normalized each question twice and re-serialized/re-tokenized the
  whole state once per question. build_sequence now accepts state_ids and
  system_one tokenizes the state once (truncated to max_len). On a 4.7k-char
  state with 5 questions this cuts preprocessing from 8.44 ms to 2.63 ms (-69%).
- autocast was CUDA-only; cpu/mps were forced to float32. MPS now runs fp16, and
  CPU bf16 is opt-in via LAYA_CPU_AMP=bf16 (off by default: bf16 is ~6x slower on
  CPUs without native BF16). The forward pass is extracted into _forward/_infer so
  an unsupported-autocast error degrades to full precision instead of failing.

Adds tests/test_runtime_fixes.py and runs it in CI.

* perf(common): cap option tokens at the tokenizer instead of slicing after

build_sequence tokenized each choice option in full and then kept [:48], so a
long option description was processed in full and thrown away. Pass
truncation=True, max_length=48 to the tokenizer; the kept tokens are identical.
Covered by a new check in tests/test_runtime_fixes.py.

* perf(agent): gate MPS fp16 by batch rows

MPS fp16 autocast is slower than fp32 on a single small row (autocast overhead
dominates) and only wins once the batch has several rows. Measured on an Apple M5
against the real multilingual checkpoint: 15.4 ms vs 11.0 ms at 1 question,
32.2 ms vs 37.3 ms at 5. So MPS fp16 now starts at mps_amp_min_rows (default 5,
override with LAYA_MPS_AMP_MIN_ROWS) instead of being enabled unconditionally.
CUDA and CPU are unchanged. Adds gate tests to test_runtime_fixes.py.

* perf(agent): never enter autocast when it is disabled

Wrap the forward pass with a shared _amp_context(device, dtype, enabled) that
returns nullcontext() when autocast is off. Entering torch.autocast on a device
torch has no autocast backend for raises even with enabled=False, which broke
MPS predict() on some torch builds, and the row gate made that path reachable.
The helper matches the one in #273 so the two rebase cleanly. Adds amp-context
tests (disabled is a no-op and does not raise on mps; enabled returns autocast).

* test: reconcile portability and training fixtures with the rebased branch

- test_portability: _amp_context now takes an enabled flag and MPS may autocast at
  high batch, so assert the disabled path never enters torch.autocast and read the
  wrapper from _infer.
- test_portability/_training fakes: accept truncation/max_length, since
  build_sequence now caps option tokens at the tokenizer.

---------

Co-authored-by: NandhaKishorM <[email protected]>
2026-09-24 08:28:45 +05:30

145 lines
5.8 KiB
Python

"""Regression: Agent.system_one preserves the newest conversation turn.
The public Agent API advertises `state` as accepting a chronological
conversation-turn list. build_sequence defaults to truncate_left=False
(st[:room]), which preserves the head and drops the tail — for a chronological
list that means the newest turn is silently lost. For list-shaped state the agent
now truncates from the left so the most recent intent survives. Strings and
dicts are unaffected (backward compatible).
"""
import json
import torch
from transformers import AutoConfig, AutoModel
from laya.agent import Agent
from laya.common import DecisionModel, build_sequence, serialize_state
class _FakeTok:
cls_token_id, sep_token_id, mask_token_id, pad_token_id = 0, 1, 4, 2
mask_token = "[MASK]"
def __call__(self, text, add_special_tokens=False, truncation=False, max_length=None):
ids = [10 + (len(w) % 90) for w in text.split() if w]
if truncation and max_length:
ids = ids[:max_length]
return {"input_ids": ids}
def _tiny_agent():
agent = object.__new__(Agent)
cfg = AutoConfig.for_model("bert", hidden_size=16, num_hidden_layers=1, num_attention_heads=1,
intermediate_size=32, vocab_size=64)
agent.cfg = {"max_len": 30, "head_max_len": 12, "encoder": "tiny"}
agent.tok = _FakeTok()
agent.model = DecisionModel(AutoModel.from_config(cfg), head_layers=1, n_act=2).eval()
agent.device = torch.device("cpu")
agent.dtype = torch.float32
agent.temperature = [1.0, 1.0, 1.0]
agent.temperature_by_options = {}
return agent
def _state_prefix_len(tok, q, max_len, head_max_len):
ref, _ = build_sequence(tok, "", q, max_len, head_max_len)
return len(ref) - 1
Q = {"t": "choice", "ins": "What action?", "crit": {"refund": "money back", "escalate": "manager", "hold": "wait"}}
def test_agent_list_vs_string_truncation_direction():
"""Agent must truncate left for lists, right for strings (via mock inspection)."""
from unittest.mock import patch
agent = _tiny_agent()
questions = {"q": {"type": "choice", "instructions": "Act?", "criteria": {"a": "", "b": ""}}}
list_capture = {}
string_capture = {}
def fake_build(tok, state, q, max_len, head_max_len, truncate_left=False, state_ids=None):
if isinstance(state, list):
list_capture["truncate_left"] = truncate_left
else:
string_capture["truncate_left"] = truncate_left
return build_sequence(tok, state, q, max_len, head_max_len,
truncate_left=truncate_left, state_ids=state_ids)
with patch("laya.agent.build_sequence", side_effect=fake_build):
agent.system_one([{"role": "user", "content": "hi"}, {"role": "user", "content": "newest"}], questions)
agent.system_one("string state", questions)
assert list_capture["truncate_left"] is True
assert string_capture["truncate_left"] is False
def test_agent_system_one_passes_truncate_left_for_list():
"""Agent.system_one must pass truncate_left=True for list state."""
from unittest.mock import patch
agent = _tiny_agent()
conversation = [{"role": "user", "content": "hello"}, {"role": "user", "content": "NEWEST"}]
questions = {"q": {"type": "choice", "instructions": "Act?", "criteria": {"a": "", "b": ""}}}
captured = {}
def fake_build(tok, state, q, max_len, head_max_len, truncate_left=False, state_ids=None):
captured["truncate_left"] = truncate_left
captured["state"] = state
return build_sequence(tok, state, q, max_len, head_max_len,
truncate_left=truncate_left, state_ids=state_ids)
with patch("laya.agent.build_sequence", side_effect=fake_build):
agent.system_one(conversation, questions)
assert captured["truncate_left"] is True, "list state must trigger truncate_left=True"
assert isinstance(captured["state"], list)
def test_agent_system_one_string_state_default_truncation():
"""Agent.system_one with string state keeps default (truncate_left=False)."""
from unittest.mock import patch
agent = _tiny_agent()
questions = {"q": {"type": "choice", "instructions": "Act?", "criteria": {"a": "", "b": ""}}}
captured = {}
def fake_build(tok, state, q, max_len, head_max_len, truncate_left=False, state_ids=None):
captured["truncate_left"] = truncate_left
return build_sequence(tok, state, q, max_len, head_max_len,
truncate_left=truncate_left, state_ids=state_ids)
with patch("laya.agent.build_sequence", side_effect=fake_build):
agent.system_one("just a string state", questions)
assert captured["truncate_left"] is False, "string state must keep default truncation"
def test_string_state_preserves_head():
"""A string state must still preserve the head (backward compatible)."""
tok = _FakeTok()
state = "HEADMARKERWORD " + "filler " * 20 + " TAILMARKER"
state_ids = tok(state.replace("[MASK]", " "), add_special_tokens=False)["input_ids"]
head_id = state_ids[0]
tail_id = state_ids[-1]
assert head_id != tail_id, "head and tail must have different ids"
seq, _ = build_sequence(tok, state, Q, 30, 12)
prefix = _state_prefix_len(tok, Q, 30, 12)
kept = seq[prefix:-1]
assert head_id in kept, "string state must preserve the head (default mode)"
assert tail_id not in kept, "string state must drop the tail (default mode)"
def test_build_sequence_default_unchanged():
"""build_sequence's default behavior is unchanged for non-list callers."""
tok = _FakeTok()
state = "OLDFRONT " + "filler " * 20 + " NEWBACK"
seq_default, _ = build_sequence(tok, state, Q, 30, 12)
seq_explicit, _ = build_sequence(tok, state, Q, 30, 12, truncate_left=False)
assert seq_default == seq_explicit, "default must remain truncate_left=False"