mirror of
https://github.com/hanzoai/laya.git
synced 2026-10-30 19:33:31 +00:00
- lang: _script_counts walks the text once; detect_script, script_profile and analyse derive from it instead of two passes with the same range scan. - agent: add Agent.__repr__ for parity with Router and ONNXAgent. - ci: run tests/test_lang_stats.py in the script lane. - tests: test_lang_stats.py pins detect/profile and the tie-break against the pre-refactor implementations. The predict_batch JSON grouping from the original commit was already dropped: main uses the order-preserving `_question_schema` (#166). The pytest-suite wiring is dropped too: #376 runs the pytest-shaped suites on Linux and Windows.
118 lines
3.5 KiB
Python
118 lines
3.5 KiB
Python
"""`detect_script` / `script_profile` must be unchanged by the single-pass refactor in #lang.
|
|
|
|
The reference implementations below are the pre-refactor versions; the new ones share one pass
|
|
over the text, so these checks pin that the counts, the fractions and the tie-break are the same.
|
|
|
|
Run: python tests/test_lang_stats.py
|
|
"""
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
from laya.lang import _SCRIPT_RANGES, analyse, detect_script, script_profile # noqa: E402
|
|
|
|
PASS, FAIL = [], []
|
|
|
|
|
|
def check(name, got, want):
|
|
if got == want:
|
|
PASS.append(name)
|
|
else:
|
|
FAIL.append("%s: got %r, want %r" % (name, got, want))
|
|
|
|
|
|
def _is_latin_cp(cp):
|
|
return cp < 0x02B0 or 0x1E00 <= cp <= 0x1EFF or 0xFF21 <= cp <= 0xFF3A or 0xFF41 <= cp <= 0xFF5A
|
|
|
|
|
|
def ref_counts(text):
|
|
counts = {}
|
|
latin = 0
|
|
for ch in text:
|
|
if not ch.isalpha():
|
|
continue
|
|
cp = ord(ch)
|
|
if _is_latin_cp(cp):
|
|
latin += 1
|
|
continue
|
|
for name, ranges in _SCRIPT_RANGES:
|
|
if any(lo <= cp <= hi for lo, hi in ranges):
|
|
counts[name] = counts.get(name, 0) + 1
|
|
break
|
|
else:
|
|
counts["other"] = counts.get("other", 0) + 1
|
|
counts["latin"] = latin
|
|
return counts
|
|
|
|
|
|
def ref_detect(text):
|
|
counts = ref_counts(text)
|
|
if sum(counts.values()) == 0:
|
|
return "unknown"
|
|
return max(counts.items(), key=lambda kv: kv[1])[0]
|
|
|
|
|
|
def ref_profile(text):
|
|
counts = {"latin": 0}
|
|
for ch in text:
|
|
if not ch.isalpha():
|
|
continue
|
|
cp = ord(ch)
|
|
if _is_latin_cp(cp):
|
|
counts["latin"] += 1
|
|
continue
|
|
for name, ranges in _SCRIPT_RANGES:
|
|
if any(lo <= cp <= hi for lo, hi in ranges):
|
|
counts[name] = counts.get(name, 0) + 1
|
|
break
|
|
else:
|
|
counts["other"] = counts.get("other", 0) + 1
|
|
total = sum(counts.values())
|
|
if not total:
|
|
return {}
|
|
return {k: v / total for k, v in counts.items() if v}
|
|
|
|
|
|
TEXTS = [
|
|
"",
|
|
" ",
|
|
"hello world",
|
|
"hello мир",
|
|
"мир",
|
|
"世界",
|
|
"日本語です",
|
|
"ΑΒΓΔ",
|
|
"abc ᚠᚢᚦ", # an unlisted script lands under "other"
|
|
"ab αβ", # latin/greek tie
|
|
"αβ аб", # greek/cyrillic tie
|
|
"ᚠᚢᚦabc", # other/latin tie
|
|
"Hallo, meine Bestellung ist zweimal abgebucht worden.",
|
|
"مرحبا كيف حالك",
|
|
"İstanbul",
|
|
]
|
|
|
|
for text in TEXTS:
|
|
check("detect_script(%r)" % text, detect_script(text), ref_detect(text))
|
|
check("script_profile(%r)" % text, script_profile(text), ref_profile(text))
|
|
|
|
# analyse uses the same counts, so its profile must match too, and it must not regress on a
|
|
# non-Latin override (a lowercase non-Latin run flips the dominant script).
|
|
for text in ("hello мир", "мир", "مرحبا كيف حالك", "hello world"):
|
|
check("analyse.profile(%r)" % text, analyse(text)["script_profile"], ref_profile(text))
|
|
check("analyse/arabic", analyse("مرحبا كيف حالك")["script"], "arabic")
|
|
check("analyse/empty is unknown", analyse("")["script"], "unknown")
|
|
|
|
# a per-script fraction vector still sums to 1
|
|
for text in ("hello мир", "abc ᚠᚢᚦ"):
|
|
total = sum(script_profile(text).values())
|
|
check("profile sums to 1 (%r)" % text, round(total, 9), 1.0)
|
|
|
|
|
|
print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
|
|
for f in FAIL:
|
|
print(" FAIL", f)
|
|
if not FAIL:
|
|
print("all lang stats tests passed")
|
|
sys.exit(1 if FAIL else 0)
|