Files

73 lines
3.6 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// laya-ts/tests/lang.test.ts
import { describe, expect, it } from "vitest";
import { analyse, detectScript, guessLatinLanguage, isEnglish } from "../src/lang.js";
describe("lang", () => {
it("detects devanagari as non-latin", () => {
expect(detectScript("मुझसे दो बार शुल्क लिया गया")).toBe("devanagari");
});
it("routes english latin to english", () => {
expect(analyse("Please refund the duplicate charge").isEnglish).toBe(true);
});
it("routes german latin to non-english", () => {
expect(isEnglish("Der Kunde wurde zweimal belastet")).toBe(false);
});
it("unknown (no letters) is english + undecided", () => {
expect(analyse("123 !!!").script).toBe("unknown");
});
// Every expectation below was generated with Python's laya.lang.analyse on the same input.
it("names romanized Bangla instead of falling back to english", () => {
const a = analyse("ami ekta ticket khulsi, kalke theke payment hocche na, ekhon ki korte parbo");
expect(a.script).toBe("latin");
expect(a.language).toBe("bn");
expect(a.isEnglish).toBe(false);
});
it("names Azerbaijani Latin text", () => {
const a = analyse("ödənişim iki dəfə tutulub, amma heç bir təsdiq almadım, nə etməliyəm");
expect(a.language).toBe("az");
expect(a.isEnglish).toBe(false);
});
it("a CJK sentence inside an English ticket is not english", () => {
const a = analyse("please check the attached logs 請重啟服務器然後再試一次 and tell me what failed");
expect(a.script).toBe("han");
expect(a.isEnglish).toBe(false);
});
it("a tiny non-latin note does not flip an english ticket", () => {
const a = analyse("please refund my order, see note 重啟 attached to the ticket");
expect(a.script).toBe("latin");
expect(a.isEnglish).toBe(true);
});
it("counts IPA extensions as latin, like Python", () => {
expect(detectScript("ɑ ɒ ɛ ɔ ɪ ʊ æ ʃ θ ð")).toBe("latin");
});
// Plain-ASCII German: no umlaut, so the diacritic rate is 0 and the function words are the only
// evidence. Python names these `de` (laya/lang.py, #130); with the 23-word list they read as
// English here and the Router sent them to the English checkpoint.
it("names German that carries no diacritics", () => {
for (const text of [
"wie lautet die temperatur in fulda in hessen",
"Mein Konto wurde zweimal belastet, bitte erstatten Sie",
"Ich brauche eine Rechnung fuer meine letzte Bestellung",
"Kann ich meine Bestellung noch heute stornieren",
"Bitte senden Sie mir eine neue Kreditkartenabrechnung",
]) {
const a = analyse(text);
expect(a.language, text).toBe("de");
expect(a.isEnglish, text).toBe(false);
expect(guessLatinLanguage(text), text).toBe("de");
}
});
// `in` and `was` count for German as well as English, and `im` and `den` are English tokens too.
// Pin the English states that flip if any of them becomes German-only.
it("keeps English that shares words with the German list", () => {
expect(analyse("turn off smart lamp in den").isEnglish).toBe(true);
expect(analyse("What was the reason for the delay").language).toBe("en");
expect(analyse("Where can I find my invoice in the app").isEnglish).toBe(true);
});
// `es` and `du` are German function words, but Spanish and French claim them, and a word two
// lists share names neither language. They stay out of the German list.
it("does not pull es or du into German", () => {
expect(analyse("que hora es en australia").language).toBe("es");
expect(analyse("baisse le volume du haut-parleur").language).toBe("fr");
});
});