hanzo.ai/e2e/gates/translated.spec.ts

import { expect, test, type Page } from '@playwright/test'
import { serveExport } from './export'
import { MODELS, PLANS, PRICING, SUMMARY, serve } from './api'
import { signIn } from '../session'
import blog from '../../lib/data/blog.json' with { type: 'json' }
import oss from '../../lib/data/oss.json' with { type: 'json' }
import { stories } from '../../lib/data/customers'
import { backerLogos } from '../../lib/constants/partner-logos'

/**
 * IN ARABIC NOTHING AUTHORED IS LEFT IN ENGLISH, AND IT READS RIGHT TO LEFT.
 *
 * Every word the site writes itself goes through @hanzo/i18n's `t()`, and the
 * catalogs it ships carry each of them in every language. This reads the shipped
 * export as an Arabic reader gets it — the landing, every menu open, the phone
 * sheet, sign-in, and the app signed in — and fails on any Latin word that is not
 * a name, a unit, code, or something the page was served (a plan's features, a
 * model's name, a repository's description: data, which no catalog holds). A
 * string someone forgot to wrap reads as English here and nowhere else.
 *
 * French is read the same way for English's commonest words, which French does
 * not share, so a Latin-script locale cannot hide a string either.
 *
 * Under Arabic the page is `dir="rtl"`, nothing scrolls sideways, and a range or
 * a figure reads in its own order: a text holding one is isolated (`t()`
 * arguments are, and a figure outside a sentence carries `data-i18n-ltr`).
 */

let base: string
let stop: () => Promise<void>
test.beforeAll(async () => {
  const served = await serveExport()
  base = served.url
  stop = served.close
})
test.afterAll(async () => stop())

const LAPTOP = { width: 1440, height: 900 }
const PHONE = { width: 390, height: 844 }

/**
 * Latin words an Arabic page may carry, and why each is not a translation's job.
 * A word is checked lower-cased; an identifier (`api.hanzo.ai`, `src/auth.ts`,
 * `Hanzo.Start`, a run id), an abbreviation in capitals and a lone letter are
 * not words a reader is asked to read, and pass without a list.
 */
const NAMES = [
  // @hanzo/i18n's terms.json: kept in Latin script in every locale.
  'Hanzo', 'Zen', 'Enso', 'Kai', 'Dev', 'Studio', 'Gen', 'Lux', 'Zoo', 'Claude', 'Code', 'Codex', 'Cursor',
  'Antigravity', 'Gemini', 'Copilot', 'GitHub', 'Google', 'Apple', 'OpenAI', 'Anthropic', 'Hugging', 'Face',
  'Traefik', 'LiveCodeBench', 'Diamond', 'Metal',
  // Hanzo's products and models, named as they ship: Hanzo App, Bot, Team, Kata, University,
  // Base, Hanzo OS; the creative models; Kai's suite.
  'App', 'Bot', 'Team', 'Kata', 'University', 'Base', 'Koma', 'Iro', 'Mori', 'Oto', 'Koe', 'Uta',
  'Insights', 'Console', 'Forge',
  // The landing's windows are named like the apps they mock: Hanzo.Start, Glider 1.0.
  'Glider',
  // Machines, chips and runtimes as their makers name them.
  'Grace', 'Blackwell', 'Strix', 'Halo', 'silicon', 'ROCm', 'Go', 'Inc',
  // Terms of art a translation keeps in Latin script.
  'shell',
  // Benchmarks by their published names: Kai Decision Suite, Zen 6 Prefill & Decode, LongMemEval & LoCoMo.
  'Decision', 'Suite', 'Prefill', 'Decode', 'LongMemEval', 'LoCoMo', 'Banking',
  // The people in the landing's sample room (Products: Team), and the signed-in
  // reader e2e/session.ts makes: names, not copy.
  'Ada', 'Lin', 'Sam', 'gate',
]

/**
 * Words drawn by another package that takes no words from the host, each named
 * with its package so the list is cut the day that package translates them.
 * @hanzo/ui's marketing `Footer` labels its legal row itself. @hanzo/build 0.2.80
 * draws the composer's controls, the row under it and the starter suggestions in
 * English, with no words prop for rooms to pass.
 */
const BUILD = ['Ask what Hanzo can do for you', 'Build a landing page with Dev', 'Get a cited brief on AI agents']
const PACKAGED = [
  'aria-label="Terms & Policies"',
  'Medium',
  'Auto',
  ...BUILD,
  ...BUILD.flatMap((s) => [`aria-label="${s}"`, `aria-label="Dismiss ${s}"`]),
  ...['Show tabs', 'Add files, agents, skills and more', 'Dictate', 'Dictation settings', 'Voice mode', 'Effort: Medium', 'Mode: Auto', 'Suggestions'].map((s) => `aria-label="${s}"`),
  ...['Show tabs (Ctrl+Shift+B)', 'Dictate', 'Voice mode'].map((s) => `title="${s}"`),
]

/** Every word in strings the page is served rather than writes: data, not copy. */
function words(value: unknown, into = new Set<string>()): Set<string> {
  if (typeof value === 'string') for (const w of value.match(/[A-Za-z][A-Za-z'’-]*/g) ?? []) into.add(w.replace(/['’-]+$/, '').toLowerCase())
  else if (Array.isArray(value)) value.forEach((v) => words(v, into))
  else if (value && typeof value === 'object') Object.values(value).forEach((v) => words(v, into))
  return into
}
/** The served answers, the repositories the landing lists, and the companies and backers it names by their marks. */
const SERVED = words([PLANS, PRICING, SUMMARY, MODELS, blog, oss, stories.map((s) => s.company), backerLogos.map((l) => l.name)])
const ALLOWED = [...new Set([...NAMES.map((w) => w.toLowerCase()), ...SERVED])]

/** Every whole string the page is served, which a page draws as sent in every language. */
const SENT: string[] = []
const strings = (value: unknown): void => {
  if (typeof value === 'string') SENT.push(value.trim())
  else if (Array.isArray(value)) value.forEach(strings)
  else if (value && typeof value === 'object') Object.values(value).forEach(strings)
}
strings([PLANS, PRICING, SUMMARY, MODELS])

/** English's commonest words, none of them French: a French page holding one holds English. */
const ENGLISH = ['the', 'and', 'your', 'with', 'for', 'of', 'to', 'is', 'you', 'our', 'every', 'from', 'this', 'that', 'what', 'how', 'more', 'about', 'get', 'see']

/** A page in a language this browser chose before, every read answered by the fixtures. */
async function open(page: Page, path: string, locale: 'en-US' | 'ar' | 'fr-FR', size = LAPTOP, who = 'guest') {
  await page.emulateMedia({ reducedMotion: 'reduce' })
  await page.setViewportSize(size)
  await serve(page)
  await page.addInitScript(([key, value]) => localStorage.setItem(key, value), [`hanzo.locale.v1:${who}`, locale] as const)
  await page.goto(`${base}${path}`)
  await page.locator('html[data-hydrated]').waitFor({ state: 'attached', timeout: 30_000 })
  await expect(page.locator('html')).toHaveAttribute('lang', locale, { timeout: 20_000 })
  await expect(page.locator('html')).toHaveAttribute('dir', locale === 'ar' ? 'rtl' : 'ltr')
  // The catalog is its own chunk, fetched after hydration: wait until the page speaks it.
  if (locale !== 'en-US') await expect.poll(() => spoken(page, locale), { timeout: 20_000 }).toBe(true)
}

/** Whether the page reads in `locale` yet: Arabic letters in Arabic, French's own small words in French. */
const spoken = (page: Page, locale: 'ar' | 'fr-FR') =>
  page.evaluate((locale) => {
    const text = document.body.innerText
    return locale === 'ar' ? (text.match(/[\u0600-\u06FF]+/g) ?? []).length > 3 : (text.match(/\b(les|des|pour|une|avec|nouvelle|nouveau)\b/gi) ?? []).length > 1
  }, locale)

/**
 * The visible text and the read-aloud attributes under `scope` that hold English:
 * for Arabic, any Latin word not in `allowed`; for French, any of `ENGLISH`.
 * Code, commands and hidden marks are not prose and are skipped.
 */
function leftovers(page: Page, scope: string, locale: 'ar' | 'fr-FR'): Promise<string[]> {
  return page.evaluate(
    ({ scope, locale, allowed, english, packaged, sent }) => {
      const allow = new Set(allowed)
      const served = new Set(sent)
      const common = new Set(english)
      const skip = 'code, pre, kbd, samp, script, style, noscript, [aria-hidden="true"], [translate="no"], [data-i18n-ltr], [data-upstream]'
      // A word a reader reads: not an argument `t()` isolated (a name, a figure: data),
      // not an identifier (a dot, slash, @ or _ inside it), not an abbreviation in
      // capitals, not a lone letter.
      const read = (text: string) =>
        text
          .replace(/\u2068[^\u2069]*\u2069/g, ' ')
          .split(/\s+/)
          .filter((raw) => !/[A-Za-z0-9][./@_][A-Za-z0-9]/.test(raw))
          .flatMap((raw) => raw.match(/[A-Za-z][A-Za-z'’]*/g) ?? [])
          .map((w) => w.replace(/['’]+$/, ''))
          .filter((w) => w.length > 1 && !/^[A-Z0-9]+s?$/.test(w))
      const bad = (text: string) =>
        read(text).some((w) => (locale === 'ar' ? !allow.has(w.toLowerCase()) : common.has(w.toLowerCase())))
      const out = new Set<string>()
      for (const root of Array.from(document.querySelectorAll(scope))) {
        const walk = document.createTreeWalker(root, NodeFilter.SHOW_TEXT)
        for (let n = walk.nextNode(); n; n = walk.nextNode()) {
          const el = n.parentElement
          const text = (n.textContent ?? '').trim()
          if (!el || !text || el.closest(skip)) continue
          const shown = el.getClientRects().length > 0 && getComputedStyle(el).visibility !== 'hidden'
          if (shown && !served.has(text) && !packaged.includes(text) && bad(text)) out.add(text)
        }
        for (const el of [root, ...Array.from(root.querySelectorAll('[aria-label], [alt], [placeholder], [title]'))]) {
          if (el.closest(skip)) continue
          for (const name of ['aria-label', 'alt', 'placeholder', 'title']) {
            const value = el.getAttribute(name)
            // An address field's example is an address.
            if (name === 'placeholder' && el.matches('input[type="email"]')) continue
            if (value && bad(value) && !packaged.includes(`${name}="${value}"`)) out.add(`${name}="${value}"`)
          }
        }
      }
      return [...out]
    },
    { scope, locale, allowed: ALLOWED, english: ENGLISH, packaged: PACKAGED, sent: SENT },
  )
}

/**
 * Every visible text holding a figure range ("1,150–1,400"), and those of them not
 * isolated, which read backwards in Arabic ("1,400–1,150").
 */
function ranges(page: Page): Promise<{ seen: number; reversed: string[] }> {
  return page.evaluate(() => {
    let seen = 0
    const reversed: string[] = []
    const walk = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT)
    for (let n = walk.nextNode(); n; n = walk.nextNode()) {
      const text = n.textContent ?? ''
      const el = n.parentElement
      if (!el || !/\d[\d,.]*\s*[–-]\s*\d/.test(text) || el.getClientRects().length === 0) continue
      seen++
      const isolated = text.includes('\u2068') || el.closest('bdi, code, pre, kbd, [dir="ltr"], [data-i18n-ltr]') !== null
      if (!isolated && getComputedStyle(el).direction !== 'ltr') reversed.push(text.trim())
    }
    return { seen, reversed }
  })
}

/** How far the page scrolls sideways. */
const sideways = (page: Page) => page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth)

/** Brings every lazy section in, so all of the landing is drawn before it is read. */
async function scroll(page: Page) {
  await page.evaluate(async () => {
    for (let y = 0; y < document.documentElement.scrollHeight; y += 600) {
      window.scrollTo(0, y)
      await new Promise((r) => setTimeout(r, 40))
    }
    window.scrollTo(0, 0)
  })
  await expect(page.locator('[data-api="loading"]')).toHaveCount(0, { timeout: 30_000 })
}

const HEADER = 'header[data-slot="masthead"]'

for (const [size, label] of [
  [LAPTOP, '1440'],
  [PHONE, '390'],
] as const) {
  test(`ar: the landing at ${label} is Arabic, right to left, figures in order`, async ({ page }, info) => {
    await open(page, '/', 'ar', size)
    await scroll(page)
    expect.soft(await leftovers(page, 'body', 'ar'), 'no authored English on the landing').toEqual([])
    // The ranges are drawn from the served runs, which a loaded runner answers late:
    // read again, after another pass down the page, until one is drawn.
    await expect
      .poll(async () => (await ranges(page)).seen || (await scroll(page), 0), {
        message: 'the landing shows figure ranges, so the next line measures something',
        timeout: 30_000,
      })
      .toBeGreaterThan(0)
    expect.soft((await ranges(page)).reversed, 'every range reads in its own order').toEqual([])
    expect.soft(await sideways(page), 'nothing scrolls sideways').toBeLessThanOrEqual(0)
    await page.screenshot({ path: info.outputPath(`landing-ar-${label}.png`), fullPage: true })
  })

  test(`en-US: the landing at ${label}, for the screenshot beside Arabic's`, async ({ page }, info) => {
    await open(page, '/', 'en-US', size)
    await scroll(page)
    expect(await sideways(page)).toBeLessThanOrEqual(0)
    await page.screenshot({ path: info.outputPath(`landing-en-${label}.png`), fullPage: true })
  })
}

test('ar: every menu in the bar, opened, is Arabic', async ({ page }, info) => {
  await open(page, '/', 'ar')
  const triggers = page.locator(`${HEADER} button[aria-controls^="menu-"]`)
  const count = await triggers.count()
  expect(count, 'the bar has its menus').toBe(2)
  for (let i = 0; i < count; i++) {
    const trigger = triggers.nth(i)
    const id = await trigger.getAttribute('aria-controls')
    await trigger.click()
    await expect(page.locator(`#${id}`)).toBeVisible()
    expect(await leftovers(page, `#${id}`, 'ar'), `${id} holds no authored English`).toEqual([])
    await page.screenshot({ path: info.outputPath(`${id}-ar-1440.png`) })
    await page.keyboard.press('Escape')
  }
  expect(await leftovers(page, HEADER, 'ar'), 'the bar holds no authored English').toEqual([])
})

test('ar: the phone sheet, every section open, is Arabic', async ({ page }, info) => {
  await open(page, '/', 'ar', PHONE)
  await page.locator(`${HEADER} button[aria-controls="sheet"]`).click()
  const sheet = page.locator('#sheet')
  await expect(sheet).toBeVisible()
  const sections = sheet.locator('summary')
  for (let i = 0; i < (await sections.count()); i++) {
    await sections.nth(i).click()
    expect(await leftovers(page, '#sheet', 'ar'), 'the sheet holds no authored English').toEqual([])
  }
  expect(await sideways(page)).toBeLessThanOrEqual(0)
  await page.screenshot({ path: info.outputPath('sheet-ar-390.png') })
})

for (const [path, name] of [
  ['/login', 'sign-in'],
  ['/signup', 'sign-up'],
] as const) {
  test(`ar: ${name} is Arabic`, async ({ page }, info) => {
    await open(page, path, 'ar')
    await expect(page.locator('input[type="email"]').first()).toBeVisible({ timeout: 20_000 })
    expect(await leftovers(page, 'body', 'ar'), `${name} holds no authored English`).toEqual([])
    await page.screenshot({ path: info.outputPath(`${name}-ar-1440.png`) })
  })
}

/** The app's own reads, answered empty: a room with nothing in it yet, rather than an error drawn as sent. The fixtures `open` serves are routed after, so they win. */
const empty = (page: Page) =>
  page.route(
    (u) => u.pathname.startsWith('/v1/') && !u.pathname.startsWith('/v1/iam/') && !u.pathname.startsWith('/v1/billing/'),
    (route) => (new URL(route.request().url()).pathname.endsWith('/stream') ? route.fulfill({ status: 200, contentType: 'text/event-stream', body: '' }) : route.fulfill({ json: {} })),
  )

test('ar: the app, signed in, is Arabic', async ({ page }, info) => {
  await signIn(page)
  await empty(page)
  await open(page, '/', 'ar', LAPTOP, 'gate')
  const app = page.locator('[data-room]')
  await expect(app).toBeVisible({ timeout: 30_000 })
  await expect(page.locator('[data-slot="rail"]').first()).toBeVisible()
  expect(await leftovers(page, '[data-room]', 'ar'), 'the app holds no authored English').toEqual([])
  expect(await sideways(page)).toBeLessThanOrEqual(0)
  await page.screenshot({ path: info.outputPath('app-ar-1440.png') })
})

test('fr-FR: the landing and a menu hold no English', async ({ page }) => {
  await open(page, '/', 'fr-FR')
  await scroll(page)
  expect(await leftovers(page, 'body', 'fr-FR'), 'the landing').toEqual([])
  await page.locator(`${HEADER} button[aria-controls="menu-platform"]`).click()
  expect(await leftovers(page, '#menu-platform', 'fr-FR'), 'a menu').toEqual([])
})

test('fr-FR: sign-in and the app hold no English', async ({ page }) => {
  await open(page, '/login', 'fr-FR')
  await expect(page.locator('input[type="email"]').first()).toBeVisible({ timeout: 20_000 })
  expect(await leftovers(page, 'body', 'fr-FR'), 'sign-in').toEqual([])
  await signIn(page)
  await empty(page)
  await open(page, '/', 'fr-FR', LAPTOP, 'gate')
  await expect(page.locator('[data-room]')).toBeVisible({ timeout: 30_000 })
  expect(await leftovers(page, '[data-room]', 'fr-FR'), 'the app').toEqual([])
})