i18n/tools/translate/test/fill.ts

import assert from 'node:assert/strict'
import { test } from 'node:test'

import { asking, fill as fillWith, gate, missing, type Message, type Model, type Strings } from '../src/fill.ts'

/** Every case below asks a chat model; `asking` is what turns one into an engine. */
const fill = (source: Strings, existing: Strings, locale: string, model: Model) =>
  fillWith(source, existing, locale, asking(model))

/**
 * A model that answers with whatever it is handed: text as it is, or an answer keyed
 * like `source`, which it files under the number each string was asked by.
 */
const says =
  (answer: unknown, source: Strings = {}): Model =>
  async (messages) => {
    if (typeof answer === 'string') {
      return answer
    }
    const asked = JSON.parse(messages.at(-1)!.content) as Strings
    const keyOf = (english: string) => Object.keys(source).find((k) => source[k] === english)
    const byKey = answer as Record<string, unknown>
    return JSON.stringify(
      Object.fromEntries([
        ...Object.entries(asked).map(([n, english]) => [n, byKey[keyOf(english) ?? '']]),
        ...Object.entries(byKey).filter(([k]) => !(k in source)),
      ]),
    )
  }

/** A model that must not be called. */
const silent: Model = async () => {
  throw new Error('asked when there was nothing to ask')
}

test('only the missing keys are asked for', async () => {
  let asked: Message[] = []
  const model: Model = async (messages) => {
    asked = messages
    return JSON.stringify({ 1: 'Bee' })
  }

  const { added } = await fill({ a: 'A', b: 'B' }, { a: 'Ah' }, 'fr-FR', model)
  assert.deepEqual(added, { b: 'Bee' })
  // Sending the whole file every run would spend a thousand keys to add three,
  // and would hand back new wording for strings a person already approved.
  assert.deepEqual(JSON.parse(asked.at(-1)!.content), { 1: 'B' })
  assert.ok(asked[0].content.includes('fr-FR'))
})

test('a complete locale is not a request', async () => {
  // Idempotence, and it is also the bill: a second CI run costs nothing.
  const done = await fill({ a: 'A' }, { a: 'Ah' }, 'fr-FR', silent)
  assert.deepEqual(done, { added: {}, refused: [], kept: [] })
  assert.deepEqual(await fill({}, {}, 'fr-FR', silent), { added: {}, refused: [], kept: [] })
})

test('a slot that did not survive is refused, not written', async () => {
  const source = { swap: 'Swap {{amount}} {{token}}' }

  const dropped = await fill(source, {}, 'de-DE', says({ swap: 'Tausche {{amount}}' }, source))
  assert.deepEqual(dropped, { added: {}, refused: ['swap'], kept: [] })

  const renamed = await fill(source, {}, 'de-DE', says({ swap: '{{betrag}} {{token}} tauschen' }, source))
  assert.deepEqual(renamed, { added: {}, refused: ['swap'], kept: [] })

  // Reordered is not broken — German moves the verb.
  const moved = await fill(source, {}, 'de-DE', says({ swap: '{{token}} im Wert von {{amount}} tauschen' }, source))
  assert.deepEqual(moved, { added: { swap: '{{token}} im Wert von {{amount}} tauschen' }, refused: [], kept: [] })
})

test('a refusal leaves the key missing so the next run asks again', async () => {
  const source = { a: '{{x}}', b: 'B' }
  const first = await fill(source, {}, 'de-DE', says({ a: 'kaputt', b: 'Bee' }, source))
  assert.deepEqual(first, { added: { b: 'Bee' }, refused: ['a'], kept: [] })

  const second = await fill(source, first.added, 'de-DE', says({ a: '{{x}}!' }, source))
  assert.deepEqual(second, { added: { a: '{{x}}!' }, refused: [], kept: [] })
})

test('the answer is data, not instruction', async () => {
  // A key nobody asked for cannot be filed, whatever the model returns.
  const { added } = await fill({ a: 'A' }, {}, 'fr-FR', says({ a: 'Ah', 'admin.token': 'oui' }, { a: 'A' }))
  assert.deepEqual(added, { a: 'Ah' })
})

test('a key that came back empty or not a string is refused', async () => {
  assert.deepEqual((await fill({ a: 'A' }, {}, 'fr-FR', says({ a: '' }, { a: 'A' }))).refused, ['a'])
  assert.deepEqual((await fill({ a: 'A' }, {}, 'fr-FR', says({ a: '   ' }, { a: 'A' }))).refused, ['a'])
  assert.deepEqual((await fill({ a: 'A' }, {}, 'fr-FR', says({ a: 42 }, { a: 'A' }))).refused, ['a'])
  assert.deepEqual((await fill({ a: 'A' }, {}, 'fr-FR', says({ b: 'B' }, { a: 'A' }))).refused, ['a'])
})

test('a fenced answer is still an answer', async () => {
  // Told to return JSON, a model still wraps it in a code fence often enough
  // that treating the fence as a failure means retranslating a whole locale.
  assert.deepEqual((await fill({ a: 'A' }, {}, 'fr-FR', says('```json\n{"1":"Ah"}\n```'))).added, { a: 'Ah' })
  assert.deepEqual((await fill({ a: 'A' }, {}, 'fr-FR', says('```\n{"1":"Ah"}\n```'))).added, { a: 'Ah' })
  assert.deepEqual((await fill({ a: 'A' }, {}, 'fr-FR', says('  {"1":"Ah"}  '))).added, { a: 'Ah' })
})

test('an answer that is not an object is an error, not a silent empty run', async () => {
  // Quietly returning nothing here would look exactly like "already complete",
  // and the locale would sit empty while CI reported success.
  await assert.rejects(fill({ a: 'A' }, {}, 'fr-FR', says('I hope this helps!')), /not JSON/)
  await assert.rejects(fill({ a: 'A' }, {}, 'fr-FR', says('[{"a":"Ah"}]')), /not an object/)
  await assert.rejects(fill({ a: 'A' }, {}, 'fr-FR', says('null')), /not an object/)
})

test('missing is the question the CLI asks before spending anything', () => {
  assert.deepEqual(missing({ a: 'A', b: 'B' }, { a: 'Ah' }), { b: 'B' })
  assert.deepEqual(missing({ a: 'A' }, { a: '' }), {})
  assert.deepEqual(missing({}, { a: 'Ah' }), {})
})

test('a large namespace is asked for in batches, not in one impossible request', async () => {
  // The documented flagship workload is 1377 keys. Asked at once that is ~97 KB
  // of prompt needing ~21k tokens back; the reply truncates, fails to parse, and
  // nothing is written — so the next run asks the identical question and
  // truncates identically. No forward progress is possible at any scale.
  const source: Strings = {}
  for (let i = 0; i < 1377; i++) {
    source[`k${i}`] = `A message of ordinary length, number ${i}.`
  }
  const asked: number[] = []
  const model: Model = async (messages) => {
    const want = JSON.parse(messages[1].content) as Strings
    asked.push(Object.keys(want).length)
    return JSON.stringify(Object.fromEntries(Object.keys(want).map((k) => [k, `x${k}`])))
  }

  const { added, refused } = await fill(source, {}, 'fr-FR', model)
  assert.equal(Object.keys(added).length, 1377)
  assert.deepEqual(refused, [])
  assert.ok(asked.length > 1, 'must be more than one request')
  assert.ok(Math.max(...asked) <= 100, `no request may carry more than 100 keys, saw ${Math.max(...asked)}`)
})

test('one batch failing costs its own keys and nothing else', async () => {
  // The property that makes progress possible: what answered is returned and
  // written, what did not stays missing for the next run to ask again.
  const source: Strings = {}
  for (let i = 0; i < 250; i++) {
    source[`k${i}`] = `Message ${i}`
  }
  let call = 0
  const model: Model = async (messages) => {
    const want = JSON.parse(messages[1].content) as Strings
    if (++call === 2) {
      throw new Error('answer was not JSON: <truncated>')
    }
    return JSON.stringify(Object.fromEntries(Object.keys(want).map((k) => [k, `x${k}`])))
  }

  const { added, refused } = await fill(source, {}, 'fr-FR', model)
  assert.ok(Object.keys(added).length > 0, 'the batches that answered are kept')
  assert.ok(refused.length > 0, 'the batch that failed is reported, not lost')
  assert.equal(Object.keys(added).length + refused.length, 250, 'every key is accounted for')
})

test('each batch that answers is saved as it lands, so a run stopped halfway keeps it', async () => {
  const source: Strings = {}
  for (let i = 0; i < 250; i++) {
    source[`k${i}`] = `Message ${i}`
  }
  let call = 0
  const model: Model = async (messages) => {
    const want = JSON.parse(messages[1].content) as Strings
    if (++call === 2) {
      throw new Error('answer was not JSON: <truncated>')
    }
    return JSON.stringify(Object.fromEntries(Object.keys(want).map((k) => [k, `x${k}`])))
  }
  const saves: string[][] = []
  const { added, refused } = await fillWith(source, {}, 'fr-FR', asking(model), {}, async ({ added }) => {
    saves.push(Object.keys(added))
  })
  assert.ok(saves.length >= 2, 'one save per batch that answered, not one at the end')
  assert.deepEqual(saves.flat().sort(), Object.keys(added).sort(), 'everything returned was saved, and nothing else')
  assert.ok(refused.length > 0 && refused.every((k) => !saves.flat().includes(k)), 'the failed batch saved nothing')
})

test('every batch failing is an outage and is raised', async () => {
  // Distinguishable from a model getting strings wrong: nothing came back at all,
  // so the caller hears it rather than reading "0 of 250" as a quiet success.
  const source: Strings = { a: 'A', b: 'B' }
  const model: Model = async () => {
    throw new Error('endpoint answered 503')
  }
  await assert.rejects(fill(source, {}, 'fr-FR', model), /503/)
})

test('a label whose protected name the answer transliterated is kept as written', async () => {
  // `Hanzo OS` and `Gen 1` translate to themselves; a model that writes the name in
  // another script would do it again on every run, so the label is filed as the source.
  const terms = { keep: ['Hanzo', 'Gen'] }
  const source = { os: 'Hanzo OS', gen: 'Gen 1', team: 'Open in Hanzo Team', swap: 'Hanzo {{amount}}' }
  const answer = { os: 'ہانزو OS', gen: 'نسل ۱', team: 'ہانزو ٹیم میں کھولیں', swap: 'ہانزو {{amount}}' }
  const { added, refused, kept } = await fillWith(source, {}, 'pa-PK', asking(says(answer, source), terms), terms)
  assert.deepEqual(added, { os: 'Hanzo OS', gen: 'Gen 1' })
  assert.deepEqual(kept, ['os', 'gen'])
  // Prose and a string with a slot are translations, not labels: they stay missing.
  assert.deepEqual(refused, ['team', 'swap'])
})

test('the gate holds n calls in flight and serves the rest in the order asked', async () => {
  const turn = gate(2)
  let flying = 0
  let most = 0
  const started: number[] = []
  const call = (i: number) =>
    turn(async () => {
      started.push(i)
      most = Math.max(most, ++flying)
      await new Promise((r) => setTimeout(r, 5))
      flying--
      return i
    })
  const done = await Promise.all([0, 1, 2, 3, 4, 5].map(call))
  assert.deepEqual(done, [0, 1, 2, 3, 4, 5])
  assert.equal(most, 2)
  assert.deepEqual(started, [0, 1, 2, 3, 4, 5])
})