mirror of
https://github.com/hanzoai/i18n.git
synced 2026-10-30 14:24:32 +00:00
zeekay
066c5220dc
--mode madlad runs MADLAD-400 through CTranslate2 on this machine (tools/translate/madlad.py). It takes no instructions, so protected names cross it as numbered placeholders and are restored after. A label, a protected name with at most one other word and no slot, translates to itself: when an answer transliterates its name it is written as the source and recorded under a provenance run of model "source". Variant notes for the new locales go with the prompt.
20 lines
1.0 KiB
Python
20 lines
1.0 KiB
Python
# MADLAD-400 translation worker for i18n-translate --mode madlad (src/madlad.ts).
|
|
# Needs ctranslate2 and sentencepiece; argv[1] is the CTranslate2 conversion directory.
|
|
# stdin, one per line: {"target": "my", "texts": [...]}; stdout, one per line: {"out": [...]}.
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
import ctranslate2
|
|
import sentencepiece
|
|
|
|
model = sys.argv[1]
|
|
pieces = sentencepiece.SentencePieceProcessor(model_file=os.path.join(model, 'spiece.model'))
|
|
translator = ctranslate2.Translator(model, device='cpu', compute_type='int8', intra_threads=int(os.environ.get('THREADS', '8')))
|
|
for line in sys.stdin:
|
|
job = json.loads(line)
|
|
# The target tag leads the text and the source ends with </s>; without </s> the decoder repeats itself.
|
|
batch = [pieces.encode(f"<2{job['target']}> {text}", out_type=str) + ['</s>'] for text in job['texts']]
|
|
results = translator.translate_batch(batch, beam_size=2, max_decoding_length=256, max_batch_size=16)
|
|
print(json.dumps({'out': [pieces.decode(r.hypotheses[0]) for r in results]}, ensure_ascii=False), flush=True)
|