speech: decode in a pool, so the process can exit while one is running

The live gate caught this on its first run: 40 tests passed and the build was
still red, exit 134. A raw daemon thread is not joined at interpreter shutdown,
so a decode still inside ctranslate2's C++ when the runtime tore down aborted —
`terminate called without an active exception`. The failure is in the exit code,
not in the report, which is why the fast suite and the summary line both looked
fine. In a pod it is a container that cannot drain on SIGTERM.

A pool fixes it because its workers ARE joined, and it is the better shape
anyway: decoding is CPU-bound and the pod is capped at 4 cores, so bounding
concurrent decodes stops many sessions from thrashing each other. Per-session
single flight is unchanged.

The test shuts a child down mid-decode and asserts it exits 0 — with the model
loaded first and a pause after pushing, so shutdown lands inside the decode by
construction rather than by luck.

Co-authored-by: Hanzo Dev <[email protected]>

test_live.py

@@ -17,6 +17,9 @@ in a stage above the weight bake and never sees this file.
 """
 
 import io
+import os
+import subprocess
+import sys
 import time
 
 import numpy
@@ -121,6 +124,40 @@ def test_the_seconds_billed_are_the_seconds_spoken(said):
     assert spoken > 4.0, "a real sentence, not a click"
 
 
+def test_the_process_exits_cleanly_mid_decode(said, tmp_path):
+    """Shut down while a decode is running and the process must still exit 0.
+
+    A thread sitting inside the model's C++ when the interpreter tears down makes
+    the C++ runtime abort — `terminate called without an active exception`,
+    SIGABRT. Every test still reports passed, because the failure is in the exit
+    code and not in the report; in a pod it is a container that cannot drain on
+    SIGTERM. Only real weights reproduce it: a stand-in has no native code to
+    abort in.
+
+    The model is loaded first and the child pauses after pushing, so shutdown
+    lands mid-decode by construction rather than by luck.
+    """
+    raw = tmp_path / "said.pcm"
+    raw.write_bytes(said)
+    child = (
+        "import time, stt, transcript;"
+        "stt.load('whisper');"
+        "live = transcript.begin('whisper', 'en');"
+        f"pcm = open({str(raw)!r}, 'rb').read();"
+        "[live.push(pcm[a:a + transcript.CHUNK])"
+        " for a in range(0, len(pcm), transcript.CHUNK)];"
+        "time.sleep(0.5)"
+    )
+    done = subprocess.run(
+        [sys.executable, "-c", child],
+        cwd=os.path.dirname(os.path.abspath(__file__)),
+        capture_output=True, timeout=600,
+    )
+    assert done.returncode == 0, (
+        f"exit {done.returncode}\n{done.stderr.decode()[-800:]}"
+    )
+
+
 def test_the_batch_route_still_reports_a_real_duration(said):
     """The quantity the plane already meters, from the real model rather than a
     stub: a wav of known length must come back with that length."""

transcript.py

@@ -21,6 +21,7 @@ reports as `duration`, measured the same way, and it is what the call is billed
 import secrets
 import threading
 import time
+from concurrent.futures import ThreadPoolExecutor
 
 import stt
 
@@ -36,6 +37,20 @@ WINDOW = 12.0  # past this the guard is dropped, so the window cannot grow forev
 IDLE = 30.0  # a session untouched this long is collectable
 LIMIT = 600.0  # audio one session will accept, in seconds
 
+# Decoding is CPU-bound and the pod is capped at 4 cores, so two at a time leaves
+# room for the request path to keep acking. Sessions beyond that queue rather than
+# thrash: more concurrent decodes than cores makes every session slower, not one
+# session faster.
+#
+# A pool, not a thread per session, for a second reason that is not about speed:
+# its workers are joined when the interpreter shuts down. Raw daemon threads are
+# not, and a thread still inside the model's C++ when the runtime tears down
+# aborts the process — `terminate called without an active exception`, SIGABRT,
+# with every test passing and the run still red. In a pod that is a container
+# that cannot drain on SIGTERM.
+WORKERS = 2
+_pool = ThreadPoolExecutor(max_workers=WORKERS, thread_name_prefix="decode")
+
 
 class Transcript:
     """One growing transcript. Everything mutable is under `_lock`."""
@@ -77,7 +92,7 @@ class Transcript:
             if start:
                 self._decoding = True
         if start:
-            threading.Thread(target=self._work, daemon=True).start()
+            _pool.submit(self._work)
         return len(pcm) / stt.SECOND
 
     def close(self) -> None: