hanzoai/frames
2
0
Fork 0
mirror of https://github.com/hanzoai/frames.git synced 2026-09-24 00:08:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(media-use): resolve every media type through api.hanzo.ai

Browse Source 
Voice is POST /v1/audio/speech and word timings come from
POST /v1/audio/transcriptions over the rendered audio. Music, sound
effects, images, and video generate through /v1/audio/music,
/v1/audio/foley, /v1/images/generations, and /v1/videos/generations. The
shared catalog is a listing under media/<type>/ in our own object store,
browsed through /v1/s3 — an empty prefix reports itself as empty rather
than reading as a bad query.

One client (scripts/lib/api.mjs) holds every route and the one credential:
a Hanzo IAM token in $HANZO_API_KEY, read from Hanzo KMS. No credential
file, no per-repo .env, no per-provider CLI to install or keep current.
$HANZO_BASE_URL points the same code at a local Hanzo Engine.

Word timings are passed through, never interpolated: a caption cut on
invented boundaries reads as correct and drifts against the voice. When
the service returns text without them, the word array is empty and the
caller is told.

Deletes the provider cascade this replaces, its model ladders, and the
account linking in telemetry — the install id stands alone.
This commit is contained in:
zooqueen
2026-08-13 17:18:37 -07:00
parent bb68594c65
commit ae80da5998
82 changed files with 1794 additions and 6006 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills-manifest.json
+28 -28
View File
@@ -2,79 +2,79 @@
"source": "hanzoai/frames",
"skills": {
"embedded-captions": {
"hash": "ed4dc7b850b92ff5",
"hash": "a8a486d53e620373",
"files": 140
},
"faceless-explainer": {
"hash": "b772a9b6c8118c2c",
"hash": "f785f30abdfd4d31",
"files": 22
},
"figma": {
"hash": "517e4dc53c13ea05",
"hash": "4863dca8d7ad16bd",
"files": 2
},
"general-video": {
"hash": "87f292b572cad3f9",
"hash": "fdde7482e76fadaf",
"files": 4
},
"frames": {
"hash": "02b48855a5321e48",
"hyperframes": {
"hash": "abce26ac789054e5",
"files": 17
},
"frames-animation": {
"hash": "3d9855346af7d51c",
"hyperframes-animation": {
"hash": "b5c6b511d6defd0c",
"files": 121
},
"frames-cli": {
"hash": "966972db5ab8f932",
"hyperframes-cli": {
"hash": "5589bfd1f7d4c22e",
"files": 11
},
"frames-core": {
"hash": "a270c70ced8cf952",
"hyperframes-core": {
"hash": "9cbd7d594a20568b",
"files": 19
},
"frames-creative": {
"hash": "bacb5205ea5eb3d8",
"hyperframes-creative": {
"hash": "6067e5377e69d48c",
"files": 78
},
"frames-keyframes": {
"hash": "a568c05c01b27461",
"hyperframes-keyframes": {
"hash": "839d23bf6479a12a",
"files": 3
},
"frames-registry": {
"hash": "3a864992e765b9bf",
"hyperframes-registry": {
"hash": "a9f271c04a01a50d",
"files": 10
},
"media-use": {
"hash": "2e02a0799754e411",
"files": 152
"hash": "d9c65ef59eff5bc9",
"files": 124
},
"motion-graphics": {
"hash": "0212a19069119e72",
"hash": "539b80792f36e267",
"files": 23
},
"music-to-video": {
"hash": "55a2b5fdcd6f892c",
"hash": "c18d52831b867725",
"files": 132
},
"pr-to-video": {
"hash": "44a9877e7ea1289e",
"hash": "74a5ebeeced75313",
"files": 29
},
"product-launch-video": {
"hash": "8cce4a32487eaf80",
"hash": "1bce2438e5baf2db",
"files": 26
},
"remotion-to-frames": {
"hash": "3a0e6c2affb9f74e",
"remotion-to-hyperframes": {
"hash": "3e6a7dac6fd01dfc",
"files": 70
},
"slideshow": {
"hash": "71368acf61074198",
"hash": "23fe4debdcb1604a",
"files": 2
},
"talking-head-recut": {
"hash": "2f5d99f823c48e75",
"hash": "006f1525a7fefdb1",
"files": 28
}
}
skills/media-use/SKILL.md
+16 -16
View File
@@ -7,7 +7,7 @@ description: Agent Media OS, the single skill for every media need in a Frames p
The media OS for Frames: resolve · generate · operate · remember — every media type, one skill, zero context noise.
First run: install and sign in to the `heygen` CLI (the free-usage path), then verify with `node <SKILL_DIR>/scripts/resolve.mjs --doctor`. Setup and providers: `references/setup-providers.md`.
First run: export `HANZO_API_KEY` (from Hanzo KMS) and `HANZO_ORG`, then verify with `node <SKILL_DIR>/scripts/resolve.mjs --doctor`. Setup and providers: `references/setup-providers.md`.
## Resolve — the one verb
@@ -19,12 +19,12 @@ Returns one line: `resolved <id> → <path> (<type>, <metadata>)`. All search no
| Type | One-line intent |
| ------- | ----------------------------------------------------------------------------------- |
| `bgm` | background music (HeyGen catalog, 10k+ tracks) |
| `sfx` | sound effects (bundled 19-file library + catalog) |
| `image` | photos, backgrounds (HeyGen asset search, 75k+ vectors) |
| `bgm` | background music (shared catalog, then generated) |
| `sfx` | sound effects (21 bundled files, then the catalog, then foley) |
| `image` | photos, backgrounds (shared catalog, then generated) |
| `icon` | icons, symbols (transparent) |
| `logo` | official brand marks (svgl → simple-icons → GitHub avatar → favicon; never redrawn) |
| `voice` | TTS voiceover (HeyGen free-usage path; optional local Kokoro) |
| `voice` | spoken narration, with word timings from the same transcript pass |
| `grade` | measured correction candidate; broad polish/stylization follows Media Treatments |
| `lut` | user-provided or explicitly chosen reusable validated `.cube` file |
@@ -76,7 +76,7 @@ Surface an opportunity only when a concrete signal is present:
| Image that is a placeholder, tiny, or upscaled-looking | a better `image` (and/or upscale — see `references/operations.md`) |
| Hard scene cuts / transitions with no sound | transition `sfx` |
| A piece over ~10s with no music bed | `bgm` |
| Footage that reads under/over-exposed or color-cast | a corrective grade (inspect it with `frames media-treatment --selector '#hero' --analyze --json`) |
| Footage that reads under/over-exposed or color-cast | a corrective grade (inspect it with `frames media-treatment --selector '#hero' --analyze --json`) |
| Photographic media that feels visually flat or off-topic | one specific source-appropriate preset or custom treatment, with the intended target named |
| A meaningful media entrance/reveal that feels static | one supported seek-safe treatment animation; preserve color unless the request also justifies a preset |
@@ -84,13 +84,13 @@ Rules that keep this a help, not nagware: **grounded, not generic** (no signal
## Where to look — read only the file your task needs
| Task | Read |
| ------------------------------------------------------------------------- | -------------------------------- |
| resolve / reuse / adopt / ingest, flags, cascade, inventory | `references/resolve.md` |
| color grading, LUTs, smart grade (`--for`), grade-compare | `references/grading.md` |
| voiceover / TTS, music, SFX, captions, transcription (audio engine) | `references/audio.md` |
| cut / reframe / transform existing media, exact error diffusion, HEVC | `references/operations.md` |
| source-aware creative treatments, realtime effects, overlays, reveals | `references/media-treatments.md` |
| install + auth, provider table, RAM ladders, `--local-only`, `--provider` | `references/setup-providers.md` |
| remembered preferences + frozen recipes (user memory) | `references/memory.md` |
| ownership matrix, usage stats, telemetry, privacy (maintainer-facing) | `references/meta.md` |
| Task | Read |
| --------------------------------------------------------------------- | -------------------------------- |
| resolve / reuse / adopt / ingest, flags, cascade, inventory | `references/resolve.md` |
| color grading, LUTs, smart grade (`--for`), grade-compare | `references/grading.md` |
| voiceover / TTS, music, SFX, captions, transcription (audio engine) | `references/audio.md` |
| cut / reframe / transform existing media, exact error diffusion, HEVC | `references/operations.md` |
| source-aware creative treatments, realtime effects, overlays, reveals | `references/media-treatments.md` |
| credential, provider table, `--local-only`, `--provider` | `references/setup-providers.md` |
| remembered preferences + frozen recipes (user memory) | `references/memory.md` |
| ownership matrix, usage stats, telemetry, privacy (maintainer-facing) | `references/meta.md` |
skills/media-use/audio/references/bgm.md
+51 -36
View File
@@ -1,50 +1,66 @@
# Background music (BGM)
One music bed per composition, produced by the shared audio engine (`scripts/audio.mjs` → `scripts/lib/bgm.mjs`). Two routes, chosen by the engine's one switch — whether a HeyGen credential is present:
One music bed per composition, produced by the shared audio engine
(`scripts/audio.mjs` -> `scripts/lib/bgm.mjs`). Two routes:
- **HeyGen retrieval — the default when credentialed.** Search HeyGen's music catalog by mood, download the top track. No generation; same `~/.heygen` / `$HEYGEN_API_KEY` credential as TTS.
- **Local generation (Lyria → MusicGen) — the fallback when there is no credential** (or when asked for explicitly). Generate a WAV from a mood prompt. There is **no `npx frames bgm` command**; the engine spawns `scripts/lyria-recipe.py` or an inline MusicGen script directly.
- **catalog** — the closest name under `media/bgm/` in the shared catalog,
downloaded to `assets/bgm/track.<ext>`. Free, instant, identical every run.
- **compose** — `POST /v1/audio/music` with a mood prompt, written to
`assets/bgm/track.wav`. Used when the catalog cannot answer.
> **Run the Preflight first — no credential is not a green light to silently generate locally.** Before generating, complete the sign-in **Preflight** (see `../SKILL.md` → Preflight): run `npx frames auth status`, recommend signing in, and **STOP for the user's choice** (sign in for HeyGen's music library, or continue offline with local generation). This applies to a one-off "generate a BGM" request just as much as inside a full workflow.
Both are synchronous: when the engine returns, the file is on disk.
## Driving it from the request
`audio_request.json` → `bgm: { mode?, query?, prompt? }`:
`audio_request.json` -> `bgm: { mode?, query?, prompt? }`:
- **`mode`** — `retrieve | generate | none`. Omit for **auto** (retrieve when credentialed, else generate). An **explicit** `retrieve` is strict: no credential ⇒ skip, never a detached generate (so a caller with no `wait-bgm` step, e.g. product-launch, can't get a pending job it won't await).
- **`query`** — the mood, used for retrieval and as a fallback prompt seed (e.g. a storyboard's `music:` field, falling back to `message` → `arc` → `"calm cinematic underscore"`).
- **`prompt`** — an explicit full prompt for generation; omit and the engine infers one (see Mood inference). Optional `blob` / `archetype` / `arc` feed that inference.
- **`mode`** — `catalog | compose | none`. Omit for **auto** (catalog, then
compose). An explicit `catalog` is honored as written: an empty or unmatched
shelf is reported and BGM is skipped, never quietly composed instead.
- **`query`** — the mood. It is the catalog query and the fallback prompt seed
(a storyboard's `music:` field, falling back to `message` -> `arc` ->
`"calm cinematic underscore"`).
- **`prompt`** — an explicit full prompt for compose; omit it and the engine
infers one. Optional `blob` / `archetype` / `arc` feed that inference.
## HeyGen retrieval (default)
## The catalog rung
`searchSounds(query, "music", { limit: 5 })` → `GET /audio/sounds?query=<mood>&type=music&limit=5`. Take the top result (ranked by `score`), download its presigned `audio_url` → `assets/bgm/track.mp3`. Synchronous. No match → skip (BGM is optional; never fail the render over it). Cue written to `audio_meta.json`:
The catalog is a listing under `media/bgm/`, browsed through
`GET /v1/s3/buckets/media/objects`. **An object key IS its description** — a
file named `calm-cinematic-underscore.mp3` answers "calm cinematic underscore" —
so stock it with names written in the words a brief would use.
**The catalog is empty until someone stocks it.** That is reported, not hidden:
an empty prefix logs `the catalog holds no music yet (bgm/)` and, under auto,
falls through to compose.
The cue written to `audio_meta.json`:
```jsonc
{
"path": "assets/bgm/track.mp3",
"volume": 0.12,
"mode": "retrieve",
"mode": "catalog",
"query": "calm cinematic underscore",
"duration_s": 42.0,
"key": "bgm/calm-cinematic-underscore.mp3",
}
```
`volume` comes from the engine's `bgmDefaultVolume()`: `BGM_BED_VOLUME` (currently `0.12` ≈ -18 dB — a bed under the voice) under narration, `BGM_SILENT_VOLUME` (currently `0.9`) for a silent film (no voice). Tune those constants in `scripts/lib/bgm.mjs`, not call sites. An explicit `volume` in `audio_meta.json` always overrides this default. `bgm_pending` is `false` — the file is on disk when the engine returns.
`volume` comes from `bgmDefaultVolume()`: `BGM_BED_VOLUME` (0.12, about -18 dB —
a bed under the voice) under narration, `BGM_SILENT_VOLUME` (0.9) for a silent
film. Tune those constants in `scripts/lib/bgm.mjs`, not at call sites. An
explicit `volume` in `audio_meta.json` always wins.
## Local generation (fallback) — Lyria → MusicGen
## The compose rung
Spawned **detached** so voice work isn't blocked; `audio_meta.bgm_pending: true` and `bgm_pid` / `bgm_log` are set until it finishes. **Run `scripts/wait-bgm.mjs` before assembling** — it polls the output file / process / log, detects crashes, and writes `bgm_status.json` (`status: ready | failed | timeout | disabled`). A failed/absent track is simply omitted; it never blocks voice/SFX.
`POST /v1/audio/music` with the inferred (or given) prompt and the total voice
duration. Output goes to `assets/bgm/track.wav`.
| Order | Provider | Env / deps | Speed | Quality |
| ----- | ------------------------------------ | ------------------------------------------------------------------------------------- | --------------------------------------- | --------------------------- |
| 1 | Google Lyria RealTime | `$GEMINI_API_KEY` or `$GOOGLE_API_KEY` + `google-genai` (auto-installed on demand) | Real-time stream (≈ requested duration) | Production-grade |
| 2 | MusicGen (`facebook/musicgen-small`) | Python `transformers + torch + soundfile + numpy` (~300 MB first run; auto-installed) | Slow on CPU; fast on Apple MPS / CUDA | Decent; prompt-only control |
## Mood inference (the compose prompt)
Output → `assets/bgm/track.wav`, target = total voice duration. MusicGen generates **one** seed clip (≤28–30s, under the decoder's positional limit) then crossfade-loops it up to the target (or trims down if shorter), avoiding per-segment seams. Backend selection is by what can actually **run**: Lyria only when `import google.genai` succeeds, else MusicGen; if neither can be made to run, BGM is skipped (voice + SFX still render).
## Mood inference (the generate prompt)
`inferBgmPrompt()` in `scripts/lib/bgm.mjs`: an explicit `prompt` wins; otherwise industry-keyword **base** → narrative-**archetype** shape → emotional-**arc** tiebreaker.
`inferBgmPrompt()` in `scripts/lib/bgm.mjs`: an explicit `prompt` wins;
otherwise industry-keyword **base** -> narrative-**archetype** shape ->
emotional-**arc** tiebreaker.
| Match in `blob` / `query` | Base prompt | BPM |
| ------------------------------------------------------ | --------------------------------------------------------------------------- | --- |
@@ -53,20 +69,19 @@ Output → `assets/bgm/track.wav`, target = total voice duration. MusicGen gener
| `creative / agency / design / studio / art / brand` | playful electronic, warm pads, light percussion | 115 |
| _(default: SaaS / tech / platform)_ | uplifting corporate tech, bright modern piano with synth pads | 108 |
Archetype then reshapes the arc — PAS → "MINOR to MAJOR" build; BAB / future-pacing → aspirational rising; feature-cascade → +10 BPM driving; demo-loop → −8 BPM minimal. The emotional arc breaks remaining ties (tension→relief, excitement, trust/reassurance).
## Lyria knobs (direct recipe use)
The engine bakes BPM / scale into the **prompt text** (via the inference above) and passes only `--output` / `--duration` / `--prompt` to the recipe. If you invoke `scripts/lyria-recipe.py` directly you can also set: `--bpm` (90–110 calm, 110–130 energetic), `--brightness` (0–1, ≥0.7 promotional), `--density` (0–1, higher = fuller), `--scale` (`MAJOR` / `MINOR` / `PENTATONIC` / …), `--negative-prompt` (styles to exclude). MusicGen ignores all of these — put the mood in the prompt.
Archetype then reshapes the arc — PAS -> "MINOR to MAJOR" build; BAB /
future-pacing -> aspirational rising; feature-cascade -> +10 BPM driving;
demo-loop -> -8 BPM minimal. The emotional arc breaks remaining ties
(tension->relief, excitement, trust/reassurance).
## Failure modes
| Failure | Behavior |
| --------------------------------------------- | ---------------------------------------------------------------------------------------- |
| No music match (retrieve) | `bgm: null`, anomaly logged. Render proceeds without BGM. |
| Explicit `retrieve`, no credential | Skipped (no silent generate fallback). Use `mode: generate` or omit `mode` for auto. |
| Neither Lyria nor MusicGen can run (generate) | `bgm` disabled with a `pip install …` hint. Voice + SFX still render. |
| Generate still rendering at assemble time | `bgm_pending: true`; `wait-bgm.mjs` waits/checks and writes `bgm_status.json` first. |
| Generate crashed | `wait-bgm.mjs` → `bgm_status.json { status: "failed" }`; the `<audio>` track is omitted. |
| Failure | Behavior |
| ------------------------------- | ----------------------------------------------------------- |
| Empty catalog, auto mode | Anomaly logged, composes instead. |
| Empty catalog, explicit catalog | Anomaly logged, `bgm: null`. Render proceeds without music. |
| No catalog match | Same as above — reported, then composed or skipped. |
| No credential | `bgm: null`, anomaly logged. |
| The music call fails | `bgm: null`, anomaly names the HTTP status. |
BGM failure never blocks a render.
skills/media-use/audio/references/requirements.md
+36 -21
View File
@@ -1,29 +1,44 @@
# Requirements & Caches
# Requirements & caches
## Credential & key priority
## Credential
Run `npx frames auth status` to see what's configured and which engines a workflow will use (see the skill's **Preflight** section). Keys resolve in this order — **first match wins**:
One credential, one host. Read it from Hanzo KMS (`kms.hanzo.ai`) and export it:
| Provider | Resolution order (first non-empty wins) | Local deps when used |
| ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ |
| **HeyGen** (TTS + BGM/SFX retrieval) | `$HEYGEN_API_KEY` → `$FRAMES_API_KEY` → `~/.heygen/credentials` (shared with heygen-cli; `$HEYGEN_CONFIG_DIR` overrides the dir; written by `frames auth login`) | none (REST) |
| **ElevenLabs** (TTS fallback) | `$ELEVENLABS_API_KEY` | `pip install elevenlabs` |
| **Lyria** (BGM fallback) | `$GEMINI_API_KEY` → `$GOOGLE_API_KEY` | `pip install google-genai` |
| **Kokoro** (TTS, no key) | always — final voice fallback | `pip install kokoro-onnx soundfile` |
| **MusicGen** (BGM, no key) | always — final music fallback | `pip install transformers torch soundfile numpy` |
```bash
export HANZO_API_KEY=... # a Hanzo IAM token, or an sk- cloud key
export HANZO_ORG=... # the tenant that owns the shared media bucket
```
`frames auth login` (browser OAuth) is the recommended setup: one sign-in, every project, no per-repo `.env`. An OAuth login is sent as `Authorization: Bearer`; an API key as `X-Api-Key`; both are tagged with `X-HeyGen-Source: cli`. OAuth CLI users can consume the web-plan free allowance for HeyGen TTS (10 min/month); API keys follow the normal API billing path. With no HeyGen credential, voice/BGM run fully locally (Kokoro / MusicGen) — `frames auth status` and `frames doctor` both report whether those local deps are installed.
There is no credential file and no per-repo `.env`. A publishable `pk-` key is
read-only and the media routes refuse it. `HANZO_BASE_URL` points the same code
at a Hanzo Engine on your own machine (default `https://api.hanzo.ai`).
## Model caches & system dependencies
`node <SKILL_DIR>/scripts/resolve.mjs --doctor` reports what is set and what is
reachable.
Each command downloads its own model on first run and caches it under `~/.cache/frames/`:
## What each capability needs
- **TTS (HeyGen)** — no local deps; needs a HeyGen credential + `ffmpeg` on PATH (to transcode the mp3 response to `.wav`). Credential resolves like the CLI: `$HEYGEN_API_KEY` → `$FRAMES_API_KEY` → `~/.heygen/credentials` (shared with heygen-cli; run `npx frames auth login`). An OAuth login is sent as `Authorization: Bearer`; an API key as `X-Api-Key`; both include `X-HeyGen-Source: cli` so the backend can apply CLI OAuth free usage.
- **TTS (ElevenLabs)** — same as HeyGen: API key + `ffmpeg`.
- **TTS (Kokoro)** — Kokoro-82M (~311 MB) + voices (~27 MB) in `tts/`. Requires Python 3.8+ with `kokoro-onnx` and `soundfile` (`pip install kokoro-onnx soundfile`). Non-English text also needs `espeak-ng` system-wide.
- **BGM (Lyria)** — needs `$GEMINI_API_KEY` or `$GOOGLE_API_KEY` + `pip install google-genai`. No local model cache.
- **BGM (MusicGen)** — `pip install transformers torch soundfile`. `facebook/musicgen-small` (~300 MB) cached under `~/.cache/huggingface/` on first run.
- **Transcribe** — Whisper model size depending on choice (75 MB – 3.1 GB) in `whisper/`, downloaded from HuggingFace on first use. `whisper.cpp` itself is NOT bundled: the CLI resolves it from PATH, installs via Homebrew (macOS), or builds it from source with git+cmake on first use (`$FRAMES_WHISPER_PATH` overrides).
- **Remove-background** — `u2net_human_seg` (~168 MB ONNX) in `background-removal/models/`. Peak inference RAM ~1.5 GB.
| Capability | Route | Local dependency |
| ------------------ | ------------------------------- | ----------------------------- |
| Voice | `POST /v1/audio/speech` | `ffmpeg` for `.wav` output |
| Word timings | `POST /v1/audio/transcriptions` | none |
| Music | `POST /v1/audio/music` | none |
| Sound effects | `POST /v1/audio/foley` | none |
| Images | `POST /v1/images/generations` | none |
| Video | `POST /v1/videos/generations` | none |
| The shared catalog | `GET /v1/s3/buckets/media/…` | none |
| Grade / LUT | built locally from `params` | `ffprobe` for measured grades |
Run `npx frames doctor` if a command fails because of a missing dependency.
No model downloads and no per-provider Python: everything but ffmpeg is an HTTP
call. `ffmpeg` and `ffprobe` are the only strict requirements
(`brew install ffmpeg`).
## Limits worth knowing
- **Speech input is capped at 4096 bytes per request.** The engine sends one
request per line, which is also how each line gets its own duration.
- **Transcription upload is capped at 25 MiB.** Longer audio is cut first.
- **A catalog listing returns at most 1000 keys per prefix.** Treat a full page
as "there may be more".
- **A catalog link is signed and expires in minutes.** Fetch it now; never write
one into a document.
skills/media-use/audio/references/sfx.md
+37 -18
View File
@@ -1,42 +1,61 @@
# Sound effects (SFX)
Named sound effects, produced by the shared audio engine (`scripts/audio.mjs` → `scripts/lib/sfx.mjs`). **Provider-gated** by the engine's one switch — whether a HeyGen credential is present, decided once (not per cue):
Named sound effects, produced by the shared audio engine (`scripts/audio.mjs` ->
`scripts/lib/sfx.mjs`). Each cue is resolved in the same order, cheapest first:
- **HeyGen credential present → retrieve every cue** from HeyGen's audio library (`/v3/audio/sounds`, `type=sound_effects`, `min_score=0.4`). Search-and-download, **not** generation. The bundled library is NOT consulted.
- **No credential → the bundled 21-file library** (`assets/sfx/` + `manifest.json`): match each cue name, copy the matched file into the project. Offline, deterministic, free.
- **library** — the 21 files bundled with this skill (`assets/sfx/` +
`manifest.json`). Offline, deterministic, free, and the same bytes every run.
- **catalog** — `media/sfx/` in the shared catalog, for the long tail the 21
miss.
- **foley** — `POST /v1/audio/foley` when neither has it.
There is no `npx frames sfx` command. SFX is never generated — it is retrieved (online) or taken from the bundled library (offline).
A cue nothing can answer is **skipped** and recorded as an anomaly; SFX never
blocks a render.
## Cues — request → meta
## Cues — request to meta
Each line names the effects it wants: `lines[].sfx: ["whoosh", "ui click"]`. The engine flattens these into cues, resolves them per the switch, dedupes identical `(id, name)` pairs (the same effect named twice downloads/copies once), and writes `audio_meta.sfx[]`:
Each line names the effects it wants: `lines[].sfx: ["whoosh", "ui click"]`. The
engine flattens these into cues, resolves them, dedupes identical `(id, name)`
pairs (the same effect named twice resolves once), and writes `audio_meta.sfx[]`:
```jsonc
{
"id": "3", // joins the cue to the caller's model (frame / scene / segment)
"name": "whoosh",
"file": "assets/sfx/whoosh.mp3", // downloaded or copied, relative to project root
"source": "heygen" | "local", // which route resolved it
"file": "assets/sfx/whoosh.mp3", // relative to project root
"source": "library" | "catalog" | "foley",
"offset_s": 0, // delay from the line's start
"duration_s": 0.57,
"volume": 0.35 // SFX sit UNDER voice + BGM
}
```
A cue that matches nothing is **skipped** (recorded as an anomaly); SFX never blocks a render.
## The bundled library
## HeyGen retrieval (credentialed)
21 curated files in `assets/sfx/`, indexed by `manifest.json` —
`{ file, duration, description }` per key (`whoosh`, `pop`, `click`, `chime`,
`riser`, `impact-bass-1`, `glitch-1`, `typing`, …). A cue name resolves by
**manifest key, file basename, or slug**, so `whoosh`, `whoosh.mp3`, and
`"ui click"` all land. Matched files are copied into the project's
`assets/sfx/`; `duration_s` comes from the manifest, so timing is known
**offline** — `riser` is 10.03s, so it fires at `climax - 10.03s`. The manifest's
`description` carries placement hints; read `assets/sfx/manifest.json` for the
full set.
`searchSounds(name, "sound_effects", { limit: 3, minScore: 0.4 })` → top hit → `assets/sfx/<slug>.mp3`. Results are ranked by `score` (each carries a presigned `audio_url`, `duration`, `description`). The floor is **0.4** because good SFX hits score ~0.5–0.67 — below the API's default `0.7`, which would silently drop most named cues (only whoosh/swoosh-family clears 0.7). `duration_s` comes from the result (else 1.0). Name effects concretely (`glass shatter`, not `dramatic sound`); a vague query returns a poor match.
## The catalog and foley rungs
## Bundled library (no credential)
Same catalog as BGM, under `media/sfx/`: a key IS its description, and an empty
prefix says so rather than reading as a bad query. When the shelf cannot answer,
`POST /v1/audio/foley` makes the effect from the cue name.
21 curated files in `assets/sfx/`, indexed by `manifest.json` — `{ file, duration, description }` per key (e.g. `whoosh`, `pop`, `click`, `chime`, `riser`, `impact-bass-1`, `glitch-1`, `typing`, …). A cue name resolves by **manifest key, file basename, or slug**, so `whoosh`, `whoosh.mp3`, or `"ui click"` (→ slug) all match. Matched files are copied into the project's `assets/sfx/`; `duration_s` comes from the manifest, so timing is known **offline** — e.g. `riser` is 10.03s, so trigger it at `climax − 10.03s`. The manifest's `description` field carries placement hints per effect; read `assets/sfx/manifest.json` for the full set and usage.
Name effects concretely — `glass shatter`, not `dramatic sound`. A vague name is
a vague match on the shelf and a vague prompt at the foley route.
## Rules
- **Volume ~0.35.** SFX must sit under narration and BGM, not fight them.
- **No match → skip, don't fail.** A missing effect logs an anomaly and moves on; never a render blocker.
- **Retrieval (credentialed) or bundled library (offline) — never generation.** You search HeyGen by text, or match a name against the 21-file manifest.
- **One asset per distinct name.** Reuse across lines is deduped to a single download/copy, many cues.
- **The switch is global, not per cue.** With a credential, retrieval handles even the long tail (effects not in the 21); without one, only the 21 bundled names resolve.
- **Volume 0.35.** SFX sit under narration and BGM, not against them.
- **No match, no failure.** A missing effect logs an anomaly and moves on.
- **One asset per distinct name.** Reuse across lines is deduped to a single
resolve, many cues.
- **Cheapest rung first.** The bundled 21 cost nothing and never change; reach
past them only for what they do not have.
skills/media-use/audio/references/transcribe.md
+30 -37
View File
@@ -1,52 +1,45 @@
# Transcription
Create normalized word-level timestamps. **Always specify `--model` explicitly** — the CLI default is `small.en`, which silently translates non-English audio into English.
Word-level timestamps from audio, through `POST /v1/audio/transcriptions`.
```bash
npx frames transcribe audio.mp3 --model small.en # known English
npx frames transcribe video.mp4 --model small --language es # known Spanish
npx frames transcribe audio.mp3 --model small # unknown language (auto-detect)
npx frames transcribe subtitles.srt # import existing
npx frames transcribe subtitles.vtt
npx frames transcribe openai-response.json
node <SKILL_DIR>/scripts/transcribe.mjs --input audio.mp3
node <SKILL_DIR>/scripts/transcribe.mjs --input video.mp4 --lang es
node <SKILL_DIR>/scripts/transcribe.mjs --input audio.mp3 --out words.json --json
```
## Language Rule (Non-Negotiable)
## Language
`.en` models (`tiny.en` / `base.en` / `small.en` / `medium.en`) **translate** non-English audio into English. This silently destroys the original language.
`--lang` defaults to `en` and is sent as the request language. Pass the real
language of the audio: a transcription asked for the wrong language returns
translated text, which silently destroys the original. For mixed-language audio,
name the one you want kept.
1. **Known English** → `--model small.en` (or `medium.en` for music / noisy audio)
2. **Known non-English** → `--model small --language <iso-code>` (no `.en` suffix)
3. **Unknown language** → `--model small` (whisper auto-detects)
## Model
**CLI default is `small.en`** — do not rely on it; always pass `--model` to make the choice explicit. `--language` also filters out non-target-language segments from mixed-language audio.
`--model` defaults to `whisper`. `whisper-small` is the faster, smaller
alternative. Both are served by our speech service; there is nothing to download
and no first-run build.
## Model Sizes
## Output shape
| Model | Size | Speed | When |
| ---------- | ------ | -------- | ------------------------------------- |
| `tiny` | 75 MB | Fastest | Quick previews, smoke tests |
| `base` | 142 MB | Fast | Short clips, clear audio |
| `small` | 466 MB | Moderate | Default for most multilingual content |
| `medium` | 1.5 GB | Slow | Music with vocals, noisy audio |
| `large-v3` | 3.1 GB | Slowest | Production quality |
### Picking a model by content type
1. Speech over silence / light background → `small.en`
2. Speech over music, or music with vocals → start with `medium.en`
3. Produced music track (vocals + full instrumentation) → start with `medium.en`; expect to need manual lyrics or an external API ([`captions/transcript-handling.md`](captions/transcript-handling.md) → "Using External Transcription APIs")
4. Multilingual → `medium` or `large-v3` (no `.en` suffix), pair with `--language`
## Output Shape
Compositions consume a flat array of word objects. The `id` (`w0`, `w1`, …) is added during normalization for stable references in caption overrides; optional for backwards compatibility.
A flat array of word objects. The `id` (`w0`, `w1`, …) is added during
normalization so caption overrides have a stable reference.
```json
[
{ "id": "w0", "text": "Hello", "start": 0.0, "end": 0.5 },
{ "id": "w1", "text": "world.", "start": 0.6, "end": 1.2 }
]
{
"text": "Hello world.",
"words": [
{ "text": "Hello", "start": 0.0, "end": 0.5 },
{ "text": "world.", "start": 0.6, "end": 1.2 }
]
}
```
For mandatory caption-quality checks, retry rules, and the OpenAI/Groq Whisper API import path, see `captions/transcript-handling.md`.
**`words` is empty when the service returns text without per-word timings**, and
the command says so on stderr. Nothing is interpolated to fill the gap. Callers
that need per-word cuts should treat an empty `words` as "captions fall back to
line timing", not as a failed transcription.
For caption-quality checks and retry rules, see
`captions/transcript-handling.md`.
skills/media-use/audio/references/tts-to-captions.md
+21 -16
View File
@@ -1,26 +1,31 @@
# TTS → Captions
# Voice to captions
When no recorded voiceover exists, generate one and obtain word-level caption timing. Two paths depending on which TTS provider is in use:
## Path A — HeyGen (single call, no Whisper)
HeyGen returns word timestamps in the same response as the audio. Use the
bundled REST helper (the `frames tts` command is Kokoro-only):
When no recorded voiceover exists, render one and take its word timing from the
same audio.
```bash
node skills/media-use/audio/scripts/heygen-tts.mjs \
node skills/media-use/audio/scripts/speak.mjs \
script.txt --output narration.wav --words narration.words.json
```
`narration.words.json` is already in the `[{ id, text, start, end }]` shape the captions pipeline consumes — no separate transcribe pass.
`speak.mjs` renders the line through `POST /v1/audio/speech`, then reads word
timings back through `POST /v1/audio/transcriptions` over the audio it just
wrote. `narration.words.json` lands in the `[{ id, text, start, end }]` shape
the captions pipeline consumes.
## Path B — ElevenLabs / Kokoro (TTS → Whisper)
Two calls, not one, because they answer two questions — a piece with no captions
never pays for the alignment pass. Aligning against the rendered audio (rather
than trusting a synthesis-time estimate) is also what keeps caption timing
matched to delivery.
These providers don't return word data. Generate the audio, then transcribe:
## When there are no timings
```bash
npx frames tts script.txt --voice af_heart --output narration.wav
npx frames transcribe narration.wav --model small.en # voice af_heart is American English
```
If the transcription returns text without per-word timings, `--words` writes
nothing and says so. Timings are never interpolated from the audio duration: a
caption cut on invented boundaries reads as correct and drifts against the
voice. Captions then fall back to line-level timing, which the audio engine
already records as `voices[].duration_s`.
Whisper extracts precise word boundaries from the generated audio, so caption timing matches delivery without hand-tuning. Match `--model` to the voice's language (use `small.en` for `a`/`b` prefixes, `small --language <code>` otherwise). Then consume `transcript.json` via the caption references in `captions/`.
For a whole script — every line, plus music and effects, with timings folded
into `audio_meta.json` — use the audio engine rather than calling `speak.mjs`
per line. See `../../references/audio.md`.
skills/media-use/audio/references/tts.md
+42 -120
View File
@@ -1,131 +1,42 @@
# Text To Speech
# Text to speech
`npx frames tts` synthesizes locally with Kokoro. It does not accept a
`--provider` or `--words` flag. For HeyGen audio plus word timestamps, use the
bundled `heygen-tts.mjs` script below.
> **Run the Preflight first — no credential is not a green light to silently use the local voice.** Before generating a voiceover, complete the sign-in **Preflight** (see `../SKILL.md` → Preflight): run `npx frames auth status`, recommend signing in, and **STOP for the user's choice** (sign in for HeyGen voices, or continue offline with local Kokoro). This applies to a one-off "generate a voiceover" request just as much as inside a full workflow.
## Available routes
| Order | Provider | Env trigger | Voice IDs | Word timestamps | Audio format |
| ----- | ----------------- | ------------------------------------------- | ------------------------------------------- | ----------------------------------------- | -------------------- |
| 1 | HeyGen (Starfish) | `$HEYGEN_API_KEY` / `~/.heygen/credentials` | UUIDs from `GET /v3/voices?engine=starfish` | **Yes** (`word_timestamps[]` in response) | mp3 → wav via ffmpeg |
| 2 | ElevenLabs | `$ELEVENLABS_API_KEY` | UUIDs from elevenlabs.io dashboard | No | mp3 → wav via ffmpeg |
| 3 | Kokoro-82M | always (local fallback) | `am_michael`, `af_heart`, … (54 voices) | No | wav direct |
One route: `POST /v1/audio/speech` through `api.hanzo.ai`. Word timings come
from a second call, `POST /v1/audio/transcriptions`, over the audio that was
just rendered.
```bash
# Local Kokoro CLI
npx frames tts "Welcome to Frames" -o narration.wav
# one line, one file
node <SKILL_DIR>/audio/scripts/speak.mjs "Welcome to Frames." -o narration.wav
# with word timings for captions
node <SKILL_DIR>/audio/scripts/speak.mjs ./script.txt \
-o narration.wav --words narration.words.json
# another language, a chosen voice
node <SKILL_DIR>/audio/scripts/speak.mjs "Bonjour" -o fr.wav --lang fr --voice <id>
```
## Self-contained HeyGen (no CLI) — `scripts/heygen-tts.mjs`
For a whole script — every line, plus music and effects — use the audio engine
(`audio/scripts/audio.mjs`) instead; `speak.mjs` is the single-line door onto the
same code.
The published `frames tts` CLI synthesizes locally with Kokoro only. When you
want HeyGen specifically — best quality **plus** word timestamps in one call — use
the skill's bundled script, which calls the HeyGen v3 REST API directly and needs
no CLI provider plumbing:
## Flags
The script resolves a HeyGen credential the same way the CLI does — first source
wins: `$HEYGEN_API_KEY` → `$FRAMES_API_KEY` → a project `.env` (auto-loaded,
walks up ≤5 dirs) → `~/.heygen/credentials` (shared with heygen-cli;
`$HEYGEN_CONFIG_DIR` overrides the dir). An OAuth login is sent as
`Authorization: Bearer`; an API key as `X-Api-Key`; both include
`X-HeyGen-Source: cli`. OAuth CLI users can consume the web-plan free allowance
(10 min/month) before paid usage; API keys follow normal API billing. If the
only credential is an expired OAuth token it stops with a hint to run
`npx frames auth refresh`.
- **`--voice <id>`** — a voice the speech service accepts. Omit it and the
service picks its own default. media-use keeps no second voice catalog to
drift out of step with theirs.
- **`--output` / `-o`** — `.wav` is transcoded to 44.1k mono through ffmpeg (what
`ffprobe` and the captions pipeline expect); any other extension keeps the
service's own bytes.
- **`--words <path>`** — runs the transcription pass and writes the flat
`[{id,text,start,end}]` shape the captions pipeline reads.
- **`--speed`** — 0.7-0.8 for tutorials and accessibility, 1.0 natural, 1.1-1.2
for intros and upbeat pieces. Past 1.5 rarely survives a listen.
- **`--lang <code>`** — anything but `en` is sent as the request language.
```bash
# Only needed if you haven't run `npx frames auth login`:
export HEYGEN_API_KEY=... # or put it in a project .env
## Word timings
# Synthesize + capture word timestamps in one call (skips a Whisper pass)
node skills/media-use/audio/scripts/heygen-tts.mjs \
"Welcome to Frames." -o narration.wav --words narration.words.json
node skills/media-use/audio/scripts/heygen-tts.mjs ./script.txt -o narration.wav
node skills/media-use/audio/scripts/heygen-tts.mjs --list # public starfish voices
```
- **Voice:** `--voice <id>` must be a **starfish** voice_id (`--list`, or `GET /v3/voices?engine=starfish`). v2-catalog ids are rejected with HTTP 400. Omit `--voice` (English) and it defaults to **Marcia** (`05f19352e8f74b0392a8f411eba40de1`, a fixed default so the choice is deterministic). Non-English with no `--voice` falls back to the first matching catalog voice.
- **Output:** `.wav` → transcoded to 44.1k mono via ffmpeg; `.mp3` → raw bytes (no ffmpeg needed).
- **Words:** `--words <path>` writes the flat `[{id,text,start,end}]` shape below, drop-in for the captions pipeline. HeyGen's `<start>`/`<end>` boundary sentinels are filtered out and ids are re-contiguous.
- **Non-English:** `--lang <code>` (anything but `en`) is sent as the request `language`.
## When to use which provider
| Goal | Use |
| --------------------------------------------------------- | --------------------------------------------------- |
| Best voice quality + word timestamps in one call | **HeyGen** |
| Drop-in cloud TTS, big voice catalog | **ElevenLabs** |
| Offline, no API key, fast iteration | **Kokoro** |
| Non-English multilingual with deterministic phonemization | **Kokoro** (`ef_dora`, `jf_alpha`, `zf_xiaobei`, …) |
## ffmpeg requirement
HeyGen + ElevenLabs return mp3. The bundled HeyGen helper transcodes to wav
when `--output` ends in `.wav` (the default and what downstream `ffprobe` +
Whisper expect). If you'd rather skip the transcode, pass `-o file.mp3`.
Without `ffmpeg` on PATH, wav output from cloud providers fails; the local
Kokoro CLI writes wav directly.
## Voice selection (Kokoro)
Default `af_heart`. Curated picks:
| Content type | Voice |
| ----------------- | ---------------------- |
| Product demo | `af_heart`, `af_nova` |
| Tutorial / how-to | `am_adam`, `bf_emma` |
| Marketing / promo | `af_sky`, `am_michael` |
| Documentation | `bf_emma`, `bm_george` |
| Casual / social | `af_heart`, `af_sky` |
Run `npx frames tts --list` for the bundled set.
## Multilingual (Kokoro voice prefix → language)
The first letter of a Kokoro voice ID picks the phonemizer language; `--lang` overrides auto-detection.
| Prefix | Language |
| ------ | -------------------- |
| `a` | American English |
| `b` | British English |
| `e` | Spanish |
| `f` | French |
| `h` | Hindi |
| `i` | Italian |
| `j` | Japanese |
| `p` | Brazilian Portuguese |
| `z` | Mandarin |
```bash
npx frames tts "La reunión empieza a las nueve" --voice ef_dora
npx frames tts "Today is a nice day" --voice af_heart
```
Valid `--lang` codes (only needed to override the voice's auto-detected language): `en-us`, `en-gb`, `es`, `fr-fr`, `hi`, `it`, `pt-br`, `ja`, `zh`.
Non-English phonemization requires `espeak-ng` system-wide (`brew install espeak-ng` / `apt-get install espeak-ng`).
## Speed
- `0.7-0.8` — tutorial, complex content, accessibility
- `1.0` — natural pace (default)
- `1.1-1.2` — intros, transitions, upbeat content
- `1.5+` — rarely appropriate, test carefully
The `frames tts` command honors `--speed` for Kokoro. Provider-specific
helpers document their own pacing controls.
## Long scripts
Past a few paragraphs, write the text to a `.txt` file and pass the path. Inputs over ~5 minutes of speech may benefit from splitting into segments.
## HeyGen word-timestamp shape
When `--words <path>` is passed to a HeyGen call, the file is written in the same flat shape `transcribe` produces — drop-in compatible with the captions pipeline:
`--words` writes the same shape `transcribe` produces, drop-in for captions:
```json
[
@@ -134,4 +45,15 @@ When `--words <path>` is passed to a HeyGen call, the file is written in the sam
]
```
For ElevenLabs / Kokoro, run `npx frames transcribe narration.wav --model small.en` to get the same shape.
**When the service returns no per-word timings, nothing is written and the
command says so.** Timings are never interpolated from the audio duration: a
caption cut on invented boundaries looks correct and drifts against the voice.
Captions then fall back to line-level timing, which the audio engine already
records as `voices[].duration_s`.
## Limits
- A single request accepts **4096 bytes** of input. Longer scripts are written
as lines in an `audio_request.json` and rendered one line per request by the
engine, which is also how each line gets its own timing.
- `.wav` output needs `ffmpeg` on PATH. Ask for `.mp3` to skip the transcode.
skills/media-use/audio/scripts/audio.mjs
+66 -128
View File
@@ -6,52 +6,40 @@
//
// node <MEDIA_DIR>/scripts/audio.mjs --request ./audio_request.json --frames . --out ./audio_meta.json
//
// The three capabilities degrade on ONE switch — whether HeyGen is configured
// (credential present, NOT the CLI). This mirrors the table in ../SKILL.md:
// All three capabilities go through api.hanzo.ai — one credential, one host:
//
// TTS : HeyGen REST → ElevenLabs → Kokoro (CLI)
// BGM : HeyGen retrieve → (no credential) Lyria/MusicGen generate
// SFX : HeyGen retrieve → (no credential) bundled 19-file library
// TTS : POST /v1/audio/speech, then /v1/audio/transcriptions for word timings
// BGM : media/bgm/ in our object store, else POST /v1/audio/music
// SFX : the 21 bundled files, else media/sfx/, else POST /v1/audio/foley
//
// ── audio_request.json (input) ────────────────────────────────────────────────
// {
// "provider": "auto", // auto|heygen|elevenlabs|kokoro (override: --provider)
// "lang": "en", "speed": 1.0,
// "voice": null, // a voice id the speech service accepts
// "lines": [ // one TTS unit each; id joins back to the caller's model
// { "id": "01", "text": "...", "sfx": ["whoosh", "ui click"] }
// ],
// "bgm": { "mode": "retrieve", // retrieve|generate|none (override: --bgm-mode / --no-bgm)
// "query": "calm cinematic underscore", // mood for retrieval
// "prompt": null, // full prompt for generation (else inferred)
// "bgm": { "mode": "catalog", // catalog|compose|none (override: --bgm-mode / --no-bgm)
// "query": "calm cinematic underscore", // mood, and the catalog query
// "prompt": null, // full prompt for compose (else inferred)
// "blob": "...", "archetype": "...", "arc": "..." } // optional mood-inference hints
// }
//
// ── audio_meta.json (output, id-keyed) ───────────────────────────────────────
// { tts_provider, voice_id,
// bgm: { path, volume, mode, query?, duration_s? } | null,
// bgm_pending, bgm_provider, bgm_pid, bgm_log, bgm_mode, bgm_target_duration_s, …,
// { voice_id,
// bgm: { path, volume, mode, query?, duration_s? } | null, bgm_mode,
// voices: [ { id, path, duration_s, words: [{id,text,start,end}] } ],
// sfx: [ { id, name, file, source, offset_s, duration_s, volume } ],
// total_duration_s }
//
// --only tts,bgm,sfx runs a subset and MERGES into an existing --out (so a
// workflow can do TTS+BGM early, then SFX later once cues exist). When BGM uses
// the generate path it is spawned detached (bgm_pending:true) — run wait-bgm.mjs
// before assembling.
// workflow can do TTS+BGM early, then SFX later once cues exist).
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { heygenAuthHeaders, heygenCredential, loadEnvFromDir } from "./lib/heygen.mjs";
import {
ffprobeDuration,
pickProvider,
resolveVoiceId,
synthesizeOne,
transcribeWav,
withWordIds,
} from "./lib/tts.mjs";
import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs";
import { align, ffprobeDuration, ready, synthesize, withWordIds } from "./lib/tts.mjs";
import { fromCatalog, fromPrompt, inferBgmPrompt } from "./lib/bgm.mjs";
import { resolveSfx } from "./lib/sfx.mjs";
import { mapWithConcurrency } from "./lib/concurrency.mjs";
@@ -82,7 +70,6 @@ const framesDir = resolve(flag("frames", "."));
const requestPath = resolve(flag("request", join(framesDir, "audio_request.json")));
const outPath = resolve(flag("out", join(framesDir, "audio_meta.json")));
const sfxLibDir = resolve(flag("sfx-lib", join(HERE, "..", "assets", "sfx")));
const lyriaRecipe = resolve(flag("lyria-recipe", join(HERE, "lyria-recipe.py")));
const onlyArg = flag("only", "tts,bgm,sfx");
const only = new Set(
onlyArg
@@ -90,13 +77,11 @@ const only = new Set(
.map((s) => s.trim())
.filter(Boolean),
);
const providerOverride = flag("provider", null);
const bgmModeOverride = flag("bgm-mode", null);
const noBgm = has("no-bgm");
const voiceOverride = flag("voice", null);
const speedOverride = flag("speed", null);
const langOverride = flag("lang", null);
const seedSeconds = Number(flag("seed-seconds", "28")) || 28;
if (!existsSync(requestPath)) die(`audio_request.json not found at ${requestPath}`);
let request;
@@ -109,10 +94,12 @@ const lines = Array.isArray(request.lines) ? request.lines : [];
const lang = langOverride || request.lang || "en";
const speed = Number(speedOverride ?? request.speed ?? 1.0) || 1.0;
// ── env + HeyGen availability (the single switch) ─────────────────────────────
loadEnvFromDir(framesDir);
const heygenOK = heygenCredential() !== null;
const headers = heygenOK ? heygenAuthHeaders() : null;
// ── credential (the single switch) ────────────────────────────────────────────
const online = ready();
if (!online)
console.error(
"· no HANZO_API_KEY — voice and generated audio are unavailable; bundled SFX still resolve",
);
// ── merge base: preserve sections not selected by --only ──────────────────────
const prev = existsSync(outPath) ? JSON.parse(readFileSync(outPath, "utf8")) : {};
@@ -120,22 +107,13 @@ const anomalies = [];
// ── TTS ───────────────────────────────────────────────────────────────────────
let voices = prev.voices ?? [];
let ttsProvider = prev.tts_provider ?? null;
let voiceId = prev.voice_id ?? null;
if (only.has("tts") && lines.length) {
try {
ttsProvider = pickProvider(
providerOverride || (request.provider === "auto" ? null : request.provider),
);
} catch (e) {
die(e.message);
}
voiceId = await resolveVoiceId({
provider: ttsProvider,
userVoice: voiceOverride || request.voice,
lang,
});
console.error(`· tts: ${ttsProvider} · voice ${voiceId} · ${lines.length} line(s)`);
if (!online) die("voice needs a Hanzo credential — export HANZO_API_KEY");
// No voice id means the speech service picks its own default. Pass one to
// choose; there is no second catalog here to keep in step with theirs.
voiceId = voiceOverride || request.voice || null;
console.error(`· tts: ${voiceId ?? "service default"} voice · ${lines.length} line(s)`);
const synthLine = async (line) => {
const id = String(line.id);
const text = String(line.text ?? "").trim();
@@ -145,27 +123,19 @@ if (only.has("tts") && lines.length) {
}
const rel = `assets/voice/${id}.wav`;
const abs = join(framesDir, rel);
const { ok, words, error } = await synthesizeOne({
provider: ttsProvider,
text,
voiceId,
lang,
speed,
wavAbs: abs,
framesDir,
});
const { ok, error } = await synthesize({ text, voice: voiceId, lang, speed, out: abs });
if (!ok) {
anomalies.push(`line ${id}: TTS failed — omitted${error ? ` (${error})` : ""}`);
return null;
}
let wordArr = words; // heygen: native; else transcribe
if (!wordArr) wordArr = await transcribeWav({ wavRel: rel, lang, framesDir });
const dur = ffprobeDuration(abs);
if (!isFinite(dur) || dur <= 0) {
anomalies.push(`line ${id}: bad voice duration — omitted`);
return null;
}
return { id, path: rel, duration_s: r3(dur), words: withWordIds(wordArr) };
const words = await align({ file: abs, lang });
if (!words) anomalies.push(`line ${id}: no word timings returned — captions fall back to line timing`);
return { id, path: rel, duration_s: r3(dur), words: withWordIds(words) };
};
const results = await mapWithConcurrency(lines, ttsConcurrency, synthLine);
voices = results.filter(Boolean);
@@ -177,77 +147,48 @@ const totalDuration = r3(voices.reduce((a, v) => a + (v.duration_s || 0), 0));
// ── BGM ─────────────────────────────────────────────────────────────────────
let bgm = prev.bgm ?? null;
const bgmFields = {
bgm_pending: prev.bgm_pending ?? false,
bgm_provider: prev.bgm_provider ?? null,
bgm_pid: prev.bgm_pid ?? null,
bgm_log: prev.bgm_log ?? null,
bgm_mode: prev.bgm_mode ?? null,
bgm_target_duration_s: prev.bgm_target_duration_s ?? null,
bgm_seed_duration_s: prev.bgm_seed_duration_s ?? null,
bgm_loop_count: prev.bgm_loop_count ?? null,
};
let bgmMode = prev.bgm_mode ?? null;
if (only.has("bgm")) {
bgm = null;
Object.keys(bgmFields).forEach((k) => (bgmFields[k] = k === "bgm_pending" ? false : null));
// Mode resolution. An EXPLICIT mode (flag or request.bgm.mode) is strict:
// "retrieve" means retrieve-or-nothing — it never silently starts a detached
// generate (a caller with no wait-bgm step, e.g. product-launch, must not get
// a pending job it can't await). Only the UNSET/auto default picks generate
// when HeyGen is absent.
bgmMode = null;
// An EXPLICIT mode (flag or request.bgm.mode) is honored as written: asking
// for the catalog and getting a generated track instead is a surprise the
// caller cannot see in the output. Only the unset default falls through.
const explicitMode = bgmModeOverride || request.bgm?.mode || null;
let mode = noBgm ? "none" : explicitMode || (heygenOK ? "retrieve" : "generate");
if (mode === "retrieve" && !heygenOK) {
anomalies.push(
"bgm: retrieve requires a HeyGen credential — skipped (no generate fallback for an explicit retrieve)",
);
mode = "none";
}
const mode = noBgm ? "none" : explicitMode || "auto";
const query = request.bgm?.query;
if (mode === "none") {
if (mode === "none" || !online) {
if (mode !== "none") anomalies.push("bgm: needs a Hanzo credential — skipped");
console.error(`· bgm: disabled`);
} else if (mode === "retrieve") {
} else {
try {
bgm = await retrieveBgm({ query: request.bgm?.query, headers, framesDir, hasVoice });
if (bgm) {
bgmFields.bgm_provider = "heygen";
bgmFields.bgm_mode = "retrieve";
console.error(` bgm: ${bgm.path} (retrieve "${bgm.query}")`);
const hit = mode === "compose" ? null : await fromCatalog({ query, framesDir, hasVoice });
if (hit?.path) {
bgm = hit;
bgmMode = "catalog";
console.error(` bgm: ${bgm.path} (catalog "${bgm.query}")`);
} else if (mode === "catalog") {
anomalies.push(
hit?.empty
? `bgm: the catalog holds no music yet (${hit.prefix} is empty) — skipped`
: `bgm: no catalog match for "${query ?? ""}" — skipped`,
);
} else {
anomalies.push(`bgm: no music match for "${request.bgm?.query ?? ""}" — skipped`);
if (hit?.empty)
anomalies.push(`bgm: the catalog holds no music yet (${hit.prefix}) — composing instead`);
const prompt = inferBgmPrompt({
userPrompt: request.bgm?.prompt,
blob: request.bgm?.blob || query,
archetype: request.bgm?.archetype,
arc: request.bgm?.arc,
});
bgm = await fromPrompt({ prompt, durationS: totalDuration || 30, framesDir, hasVoice });
bgmMode = "compose";
console.error(` bgm: ${bgm.path} (composed)`);
}
} catch (e) {
anomalies.push(`bgm retrieve failed: ${e.message} — skipped`);
}
} else {
// generate
const prompt = inferBgmPrompt({
userPrompt: request.bgm?.prompt,
blob: request.bgm?.blob || request.bgm?.query,
archetype: request.bgm?.archetype,
arc: request.bgm?.arc,
});
const gen = generateBgmDetached({
prompt,
durationS: totalDuration || 30,
framesDir,
lyriaRecipe: existsSync(lyriaRecipe) ? lyriaRecipe : null,
seedSeconds,
hasVoice,
});
if (gen.disabled) {
anomalies.push(`bgm: ${gen.reason}`);
} else {
bgm = { path: gen.path, volume: gen.volume, mode: gen.mode, duration_s: null };
bgmFields.bgm_pending = true;
bgmFields.bgm_provider = gen.provider;
bgmFields.bgm_pid = gen.pid;
bgmFields.bgm_log = gen.log;
bgmFields.bgm_mode = gen.mode;
bgmFields.bgm_target_duration_s = gen.target_duration_s ?? null;
bgmFields.bgm_seed_duration_s = gen.seed_duration_s ?? null;
bgmFields.bgm_loop_count = gen.loop_count ?? null;
console.error(` bgm: launched ${gen.provider} (detached, pid ${gen.pid}) → ${gen.path}`);
anomalies.push(`bgm failed: ${e.message} — skipped`);
}
}
}
@@ -260,20 +201,17 @@ if (only.has("sfx")) {
.map((name) => ({ id: String(l.id), name: String(name).trim() }))
.filter((c) => c.name),
);
const res = await resolveSfx({ cues, heygenOK, headers, framesDir, sfxLibDir });
const res = await resolveSfx({ cues, framesDir, sfxLibDir });
sfx = res.sfx;
anomalies.push(...res.anomalies);
console.error(
`· sfx: ${sfx.length} cue(s) resolved (${heygenOK ? "heygen retrieval" : "bundled library"})`,
);
console.error(`· sfx: ${sfx.length} cue(s) resolved`);
}
// ── write audio_meta.json ─────────────────────────────────────────────────────
const meta = {
tts_provider: ttsProvider,
voice_id: voiceId,
bgm,
...bgmFields,
bgm_mode: bgmMode,
voices,
sfx,
total_duration_s: totalDuration,
@@ -282,9 +220,9 @@ mkdirSync(dirname(outPath), { recursive: true });
writeFileSync(outPath, JSON.stringify(meta, null, 2));
console.log(`✓ audio engine → ${outPath}`);
console.log(` heygen: ${heygenOK ? "yes" : "no"} · ran: ${[...only].join(",")}`);
console.log(` ran: ${[...only].join(",")}`);
console.log(
` voices: ${voices.length} · bgm: ${bgm ? `${bgmFields.bgm_provider}${bgmFields.bgm_pending ? " (pending)" : ""}` : "none"} · sfx: ${sfx.length}`,
` voices: ${voices.length} · bgm: ${bgm ? bgmMode : "none"} · sfx: ${sfx.length}`,
);
console.log(` total voice duration: ${totalDuration}s`);
if (anomalies.length) {
skills/media-use/audio/scripts/audio.test.mjs
+3 -5
View File
@@ -8,7 +8,7 @@ import { resolveSfx } from "./lib/sfx.mjs";
// Proves the relocated engine (skills/media-use/audio/) still resolves its
// bundled SFX library from the moved location — the path most likely to break
// on a subtree move. Offline (heygenOK:false), no network.
// on a subtree move. No credential is set, so no network rung is reached.
const HERE = dirname(fileURLToPath(import.meta.url));
const sfxLibDir = join(HERE, "..", "assets", "sfx"); // same offset the engine uses
@@ -19,12 +19,11 @@ test("bundled SFX library resolves from the relocated path", async () => {
try {
const { sfx, anomalies } = await resolveSfx({
cues: [{ id: "1", name: "whoosh" }],
heygenOK: false,
framesDir: dir,
sfxLibDir,
});
assert.equal(sfx.length, 1, `expected 1 resolved cue, got anomalies: ${anomalies.join("; ")}`);
assert.equal(sfx[0].source, "local");
assert.equal(sfx[0].source, "library");
assert.match(sfx[0].file, /assets\/sfx\//);
assert.ok(existsSync(join(dir, sfx[0].file)), "matched SFX copied into the project");
} finally {
@@ -37,12 +36,11 @@ test("an unknown cue is reported, not fatal", async () => {
try {
const { sfx, anomalies } = await resolveSfx({
cues: [{ id: "1", name: "definitely-not-a-real-sfx" }],
heygenOK: false,
framesDir: dir,
sfxLibDir,
});
assert.equal(sfx.length, 0);
assert.ok(anomalies.some((a) => /not in bundled library/.test(a)));
assert.ok(anomalies.some((a) => /none could be made/.test(a)));
} finally {
rmSync(dir, { recursive: true, force: true });
}
skills/media-use/audio/scripts/heygen-tts.mjs
-121
View File
@@ -1,121 +0,0 @@
#!/usr/bin/env node
// Self-contained HeyGen TTS — single text in → one wav (+ optional words JSON)
// out. A thin CLI over lib/tts.mjs (the same code the audio engine uses), so the
// HeyGen REST call, starfish voice pick, mp3→wav transcode, and word-timestamp
// filtering live in exactly one place. Bypasses the `frames` CLI, which in
// the published build is Kokoro-only.
//
// Usage:
// node heygen-tts.mjs "Text to speak" -o narration.wav [--words narration.words.json]
// node heygen-tts.mjs ./script.txt -o narration.wav --words narration.words.json
// node heygen-tts.mjs "Bonjour" -o fr.wav --lang fr --voice <id>
// node heygen-tts.mjs --list # list starfish voices and exit
//
// Flags: -o/--output (.wav → ffmpeg transcode; .mp3 → raw bytes), --words,
// --voice (starfish id), --speed, --lang, --list.
// Requires: $HEYGEN_API_KEY / OAuth ~/.heygen credentials and ffmpeg for .wav output.
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { heygenAuthHeaders, heygenJSON, loadEnvFromDir } from "./lib/heygen.mjs";
import { ffprobeDuration, resolveVoiceId, synthesizeOne, withWordIds } from "./lib/tts.mjs";
const argv = process.argv.slice(2);
function flag(name, def) {
const i = argv.indexOf(`--${name}`);
if (i < 0) return def;
if (i + 1 >= argv.length) return true;
const v = argv[i + 1];
return v.startsWith("--") ? true : v;
}
const die = (m) => {
console.error(`✗ heygen-tts: ${m}`);
process.exit(1);
};
// First arg that isn't a flag or the -o value is the text / .txt path.
const positional = (() => {
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
if (a.startsWith("--")) {
const next = argv[i + 1];
if (next && !next.startsWith("--")) i++;
continue;
}
if (a === "-o") {
i++;
continue;
}
return a;
}
return null;
})();
const output = resolve(
(typeof flag("output") === "string" && flag("output")) ||
(argv.includes("-o") && argv[argv.indexOf("-o") + 1]) ||
"narration.wav",
);
const wordsPath = typeof flag("words") === "string" ? resolve(flag("words")) : null;
const userVoice = typeof flag("voice") === "string" ? flag("voice") : null;
const speedRaw = typeof flag("speed") === "string" ? Number(flag("speed")) : 1.0;
const speed = isFinite(speedRaw) && speedRaw > 0 ? speedRaw : 1.0;
const lang = typeof flag("lang") === "string" ? flag("lang") : "en";
const listOnly = flag("list") === true;
loadEnvFromDir(process.cwd());
let authHeaders;
try {
authHeaders = heygenAuthHeaders();
} catch (e) {
die(e.message);
}
// ---------- --list ----------
if (listOnly) {
const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, {
headers: authHeaders,
});
for (const v of payload.data ?? payload.voices ?? []) {
console.log(`${v.voice_id}\t${v.name}\t${v.language ?? ""}`);
}
process.exit(0);
}
// ---------- resolve text + voice ----------
if (!positional) die("no text given. Pass a string or a .txt path, or use --list.");
const text =
positional.endsWith(".txt") && existsSync(resolve(positional))
? readFileSync(resolve(positional), "utf8").trim()
: positional;
if (!text) die("input text is empty");
const voiceId = await resolveVoiceId({ provider: "heygen", userVoice, lang });
if (!userVoice) console.error(`· using voice ${voiceId}`);
// ---------- synthesize (shared engine code) ----------
const { ok, words } = await synthesizeOne({
provider: "heygen",
text,
voiceId,
lang,
speed,
wavAbs: output,
framesDir: process.cwd(),
});
if (!ok) die("synthesis failed (HeyGen request/transcode error)");
let wordCount = 0;
if (wordsPath) {
if (words && words.length) {
mkdirSync(dirname(wordsPath), { recursive: true });
writeFileSync(wordsPath, JSON.stringify(withWordIds(words), null, 2));
wordCount = words.length;
} else {
console.error("⚠ no word_timestamps in response — run `frames transcribe` instead");
}
}
const dur = ffprobeDuration(output);
const durStr = isFinite(dur) ? ` (${dur.toFixed(2)}s)` : "";
console.log(`✓ ${output}${durStr}${wordCount ? ` · ${wordsPath} (${wordCount} words)` : ""}`);
skills/media-use/audio/scripts/lib/bgm.mjs
+50 -203
View File
@@ -1,74 +1,72 @@
// bgm.mjs — background music for the media audio engine. Two routes, gated the
// same way as TTS/SFX:
// bgm.mjs — background music for the media audio engine. Two routes, one
// switch, both ours:
//
// retrieve (default when HeyGen is configured) — search HeyGen's music library
// by mood, download the top track. Synchronous. assets/bgm/track.mp3.
// generate (the alternative; the automatic choice when HeyGen is absent) —
// Lyria (cloud, $GEMINI_API_KEY/$GOOGLE_API_KEY + google-genai) preferred,
// else local MusicGen (facebook/musicgen-small via transformers). Spawned
// DETACHED so the engine can return while audio renders; the caller marks
// bgm_pending and runs wait-bgm.mjs before assembling. assets/bgm/track.wav.
// catalog look under media/bgm/ in our object store and take the closest
// name. Free, instant, and the same track every run.
// compose POST /v1/audio/music with a mood prompt when the catalog has
// nothing that fits.
//
// Missing/failed BGM never blocks a render.
// The catalog is empty until someone fills it, and that is reported plainly
// rather than papered over: an empty prefix falls through to compose, and an
// explicit `catalog` request says the shelf is bare instead of quietly
// generating something the caller did not ask for.
//
// Missing or failed BGM never blocks a render.
import { spawn, spawnSync } from "node:child_process";
import { existsSync, mkdirSync, openSync, closeSync } from "node:fs";
import { join } from "node:path";
import { downloadTo, searchSounds } from "./heygen.mjs";
import { pythonInvocation } from "./python.mjs";
import { copyFileSync, mkdirSync, rmSync } from "node:fs";
import { dirname, join } from "node:path";
import { compose, find } from "../../../scripts/lib/catalog.mjs";
import { save } from "../../../scripts/lib/api.mjs";
const r3 = (x) => Number(x.toFixed(3));
const lyriaKey = () => process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || "";
// Default BGM level. Under narration music is a bed that must stay under the
// voice — 0.12 linear ≈ -18 dB. A silent film (no voice) has no voice to duck
// beneath, so BGM sits forward at 0.9. Callers may override per composition.
// voice — 0.12 linear is about -18 dB. A silent film (no voice) has no voice to
// duck beneath, so BGM sits forward at 0.9. Callers may override per composition.
export const BGM_BED_VOLUME = 0.12;
export const BGM_SILENT_VOLUME = 0.9;
export const bgmDefaultVolume = (hasVoice) => (hasVoice ? BGM_BED_VOLUME : BGM_SILENT_VOLUME);
const BGM_PY_DEPS = ["transformers", "torch", "soundfile", "numpy"];
const BGM_PY_PROBE =
"import transformers, soundfile, torch, numpy; from transformers import MusicgenForConditionalGeneration";
const LYRIA_PY_DEPS = ["google-genai", "python-dotenv"];
const LYRIA_PY_PROBE = "import google.genai";
function pyOk(probe) {
const { cmd, args } = pythonInvocation(["-c", probe]);
return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;
}
// `python -m pip`, not a bare `pip` binary: a Homebrew/system Python often
// exposes only `python3`/`pip3` on PATH, so a plain `pip` spawn silently
// no-ops (ENOENT) and the documented "auto-installed on demand" path never
// actually installs. `-m pip` also guarantees the packages land in the SAME
// interpreter pyOk() probes — a bare `pip`/`pip3` could resolve to a
// different Python installation than `python3` if more than one is on PATH.
function pipInstall(deps) {
const { cmd, args } = pythonInvocation(["-m", "pip", "install", "-q", ...deps]);
return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;
}
// ── retrieval (HeyGen music library) ──────────────────────────────────────────
export async function retrieveBgm({ query, headers, framesDir, hasVoice }) {
/** Take the closest catalog track. Returns null on a miss, `{ empty }` on a bare shelf. */
export async function fromCatalog({ query, framesDir, hasVoice }) {
const q = query || "calm cinematic underscore";
const results = await searchSounds(q, "music", headers, { limit: 5 });
if (!results.length) return null;
const top = results[0];
const rel = "assets/bgm/track.mp3";
await downloadTo(top.audio_url, join(framesDir, rel));
const hit = await find("bgm", q);
if (!hit || hit.empty) return hit;
const rel = `assets/bgm/track${extOf(hit.key)}`;
await save(hit.url, join(framesDir, rel));
return {
path: rel,
volume: bgmDefaultVolume(hasVoice),
query: q,
mode: "retrieve",
duration_s: typeof top.duration === "number" ? r3(top.duration) : null,
mode: "catalog",
key: hit.prefix + hit.key,
duration_s: null,
};
}
// ── mood inference (for the generate path's prompt) ──────────────────────────
// Industry base → archetype shape → emotional-arc tiebreaker. Exported so a
/** Generate a bed for the whole piece. */
export async function fromPrompt({ prompt, durationS, framesDir, hasVoice }) {
const rel = "assets/bgm/track.wav";
const made = await compose(prompt, { kind: "bgm", seconds: Math.max(1, durationS) });
const dest = join(framesDir, rel);
mkdirSync(dirname(dest), { recursive: true });
copyFileSync(made.localPath, dest);
rmSync(made.localPath, { force: true });
return {
path: rel,
volume: bgmDefaultVolume(hasVoice),
query: prompt,
mode: "compose",
duration_s: r3(Math.max(1, durationS)),
};
}
const extOf = (key) => (/\.\w+$/.exec(key) ?? [".mp3"])[0];
// ── mood inference (for the compose prompt) ──────────────────────────────────
// Industry base -> archetype shape -> emotional-arc tiebreaker. Exported so a
// workflow adapter can build a rich prompt from its own narrative metadata; the
// engine also calls it when generate has only a plain mood query.
// engine also calls it when compose has only a plain mood query.
export function inferBgmPrompt({ blob = "", archetype = "", arc = "", userPrompt = "" } = {}) {
if (userPrompt) return userPrompt;
const b = String(blob).toLowerCase();
@@ -99,159 +97,8 @@ export function inferBgmPrompt({ blob = "", archetype = "", arc = "", userPrompt
return `${base}, clean and focused, minimal arrangement, BPM ${Math.max(bpm - 8, 88)}`;
if (/frustrat|anxiety|overwhelm|tension/.test(ar) && /relief|excite|triumph/.test(ar))
return `${base}, builds from understated tension to uplifting resolution, BPM ${bpm}, MINOR to MAJOR`;
if (/excit|awe|power|triumph/.test(ar))
return `${base}, energetic and confident, BPM ${bpm}, MAJOR`;
if (/excit|awe|power|triumph/.test(ar)) return `${base}, energetic and confident, BPM ${bpm}, MAJOR`;
if (/trust|ease|clarity|reassur/.test(ar))
return `${base}, warm and reassuring, BPM ${Math.max(bpm - 5, 85)}`;
return `${base}, BPM ${bpm}, MAJOR`;
}
// ── generation (Lyria → MusicGen, detached) ──────────────────────────────────
// Returns a bgmMeta the caller folds into audio_meta:
// { path, mode, volume, provider, pid, log, target_duration_s, seed_duration_s,
// loop_count, pending:true } on success, or { disabled:true, reason }.
export function generateBgmDetached({
prompt,
durationS,
framesDir,
lyriaRecipe,
seedSeconds = 28,
hasVoice,
}) {
const rel = "assets/bgm/track.wav";
const abs = join(framesDir, rel);
mkdirSync(join(framesDir, "assets", "bgm"), { recursive: true });
const log = join(framesDir, "assets", "bgm", `bgm-${Date.now()}.log`);
const targetS = Math.max(1, durationS);
const baseMeta = { path: rel, mode: null, volume: bgmDefaultVolume(hasVoice), pending: true };
const lyriaConfigured = !!lyriaKey() && !!lyriaRecipe && existsSync(lyriaRecipe);
// Make a backend runnable: prefer Lyria when configured (install google-genai
// on demand), else ensure local MusicGen deps. Installs are synchronous here —
// generation itself is detached, so the engine still returns promptly.
if (lyriaConfigured && !pyOk(LYRIA_PY_PROBE)) pipInstall(LYRIA_PY_DEPS);
const useLyria = lyriaConfigured && pyOk(LYRIA_PY_PROBE);
if (!useLyria && !pyOk(BGM_PY_PROBE)) pipInstall(BGM_PY_DEPS);
const fd = openSync(log, "w");
if (useLyria) {
const { cmd, args } = pythonInvocation([
lyriaRecipe,
"--output",
abs,
"--duration",
String(targetS),
"--prompt",
prompt,
]);
const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] });
proc.unref();
closeSync(fd);
return {
...baseMeta,
mode: "detached-single",
provider: "lyria",
pid: proc.pid,
log,
target_duration_s: r3(targetS),
};
}
if (pyOk(BGM_PY_PROBE)) {
const seedS = Math.min(Math.max(seedSeconds, 10), 30);
const loops = targetS > seedS ? Math.ceil(targetS / seedS) : 1;
const script = musicgenScript({ prompt, abs, targetS, seedS });
const { cmd, args } = pythonInvocation(["-c", script]);
const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] });
proc.unref();
closeSync(fd);
return {
...baseMeta,
mode: targetS > seedS ? "detached-seed-loop" : "detached-seed-trim",
provider: "musicgen",
pid: proc.pid,
log,
target_duration_s: r3(targetS),
seed_duration_s: seedS,
loop_count: loops,
};
}
closeSync(fd);
return {
disabled: true,
reason: lyriaConfigured
? `Lyria configured but google-genai uninstallable, and local MusicGen unavailable (pip install ${BGM_PY_DEPS.join(" ")})`
: `no Lyria key/recipe and local MusicGen deps unavailable (pip install ${BGM_PY_DEPS.join(" ")})`,
};
}
// Inline MusicGen: generate ONE seed clip (≤30s to stay under the decoder's
// positional limit), then trim it down or crossfade-loop it up to the target.
function musicgenScript({ prompt, abs, targetS, seedS }) {
return `
import math, os, sys, traceback
from pathlib import Path
import numpy as np
import soundfile as sf
from transformers import MusicgenForConditionalGeneration, AutoProcessor
prompt = ${JSON.stringify(prompt)}
out_path = ${JSON.stringify(abs)}
target_s = float(${targetS.toFixed(3)})
seed_s = float(${seedS.toFixed(3)})
token_rate = 50
crossfade_s = 0.3
def apply_fade(arr, sr, fade_in_s=0.08, fade_out_s=0.5):
n_in = min(int(round(fade_in_s * sr)), arr.shape[0] // 2)
n_out = min(int(round(fade_out_s * sr)), arr.shape[0] // 2)
if n_in > 1: arr[:n_in] *= np.linspace(0.0, 1.0, n_in, dtype="float32")
if n_out > 1: arr[-n_out:] *= np.linspace(1.0, 0.0, n_out, dtype="float32")
return arr
def loop_crossfade(seed, target_len, xf):
if seed.shape[0] >= target_len: return seed[:target_len]
xf = min(xf, seed.shape[0] // 2)
if xf < 1:
reps = int(math.ceil(target_len / seed.shape[0]))
return np.tile(seed, reps)[:target_len]
t = np.linspace(0.0, 1.0, xf, dtype="float32")
fade_out = np.cos(t * (math.pi / 2)); fade_in = np.sin(t * (math.pi / 2))
out = seed.copy()
while out.shape[0] < target_len:
tail = out[-xf:] * fade_out; head = seed[:xf] * fade_in
out = np.concatenate([out[:-xf], tail + head, seed[xf:]])
return out[:target_len]
try:
Path(os.path.dirname(out_path)).mkdir(parents=True, exist_ok=True)
processor = AutoProcessor.from_pretrained("facebook/musicgen-small")
model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small")
model.eval()
sr = int(model.config.audio_encoder.sampling_rate)
gen_s = min(seed_s, target_s)
tokens = max(1, int(math.ceil(gen_s * token_rate)))
print(f"[musicgen] seed dur={gen_s:.2f}s tokens={tokens}", flush=True)
inputs = processor(text=[prompt], padding=True, return_tensors="pt")
audio = model.generate(**inputs, max_new_tokens=tokens)
seed = audio[0, 0].detach().cpu().numpy().astype("float32")
peak = float(np.max(np.abs(seed)))
if peak > 1e-6: seed = seed * (0.89 / peak)
want = max(1, int(round(target_s * sr)))
if seed.shape[0] >= want:
final = seed[:want].copy()
else:
final = loop_crossfade(seed, want, int(round(crossfade_s * sr)))
if final.shape[0] < want: final = np.pad(final, (0, want - final.shape[0]))
else: final = final[:want]
final = apply_fade(final, sr)
peak = float(np.max(np.abs(final)))
if peak > 1.0: final = final / peak
sf.write(out_path, final, sr)
print(f"[musicgen] wrote {out_path} samples={final.shape[0]} sr={sr}", flush=True)
except Exception:
traceback.print_exc(); sys.exit(1)
`;
}
skills/media-use/audio/scripts/lib/bgm.test.mjs
-30
View File
@@ -1,30 +0,0 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { BGM_BED_VOLUME, BGM_SILENT_VOLUME, bgmDefaultVolume } from "./bgm.mjs";
// Regression: narrated pipelines used to ship BGM at 0.8 (≈ -2 dB), ~16 dB
// hotter than a music bed under a voice should be. The default under narration
// must be a proper bed (≈ -18 dB); a silent film keeps the louder default.
const dbfs = (linear) => 20 * Math.log10(linear);
test("BGM under narration is a bed near -18 dB", () => {
assert.equal(bgmDefaultVolume(true), BGM_BED_VOLUME);
assert.equal(BGM_BED_VOLUME, 0.12);
const db = dbfs(BGM_BED_VOLUME);
assert.ok(db < -17 && db > -19, `bed should be ≈ -18 dB, got ${db.toFixed(1)} dB`);
});
test("a silent film (no voice) keeps BGM forward", () => {
assert.equal(bgmDefaultVolume(false), BGM_SILENT_VOLUME);
assert.equal(BGM_SILENT_VOLUME, 0.9);
});
test("the narrated default is well below the voice (≈ 0 dBFS)", () => {
// Voice sits at data-volume="1" (0 dBFS); the bed must be ~16+ dB under it.
const separation = dbfs(1) - dbfs(bgmDefaultVolume(true));
assert.ok(
separation >= 16,
`bed should sit ≥16 dB under the voice, got ${separation.toFixed(1)} dB`,
);
});
skills/media-use/audio/scripts/lib/heygen.mjs
-155
View File
@@ -1,155 +0,0 @@
// heygen.mjs — vendored HeyGen REST helpers (auth + transport) for the audio
// pipeline. The credential resolver matches the frames CLI auth: first
// usable source wins — $HEYGEN_API_KEY / $FRAMES_API_KEY → a nearby .env → ~/.heygen/
// credentials (oauth → Bearer, else api_key → X-Api-Key; $HEYGEN_CONFIG_DIR
// overrides the dir). Vendored so the skill ships standalone. Pure node.
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { homedir } from "node:os";
import { dirname, join, resolve } from "node:path";
export const HEYGEN_BASE = "https://api.heygen.com/v3";
export const HEYGEN_CLI_SOURCE_HEADERS = { "X-HeyGen-Source": "cli" };
// Tool-attribution sent on EVERY media-use HeyGen call regardless of auth type, so
// the backend can isolate media-use consumption from other free TTS / avatar video.
// Unconditional — a paying user's media-use call is still media-use — unlike the
// OAuth-only cli-source header above, which also gates the free allowance.
export const HEYGEN_CLIENT_SOURCE_HEADERS = { "X-HeyGen-Client-Source": "media-use" };
// Walk up ≤5 dirs from startDir; load the first .env (shell env always wins).
export function loadEnvFromDir(startDir) {
let dir = resolve(startDir);
for (let i = 0; i < 5; i++) {
const envPath = join(dir, ".env");
if (existsSync(envPath)) {
for (const raw of readFileSync(envPath, "utf8").split("\n")) {
let line = raw.trim();
if (!line || line.startsWith("#")) continue;
if (line.startsWith("export ")) line = line.slice(7).trim();
const eq = line.indexOf("=");
if (eq < 1) continue;
const key = line.slice(0, eq).trim();
let val = line.slice(eq + 1).trim();
if (val.startsWith('"') || val.startsWith("'")) {
const q = val[0];
const end = val.indexOf(q, 1);
val = end > 0 ? val.slice(1, end) : val.slice(1);
}
if (!(key in process.env)) process.env[key] = val;
}
return;
}
const parent = dirname(dir);
if (parent === dir) break;
dir = parent;
}
}
// → { headers } | { expired: true } | null. Never throws.
export function heygenCredential() {
const envKey = process.env.HEYGEN_API_KEY || process.env.FRAMES_API_KEY;
if (envKey) return { headers: { "X-Api-Key": envKey } };
const file = join(process.env.HEYGEN_CONFIG_DIR || join(homedir(), ".heygen"), "credentials");
if (!existsSync(file)) return null;
const raw = readFileSync(file, "utf8").trim();
if (!raw) return null;
if (!raw.startsWith("{")) return { headers: { "X-Api-Key": raw } };
// A malformed credentials file (partial write / wrong shape) must degrade to
// "no credential", not crash the engine at startup — this function never throws.
let cred;
try {
cred = JSON.parse(raw);
} catch {
return null;
}
const oauth = cred.oauth;
if (oauth?.access_token) {
const expired = oauth.expires_at && new Date(oauth.expires_at).getTime() - 60_000 < Date.now();
if (!expired) return { headers: { Authorization: `Bearer ${oauth.access_token}` } };
if (!cred.api_key) return { expired: true };
}
if (cred.api_key) return { headers: { "X-Api-Key": cred.api_key } };
return null;
}
// → "oauth" | "api_key" | null. Same oauth-vs-api-key check heygenAuthHeaders()
// makes internally, exposed on its own so callers that only need to *tag* the
// auth path (telemetry) don't have to parse headers back apart. Never throws:
// no credential (or an expired one) is just `null`, same as a fresh resolve
// with nothing to tag.
export function heygenAuthMethod() {
const cred = heygenCredential();
if (!cred?.headers) return null;
return "Authorization" in cred.headers ? "oauth" : "api_key";
}
// → auth headers object, or throw with a fix hint.
export function heygenAuthHeaders() {
const cred = heygenCredential();
if (cred?.headers) {
// Only tag OAuth (Bearer) traffic as cli-source — the backend uses it to
// grant the free allowance for OAuth requests and ignores it for API-key
// (X-Api-Key) traffic, where it's dead metadata.
const isOauth = "Authorization" in cred.headers;
return isOauth
? { ...cred.headers, ...HEYGEN_CLI_SOURCE_HEADERS, ...HEYGEN_CLIENT_SOURCE_HEADERS }
: { ...cred.headers, ...HEYGEN_CLIENT_SOURCE_HEADERS };
}
if (cred?.expired)
throw new Error(
"HeyGen OAuth token expired — run `npx frames auth refresh` (or `npx frames auth login`)",
);
throw new Error(
"no HeyGen credentials — set $HEYGEN_API_KEY, or run `npx frames auth login` (writes ~/.heygen/credentials)",
);
}
// Authed JSON request against the v3 API; throws on a non-OK status.
export async function heygenJSON(path, { method = "GET", headers = {}, body } = {}) {
const opts = { method, headers: { ...HEYGEN_CLIENT_SOURCE_HEADERS, ...headers } };
if (body !== undefined) {
opts.headers["Content-Type"] = "application/json";
opts.body = JSON.stringify(body);
}
const res = await fetch(`${HEYGEN_BASE}${path}`, opts);
if (!res.ok) {
const detail = await res.text().catch(() => "");
throw new Error(
`HeyGen ${method} ${path} → HTTP ${res.status}${detail ? `\n${detail.slice(0, 300)}` : ""}`,
);
}
return res.json();
}
// Download a (presigned) URL to destPath; returns byte length.
export async function downloadTo(url, destPath) {
const res = await fetch(url);
if (!res.ok) throw new Error(`download HTTP ${res.status}: ${String(url).slice(0, 80)}`);
const bytes = Buffer.from(await res.arrayBuffer());
mkdirSync(dirname(destPath), { recursive: true });
writeFileSync(destPath, bytes);
return bytes.length;
}
// Retrieval search over HeyGen's audio catalog (NOT generation). type =
// "music" | "sound_effects". Returns the ranked results array (best first); each
// item has a presigned `audio_url` (+ `duration`, `description`, `name`, `score`).
// `query` is required (≥1 char, empty → HTTP 400) and `limit` is capped at 50.
// `minScore`: omit to use the server default (0.7). That default is TOO HIGH for
// sound_effects — good SFX hits score ~0.5–0.67, so callers wanting SFX should
// pass a lower floor (~0.4); music scores high and is fine at the default.
export async function searchSounds(query, type, headers, { limit = 5, minScore } = {}) {
const params = new URLSearchParams({ query, type, limit: String(limit) });
if (minScore != null) params.set("min_score", String(minScore));
const payload = await heygenJSON(`/audio/sounds?${params.toString()}`, { headers });
// `data` comes back as a ranked array (best first). Older responses keyed it by
// numeric index ("0","1",…); normalize both shapes to an array (empty → []).
const data = payload?.data ?? payload;
if (Array.isArray(data)) return data;
if (data && typeof data === "object") return Object.values(data);
throw new Error(
`unexpected /audio/sounds shape — top keys: ${Object.keys(payload ?? {}).join(", ")}`,
);
}
skills/media-use/audio/scripts/lib/heygen.test.mjs
-103
View File
@@ -1,103 +0,0 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { heygenAuthHeaders, heygenAuthMethod } from "./heygen.mjs";
function withCleanHeygenEnv(fn) {
const previousApiKey = process.env.HEYGEN_API_KEY;
const previousHyperframesApiKey = process.env.FRAMES_API_KEY;
const previousConfigDir = process.env.HEYGEN_CONFIG_DIR;
try {
delete process.env.HEYGEN_API_KEY;
delete process.env.FRAMES_API_KEY;
delete process.env.HEYGEN_CONFIG_DIR;
return fn();
} finally {
if (previousApiKey === undefined) delete process.env.HEYGEN_API_KEY;
else process.env.HEYGEN_API_KEY = previousApiKey;
if (previousHyperframesApiKey === undefined) delete process.env.FRAMES_API_KEY;
else process.env.FRAMES_API_KEY = previousHyperframesApiKey;
if (previousConfigDir === undefined) delete process.env.HEYGEN_CONFIG_DIR;
else process.env.HEYGEN_CONFIG_DIR = previousConfigDir;
}
}
test("heygenAuthHeaders does not tag API-key requests as CLI traffic, but still carries the media-use tool tag", () => {
withCleanHeygenEnv(() => {
process.env.HEYGEN_API_KEY = "hg_test";
// API-key requests use normal billing; the backend ignores the cli-source
// header for them, so it's not sent. The tool-attribution header IS sent on
// every media-use call (any auth type) so the backend can isolate media-use.
assert.deepEqual(heygenAuthHeaders(), {
"X-Api-Key": "hg_test",
"X-HeyGen-Client-Source": "media-use",
});
});
});
test("heygenAuthHeaders tags OAuth requests as CLI traffic and with the media-use tool tag", () => {
withCleanHeygenEnv(() => {
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
try {
process.env.HEYGEN_CONFIG_DIR = dir;
writeFileSync(
join(dir, "credentials"),
JSON.stringify({
oauth: {
access_token: "at_test",
expires_at: "2099-01-01T00:00:00Z",
},
}),
);
assert.deepEqual(heygenAuthHeaders(), {
Authorization: "Bearer at_test",
"X-HeyGen-Source": "cli",
"X-HeyGen-Client-Source": "media-use",
});
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
});
test("heygenAuthMethod returns api_key for an env API key, without tagging headers", () => {
withCleanHeygenEnv(() => {
process.env.HEYGEN_API_KEY = "hg_test";
assert.equal(heygenAuthMethod(), "api_key");
});
});
test("heygenAuthMethod returns oauth for a live OAuth credential", () => {
withCleanHeygenEnv(() => {
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
try {
process.env.HEYGEN_CONFIG_DIR = dir;
writeFileSync(
join(dir, "credentials"),
JSON.stringify({
oauth: {
access_token: "at_test",
expires_at: "2099-01-01T00:00:00Z",
},
}),
);
assert.equal(heygenAuthMethod(), "oauth");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
});
test("heygenAuthMethod returns null with no credential at all", () => {
withCleanHeygenEnv(() => {
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
try {
process.env.HEYGEN_CONFIG_DIR = dir; // no credentials file written
assert.equal(heygenAuthMethod(), null);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
});
skills/media-use/audio/scripts/lib/python.mjs
-63
View File
@@ -1,63 +0,0 @@
// python.mjs — resolve which Python 3 executable to spawn, per platform.
//
// The audio engine (tts.mjs, bgm.mjs) shells out to `python3` for ElevenLabs
// TTS and the local Lyria/MusicGen BGM paths. `python3` is the right name on
// macOS/Linux, but on Windows the python.org installer only creates
// `python.exe` plus the `py` launcher — there is no `python3.exe` (only the
// Microsoft Store build adds one). So a bare `spawn("python3", …)` ENOENTs on a
// standard Windows Python install, silently disabling every Python-backed audio
// feature until the user hand-creates a `python3.exe` shim (reported twice).
//
// Resolve once, per process: probe the platform's candidates in order and take
// the first that actually runs. `py` is the launcher, so it needs a `-3` arg to
// select Python 3 — hence candidates are argv PREFIXES, not bare names.
import { spawnSync } from "node:child_process";
function defaultProbe(cmd, args) {
try {
return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;
} catch {
return false;
}
}
/**
* Pick the argv prefix that launches Python 3 on this platform.
* Returns e.g. `["python3"]`, `["python"]`, or `["py", "-3"]`.
*
* Pure except for `probe` (which runs `<cmd> … --version`); both `platform`
* and `probe` are injectable so every branch is unit-testable without spawning.
* If nothing probes OK, falls back to the canonical name for the platform so
* the eventual spawn fails loudly exactly as it did before — never worse.
*/
export function resolvePythonCommand(platform = process.platform, probe = defaultProbe) {
const candidates =
platform === "win32" ? [["python3"], ["python"], ["py", "-3"]] : [["python3"], ["python"]];
for (const prefix of candidates) {
if (probe(prefix[0], [...prefix.slice(1), "--version"])) return prefix;
}
return candidates[0];
}
let cached = null;
/** Cached `resolvePythonCommand()` — probing spawns, so resolve at most once. */
export function pythonCommand() {
if (!cached) cached = resolvePythonCommand();
return cached;
}
/**
* Build a `{ cmd, args }` for running Python 3 with `extraArgs`, using the
* resolved (or supplied) prefix. Keeps the launcher's `-3` (and any future
* prefix args) ahead of the caller's own arguments.
*/
export function pythonInvocation(extraArgs, prefix = pythonCommand()) {
return { cmd: prefix[0], args: [...prefix.slice(1), ...extraArgs] };
}
/** Test-only: clear the cached resolution so a test can re-probe. */
export function _resetPythonCommandCacheForTests() {
cached = null;
}
skills/media-use/audio/scripts/lib/python.test.mjs
-68
View File
@@ -1,68 +0,0 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { resolvePythonCommand, pythonInvocation } from "./python.mjs";
// Regression: on Windows a standard python.org install has no `python3.exe`
// (only `python.exe` + the `py` launcher), so `spawn("python3", …)` ENOENTs and
// every Python-backed audio feature silently no-ops. resolvePythonCommand takes
// injectable platform/probe params so all branches are testable without
// spawning a real interpreter.
// probeFor(names): a probe that reports success only for the given argv-0 names.
function probeFor(...names) {
const ok = new Set(names);
return (cmd) => ok.has(cmd);
}
test("non-win32 uses python3 when it runs", () => {
assert.deepEqual(resolvePythonCommand("linux", probeFor("python3")), ["python3"]);
assert.deepEqual(resolvePythonCommand("darwin", probeFor("python3")), ["python3"]);
});
test("win32 prefers python3 when the Microsoft Store build provides it", () => {
assert.deepEqual(resolvePythonCommand("win32", probeFor("python3", "python", "py")), ["python3"]);
});
test("win32 falls back to python.exe when python3 is absent (python.org install)", () => {
// The exact reported scenario: no python3, but `python` exists.
assert.deepEqual(resolvePythonCommand("win32", probeFor("python", "py")), ["python"]);
});
test("win32 falls back to the py launcher with -3 when only py exists", () => {
assert.deepEqual(resolvePythonCommand("win32", probeFor("py")), ["py", "-3"]);
});
test("py launcher is probed as `py -3 --version`, not bare `py`", () => {
const seen = [];
const probe = (cmd, args) => {
seen.push([cmd, ...args]);
return cmd === "py";
};
resolvePythonCommand("win32", probe);
assert.deepEqual(seen.at(-1), ["py", "-3", "--version"]);
});
test("falls back to the canonical name (loud failure, unchanged) when nothing runs", () => {
// No interpreter anywhere — must not throw, and must return python3 so the
// eventual spawn fails exactly as it did before this fix, never worse.
assert.deepEqual(
resolvePythonCommand("win32", () => false),
["python3"],
);
assert.deepEqual(
resolvePythonCommand("linux", () => false),
["python3"],
);
});
test("pythonInvocation prepends the resolved prefix ahead of caller args", () => {
assert.deepEqual(pythonInvocation(["-c", "import x"], ["python"]), {
cmd: "python",
args: ["-c", "import x"],
});
// The py launcher's -3 must stay ahead of the caller's own arguments.
assert.deepEqual(pythonInvocation(["-c", "import x"], ["py", "-3"]), {
cmd: "py",
args: ["-3", "-c", "import x"],
});
});
skills/media-use/audio/scripts/lib/sfx.mjs
+99 -100
View File
@@ -1,20 +1,18 @@
// sfx.mjs — sound effects for the media audio engine. Provider-gated (NOT a
// per-cue merge): the decision is made once, by whether HeyGen is configured —
// mirroring how TTS and BGM degrade.
// sfx.mjs — sound effects for the media audio engine. Each cue is resolved in
// the same order, cheapest first:
//
// HeyGen credential present → retrieve EVERY cue from HeyGen's audio library
// (/v3/audio/sounds, type=sound_effects, min_score=0.4). The bundled
// library is NOT consulted.
// HeyGen credential absent → resolve cues against the bundled 21-file
// library (assets/sfx/manifest.json), copying matched files into the
// project. Offline, deterministic, free.
// library the 21 files bundled with this skill (assets/sfx/manifest.json).
// Offline, deterministic, free — and the same bytes every run.
// catalog media/sfx/ in our object store, for the long tail the 21 miss.
// foley POST /v1/audio/foley when neither has it.
//
// A cue that matches nothing is skipped (recorded as an anomaly); SFX never
// blocks a render. Every cue sits at volume ~0.35, under voice + BGM.
// A cue nothing can answer is skipped and recorded as an anomaly; SFX never
// blocks a render. Every cue sits at volume 0.35, under voice and BGM.
import { copyFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync } from "node:fs";
import { join } from "node:path";
import { downloadTo, searchSounds } from "./heygen.mjs";
import { save } from "../../../scripts/lib/api.mjs";
import { compose, find } from "../../../scripts/lib/catalog.mjs";
const SFX_VOLUME = 0.35;
const slug = (s) =>
@@ -25,15 +23,47 @@ const slug = (s) =>
.slice(0, 40) || "x";
const r3 = (x) => Number(x.toFixed(3));
/**
* Build the lookup for the bundled library: by manifest key, by file basename,
* and by slug of either, so a cue can name "whoosh", "whoosh.mp3", or
* "ui click" and all three land.
*/
export function indexLibrary(manifest) {
const byName = new Map();
for (const [key, entry] of Object.entries(manifest)) {
if (!entry?.file || !isFinite(entry.duration)) continue;
const rec = { key, file: entry.file, duration: entry.duration };
byName.set(key, rec);
byName.set(entry.file, rec);
byName.set(slug(key), rec);
byName.set(slug(entry.file.replace(/\.\w+$/, "")), rec);
}
return byName;
}
function readLibrary(sfxLibDir, anomalies) {
const manifestPath = join(sfxLibDir, "manifest.json");
if (!existsSync(manifestPath)) {
anomalies.push(`no SFX library at ${sfxLibDir} — the bundled rung is unavailable`);
return new Map();
}
try {
return indexLibrary(JSON.parse(readFileSync(manifestPath, "utf8")));
} catch (e) {
anomalies.push(`SFX manifest parse failed (${e.message}) — the bundled rung is unavailable`);
return new Map();
}
}
// cues: [{ id, name }] (id = the line/frame/scene the cue fires in). Returns
// { sfx: [{ id, name, file, source, offset_s, duration_s, volume }], anomalies }.
export async function resolveSfx({ cues, heygenOK, headers, framesDir, sfxLibDir }) {
export async function resolveSfx({ cues, framesDir, sfxLibDir }) {
const sfx = [];
const anomalies = [];
const destDir = join(framesDir, "assets", "sfx");
// Dedupe identical (id,name) cues — the same effect named twice in one line
// downloads/copies once.
// resolves once.
const seen = new Set();
const uniq = cues.filter((c) => {
const k = `${c.id}:${c.name}`;
@@ -41,103 +71,72 @@ export async function resolveSfx({ cues, heygenOK, headers, framesDir, sfxLibDir
seen.add(k);
return true;
});
if (!uniq.length) return { sfx, anomalies };
if (heygenOK) {
for (const { id, name } of uniq) {
try {
// SFX hits score low (~0.5–0.67), below the API's default 0.7 which
// silently drops most named cues — floor to 0.4. (BGM/music score high
// and keep the default.)
const results = await searchSounds(name, "sound_effects", headers, {
limit: 3,
minScore: 0.4,
const library = readLibrary(sfxLibDir, anomalies);
mkdirSync(destDir, { recursive: true });
for (const { id, name } of uniq) {
const hit = library.get(name) ?? library.get(slug(name));
if (hit) {
const src = join(sfxLibDir, hit.file);
const rel = `assets/sfx/${hit.file}`;
const dest = join(framesDir, rel);
// The bundled library may be incomplete: some installs ship manifest.json
// without the mp3s. An entry pointing at a file we never copied is a
// dangling reference that drops silently downstream, so say so and try
// the next rung instead.
if (existsSync(dest) || existsSync(src)) {
if (!existsSync(dest)) copyFileSync(src, dest);
sfx.push({
id,
name,
file: rel,
source: "library",
offset_s: 0,
duration_s: r3(hit.duration),
volume: SFX_VOLUME,
});
if (!results.length) {
anomalies.push(`sfx "${name}" (id ${id}): no HeyGen match — skipped`);
continue;
}
const top = results[0];
const file = `assets/sfx/${slug(name)}.mp3`;
await downloadTo(top.audio_url, join(framesDir, file));
continue;
}
anomalies.push(
`sfx "${name}" (id ${id}): bundled file ${hit.file} is missing from ${sfxLibDir}`,
);
}
const rel = `assets/sfx/${slug(name)}`;
try {
const found = await find("sfx", name);
if (found && !found.empty) {
const file = `${rel}${(/\.\w+$/.exec(found.key) ?? [".mp3"])[0]}`;
await save(found.url, join(framesDir, file));
sfx.push({
id,
name,
file,
source: "heygen",
source: "catalog",
offset_s: 0,
duration_s: typeof top.duration === "number" ? r3(top.duration) : 1.0,
duration_s: null,
volume: SFX_VOLUME,
});
} catch (e) {
anomalies.push(`sfx "${name}" (id ${id}): retrieval failed — ${e.message}`);
}
}
return { sfx, anomalies };
}
// ── offline: bundled library ──
const manifestPath = join(sfxLibDir, "manifest.json");
if (!existsSync(manifestPath)) {
if (uniq.length)
anomalies.push(`no HeyGen credential and no SFX library at ${sfxLibDir} — all cues dropped`);
return { sfx, anomalies };
}
let manifest;
try {
manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
} catch (e) {
anomalies.push(`SFX manifest parse failed (${e.message}) — all cues dropped`);
return { sfx, anomalies };
}
// Build lookups: by manifest key, by file basename, and by slug of either, so
// a cue can name "whoosh", "whoosh.mp3", or "ui click" (→ slug match).
const byKey = new Map();
for (const [key, entry] of Object.entries(manifest)) {
if (!entry?.file || !isFinite(entry.duration)) continue;
const rec = { key, file: entry.file, duration: entry.duration };
byKey.set(key, rec);
byKey.set(entry.file, rec);
byKey.set(slug(key), rec);
byKey.set(slug(entry.file.replace(/\.\w+$/, "")), rec);
}
mkdirSync(destDir, { recursive: true });
for (const { id, name } of uniq) {
const hit = byKey.get(name) ?? byKey.get(slug(name));
if (!hit) {
const known = [...new Set([...byKey.values()].map((v) => v.key))].slice(0, 8).join(", ");
anomalies.push(
`sfx "${name}" (id ${id}): not in bundled library — skipped (have: ${known}…)`,
);
continue;
}
const src = join(sfxLibDir, hit.file);
const destRel = `assets/sfx/${hit.file}`;
const dest = join(framesDir, destRel);
// The bundled library may be incomplete: some installs of the skill ship
// manifest.json without the actual mp3s. Pushing an sfx entry that points at
// a file we never copied produces a dangling reference that silently drops
// downstream ("not on disk"). Surface it as a loud anomaly and skip the cue
// instead, so the audio_meta never references a missing file.
if (!existsSync(dest)) {
if (!existsSync(src)) {
anomalies.push(
`sfx "${name}" (id ${id}): bundled file ${hit.file} missing from the offline ` +
`library (${sfxLibDir}) — skipped. Reinstall the media-use skill to ` +
`restore assets/sfx/*.mp3, or configure a HeyGen credential for retrieval.`,
);
continue;
}
copyFileSync(src, dest);
const made = await compose(name, { kind: "sfx" });
const file = `${rel}.wav`;
copyFileSync(made.localPath, join(framesDir, file));
rmSync(made.localPath, { force: true });
sfx.push({
id,
name,
file,
source: "foley",
offset_s: 0,
duration_s: null,
volume: SFX_VOLUME,
});
} catch (e) {
anomalies.push(`sfx "${name}" (id ${id}): not in the library and none could be made — ${e.message}`);
}
sfx.push({
id,
name,
file: destRel,
source: "local",
offset_s: 0,
duration_s: r3(hit.duration),
volume: SFX_VOLUME,
});
}
return { sfx, anomalies };
}
skills/media-use/audio/scripts/lib/sfx.test.mjs
+11 -13
View File
@@ -5,12 +5,11 @@ import { join } from "node:path";
import { tmpdir } from "node:os";
import { resolveSfx } from "./sfx.mjs";
// Offline (no HeyGen) SFX resolution: the bundled library may ship manifest.json
// without the actual mp3s. The old code copied only when the source existed but
// pushed the sfx entry unconditionally — producing a dangling reference that
// silently dropped downstream ("not on disk"). These tests lock in the loud
// behavior: a present file is copied + referenced; a missing file yields an
// anomaly and NO dangling entry.
// The bundled SFX library may ship manifest.json without the actual mp3s. An
// entry pushed for a file that was never copied is a dangling reference that
// drops silently downstream ("not on disk"). These tests lock in the loud
// behavior: a present file is copied and referenced; a missing one says so.
// No credential is set here, so the catalog and foley rungs are never reached.
async function withDirs(fn) {
const root = mkdtempSync(join(tmpdir(), "hf-sfx-"));
@@ -27,7 +26,7 @@ async function withDirs(fn) {
}
}
test("offline: copies and references a present bundled file", async () => {
test("copies and references a present bundled file", async () => {
await withDirs(async ({ libDir, projDir }) => {
writeFileSync(
join(libDir, "manifest.json"),
@@ -36,19 +35,18 @@ test("offline: copies and references a present bundled file", async () => {
writeFileSync(join(libDir, "whoosh.mp3"), "ID3-fake-bytes");
const { sfx, anomalies } = await resolveSfx({
cues: [{ id: "s1", name: "whoosh" }],
heygenOK: false,
framesDir: projDir,
sfxLibDir: libDir,
});
assert.equal(sfx.length, 1);
assert.equal(sfx[0].file, "assets/sfx/whoosh.mp3");
assert.equal(sfx[0].source, "local");
assert.equal(sfx[0].source, "library");
assert.ok(existsSync(join(projDir, "assets/sfx/whoosh.mp3")), "mp3 copied into project");
assert.equal(anomalies.length, 0);
});
});
test("offline: a matched-but-missing bundled file yields an anomaly and NO dangling entry", async () => {
test("a matched-but-missing bundled file yields an anomaly and NO dangling entry", async () => {
await withDirs(async ({ libDir, projDir }) => {
// Manifest names whoosh.mp3, but the mp3 was never shipped (the reported bug).
writeFileSync(
@@ -57,13 +55,13 @@ test("offline: a matched-but-missing bundled file yields an anomaly and NO dangl
);
const { sfx, anomalies } = await resolveSfx({
cues: [{ id: "s1", name: "whoosh" }],
heygenOK: false,
framesDir: projDir,
sfxLibDir: libDir,
});
assert.equal(sfx.length, 0, "no dangling entry for a file that was never copied");
assert.equal(anomalies.length, 1);
assert.match(anomalies[0], /missing from the offline library/);
assert.equal(anomalies.length, 2);
assert.match(anomalies[0], /is missing from/);
assert.match(anomalies[1], /none could be made/);
assert.ok(!existsSync(join(projDir, "assets/sfx/whoosh.mp3")), "nothing copied");
});
});
skills/media-use/audio/scripts/lib/tts.mjs
+58 -323
View File
@@ -1,94 +1,34 @@
// tts.mjs — multi-provider TTS for the media audio engine. The provider chain,
// auto-detected from env, is the one documented in ../SKILL.md:
// tts.mjs — voice for the media audio engine. One route, not a cascade:
//
// 1. HeyGen (Starfish) — $HEYGEN_API_KEY / $FRAMES_API_KEY / ~/.heygen.
// Direct v3 REST (NOT `frames tts`, which in the published build is
// Kokoro-only and silently ignores a HeyGen key). Returns word_timestamps
// in the same call, so no separate transcribe pass.
// 2. ElevenLabs — $ELEVENLABS_API_KEY + `pip install elevenlabs`. No
// word timings → caller chains transcribeWav().
// 3. Kokoro-82M (local) — always available, via the published `frames tts`
// CLI. No word timings → caller chains transcribeWav().
// POST /v1/audio/speech the text becomes audio
// POST /v1/audio/transcriptions the audio becomes text, and word timings
// when the service measures them
//
// "HeyGen available" is decided by CREDENTIAL presence (heygenCredential), never
// by the CLI — see the note above.
// Both live behind api.hanzo.ai (see ../../../scripts/lib/api.mjs). Point
// $HANZO_BASE_URL at a local Hanzo Engine to run the same code offline — the
// endpoint moves, the code does not.
//
// Synthesis and word timings are two calls because they are two questions. The
// engine asks the first for every line and the second only when captions need
// per-word cuts, so a piece with no captions never pays for alignment.
import { spawn, spawnSync } from "node:child_process";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { spawnSync } from "node:child_process";
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { heygenAuthHeaders, heygenCredential, heygenJSON } from "./heygen.mjs";
import { pythonInvocation } from "./python.mjs";
import { MAX_SPEECH_INPUT, ready, speech, transcript, write } from "../../../scripts/lib/api.mjs";
// ── provider detection ────────────────────────────────────────────────────────
export function heygenAvailable() {
return heygenCredential() !== null;
}
export function elevenlabsAvailable() {
if (!process.env.ELEVENLABS_API_KEY) return false;
const { cmd, args } = pythonInvocation(["-c", "import elevenlabs"]);
const r = spawnSync(cmd, args, {
stdio: "ignore",
});
return r.status === 0;
}
export { ready };
// First available provider wins; an explicit choice is honored (and validated).
export function pickProvider(userProvider) {
if (userProvider) {
if (!["heygen", "elevenlabs", "kokoro"].includes(userProvider))
throw new Error(`invalid provider "${userProvider}" (heygen | elevenlabs | kokoro)`);
if (userProvider === "heygen" && !heygenAvailable())
throw new Error(
"provider=heygen but no HeyGen credentials (set $HEYGEN_API_KEY or run `npx frames auth login`)",
);
if (userProvider === "elevenlabs" && !process.env.ELEVENLABS_API_KEY)
throw new Error("provider=elevenlabs but $ELEVENLABS_API_KEY is not set");
return userProvider;
}
return heygenAvailable() ? "heygen" : elevenlabsAvailable() ? "elevenlabs" : "kokoro";
}
// ── voice resolution ──────────────────────────────────────────────────────────
// HeyGen /v3/voices/speech only accepts STARFISH voice_ids; auto-pick the first
// English public starfish voice when none is pinned. ElevenLabs/Kokoro have
// their own defaults.
export async function resolveVoiceId({ provider, userVoice, lang = "en" }) {
if (userVoice) return userVoice;
if (provider === "elevenlabs") return "21m00Tcm4TlvDq8ikWAM"; // Rachel
if (provider === "kokoro") {
if (lang === "en") return "am_michael";
throw new Error("Kokoro non-English needs an explicit --voice (see references/tts.md)");
}
// heygen — pin a fixed English default so the choice is deterministic. The old
// "first English voice the API returns" drifts whenever HeyGen re-sorts the
// public catalog. Marcia (mature, low female). Override with --voice / request.voice.
if (lang === "en") return "05f19352e8f74b0392a8f411eba40de1"; // Marcia · English · female
// Non-English: no fixed default — fall back to the first matching catalog voice.
const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, {
headers: heygenAuthHeaders(),
});
const voices = payload.data ?? payload.voices ?? [];
const pick = voices.find((v) => v.language === "English") ?? voices[0];
if (!pick) throw new Error("no public starfish voice to default to — pass --voice");
return pick.voice_id;
}
// ── helpers ─────────────────────────────────────────────────────────────────
/** Give each word a stable id so captions can key off it. */
export function withWordIds(words) {
return (words ?? []).map((w, i) => ({
id: `w${i}`,
text: w.text,
start: w.start,
end: w.end,
}));
return (words ?? []).map((w, i) => ({ id: `w${i}`, text: w.text, start: w.start, end: w.end }));
}
// `ffmpeg -i <file>` prints a `Duration: HH:MM:SS.ms` line to stderr even
// though it exits non-zero with no output requested. Parsing pulled out as
// a pure function so the ENOENT fallback below can be tested without
// depending on whether ffprobe/ffmpeg are actually installed on the
// machine running the tests.
// though it exits non-zero with no output requested. Parsing pulled out as a
// pure function so the ENOENT fallback below can be tested without depending on
// whether ffprobe/ffmpeg are actually installed on the machine running tests.
export function parseFfmpegDurationBanner(stderrText) {
const match = /Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/.exec(stderrText ?? "");
if (!match) return NaN;
@@ -99,8 +39,7 @@ export function parseFfmpegDurationBanner(stderrText) {
// Some "essentials"-style ffmpeg distributions (common on Windows) ship
// ffmpeg.exe without ffprobe.exe. ffprobeDuration's caller (audio.mjs)
// otherwise reads a spurious NaN as "the WAV file is corrupt" and drops an
// already-successfully-synthesized TTS line, rather than "the tool for
// measuring it is missing".
// already-synthesized line, rather than "the tool for measuring it is missing".
function ffmpegDurationFallback(absPath) {
const r = spawnSync("ffmpeg", ["-i", absPath], { encoding: "utf8" });
return parseFfmpegDurationBanner(r.stderr);
@@ -117,262 +56,58 @@ export function ffprobeDuration(absPath) {
return parseFloat(String(r.stdout).trim());
}
export function resolveNpxCliFromNpmExecPath(
npmExecPath = process.env.npm_execpath,
pathExists = existsSync,
) {
if (!npmExecPath) return null;
const fileName = npmExecPath.replace(/\\/g, "/").split("/").pop()?.toLowerCase();
const npxCliPath =
fileName === "npx-cli.js" ? npmExecPath : join(dirname(npmExecPath), "npx-cli.js");
return pathExists(npxCliPath) ? npxCliPath : null;
}
export function resolveNpxCliPath(
npmExecPath = process.env.npm_execpath,
nodeExecPath = process.env.npm_node_execpath || process.execPath,
pathExists = existsSync,
) {
const fromNpm = resolveNpxCliFromNpmExecPath(npmExecPath, pathExists);
if (fromNpm) return fromNpm;
const besideNode = join(dirname(nodeExecPath), "node_modules", "npm", "bin", "npx-cli.js");
return pathExists(besideNode) ? besideNode : null;
}
export function resolveSpawnCommand(
cmd,
args,
opts = {},
platform = process.platform,
env = process.env,
pathExists = existsSync,
) {
if (cmd !== "npx" || platform !== "win32") {
return { cmd, args, opts: { stdio: "ignore", ...opts } };
}
// On Windows, npx resolves to npx.cmd, which Node cannot execute directly.
// Avoid `shell:true` and the .cmd shim entirely by invoking npm's JS CLI with
// node, preserving request-provided values as argv data instead of shell text.
const nodeExecPath = env.npm_node_execpath || process.execPath;
const npxCliPath = resolveNpxCliPath(env.npm_execpath, nodeExecPath, pathExists);
if (!npxCliPath) return null;
return {
cmd: nodeExecPath,
args: [npxCliPath, ...args.map((arg) => String(arg))],
opts: { stdio: "ignore", windowsHide: true, ...opts },
};
}
// `platform`/`spawnFn` params (default process.platform / the real spawn)
// exist so tests can exercise the win32 branch without mocking node:child_process
// (its ESM exports are non-configurable, so mock.method can't patch it).
// One-shot so a whole batch of TTS lines doesn't repeat the same diagnostic.
let _warnedNpxResolution = false;
/** Test-only: reset the one-shot npx-resolution warning latch. */
export function _resetNpxResolutionWarnForTests() {
_warnedNpxResolution = false;
}
export function spawnP(
cmd,
args,
opts = {},
platform = process.platform,
spawnFn = spawn,
env = process.env,
pathExists = existsSync,
) {
const resolved = resolveSpawnCommand(cmd, args, opts, platform, env, pathExists);
if (!resolved) {
// resolveSpawnCommand only returns null for the npx-on-win32 case where
// neither npm's configured CLI nor the beside-node fallback exists. Without
// this, every call silently returns status:-1 and stdio:"ignore" hides why.
if (!_warnedNpxResolution) {
_warnedNpxResolution = true;
const reason = env.npm_execpath
? `npm_execpath (${env.npm_execpath}) and the beside-node npm fallback could not be found`
: "npm_execpath is unset and the beside-node npm fallback could not be found";
console.error(
`[media-use] Cannot run "${cmd}" on Windows: ${reason}. ` +
`Every "${cmd}" call is being skipped. Install npm with Node, or run via ` +
`\`npx\`/\`npm run\` with a valid npm_execpath.`,
);
}
return Promise.resolve({ status: -1 });
}
return new Promise((resolve) => {
const p = spawnFn(resolved.cmd, resolved.args, resolved.opts);
p.on("exit", (code) => resolve({ status: code ?? -1 }));
p.on("error", () => resolve({ status: -1 }));
});
}
// mp3/whatever bytes → wav 44.1k mono at destWav (ffmpeg detects true format).
function transcodeToWav(bytes, destWav) {
const td = mkdtempSync(join(tmpdir(), "hf-tts-"));
const tmp = join(td, "a.mp3");
writeFileSync(tmp, bytes);
// mp3 bytes -> wav 44.1k mono at destWav (ffmpeg detects the true format).
export function toWav(buf, destWav) {
const dir = mkdtempSync(join(tmpdir(), "media-tts-"));
const src = join(dir, "a.mp3");
writeFileSync(src, buf);
mkdirSync(dirname(destWav), { recursive: true });
const ff = spawnSync(
"ffmpeg",
["-y", "-loglevel", "error", "-i", tmp, "-ar", "44100", "-ac", "1", destWav],
["-y", "-loglevel", "error", "-i", src, "-ar", "44100", "-ac", "1", destWav],
{ stdio: "ignore" },
);
rmSync(td, { recursive: true, force: true });
rmSync(dir, { recursive: true, force: true });
return ff.status === 0 && existsSync(destWav);
}
const ELEVENLABS_PY = `
import os, sys
from elevenlabs.client import ElevenLabs
from elevenlabs import save
client = ElevenLabs(api_key=os.environ["ELEVENLABS_API_KEY"])
text = open(sys.argv[1]).read()
audio = client.text_to_speech.convert(
text=text, voice_id=sys.argv[2],
model_id="eleven_multilingual_v2", output_format="mp3_44100_128",
)
save(audio, sys.argv[3])
`;
// ── synthesize one line ───────────────────────────────────────────────────────
// Writes wav at wavAbs. Returns { ok, words, error } — words is the raw
// [{text,start,end}] array for HeyGen (native), or null for ElevenLabs/Kokoro
// (caller must transcribeWav). Never throws; failures return { ok:false, error }
// where `error` states WHY (so the caller can surface it, not a bare "TTS failed").
export async function synthesizeOne({
provider,
text,
voiceId,
lang = "en",
speed = 1.0,
wavAbs,
framesDir,
}) {
if (provider === "heygen") return synthesizeHeygen({ text, voiceId, lang, speed, wavAbs });
if (provider === "elevenlabs") {
// The Python helper writes straight to wavAbs; unlike heygen (transcodeToWav)
// and kokoro (the `frames tts` CLI), it does NOT create the parent dir,
// so on a fresh project (no assets/voice/ yet) the save fails and the line is
// silently dropped as "TTS failed - omitted". Create it first, like the other
// providers do. Guarded so a mkdir failure (EACCES/EROFS) returns
// { ok:false } like the rest of this branch rather than throwing (the
// function's contract is "never throws; failures return { ok:false }").
try {
mkdirSync(dirname(wavAbs), { recursive: true });
} catch {
return { ok: false, words: null };
}
const { cmd, args } = pythonInvocation([
"-c",
ELEVENLABS_PY,
writeTmpText(text),
voiceId,
wavAbs,
]);
const r = await spawnP(cmd, args, {});
return synthResult(r, wavAbs, "elevenlabs (python)");
/**
* Synthesize one line to `out`. `.wav` is transcoded to 44.1k mono (what the
* rest of the pipeline expects); any other extension keeps the service's bytes.
* Never throws: a failure returns `{ ok: false, error }` naming the cause, so
* the caller can say WHY a line was dropped instead of "TTS failed".
*/
export async function synthesize({ text, voice, speed = 1.0, lang = "en", out }, deps = {}) {
const say = deps.speech ?? speech;
const wav = deps.toWav ?? toWav;
if (!ready()) return { ok: false, error: "no Hanzo credential — export HANZO_API_KEY" };
if (Buffer.byteLength(text, "utf8") > MAX_SPEECH_INPUT) {
return { ok: false, error: `line exceeds the ${MAX_SPEECH_INPUT}-byte speech limit — split it` };
}
// kokoro — via the published CLI; --output is relative to the project dir.
const wavRel = relTo(framesDir, wavAbs);
const args = ["frames", "tts", writeTmpText(text), "--voice", voiceId, "--output", wavRel];
if (lang !== "en") args.push("--lang", lang);
const r = await spawnP("npx", args, { cwd: framesDir });
return synthResult(r, wavAbs, "kokoro (npx frames tts)");
}
// Shape a spawn result into { ok, words, error }, naming why on failure so the
// caller surfaces it instead of a bare "TTS failed".
export function synthResult(r, wavAbs, label) {
if (r.status === 0 && existsSync(wavAbs)) return { ok: true, words: null };
const why =
r.status !== 0 ? `${label} exited with status ${r.status}` : `${label} produced no wav file`;
return { ok: false, words: null, error: why };
}
// `deps` is injectable for tests; production uses the real network/ffmpeg impls.
// Every failure path returns an `error` string so the caller can surface WHY a
// line was dropped instead of the bare "TTS failed" that hid the real cause
// (e.g. an HTTP 402 plan_upgrade_required thrown by heygenJSON was swallowed).
export async function synthesizeHeygen({ text, voiceId, lang, speed, wavAbs }, deps = {}) {
const requestJSON = deps.heygenJSON ?? heygenJSON;
const authHeaders = deps.heygenAuthHeaders ?? heygenAuthHeaders;
const fetchImpl = deps.fetch ?? fetch;
const transcode = deps.transcodeToWav ?? transcodeToWav;
try {
const body = { text, voice_id: voiceId, speed };
if (lang !== "en") body.language = lang;
const payload = await requestJSON(`/voices/speech`, {
method: "POST",
headers: authHeaders(),
body,
});
const inner = payload.data ?? payload;
if (!inner.audio_url) {
return { ok: false, words: null, error: "HeyGen /voices/speech returned no audio_url" };
}
const res = await fetchImpl(inner.audio_url);
if (!res.ok) {
return { ok: false, words: null, error: `audio_url fetch failed: HTTP ${res.status}` };
}
const bytes = Buffer.from(await res.arrayBuffer());
// .wav output → transcode to 44.1k mono; .mp3 → raw bytes (no ffmpeg). The
// engine always asks for .wav; the standalone heygen-tts CLI may ask for .mp3.
if (wavAbs.endsWith(".wav")) {
if (!transcode(bytes, wavAbs)) {
return {
ok: false,
words: null,
error: "wav transcode failed (ffmpeg)",
};
}
const buf = await say({ text, voice, speed, lang, format: out.endsWith(".wav") ? "wav" : "mp3" });
if (out.endsWith(".wav")) {
if (!wav(buf, out)) return { ok: false, error: "wav transcode failed (ffmpeg)" };
} else {
mkdirSync(dirname(wavAbs), { recursive: true });
writeFileSync(wavAbs, bytes);
write(buf, out);
}
const words = Array.isArray(inner.word_timestamps)
? inner.word_timestamps
.filter((w) => w && typeof w.word === "string" && isFinite(w.start) && isFinite(w.end))
.filter((w) => !/^<.*>$/.test(w.word.trim())) // drop <start>/<end> sentinels
.map((w) => ({ text: w.word, start: w.start, end: w.end }))
: [];
return { ok: true, words };
return { ok: true };
} catch (e) {
return { ok: false, words: null, error: e?.message ? String(e.message) : String(e) };
return { ok: false, error: e?.message ? String(e.message) : String(e) };
}
}
// ElevenLabs/Kokoro have no word timings — run Whisper over the wav. Returns the
// flat [{id,text,start,end}] word array, or null. Each call uses a throwaway
// --dir so parallel scenes don't collide on transcript.json.
export async function transcribeWav({ wavRel, lang = "en", framesDir }) {
const model = lang === "en" ? "small.en" : "small";
const td = mkdtempSync(join(tmpdir(), "hf-trans-"));
const args = ["frames", "transcribe", wavRel, "--model", model, "--dir", td];
if (lang !== "en") args.push("--language", lang);
const r = await spawnP("npx", args, { cwd: framesDir });
let words = null;
if (r.status === 0) {
const src = join(td, "transcript.json");
if (existsSync(src)) {
try {
const arr = JSON.parse(readFileSync(src, "utf8"));
if (Array.isArray(arr) && arr.length) words = arr;
} catch {}
}
/**
* Word timings for a rendered audio file, or `null` when the service returned
* none. Callers treat `null` as "captions fall back to line timing" — never as
* a reason to invent word boundaries.
*/
export async function align({ file, lang = "en" }, deps = {}) {
const read = deps.transcript ?? transcript;
try {
const { words } = await read({ file, lang });
return words.length ? words : null;
} catch {
return null;
}
rmSync(td, { recursive: true, force: true });
return words;
}
// ── tiny local utils ──────────────────────────────────────────────────────────
function writeTmpText(text) {
const td = mkdtempSync(join(tmpdir(), "hf-txt-"));
const p = join(td, "line.txt");
writeFileSync(p, text);
return p;
}
function relTo(base, abs) {
return abs.startsWith(base + "/") ? abs.slice(base.length + 1) : abs;
}
skills/media-use/audio/scripts/lib/tts.spawn.test.mjs
-158
View File
@@ -1,158 +0,0 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { EventEmitter } from "node:events";
import {
resolveNpxCliFromNpmExecPath,
resolveNpxCliPath,
resolveSpawnCommand,
spawnP,
_resetNpxResolutionWarnForTests,
} from "./tts.mjs";
// Regression: on Windows, npx resolves to npx.cmd, which spawn() cannot exec
// without shell:true — it fails ENOENT, silently swallowed as ok:false by the
// caller. spawnP takes injectable platform/spawnFn params so this doesn't
// need to touch the real process.platform or mock node:child_process (whose
// ESM exports are non-configurable).
function fakeSpawn(captured) {
return (cmd, args, opts) => {
captured.push({ cmd, args, opts });
const p = new EventEmitter();
setImmediate(() => p.emit("exit", 0));
return p;
};
}
const envWithNpxCli = {
npm_execpath: "/opt/node/lib/node_modules/npm/bin/npm-cli.js",
npm_node_execpath: "/opt/node/bin/node",
};
const npxCliPath = "/opt/node/lib/node_modules/npm/bin/npx-cli.js";
const pathExists = (path) => path === npxCliPath;
test("resolveNpxCliFromNpmExecPath finds npx-cli next to npm-cli", () => {
assert.equal(resolveNpxCliFromNpmExecPath(envWithNpxCli.npm_execpath, pathExists), npxCliPath);
});
test("resolveNpxCliPath finds npx-cli beside node when npm_execpath is unset", () => {
const node = "C:/Program Files/nodejs/node.exe";
const expected = "C:/Program Files/nodejs/node_modules/npm/bin/npx-cli.js";
assert.equal(
resolveNpxCliPath(undefined, node, (path) => path === expected),
expected,
);
});
test("resolveSpawnCommand routes npx through node+npx-cli on win32 without shell:true", () => {
const resolved = resolveSpawnCommand(
"npx",
["frames", "tts", "C:\\Users\\Test User\\line.txt", "--voice", "am_michael"],
{},
"win32",
envWithNpxCli,
pathExists,
);
assert.ok(resolved);
assert.equal(resolved.cmd, envWithNpxCli.npm_node_execpath);
assert.deepEqual(resolved.args, [
npxCliPath,
"frames",
"tts",
"C:\\Users\\Test User\\line.txt",
"--voice",
"am_michael",
]);
assert.equal(resolved.opts.shell, undefined);
});
test("resolveSpawnCommand preserves Windows npx shell metacharacters as argv data", () => {
const resolved = resolveSpawnCommand(
"npx",
["frames", "tts", "hello & calc"],
{},
"win32",
envWithNpxCli,
pathExists,
);
assert.ok(resolved);
assert.deepEqual(resolved.args, [npxCliPath, "frames", "tts", "hello & calc"]);
});
test("spawnP uses the resolved node+npx-cli command for npx on win32", async () => {
const captured = [];
await spawnP(
"npx",
["frames", "tts"],
{},
"win32",
fakeSpawn(captured),
envWithNpxCli,
pathExists,
);
assert.equal(captured.length, 1);
assert.equal(captured[0].cmd, envWithNpxCli.npm_node_execpath);
assert.deepEqual(captured[0].args, [npxCliPath, "frames", "tts"]);
assert.equal(captured[0].opts.shell, undefined);
});
test("spawnP does not enable shell for npx on darwin/linux", async () => {
const captured = [];
await spawnP("npx", ["frames", "tts"], {}, "darwin", fakeSpawn(captured));
assert.equal(captured[0].cmd, "npx");
assert.deepEqual(captured[0].args, ["frames", "tts"]);
assert.equal(captured[0].opts.shell, undefined);
});
test("spawnP does not enable shell for non-npx commands even on win32", async () => {
const captured = [];
await spawnP("python3", ["-c", "pass"], {}, "win32", fakeSpawn(captured));
assert.equal(captured[0].cmd, "python3");
assert.deepEqual(captured[0].args, ["-c", "pass"]);
assert.equal(captured[0].opts.shell, undefined);
});
test("spawnP resolves npx beside node when npm_execpath is unset on win32", async () => {
_resetNpxResolutionWarnForTests();
const captured = [];
const node = "C:/Program Files/nodejs/node.exe";
const npxCli = "C:/Program Files/nodejs/node_modules/npm/bin/npx-cli.js";
const result = await spawnP(
"npx",
["frames", "tts"],
{},
"win32",
fakeSpawn(captured),
{ npm_node_execpath: node },
(path) => path === npxCli,
);
assert.equal(result.status, 0);
assert.equal(captured.length, 1);
assert.equal(captured[0].cmd, node);
assert.deepEqual(captured[0].args, [npxCli, "frames", "tts"]);
});
test("spawnP warns once with an accurate diagnostic when neither npx path exists", async () => {
_resetNpxResolutionWarnForTests();
const errors = [];
const originalError = console.error;
console.error = (message) => errors.push(String(message));
try {
const env = { npm_execpath: "C:/missing/npm-cli.js", npm_node_execpath: "C:/node/node.exe" };
const missing = () => false;
assert.equal(
(await spawnP("npx", ["frames", "tts"], {}, "win32", fakeSpawn([]), env, missing))
.status,
-1,
);
assert.equal(
(await spawnP("npx", ["frames", "tts"], {}, "win32", fakeSpawn([]), env, missing))
.status,
-1,
);
} finally {
console.error = originalError;
}
assert.equal(errors.length, 1);
assert.match(errors[0], /npm_execpath \(C:\/missing\/npm-cli\.js\)/);
assert.doesNotMatch(errors[0], /npm_execpath is not set/);
});
skills/media-use/audio/scripts/lib/tts.test.mjs
-157
View File
@@ -1,157 +0,0 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, writeFileSync, chmodSync, rmSync, existsSync } from "node:fs";
import { join, dirname } from "node:path";
import { tmpdir } from "node:os";
import {
parseFfmpegDurationBanner,
ffprobeDuration,
synthesizeOne,
synthesizeHeygen,
synthResult,
} from "./tts.mjs";
test("parseFfmpegDurationBanner reads ffmpeg's stderr Duration line", () => {
const stderr = [
"ffmpeg version 6.0",
"Input #0, wav, from 'a.wav':",
" Duration: 00:00:03.42, bitrate: 705 kb/s",
"At least one output file must be specified",
].join("\n");
assert.equal(parseFfmpegDurationBanner(stderr), 3.42);
});
test("parseFfmpegDurationBanner handles an hours component", () => {
const stderr = " Duration: 01:02:03.50, start: 0.000000, bitrate: 128 kb/s";
assert.equal(parseFfmpegDurationBanner(stderr), 3723.5);
});
test("parseFfmpegDurationBanner returns NaN when there is no Duration line", () => {
assert.ok(Number.isNaN(parseFfmpegDurationBanner("ffmpeg: command not found")));
assert.ok(Number.isNaN(parseFfmpegDurationBanner("")));
assert.ok(Number.isNaN(parseFfmpegDurationBanner(undefined)));
});
// Regression for the actual bug: ffprobeDuration used to collapse "ffprobe
// binary is missing" (ENOENT — the "essentials"-style Windows ffmpeg build
// with no ffprobe.exe) and "file is genuinely unreadable" into the same NaN,
// giving audio.mjs no way to tell "measure differently" from "give up".
//
// Builds an isolated PATH containing only a fake `ffmpeg` stub (no `ffprobe`
// at all) so ffprobeDuration's spawnSync("ffprobe", ...) call ENOENTs for
// real, then verifies it recovers the duration via the ffmpeg fallback
// instead of returning NaN.
test("ffprobeDuration falls back to ffmpeg when the ffprobe binary itself is missing", () => {
const dir = mkdtempSync(join(tmpdir(), "tts-ffprobe-fallback-"));
const fakeFfmpeg = join(dir, "ffmpeg");
writeFileSync(
fakeFfmpeg,
"#!/bin/sh\necho 'Duration: 00:00:02.50, start: 0.000000, bitrate: 128 kb/s' 1>&2\nexit 1\n",
);
chmodSync(fakeFfmpeg, 0o755);
const originalPath = process.env.PATH;
try {
process.env.PATH = dir; // only the fake ffmpeg resolves; no real ffprobe on this PATH
assert.equal(ffprobeDuration("/does/not/matter.wav"), 2.5);
} finally {
process.env.PATH = originalPath;
rmSync(dir, { recursive: true, force: true });
}
});
test("ffprobeDuration returns NaN when neither ffprobe nor ffmpeg resolve", () => {
const dir = mkdtempSync(join(tmpdir(), "tts-no-binaries-"));
const originalPath = process.env.PATH;
try {
process.env.PATH = dir; // empty directory — nothing resolves
assert.ok(Number.isNaN(ffprobeDuration("/does/not/matter.wav")));
} finally {
process.env.PATH = originalPath;
rmSync(dir, { recursive: true, force: true });
}
});
test("synthesizeOne(elevenlabs) creates the output dir before writing", async () => {
const dir = mkdtempSync(join(tmpdir(), "tts-el-mkdir-"));
const wavAbs = join(dir, "assets", "voice", "line-0.wav"); // nested, not yet created
const savedKey = process.env.ELEVENLABS_API_KEY;
try {
// Unset the key so the Python side fails fast — the mkdir must run before
// the spawn regardless, which is what this guards.
delete process.env.ELEVENLABS_API_KEY;
await synthesizeOne({
provider: "elevenlabs",
text: "hi",
voiceId: "v",
wavAbs,
framesDir: dir,
});
assert.ok(existsSync(dirname(wavAbs)), "output directory should be created");
} finally {
if (savedKey === undefined) delete process.env.ELEVENLABS_API_KEY;
else process.env.ELEVENLABS_API_KEY = savedKey;
rmSync(dir, { recursive: true, force: true });
}
});
test("synthesizeHeygen surfaces a thrown HTTP error (e.g. 402) instead of swallowing it", async () => {
const res = await synthesizeHeygen(
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: "/tmp/x.wav" },
{
heygenAuthHeaders: () => ({}),
heygenJSON: async () => {
throw new Error("HeyGen POST /voices/speech → HTTP 402\nplan_upgrade_required");
},
},
);
assert.equal(res.ok, false);
assert.match(res.error, /402/);
assert.match(res.error, /plan_upgrade_required/);
});
test("synthesizeHeygen surfaces a failed audio_url fetch with its status", async () => {
const res = await synthesizeHeygen(
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: "/tmp/x.wav" },
{
heygenAuthHeaders: () => ({}),
heygenJSON: async () => ({ data: { audio_url: "http://audio.example/x" } }),
fetch: async () => ({ ok: false, status: 403 }),
},
);
assert.equal(res.ok, false);
assert.match(res.error, /HTTP 403/);
});
test("synthesizeHeygen reports a missing audio_url", async () => {
const res = await synthesizeHeygen(
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: "/tmp/x.wav" },
{ heygenAuthHeaders: () => ({}), heygenJSON: async () => ({}) },
);
assert.equal(res.ok, false);
assert.match(res.error, /no audio_url/);
});
test("synthesizeHeygen reports wav transcode failures", async () => {
const dir = mkdtempSync(join(tmpdir(), "hf-tts-test-"));
try {
const res = await synthesizeHeygen(
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: join(dir, "voice.wav") },
{
heygenAuthHeaders: () => ({}),
heygenJSON: async () => ({ data: { audio_url: "http://audio.example/x" } }),
fetch: async () => ({ ok: true, status: 200, arrayBuffer: async () => new ArrayBuffer(0) }),
transcodeToWav: () => false,
},
);
assert.equal(res.ok, false);
assert.equal(res.error, "wav transcode failed (ffmpeg)");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test("synthResult names a non-zero subprocess exit", () => {
const res = synthResult({ status: 2 }, "/tmp/none.wav", "kokoro (npx frames tts)");
assert.equal(res.ok, false);
assert.match(res.error, /kokoro .* exited with status 2/);
});
skills/media-use/audio/scripts/lyria-recipe.py
-128
View File
@@ -1,128 +0,0 @@
#!/usr/bin/env python3
"""Generate BGM using Google Lyria RealTime API.
Usage:
python lyria-recipe.py --output <path> --duration <seconds> [tuning flags]
Requires:
$GOOGLE_API_KEY or $GEMINI_API_KEY environment variable (treated as aliases).
pip install google-genai python-dotenv. audio.mjs Step 4b installs these on
demand when a key is set but google.genai is not importable; if that install
fails it falls back to local MusicGen rather than leaving the video with no BGM.
"""
from __future__ import annotations
import argparse
import asyncio
import os
import sys
import wave
from pathlib import Path
DEFAULT_PROMPT = "Uplifting corporate tech, bright and modern, gentle piano with synth pads"
SAMPLE_RATE = 48000
CHANNELS = 2
SAMPLE_WIDTH = 2 # 16-bit
def parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description="Generate BGM via Google Lyria RealTime.")
p.add_argument("--output", required=True, help="Output WAV path.")
p.add_argument("--duration", type=float, required=True, help="Target duration in seconds.")
p.add_argument("--prompt", default=DEFAULT_PROMPT, help="Mood / instrumentation prompt.")
p.add_argument("--negative-prompt", default=None, help="Styles to exclude (optional).")
p.add_argument("--bpm", type=int, default=110)
p.add_argument("--brightness", type=float, default=0.8, help="0-1, higher = brighter mood.")
p.add_argument("--density", type=float, default=0.5, help="0-1, higher = fuller mix.")
p.add_argument(
"--scale",
default="MAJOR",
help="MAJOR / MINOR / PENTATONIC / etc. — see google.genai.types.Scale. Pass empty string for none.",
)
return p.parse_args()
async def generate_bgm(args: argparse.Namespace) -> dict:
from google import genai
from google.genai import types
api_key = os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY") or ""
if not api_key:
raise RuntimeError("Neither GOOGLE_API_KEY nor GEMINI_API_KEY is set.")
client = genai.Client(
api_key=api_key,
http_options={"api_version": "v1alpha"},
)
out_path = Path(args.output)
out_path.parent.mkdir(parents=True, exist_ok=True)
target_bytes = int(args.duration * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)
cfg: dict = {"bpm": args.bpm, "temperature": 1.0}
if args.density is not None:
cfg["density"] = args.density
if args.brightness is not None:
cfg["brightness"] = args.brightness
if args.scale:
scale_enum = getattr(types.Scale, args.scale, None)
if scale_enum:
cfg["scale"] = scale_enum
prompts = [types.WeightedPrompt(text=args.prompt, weight=1.0)]
if args.negative_prompt:
prompts.append(types.WeightedPrompt(text=args.negative_prompt, weight=-1.0))
buf = bytearray()
timeout = args.duration + 8
async with client.aio.live.music.connect(
model="models/lyria-realtime-exp",
) as session:
await session.set_weighted_prompts(prompts=prompts)
await session.set_music_generation_config(
config=types.LiveMusicGenerationConfig(**cfg),
)
await session.play()
async def collect():
while len(buf) < target_bytes:
async for msg in session.receive():
sc = msg.server_content
if sc and sc.audio_chunks:
for chunk in sc.audio_chunks:
buf.extend(chunk.data)
if len(buf) >= target_bytes:
return
await asyncio.sleep(1e-6)
try:
await asyncio.wait_for(collect(), timeout=timeout)
except TimeoutError:
print(f"Timeout after {timeout:.0f}s, collected {len(buf)} bytes", file=sys.stderr)
audio = bytes(buf[:target_bytes])
with wave.open(str(out_path), "wb") as wf:
wf.setnchannels(CHANNELS)
wf.setsampwidth(SAMPLE_WIDTH)
wf.setframerate(SAMPLE_RATE)
wf.writeframes(audio)
actual_duration = len(audio) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)
print(f"BGM: {out_path} ({actual_duration:.2f}s)")
return {"file": str(out_path), "duration_sec": round(actual_duration, 2)}
def main() -> None:
args = parse_args()
try:
asyncio.run(generate_bgm(args))
except RuntimeError as exc:
print(f"BGM generation failed: {exc}", file=sys.stderr)
sys.exit(1)
if __name__ == "__main__":
main()
skills/media-use/audio/scripts/speak.mjs
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env node
// speak.mjs — one text in, one audio file out (plus optional word timings).
// A thin CLI over lib/tts.mjs, the same code the audio engine runs, so the
// request, the mp3->wav transcode, and the alignment pass live in one place.
//
// Usage:
// node speak.mjs "Text to speak" -o narration.wav [--words narration.words.json]
// node speak.mjs ./script.txt -o narration.wav --words narration.words.json
// node speak.mjs "Bonjour" -o fr.wav --lang fr --voice <id>
//
// Flags: -o/--output (.wav transcodes through ffmpeg; anything else keeps the
// service's bytes), --words, --voice, --speed, --lang.
// Needs $HANZO_API_KEY, and ffmpeg for .wav output.
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { align, ffprobeDuration, synthesize, withWordIds } from "./lib/tts.mjs";
const argv = process.argv.slice(2);
function flag(name, def) {
const i = argv.indexOf(`--${name}`);
if (i < 0) return def;
if (i + 1 >= argv.length) return true;
const v = argv[i + 1];
return v.startsWith("--") ? true : v;
}
const die = (m) => {
console.error(`✗ speak: ${m}`);
process.exit(1);
};
// First arg that isn't a flag or the -o value is the text / .txt path.
const positional = (() => {
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
if (a.startsWith("--")) {
const next = argv[i + 1];
if (next && !next.startsWith("--")) i++;
continue;
}
if (a === "-o") {
i++;
continue;
}
return a;
}
return null;
})();
const output = resolve(
(typeof flag("output") === "string" && flag("output")) ||
(argv.includes("-o") && argv[argv.indexOf("-o") + 1]) ||
"narration.wav",
);
const wordsPath = typeof flag("words") === "string" ? resolve(flag("words")) : null;
const voice = typeof flag("voice") === "string" ? flag("voice") : null;
const speedRaw = typeof flag("speed") === "string" ? Number(flag("speed")) : 1.0;
const speed = isFinite(speedRaw) && speedRaw > 0 ? speedRaw : 1.0;
const lang = typeof flag("lang") === "string" ? flag("lang") : "en";
if (!positional) die("no text given. Pass a string or a .txt path.");
const text =
positional.endsWith(".txt") && existsSync(resolve(positional))
? readFileSync(resolve(positional), "utf8").trim()
: positional;
if (!text) die("input text is empty");
const { ok, error } = await synthesize({ text, voice, speed, lang, out: output });
if (!ok) die(error);
let wordCount = 0;
if (wordsPath) {
const words = await align({ file: output, lang });
if (words?.length) {
mkdirSync(dirname(wordsPath), { recursive: true });
writeFileSync(wordsPath, JSON.stringify(withWordIds(words), null, 2));
wordCount = words.length;
} else {
console.error("⚠ the transcription returned no word timings — nothing written to --words");
}
}
const dur = ffprobeDuration(output);
const durStr = isFinite(dur) ? ` (${dur.toFixed(2)}s)` : "";
console.log(`✓ ${output}${durStr}${wordCount ? ` · ${wordsPath} (${wordCount} words)` : ""}`);
skills/media-use/audio/scripts/wait-bgm.mjs
-167
View File
@@ -1,167 +0,0 @@
#!/usr/bin/env node
// Phase 4c pre-assemble helper — wait for detached BGM, then write status.
//
// audio.mjs may launch Lyria / MusicGen in a detached process so voice work can
// keep moving. Before assemble-index.mjs decides whether to emit the BGM audio
// track, this script gives the background renderer a bounded chance to finish
// and converts log/process state into a small bgm_status.json file.
//
// Always exits 0 for normal pipeline use: missing/failed BGM should not block a
// voice/captions/SFX render. Structural invocation errors still exit 1.
//
// Usage:
// node wait-bgm.mjs --audio-meta ./audio_meta.json --frames . \
// [--timeout-ms 120000] [--interval-ms 2000] [--out ./bgm_status.json]
import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
import { join, resolve } from "node:path";
const argv = process.argv.slice(2);
const flag = (name, def) => {
const i = argv.indexOf(`--${name}`);
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : def;
};
function die(msg) {
console.error(`✗ wait-bgm.mjs: ${msg}`);
process.exit(1);
}
const audioMetaPath = resolve(flag("audio-meta", "./audio_meta.json"));
const framesDir = resolve(flag("frames", "."));
const outPath = resolve(flag("out", join(framesDir, "bgm_status.json")));
const timeoutMs = Math.max(0, Number(flag("timeout-ms", "120000")) || 0);
const intervalMs = Math.max(250, Number(flag("interval-ms", "2000")) || 2000);
function sleep(ms) {
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
}
function isProcessAlive(pid) {
if (!pid || !Number.isFinite(Number(pid))) return false;
try {
process.kill(Number(pid), 0);
return true;
} catch {
return false;
}
}
function readTail(path, maxChars = 6000) {
if (!path || !existsSync(path)) return "";
const s = statSync(path);
const txt = readFileSync(path, "utf8");
return txt.slice(Math.max(0, txt.length - Math.min(maxChars, s.size)));
}
function detectFailure(logTail) {
if (!logTail) return "";
const lines = logTail.split("\n");
// Bare "out of range" over-matched benign BGM-renderer logs (e.g. a "sample rate
// out of range, resampling" notice), mislabelling a healthy track as failed and
// silently dropping the music. Anchor to the actual crash strings instead:
// Python "(list) index out of range" and torch "index … out of bounds".
const idx = lines.findIndex((line) =>
/(Traceback|IndexError|RuntimeError|Exception|Killed|No space left|Cannot allocate|index out of range|out of bounds)/i.test(
line,
),
);
if (idx < 0) return "";
return lines.slice(idx).join("\n").trim();
}
function writeStatus(status) {
const payload = {
generated_at: new Date().toISOString(),
...status,
};
writeFileSync(outPath, JSON.stringify(payload, null, 2) + "\n");
return payload;
}
if (!existsSync(audioMetaPath)) die(`audio_meta.json missing at ${audioMetaPath}`);
const audioMeta = JSON.parse(readFileSync(audioMetaPath, "utf8"));
const bgmPath = audioMeta.bgm?.path || "";
const bgmAbsPath = bgmPath ? join(framesDir, bgmPath) : "";
const logPath = audioMeta.bgm_log || "";
const pid = audioMeta.bgm_pid || null;
const base = {
enabled: Boolean(audioMeta.bgm_pending && bgmPath),
provider: audioMeta.bgm_provider || null,
mode: audioMeta.bgm_mode || null,
path: bgmPath || null,
log: logPath || null,
pid,
target_duration_s: audioMeta.bgm_target_duration_s || null,
seed_duration_s: audioMeta.bgm_seed_duration_s || null,
loop_count: audioMeta.bgm_loop_count || null,
timeout_ms: timeoutMs,
};
if (!base.enabled) {
const status = writeStatus({
...base,
status: "disabled",
ready: false,
waited_ms: 0,
message: "BGM not requested or disabled in audio_meta.json.",
});
console.log(`✓ bgm: ${status.status} (${status.message})`);
process.exit(0);
}
const started = Date.now();
let lastFailure = "";
let lastTail = "";
while (Date.now() - started <= timeoutMs) {
if (existsSync(bgmAbsPath)) {
const size = statSync(bgmAbsPath).size;
writeStatus({
...base,
status: "ready",
ready: true,
waited_ms: Date.now() - started,
size_bytes: size,
message: `BGM ready at ${bgmPath}.`,
});
console.log(`✓ bgm: ready (${bgmPath}, ${size}B)`);
process.exit(0);
}
lastTail = readTail(logPath);
lastFailure = detectFailure(lastTail);
const alive = isProcessAlive(pid);
if (lastFailure || (!alive && logPath && existsSync(logPath))) {
const message = lastFailure
? `BGM renderer failed; see ${logPath}.`
: `BGM renderer exited without writing ${bgmPath}; see ${logPath}.`;
const status = writeStatus({
...base,
status: "failed",
ready: false,
waited_ms: Date.now() - started,
process_alive: alive,
message,
error_tail: lastFailure || lastTail.slice(-2000),
});
console.log(`! bgm: failed (${status.message})`);
process.exit(0);
}
if (timeoutMs === 0) break;
await sleep(Math.min(intervalMs, Math.max(0, timeoutMs - (Date.now() - started))));
}
const status = writeStatus({
...base,
status: "timeout",
ready: false,
waited_ms: Date.now() - started,
process_alive: isProcessAlive(pid),
message: `Timed out waiting for ${bgmPath}; assemble-index will skip BGM if still absent.`,
log_tail: lastTail.slice(-2000),
});
console.log(`! bgm: timeout after ${status.waited_ms}ms (${bgmPath})`);
skills/media-use/luts/README.md
+11 -19
View File
@@ -1,28 +1,20 @@
# LUT library (authoring)
`index.json` is the agent-consumed catalog of color-grade looks. Each entry resolves
on demand — no `.cube` bodies are committed to the repo.
`index.json` is the agent-consumed catalog of color-grade looks. Each entry
builds on demand — no `.cube` bodies are committed to the repo.
Each look has:
- `id`, `description`, `tags`, `intensity` — matching + application metadata.
- `url` (optional) — a hosted `.cube` downloaded, validated, and frozen at resolve
time, exactly like bgm/image assets.
- `params` (optional) — a deterministic `buildCube` spec used offline (`--local-only`)
or as a fallback if the `url` download/validation fails.
- `params` — a deterministic `buildCube` spec. The same look builds the same
bytes on every machine, offline, with no host to keep alive.
An entry needs at least one of `url` or `params`; prefer both (CDN url with a params
fallback) so resolution is never blocked on the network.
## Authoring a new look
## Hosting a new look (operators)
1. Find the params: `resolve -t lut --params '{...}'` writes a `.cube` you can
inspect and iterate on.
2. Add an entry to `index.json` with those `params`, an `id`, a `description`,
and the `tags` an intent is likely to use.
1. Generate the `.cube` (e.g. `resolve -t lut --params '{...}'` or a graded export).
2. Upload it to the public CDN origin bucket:
```
aws s3 cp <id>.cube s3://heygen-public/luts/<id>.cube
```
It is then served at `https://static.heygen.ai/luts/<id>.cube` (CloudFront).
3. Add an entry to `index.json` with that `url` (and ideally a `params` fallback).
A look is matched lexically against the intent, so write the description and
tags in the words a person would actually say.
skills/media-use/luts/index.json
+1 -4
View File
@@ -1,12 +1,11 @@
{
"notes": "Look entries resolve on-demand from a CDN .cube `url` (downloaded + frozen, like bgm/image); `params` is a deterministic buildCube fallback used offline (--local-only) or if the download/validation fails. No .cube bodies are committed. See README.md to author or host a new look.",
"notes": "Look entries build deterministically from `params` via buildCube \u2014 the same bytes every run, offline, with nothing to host. No .cube bodies are committed. See README.md to author a new look.",
"looks": [
{
"id": "teal-orange-blockbuster",
"description": "Teal shadows and warm orange highlights for blockbuster-style cinematic footage.",
"tags": ["teal", "orange", "blockbuster", "cinematic", "split tone", "movie"],
"intensity": 0.85,
"url": "https://static.heygen.ai/luts/teal-orange-blockbuster.cube",
"params": {
"contrast": 0.18,
"saturation": 0.08,
@@ -24,7 +23,6 @@
"description": "High-contrast desaturated bleach bypass look with strong blacks.",
"tags": ["bleach", "bypass", "desaturated", "high contrast", "gritty", "film"],
"intensity": 0.8,
"url": "https://static.heygen.ai/luts/bleach-bypass.cube",
"params": {
"blacks": 0.04,
"shadows": -0.08,
@@ -41,7 +39,6 @@
"description": "Soft faded film wash with lifted blacks and warm highlights.",
"tags": ["film", "fade", "faded", "wash", "warm", "vintage"],
"intensity": 0.75,
"url": "https://static.heygen.ai/luts/film-fade.cube",
"params": {
"blacks": 0.35,
"shadows": 0.18,
skills/media-use/references/audio.md
+5 -5
View File
@@ -9,11 +9,11 @@ shot), use the shared engine at `audio/scripts/audio.mjs`. It takes a neutral
node <SKILL_DIR>/audio/scripts/audio.mjs --request ./audio_request.json --out ./audio_meta.json
```
- **Request** `{ provider?, lang?, speed?, lines: [{ id, text, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `retrieve | generate | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`.
- **Request** `{ lang?, speed?, voice?, lines: [{ id, text, sfx?: [names] }], bgm: { mode?, query?, prompt? } }`: `id` joins each line back to your model; `bgm.mode` = `catalog | compose | none` (omit for auto). `--only tts,bgm,sfx` runs a subset and merges into an existing `--out`.
- **Output** `audio_meta.json` (id-keyed): `voices[].{path,duration_s,words[]}` (word timestamps for captions), `sfx[]`, `bgm`, `total_duration_s`.
- **HeyGen free-usage path**: HeyGen CLI auth unlocks TTS plus music/SFX retrieval. Local/provider-specific generators are explicit alternatives where installed; run `node <SKILL_DIR>/scripts/resolve.mjs --doctor` before assuming retrieval or TTS will work.
- If BGM took the generate path (`bgm_pending: true`), run `audio/scripts/wait-bgm.mjs` before final render.
- **One credential**: `$HANZO_API_KEY` (and `$HANZO_ORG` for the shared catalog) unlocks voice, music, foley, and catalog reads. Run `node <SKILL_DIR>/scripts/resolve.mjs --doctor` before assuming any of them will work.
- Every route is synchronous: when the engine returns, every file it reports is on disk.
Single-shot helpers: `audio/scripts/heygen-tts.mjs` (one voice file). Transcription / background removal / captions use the `frames` CLI (`transcribe`, `remove-background`), see the per-topic guides in `audio/references/` (`tts.md`, `bgm.md`, `sfx.md`, `transcribe.md`, `remove-background.md`, `captions/`).
Single-shot helper: `audio/scripts/speak.mjs` (one line, one file, optional word timings). Per-topic guides live in `audio/references/` (`tts.md`, `bgm.md`, `sfx.md`, `transcribe.md`, `remove-background.md`, `captions/`).
Transcription defaults to Parakeet (better than whisper.cpp: 6.05% vs 7.44% WER, 5-10x faster) via `scripts/transcribe.mjs`, with whisper.cpp auto-fallback (see `references/operations.md`).
Word timings come from `scripts/transcribe.mjs` -> `POST /v1/audio/transcriptions`. When the service returns text without per-word timings, the word array is empty and the engine records an anomaly; captions then fall back to line timing. Nothing is interpolated.
skills/media-use/references/meta.md
+18 -25
View File
@@ -6,20 +6,20 @@ Maintainer-facing reference. Nothing here changes how you resolve or operate on
Frames owns media _playback_; media-use owns everything else. Each row is enforced by `scripts/lib/coverage.test.mjs` so the claim can't rot.
| Frames gap | media-use owns it via |
| ------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Audio-only, no image/icon | `resolve --type image\|icon` (heygen asset search) |
| No third-party brand logos | `resolve --type logo` (svgl → simple-icons → GitHub org avatar → domain favicon) |
| No voice / audio generation | `resolve --type voice` (HeyGen TTS free-usage path; optional local Kokoro) + the audio engine (`audio/scripts/audio.mjs`) |
| Scattered/duplicated audio engine | one consolidated engine under `audio/` (frames-media retired) |
| No agent media-ops (cut/reframe/transform) | `references/operations.md` + `resolve --from` to register outputs |
| No transcript-driven cutting | `scripts/transcript-cut.mjs` compiles word-timestamp edits into cut lists |
| No auto-duck / publish loudness | `scripts/audio-duck.mjs` + `references/operations.md` loudnorm/sidechain recipes |
| No cross-project memory | global content-addressed cache + auto-promote (`~/.media`) |
| Grade recipes and LUT freezing | `resolve --type grade` emits a paste-ready recipe and `resolve --type lut` freezes validated `.cube` files; direct element analysis/authoring lives in `frames media-treatment` |
| No image generation | RAM-graded local mflux (FLUX) via `scripts/lib/mflux-provider.mjs`, codex `image_gen` upsell (`scripts/lib/codex-provider.mjs`) |
| No video generation | `resolve --type video` — HeyGen avatar video first (free-usage path, sign-in nudge on auth failure), local LTX fallback (`videogen` in `scripts/lib/local-models.mjs`); image-to-video, photo-avatar, dub/translate remain manual `heygen` CLI recipes (`references/operations.md`) |
| Weak local-model defaults | HeyGen free-usage path via the `heygen` CLI; local open-source tools only as opt-in alternatives (`scripts/lib/local-run.mjs`) |
| Frames gap | media-use owns it via |
| ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Audio-only, no image/icon | `resolve --type image\|icon` (shared catalog, then generated) |
| No third-party brand logos | `resolve --type logo` (svgl → simple-icons → GitHub org avatar → domain favicon) |
| No voice / audio generation | `resolve --type voice` (`POST /v1/audio/speech`) + the audio engine (`audio/scripts/audio.mjs`) |
| Scattered/duplicated audio engine | one consolidated engine under `audio/` (frames-media retired) |
| No agent media-ops (cut/reframe/transform) | `references/operations.md` + `resolve --from` to register outputs |
| No transcript-driven cutting | `scripts/transcript-cut.mjs` compiles word-timestamp edits into cut lists |
| No auto-duck / publish loudness | `scripts/audio-duck.mjs` + `references/operations.md` loudnorm/sidechain recipes |
| No cross-project memory | global content-addressed cache + auto-promote (`~/.media`) |
| Grade recipes and LUT freezing | `resolve --type grade` emits a paste-ready recipe and `resolve --type lut` freezes validated `.cube` files; direct element analysis/authoring lives in `frames media-treatment` |
| No image generation | RAM-graded local mflux (FLUX) via `scripts/lib/mflux-provider.mjs`, codex `image_gen` upsell (`scripts/lib/codex-provider.mjs`) |
| No video generation | `resolve --type video` (`POST /v1/videos/generations`), frozen and ledgered like any other asset (`references/operations.md`) |
| Scattered provider setup | One credential, one host: `$HANZO_API_KEY` against `api.hanzo.ai` (`scripts/lib/api.mjs`) |
## Usage stats
@@ -45,17 +45,10 @@ resolve never waits on or fails from telemetry).
Opt out with `DO_NOT_TRACK=1` or `FRAMES_NO_TELEMETRY=1` (also off in CI and
dev). Same public PostHog project key and opt-outs as the `frames` CLI.
HeyGen request tagging: every generating `heygen` call (TTS, avatar video, catalog
search) carries the allowlisted `X-HeyGen-Client-Source: media-use` header, sourced
from one shared constant (`HEYGEN_CLIENT_SOURCE_ARGV` in `scripts/lib/heygen-cli.mjs`)
so a future call site can't silently ship untagged. Read-only discovery calls
(`voice list`, `avatar list`) are intentionally left untagged.
## Privacy
media-use uses the same shared install id as the `frames` CLI/studio
(`~/.frames/config.json`). When you are signed in to HeyGen, usage is
linked to your account email, or username when email is unavailable, matching
the CLI behavior. The events stay coarse: media type, source, provider, and
small counts only; intent text and paths stay local. Disable telemetry with
media-use uses the same shared install id as the CLI and studio
(`~/.frames/config.json`) and nothing else — no account, no credential, no
email. The events stay coarse: media type, source, provider, and small counts
only; intent text and paths stay local. Disable telemetry with
`FRAMES_NO_TELEMETRY=1` or `DO_NOT_TRACK=1`.
skills/media-use/references/operations.md
+38 -139
View File
@@ -43,26 +43,19 @@ auto-editor in.mp4 --edit audio:threshold=4% -o tight.mp4 # pip install auto-e
scenedetect -i in.mp4 detect-adaptive list-scenes # pip install scenedetect
```
## Transforms with a quality choice (process)
## Transforms that run on this machine (process)
These have a local option AND a higher-quality HeyGen-CLI option. Run the local
one for free/offline; use the HeyGen CLI when quality matters. Showing the user
a **side-by-side** (local vs HeyGen) is the honest way to let them choose.
| Op | Local (free) | HeyGen CLI (quality) |
| ------------------ | -------------------------------------------------- | --------------------------- |
| Background removal | `frames remove-background in.png` (u2net) | `heygen background-removal` |
| Upscale | `realesrgan-ncnn-vulkan -i in.png -o out.png -s 4` | n/a |
| Lipsync (dub) | n/a | `heygen lipsync` |
| Translate | n/a | `heygen video-translate` |
| Op | Command |
| ------------------ | -------------------------------------------------- |
| Background removal | `hanzo frame remove-background in.png` (u2net) |
| Upscale | `realesrgan-ncnn-vulkan -i in.png -o out.png -s 4` |
After any op: `resolve --from out.ext --type <type>` to register the derived
asset (it records provenance and auto-promotes to the global cache).
> ponytail: media-use doesn't re-wrap ffmpeg/heygen here, that's deliberate
> (OP1). The value it adds is the ledger + global reuse on the _output_, via
> `--from`. Add a thin `process` verb only if agents repeatedly fumble these
> recipes.
> media-use does not re-wrap ffmpeg here, and that is deliberate (OP1). The
> value it adds is the ledger + global reuse on the _output_, via `--from`. Add
> a thin `process` verb only if agents repeatedly fumble these recipes.
## Exact error-diffusion dither
@@ -98,30 +91,24 @@ files as two real media layers and use the seek-safe GSAP timeline to reveal or
crossfade between them. Use the realtime Bayer shader instead when the dither
amount itself must animate continuously.
## Transcription (default: Parakeet, better than whisper.cpp)
## Transcription
`transcribe.mjs` is the default local transcription path. It runs **NVIDIA
Parakeet-TDT via parakeet-mlx**, which beats whisper.cpp on the Open ASR
Leaderboard (avg WER ~6.05% vs 7.44%; on NOISY audio 4.73% vs 5.96%, where
whisper-large-v3 hallucinated to 308% WER on meetings) and is 5-10x faster.
It emits `{ text, words:[{text,start,end}] }` with word timestamps (merged from
Parakeet's sub-word tokens), feeding transcript-cut, captions, and the audio
engine directly.
`transcribe.mjs` is the one transcription path: `POST /v1/audio/transcriptions`
through `api.hanzo.ai`. It emits `{ text, words: [{text,start,end}] }`, feeding
transcript-cut, captions, and the audio engine directly.
```bash
# install once: uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx
node <SKILL_DIR>/scripts/transcribe.mjs --input talk.mp4 --out talk.transcribe.json
# equivalently, the frames CLI has Parakeet built in (auto-detects it, whisper fallback):
npx frames transcribe talk.mp4 --engine parakeet # or --engine auto (default)
node <SKILL_DIR>/scripts/transcribe.mjs --input talk.mp4 --lang es
```
VERIFIED on 24GB: accurate, ~3s (cached) for 8s audio. Parakeet covers English +
25 European languages. For other languages, or when parakeet-mlx is not
installed, transcribe.mjs auto-falls-back to whisper.cpp (99 languages) via
`frames transcribe`. `--engine parakeet|whisper` forces one. (Cohere
Transcribe tops the leaderboard on paper but its mlx-audio quants produced
garbage and ran 40-70x slower on a Mac in testing, so it is not wired in.)
`--model` selects `whisper` (default) or the smaller, faster `whisper-small`.
Nothing is downloaded and nothing is built on first use.
**`words` is empty when the service returns text without per-word timings**, and
the command says so. Timings are never interpolated: a cut made on invented word
boundaries reads as correct and drifts against the audio. A caller that needs
per-word cuts should treat an empty `words` as "fall back to line timing".
## Text-based editing (transcript cut)
@@ -202,118 +189,30 @@ ffmpeg -i mix.wav \
mix.podcast.wav
```
## Generate: images (local first, cloud upsell)
## Generate: images
`resolve --type image` retrieves from the HeyGen catalog first; on a miss it
GENERATES. Two paths, best-for-the-machine picked automatically:
`resolve --type image` looks in the shared catalog first (`media/image/`); on a
miss it generates through `POST /v1/images/generations`. Both rungs are one
credential and one host — nothing to install, no model to cache, no RAM ladder
to satisfy.
1. **Local (default, free, private): mflux** (FLUX-on-MLX). `resolve` spec-checks
AVAILABLE RAM and runs the best FLUX-class model that fits, via
`scripts/lib/local-models.mjs` (`imagegen` ladder) + `mflux-provider.mjs`.
The RAM ladder (agent sees it via `describeModelLadder("imagegen", specs)`):
To skip the shelf and go straight to generation, pass `--provider hanzo`; to
stay on the shelf, `--provider catalog`. `--local-only` skips both and leaves
the project and global caches.
| Tier | Model | Needs (available RAM) | Notes |
| ------ | -------------------- | --------------------- | ----------------------------------- |
| medium | FLUX.1 schnell int4 | ~8GB (`--low-ram`) | ~20s/512px on 24GB. VERIFIED. Fast. |
| large | FLUX.2 Klein 4B int4 | ~32GB | higher quality, full-resident |
| xlarge | Qwen-Image | ~64GB | top quality, 64GB+ Macs only |
The catalog is empty until it is stocked. An empty prefix is reported as such
and image falls through to generation.
Gotchas baked into the table: the official FLUX repos are HF-gated, so it
points at non-gated community 4-bit re-uploads; and `--low-ram` is MANDATORY
at the medium tier (without it a 768x512 run swap-thrashed to 90 minutes on
24GB; with it, 20 seconds).
## Generate: video
2. **Cloud upsell (better quality): the `codex` CLI** `image_gen` tool, on the
user's ChatGPT subscription (codex owns auth, no key here, no per-call
charge). It is the automatic fallback when no local model fits AND the
explicit "make it better" choice on any machine. Users who just want codex
can ask for it directly. Verified: prompt -> raster -> frozen + ledgered.
`resolve --type video "<intent>"` generates through
`POST /v1/videos/generations` and freezes the result into the project ledger.
Pass `--voice-id` when the piece speaks and the default voice is wrong.
`--local-only` keeps mflux (once cached) and skips codex (network).
## Generate: video (`resolve --type video`, HeyGen avatar first)
`resolve --type video "<intent>"` is the default path. It generates a
script-driven HeyGen avatar video first (the free-usage allowance — OAuth
sessions ride the web-plan free avatar-video quota where eligible, API keys
follow normal API billing), falling back to local generative LTX only when
HeyGen is unavailable, uncredentialed, or `--local-only` is passed. The two
are non-substitutable outputs (a real presenter vs. a generic generative
clip), so treat the fallback as "HeyGen wasn't reachable," not "upgrade the
quality":
- **HeyGen avatar video (default, free for new API users):**
`heygenVideoGenerate` (`scripts/lib/heygen-video-provider.mjs`) shells the
`heygen` CLI — never the raw API — auto-picking a public avatar and a
starfish voice (override with `--avatar-id`/`--voice-id`, threaded through
as `ctx.avatarId`/`ctx.voiceId`). If the CLI reports `not_authenticated`,
the provider prints an onboarding recommendation (avatar video is free for
new API users — sign in) to stderr and falls through to LTX instead of
hard-failing.
- **Local fallback: LTX 2.3 on MLX** via `dgrauet/ltx-2-mlx`, the `videogen`
ladder in `local-models.mjs` (`ltx-video-provider.mjs`). Generative clips
(t2v), spec-gated to RAM. Verified on 24GB: 512x320 x 33f with audio.
Every generating `heygen` call from media-use — TTS, avatar video, and
catalog search — sends the allowlisted `X-HeyGen-Client-Source: media-use`
header (persistent flag, works on every subcommand) via the shared
`HEYGEN_CLIENT_SOURCE_ARGV` constant (`scripts/lib/heygen-cli.mjs`), so usage
tags correctly in billing/resource meta and shows up in the API dashboards.
Read-only discovery (`avatar list`, `voice list`) doesn't need it.
For structured bodies `resolve --type video` doesn't expose yet (a specific
`avatar_id`/`voice_id` combination beyond the ctx overrides, or a
pre-recorded `audio_url` instead of a script), the raw `heygen video create`
recipe below remains the escape hatch:
```bash
# discover an avatar + a starfish voice, then create + wait
heygen avatar list --ownership public --limit 5
heygen voice list --engine starfish --limit 5
heygen video create --headers "X-HeyGen-Client-Source: media-use" --wait -d '{
"type": "avatar",
"avatar_id": "<avatar-id>",
"script": "Your narration here.",
"voice_id": "<voice-id>"
}'
```
Avatar videos are deterministic + script-driven (lip-sync from a script or a
pre-recorded `audio_url`), distinct from the generative LTX clips. After a
manual recipe renders, `resolve --from <downloaded.mp4> --type video` to
ledger it (not needed when generating via `resolve --type video` directly —
that already ledgers the result).
### Image-to-video (animate any still into a talking clip)
Not wired into `resolve --type video` (deferred — the `avatar` type covers
the default script-driven case). `heygen video create` takes the raw
`POST /v3/videos` body, so switching `type`
from `avatar` to `image` animates **any image of a person** into a lip-synced
talking video, with no avatar/photo-avatar creation step first. Point `image` at a
public URL or an uploaded `asset_id`, and drive speech with a `script`+`voice_id`
or a pre-recorded `audio_url`:
```bash
heygen video create --headers "X-HeyGen-Client-Source: media-use" --wait -d '{
"type": "image",
"image": { "type": "url", "url": "https://example.com/person.jpg" },
"script": "Your narration here.",
"voice_id": "<voice-id>"
}'
```
Common optional fields: `title`, `resolution` (`4k`/`1080p`/`720p`),
`aspect_ratio`, `remove_background`, `background`, `voice_settings`,
`motion_prompt` + `expressiveness` (photo-avatar animation), and
`callback_url`/`callback_id` for webhooks. Don't hardcode these from memory: the
CLI self-documents the full, current body with
`heygen video create --request-schema` (a discriminated union keyed on `type`),
so read the schema rather than trusting a stale field list. For a still you'll
reuse across many scripts, create a reusable **Photo Avatar** once instead
(`heygen avatar create`). Ledger the result with
`resolve --from <downloaded.mp4> --type video`. Docs:
<https://developers.heygen.com/image-to-video>.
There is no catalog rung for video: a clip is made for the piece it is in, and
a shelf of generic clips is a worse answer than a fresh one. Under
`--local-only` there is no video rung at all, which resolve reports as a clean
miss rather than a silent skip.
## HEVC / H.265 sources
skills/media-use/references/resolve.md
+13 -13
View File
@@ -8,16 +8,16 @@ Returns one line: `resolved <id> → <path> (<type>, <metadata>)`
## Types
| Type | What it finds | Provider / cascade |
| ------- | -------------------------------- | ------------------------------------------------------------ |
| `bgm` | Background music | HeyGen audio catalog (10k+ tracks) |
| `sfx` | Sound effects | Bundled 19-file library + HeyGen catalog |
| `image` | Photos, backgrounds | HeyGen asset search (75k+ vectors) |
| `icon` | Icons, symbols | HeyGen asset search (type=icon) |
| `logo` | Official brand marks | svgl → simple-icons → GitHub org avatar → domain favicon |
| `voice` | TTS voiceover | HeyGen TTS free-usage path; optional local Kokoro |
| Type | What it finds | Provider / cascade |
| ------- | --------------------------- | ------------------------------------------------------------ |
| `bgm` | Background music | Shared catalog (`media/bgm/`), then generated |
| `sfx` | Sound effects | 21 bundled files, then the catalog, then foley |
| `image` | Photos, backgrounds | Shared catalog (`media/image/`), then generated |
| `icon` | Icons, symbols | Shared catalog (`media/icon/`) |
| `logo` | Official brand marks | svgl → simple-icons → GitHub org avatar → domain favicon |
| `voice` | Spoken narration | `POST /v1/audio/speech`, timings from the transcript pass |
| `grade` | Frames color-grading blocks | Core preset → look index params/CDN LUT → deterministic cube |
| `lut` | Reusable `.cube` LUT files | Look index params/CDN LUT → deterministic cube |
| `lut` | Reusable `.cube` LUT files | Look index params/CDN LUT → deterministic cube |
## Examples
@@ -64,7 +64,7 @@ node <SKILL_DIR>/scripts/resolve.mjs --type lut --intent "teal orange blockbuste
| `--from` | Freeze a local file or direct public URL (ingest) |
| `--for` | Analyze a local image/video and add measured adjust suggestions (`grade` only) |
| `--local-only` | Offline: skip every network provider (cache + local only) |
| `--provider` | Force one generator (e.g. `codex`, `mflux`, `kokoro`, `heygen`) |
| `--provider` | Pin one rung (e.g. `catalog` to stay on the shelf, `hanzo` to generate) |
| `--adopt` | Bulk-import existing assets/ into manifest |
| `--doctor` | Check local CLI dependencies; no manifest changes |
| `--stats` | Print local usage stats from `.media/` and `~/.media`; no manifest changes |
@@ -77,9 +77,9 @@ Before resolving bgm/sfx/image/icon/logo/grade/lut, **check what already exists
```bash
node <SKILL_DIR>/scripts/resolve.mjs --type bgm --intent "upbeat tech launch" --candidates --project .
# [project] upbeat tech launch (25s, heygen.audio.sounds)
# [project] upbeat tech launch (25s, catalog.bgm)
# .media/audio/bgm/bgm_001.wav
# [global] energetic tech intro (22s, heygen.audio.sounds)
# [global] energetic tech intro (22s, catalog.bgm)
# --reuse 06e052c075fd2b80
```
@@ -100,7 +100,7 @@ The deterministic floor still runs automatically: an identical (case/whitespace-
1. Check project `.media/manifest.jsonl` for a prompt match (case- and whitespace-insensitive) — auto-reuse
2. Scan existing `assets/` directory for unregistered files that share a word with the need
3. Check global cache `~/.media/` for a reusable asset matched on the same normalized prompt — auto-reuse
4. Search via provider (HeyGen audio catalog, HeyGen asset search), or resolve color locally
4. Search the shared catalog, then generate, or resolve color locally
5. Freeze file to `.media/<type>/`, register in manifest, regenerate `index.md`, auto-promote to `~/.media/`
Steps 1 and 3 are the **deterministic floor**: they only auto-reuse an exact-normalized match, never a fuzzy one. Semantic reuse ("close enough") is the agent's explicit call via [Reuse before you resolve](#reuse-before-you-resolve) — it never happens automatically. The agent gets back **one line**; candidates, scores, provenance stay on disk.
skills/media-use/references/setup-providers.md
+42 -59
View File
@@ -1,80 +1,63 @@
# Setup and providers — install, auth, RAM ladders, forcing a provider
# Setup and providers — credential, cascade, forcing a provider
## Setup — install heygen first (free-usage path)
## Setup — one credential
Install the HeyGen CLI through its [verified release instructions](https://developers.heygen.com/cli), then run:
Everything remote goes through `api.hanzo.ai` on a Hanzo IAM token. Read it from
Hanzo KMS (`kms.hanzo.ai`) and export it; there is no credential file and no
per-repo `.env`.
```bash
heygen update # free usage needs the OAuth-capable CLI (v0.3.0+)
heygen auth login --oauth # OAuth = free subscription credits; --api-key bills API credits
export HANZO_API_KEY=... # an IAM token, or an sk- cloud key
export HANZO_ORG=... # the tenant that owns the shared media bucket
```
This unlocks the FREE path for bgm/sfx/image/icon catalog search, TTS (voice), and avatar videos. Sign in with `--oauth` — the free allowance rides on the OAuth session (an API key bills API credits instead). **media-use requires heygen >= v0.3.0 uniformly** (the OAuth free-usage path needs it), so `--doctor` nudges older CLIs to update even for API-key-only use. Before resolving anything, verify setup with:
A publishable `pk-` key is read-only and the media routes refuse it. Verify the
whole setup before resolving anything:
```bash
node <SKILL_DIR>/scripts/resolve.mjs --doctor
```
To run against a Hanzo Engine on your own machine instead of the cloud, point
`HANZO_BASE_URL` at it. The endpoint moves; nothing else changes.
## Providers
media-use holds no keys; every external tool owns its auth. Generation is
centered on the HeyGen CLI free-usage path. Install and authenticate `heygen`
before resolving bgm/sfx/image/icon/voice/avatar-video. Local tools are opt-in
alternatives where they exist: mflux for image, Kokoro for voice, Parakeet for
transcription, and LTX for local video generation. `resolve` spec-checks
AVAILABLE RAM for those local ladders (`describeModelLadder`); the agent can
see the ladder and override.
Two rungs, everywhere they both apply: look in the shared catalog first, make it
when the catalog cannot answer.
| Type | Provider / path |
| --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| bgm/sfx | heygen catalog free-usage path |
| image | heygen search free-usage path; optional local mflux; codex `image_gen` upsell |
| voice | heygen tts free-usage path; optional local **Kokoro** (free, on-device) |
| icon | heygen asset search free-usage path |
| logo | svgl, then simple-icons, then GitHub org avatar, then domain favicon (all free) |
| grade/lut | local core-preset map, params/CDN look index, deterministic `buildCube` fallback |
| video | heygen avatar video free-usage path (sign-in nudge on auth failure); optional local LTX (`videogen` ladder). Image-to-video / photo-avatar / dub stay manual `heygen` recipes |
| Type | Provider / path |
| --------- | ---------------------------------------------------------------------------- |
| bgm | `media/bgm/` in the shared catalog, then `POST /v1/audio/music` |
| sfx | the 21 bundled files, then `media/sfx/`, then `POST /v1/audio/foley` |
| image | `media/image/` in the shared catalog, then `POST /v1/images/generations` |
| icon | `media/icon/` in the shared catalog |
| voice | `POST /v1/audio/speech`; word timings from `POST /v1/audio/transcriptions` |
| video | `POST /v1/videos/generations` |
| logo | svgl, then simple-icons, then the GitHub org avatar, then the domain favicon |
| grade/lut | local core-preset map, the `params` look index, deterministic `buildCube` |
Local Kokoro (voice), mflux (image), and LTX (video) run on-device (free,
private, offline once cached). The `codex` CLI remains the ChatGPT-sub image
upsell. Cost rule (X4): the agent confirms before an agent-initiated paid call;
a user-requested one just runs — `heygen.video` is flagged paid (metered free
allowance) so an agent-initiated `resolve --type video` confirms first.
The catalog is a listing, not an index: an object key IS its description, so a
file named `calm-cinematic-underscore.mp3` answers "calm cinematic underscore".
**The catalog is empty until it is stocked.** An empty prefix says so — bgm and
image fall through to generation, icon reports the shelf is bare.
To force a specific generator (e.g. a user says "make this image with codex"),
pass `--provider codex`: it pins resolution to that provider and skips the
free-usage default. See `references/operations.md` for the RAM ladders and
provider recipes.
Cost rule (X4): the agent confirms before an agent-initiated generating call; a
user-requested one just runs. Catalog reads are metered per storage operation,
generation per model call, so the generating rungs are marked `paid`.
`--local-only` skips every network provider, including the free HeyGen ones,
leaving the project + global cache and any installed local provider. For
HeyGen-only types, that means no fresh resolve.
To pin one rung (e.g. a user says "generate it, don't search"), pass
`--provider hanzo`; to stay on the shelf, `--provider catalog`. `--local-only`
skips every network provider, leaving the project and global caches plus the
bundled SFX library and the local grade/LUT builders.
## CLI tools used (what to run, and how to enable each)
## What must be installed
`resolve` auto-cascades; each provider shells one CLI. HeyGen is the
free-usage path for bgm/sfx/image/icon catalog search, TTS (voice), and avatar
video, so those capabilities need `heygen` installed and authenticated. Local
tools are OPT-IN alternatives where they exist; install one to unlock its free,
private, on-device path instead of or ahead of HeyGen for that type. Only
`ffmpeg`/`ffprobe` are strictly required for the tool to run at all.
Only `ffmpeg`/`ffprobe` are strictly required for the tool to run at all.
| Tool | Serves | Install |
| ------------------ | ------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
| `ffmpeg`/`ffprobe` | adopt probing, smart-grade signalstats, cut, duck bake, loudnorm | system package (`brew install ffmpeg`) |
| `heygen` | catalog (bgm/sfx/image/icon) + TTS (voice) + avatar video — the free-usage path | install through [verified HeyGen release instructions](https://developers.heygen.com/cli), then `heygen auth login --oauth` (needs >= v0.3.0) |
| `mflux-generate` | local image gen (FLUX), best-for-RAM | `uv venv ~/.venvs/mflux && VIRTUAL_ENV=~/.venvs/mflux uv pip install mflux==0.9.6` |
| `codex` | image gen upsell (ChatGPT sub) | Codex CLI, logged in via ChatGPT (owns its own auth) |
| `parakeet-mlx` | local transcription (default ASR, best) | `uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx` |
| `ltx-2-mlx` | local video gen | `git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras` |
| `npx frames` | Kokoro TTS (voice), whisper.cpp (transcribe fallback), remove-background | via the frames CLI; whisper.cpp is built on first use (Homebrew on macOS, else git+cmake), models download from HuggingFace |
| Tool | Serves | Install |
| ------------------ | ---------------------------------------------------------------- | -------------------------------------- |
| `ffmpeg`/`ffprobe` | adopt probing, smart-grade signalstats, cut, duck bake, loudnorm | system package (`brew install ffmpeg`) |
The RAM-graded local-model shortlist + exact per-tier install/invoke lives in
`scripts/lib/local-models.mjs` (the agent can read `describeModelLadder(cap, specs)`
to see which model fits this machine). Without a tool on PATH, its provider
prints a one-line diagnostic to stderr and resolve falls through where another
provider exists (e.g. no `mflux` -> codex image upsell; no `parakeet-mlx` -> whisper.cpp).
`heygen asset search` is a pre-launch command hidden from `heygen --help`, but it
runs; providers tag requests with the allowlisted `X-HeyGen-Client-Source` header
(v0.3.0+).
Everything else is a network call to `api.hanzo.ai` — no per-provider CLI, no
per-provider key, nothing to keep up to date on the machine.
skills/media-use/references/telemetry-dashboard.md
+4 -5
View File
@@ -9,11 +9,10 @@ from `.media/` + `~/.media`, no dashboard access needed).
## Identity (see `scripts/lib/telemetry.mjs`)
Events attribute to the **same person as the frames CLI and studio**
— the shared install id in `~/.frames/config.json` (`anonymousId`), stitched
to the HeyGen account (`$identify`, `distinct_id` = email/username) on sign-in.
Not fully anonymous by design; pseudonymous before sign-in, account-linked after.
`$ip:null`. Opt-out: `FRAMES_NO_TELEMETRY=1` / `DO_NOT_TRACK=1` (also CI, dev).
Events attribute to the **same install as the CLI and studio** — the shared
install id in `~/.frames/config.json` (`anonymousId`) and nothing else. No
account is stitched in: pseudonymous, always. `$ip:null`. Opt-out:
`FRAMES_NO_TELEMETRY=1` / `DO_NOT_TRACK=1` (also CI, dev).
## Event catalog (verified present in-project)
skills/media-use/scripts/eval.mjs
+1 -1
View File
@@ -35,7 +35,7 @@ const TEST_BLOCKS = [
// Run resolve.mjs with args as a literal argv array (no shell), so values
// interpolated from manifest metadata (--intent prompt, --type) can't inject
// shell. Mirrors the execFileSync fix in probe.mjs / heygen-search.mjs.
// shell. Mirrors the execFileSync fix in probe.mjs.
function run(args, opts = {}) {
try {
return {
skills/media-use/scripts/lib/api.mjs
+223
View File
@@ -0,0 +1,223 @@
// api.mjs — the one client for Hanzo's media services.
//
// Every remote call media-use makes goes through api.hanzo.ai: one host, one
// credential, one file to change. The routes used here:
//
// POST /v1/audio/speech text -> spoken audio
// POST /v1/audio/transcriptions audio -> text
// POST /v1/audio/music prompt -> a music bed
// POST /v1/audio/foley prompt -> a sound effect
// POST /v1/images/generations prompt -> an image
// POST /v1/videos/generations prompt -> a video
// GET /v1/s3/buckets/<b>/objects browse the shared media catalog
// GET /v1/s3/buckets/<b>/objects/<key> a short-lived link to one object
//
// Auth is a Hanzo IAM token in $HANZO_API_KEY, held in Hanzo KMS
// (kms.hanzo.ai) and handed to the process as an environment variable. There is
// no credential file and no per-repo .env. A publishable `pk-` key is read-only
// and is refused by the media routes; use an IAM token or an `sk-` key.
// Storage routes are tenant-scoped and additionally read $HANZO_ORG as X-Org-Id.
import { mkdirSync, openAsBlob, writeFileSync } from "node:fs";
import { dirname } from "node:path";
export const DEFAULT_BASE = "https://api.hanzo.ai";
// The models behind each capability. Values, not places: change one here and
// every caller follows.
export const MODEL = {
speech: "zen-voice",
words: "whisper",
music: "zen-music",
sound: "zen-foley",
image: "zen-image",
video: "zen-video",
};
// The bucket holding shared media, and the prefix per media type inside it.
export const BUCKET = "media";
export const PREFIX = { bgm: "bgm/", sfx: "sfx/", image: "image/", icon: "icon/" };
export const base = () => (process.env.HANZO_BASE_URL || DEFAULT_BASE).replace(/\/+$/, "");
export const key = () => process.env.HANZO_API_KEY || "";
export const org = () => process.env.HANZO_ORG || "";
/** Is a credential present? Providers ask before reaching for the network. */
export const ready = () => key() !== "";
const MISSING =
"no Hanzo credential — export HANZO_API_KEY (read it from Hanzo KMS at kms.hanzo.ai)";
/** Auth headers, or throw naming the fix. `tenant` adds the storage org scope. */
export function auth({ tenant = false } = {}) {
if (!ready()) throw new Error(MISSING);
const headers = { Authorization: `Bearer ${key()}` };
if (tenant) {
if (!org()) throw new Error("no Hanzo org — export HANZO_ORG (the tenant that owns the media)");
headers["X-Org-Id"] = org();
}
return headers;
}
async function fail(res, path) {
const detail = await res.text().catch(() => "");
return new Error(`${path} -> HTTP ${res.status}${detail ? `\n${detail.slice(0, 300)}` : ""}`);
}
/** JSON in, JSON out. */
export async function json(path, { method = "GET", body, tenant = false, signal } = {}) {
const init = { method, headers: auth({ tenant }), signal };
if (body !== undefined) {
init.headers["Content-Type"] = "application/json";
init.body = JSON.stringify(body);
}
const res = await fetch(`${base()}${path}`, init);
if (!res.ok) throw await fail(res, `${method} ${path}`);
return res.json();
}
/** JSON in, bytes out — the shape of every media-producing route. */
export async function bytes(path, body, { signal } = {}) {
const res = await fetch(`${base()}${path}`, {
method: "POST",
headers: { ...auth(), "Content-Type": "application/json" },
body: JSON.stringify(body),
signal,
});
if (!res.ok) throw await fail(res, `POST ${path}`);
return Buffer.from(await res.arrayBuffer());
}
/** A file plus fields in, JSON out. */
export async function form(path, fields) {
const body = new FormData();
for (const [name, value] of Object.entries(fields)) {
if (value === undefined || value === null) continue;
body.append(name, value);
}
const res = await fetch(`${base()}${path}`, { method: "POST", headers: auth(), body });
if (!res.ok) throw await fail(res, `POST ${path}`);
return res.json();
}
/** Read a URL into memory. */
export async function fetchBytes(url) {
const res = await fetch(url);
if (!res.ok) throw new Error(`download HTTP ${res.status}: ${String(url).slice(0, 80)}`);
return Buffer.from(await res.arrayBuffer());
}
/** Write bytes to disk, making the parent directory; returns the byte length. */
export function write(buf, destPath) {
mkdirSync(dirname(destPath), { recursive: true });
writeFileSync(destPath, buf);
return buf.length;
}
/** Copy a URL to disk; returns the byte length. */
export async function save(url, destPath) {
return write(await fetchBytes(url), destPath);
}
// ── media routes ─────────────────────────────────────────────────────────────
// The speech route caps `input` at 4096 bytes; split longer scripts per line.
export const MAX_SPEECH_INPUT = 4096;
/** Speak `text`. Returns the audio bytes in `format` (mp3 unless asked). */
export function speech({ text, voice, speed = 1.0, lang, model = MODEL.speech, format = "mp3" }) {
const body = { model, input: text, response_format: format, speed };
if (voice) body.voice = voice;
if (lang && lang !== "en") body.language = lang;
return bytes("/v1/audio/speech", body);
}
/**
* Transcribe an audio file to `{ text, words: [{ text, start, end }] }`.
* The one source of transcripts and, when the service returns them, of word
* timings: captions, transcript cuts, and the audio engine all read this shape.
*
* `words` is EMPTY unless the reply carries per-word timings. The route asks for
* them (verbose_json + word granularity) and passes through whatever comes back;
* it never interpolates timings the service did not measure, because a caption
* cut on invented timings looks correct and is wrong.
*/
export async function transcript({ file, lang, model = MODEL.words }) {
const payload = await form("/v1/audio/transcriptions", {
file: await openAsBlob(file),
model,
response_format: "verbose_json",
"timestamp_granularities[]": "word",
language: lang && lang !== "en" ? lang : undefined,
});
const words = (payload.words ?? [])
.filter((w) => w && Number.isFinite(w.start) && Number.isFinite(w.end))
.map((w) => ({ text: String(w.word ?? w.text ?? "").trim(), start: w.start, end: w.end }))
.filter((w) => w.text);
return { text: String(payload.text ?? "").trim(), words };
}
/** Generate a music bed. */
export function music({ prompt, seconds, model = MODEL.music, format = "wav" }) {
return audio("/v1/audio/music", { prompt, seconds, model, format });
}
/** Generate one sound effect. */
export function foley({ prompt, seconds, model = MODEL.sound, format = "wav" }) {
return audio("/v1/audio/foley", { prompt, seconds, model, format });
}
function audio(path, { prompt, seconds, model, format }) {
const body = { model, prompt, response_format: format };
if (Number.isFinite(seconds)) body.duration = Math.max(1, Math.round(seconds));
return bytes(path, body);
}
/** The first item of an OpenAI-shaped `{ data: [...] }` reply. */
export function first(payload, path) {
const item = (payload?.data ?? [])[0];
if (!item) throw new Error(`${path} returned no result`);
return item;
}
/** Generate an image. Returns the image bytes. */
export async function image({ prompt, size = "1024x1024", model = MODEL.image }) {
const path = "/v1/images/generations";
const item = first(
await json(path, { method: "POST", body: { model, prompt, size, n: 1 } }),
path,
);
if (item.b64_json) return Buffer.from(item.b64_json, "base64");
if (item.url) return fetchBytes(item.url);
throw new Error(`${path} returned neither b64_json nor url`);
}
/** Generate a video. Returns the URL of the finished clip. */
export async function video({ prompt, seconds, model = MODEL.video }) {
const path = "/v1/videos/generations";
const body = { model, prompt };
if (Number.isFinite(seconds)) body.duration = Math.max(1, Math.round(seconds));
return first(await json(path, { method: "POST", body }), path).url;
}
// ── storage ──────────────────────────────────────────────────────────────────
/**
* One flat listing under `prefix`. Keys come back RELATIVE to the prefix and the
* server caps a page at 1000 entries.
*/
export async function objects(prefix, { bucket = BUCKET } = {}) {
const query = new URLSearchParams({ prefix, recursive: "true" });
const payload = await json(`/v1/s3/buckets/${bucket}/objects?${query}`, { tenant: true });
return (payload.objects ?? []).filter((o) => !o.isDir);
}
/**
* A link to one object. The store hands back a signed URL that expires in
* minutes, so fetch it now — do not write one into a document.
*/
export async function link(key, { bucket = BUCKET } = {}) {
const path = key.split("/").map(encodeURIComponent).join("/");
const payload = await json(`/v1/s3/buckets/${bucket}/objects/${path}`, { tenant: true });
return payload.url;
}
skills/media-use/scripts/lib/api.test.mjs
+299
View File
@@ -0,0 +1,299 @@
import { strict as assert } from "node:assert";
import { test } from "node:test";
import { createServer } from "node:http";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import * as api from "./api.mjs";
// The one client. These exercise the wire shape against a local stand-in for
// api.hanzo.ai — request path, method, headers, and how each reply is read —
// so a change to a route or an envelope fails here rather than in a render.
// `await` is load-bearing: without it the restore runs before an async body
// finishes and the next test inherits this one's environment.
async function withEnv(env, fn) {
const saved = { ...process.env };
Object.assign(process.env, env);
try {
return await fn();
} finally {
for (const k of Object.keys(process.env)) if (!(k in saved)) delete process.env[k];
Object.assign(process.env, saved);
}
}
/** Serve `routes` (path prefix -> handler) and hand the base URL to `fn`. */
async function serving(routes, fn) {
const seen = [];
const server = createServer((req, res) => {
const chunks = [];
req.on("data", (c) => chunks.push(c));
req.on("end", () => {
const path = req.url.split("?")[0];
seen.push({ method: req.method, url: req.url, headers: req.headers, body: Buffer.concat(chunks) });
const route = Object.entries(routes).find(([prefix]) => path.startsWith(prefix));
if (!route) {
res.writeHead(404);
res.end("no route");
return;
}
route[1](res, seen.at(-1));
});
});
await new Promise((r) => server.listen(0, "127.0.0.1", r));
const base = `http://127.0.0.1:${server.address().port}`;
try {
return await fn(base, seen);
} finally {
await new Promise((r) => server.close(r));
}
}
const jsonRoute = (payload) => (res) => {
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify(payload));
};
test("the base URL defaults to api.hanzo.ai and $HANZO_BASE_URL overrides it", async () => {
await withEnv({ HANZO_BASE_URL: "" }, () => {
delete process.env.HANZO_BASE_URL;
assert.equal(api.base(), "https://api.hanzo.ai");
});
await withEnv({ HANZO_BASE_URL: "http://localhost:9000/" }, () => {
assert.equal(api.base(), "http://localhost:9000");
});
});
test("no credential is a named failure, never a silent one", async () => {
await withEnv({ HANZO_API_KEY: "" }, () => {
delete process.env.HANZO_API_KEY;
assert.equal(api.ready(), false);
assert.throws(() => api.auth(), /HANZO_API_KEY/);
assert.throws(() => api.auth(), /kms\.hanzo\.ai/);
});
});
test("a storage call without an org says so rather than reading another tenant", async () => {
await withEnv({ HANZO_API_KEY: "k", HANZO_ORG: "" }, () => {
delete process.env.HANZO_ORG;
assert.deepEqual(api.auth(), { Authorization: "Bearer k" });
assert.throws(() => api.auth({ tenant: true }), /HANZO_ORG/);
});
await withEnv({ HANZO_API_KEY: "k", HANZO_ORG: "acme" }, () => {
assert.deepEqual(api.auth({ tenant: true }), {
Authorization: "Bearer k",
"X-Org-Id": "acme",
});
});
});
test("speech posts to /v1/audio/speech and returns the bytes", async () => {
await serving(
{
"/v1/audio/speech": (res) => {
res.writeHead(200, { "content-type": "audio/mpeg" });
res.end(Buffer.from("ID3fake"));
},
},
async (base, seen) => {
await withEnv({ HANZO_BASE_URL: base, HANZO_API_KEY: "k" }, async () => {
const bytes = await api.speech({ text: "hello", voice: "v1", speed: 1.1 });
assert.equal(bytes.toString(), "ID3fake");
const [call] = seen;
assert.equal(call.method, "POST");
assert.equal(call.url, "/v1/audio/speech");
assert.equal(call.headers.authorization, "Bearer k");
assert.deepEqual(JSON.parse(call.body.toString()), {
model: "zen-voice",
input: "hello",
response_format: "mp3",
speed: 1.1,
voice: "v1",
});
});
},
);
});
test("transcript asks for word timings and passes through the ones it gets", async () => {
const dir = mkdtempSync(join(tmpdir(), "mu-api-"));
const file = join(dir, "a.wav");
writeFileSync(file, "RIFFfake");
try {
await serving(
{
"/v1/audio/transcriptions": jsonRoute({
text: "hi there",
words: [
{ word: "hi", start: 0, end: 0.2 },
{ word: "there", start: 0.2, end: 0.5 },
{ word: "dropped", start: null, end: 1 },
],
}),
},
async (base, seen) => {
await withEnv({ HANZO_BASE_URL: base, HANZO_API_KEY: "k" }, async () => {
const out = await api.transcript({ file });
assert.equal(out.text, "hi there");
assert.deepEqual(out.words, [
{ text: "hi", start: 0, end: 0.2 },
{ text: "there", start: 0.2, end: 0.5 },
]);
const body = seen[0].body.toString();
assert.match(body, /name="model"/);
assert.match(body, /timestamp_granularities\[\]/);
assert.match(body, /verbose_json/);
});
},
);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test("a transcript with no timings yields no words, never invented ones", async () => {
const dir = mkdtempSync(join(tmpdir(), "mu-api-"));
const file = join(dir, "a.wav");
writeFileSync(file, "RIFFfake");
try {
await serving({ "/v1/audio/transcriptions": jsonRoute({ text: "hi there" }) }, async (base) => {
await withEnv({ HANZO_BASE_URL: base, HANZO_API_KEY: "k" }, async () => {
const out = await api.transcript({ file });
assert.equal(out.text, "hi there");
assert.deepEqual(out.words, []);
});
});
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
test("music and foley are separate routes", async () => {
await serving(
{
"/v1/audio/music": (res) => res.end("music-bytes"),
"/v1/audio/foley": (res) => res.end("foley-bytes"),
},
async (base, seen) => {
await withEnv({ HANZO_BASE_URL: base, HANZO_API_KEY: "k" }, async () => {
assert.equal((await api.music({ prompt: "calm", seconds: 12 })).toString(), "music-bytes");
assert.equal((await api.foley({ prompt: "whoosh" })).toString(), "foley-bytes");
assert.deepEqual(JSON.parse(seen[0].body.toString()), {
model: "zen-music",
prompt: "calm",
response_format: "wav",
duration: 12,
});
assert.equal(JSON.parse(seen[1].body.toString()).model, "zen-foley");
assert.equal("duration" in JSON.parse(seen[1].body.toString()), false);
});
},
);
});
test("image reads b64 when offered and downloads a url otherwise", async () => {
const png = Buffer.from("PNGfake");
await serving(
{
"/v1/images/generations": jsonRoute({ data: [{ b64_json: png.toString("base64") }] }),
},
async (base) => {
await withEnv({ HANZO_BASE_URL: base, HANZO_API_KEY: "k" }, async () => {
assert.equal((await api.image({ prompt: "a cat" })).toString(), "PNGfake");
});
},
);
await serving(
{
"/v1/images/generations": (res, call) => {
const here = `http://127.0.0.1:${call.headers.host.split(":")[1]}`;
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ data: [{ url: `${here}/blob.png` }] }));
},
"/blob.png": (res) => res.end("PNGfromurl"),
},
async (base) => {
await withEnv({ HANZO_BASE_URL: base, HANZO_API_KEY: "k" }, async () => {
assert.equal((await api.image({ prompt: "a cat" })).toString(), "PNGfromurl");
});
},
);
});
test("an empty data array is an error naming the route", async () => {
await serving({ "/v1/videos/generations": jsonRoute({ data: [] }) }, async (base) => {
await withEnv({ HANZO_BASE_URL: base, HANZO_API_KEY: "k" }, async () => {
await assert.rejects(api.video({ prompt: "x" }), /\/v1\/videos\/generations returned no result/);
});
});
});
test("a non-OK reply carries the status and the body into the error", async () => {
await serving(
{
"/v1/audio/speech": (res) => {
res.writeHead(402);
res.end("insufficient balance");
},
},
async (base) => {
await withEnv({ HANZO_BASE_URL: base, HANZO_API_KEY: "k" }, async () => {
await assert.rejects(api.speech({ text: "x" }), /HTTP 402/);
await assert.rejects(api.speech({ text: "x" }), /insufficient balance/);
});
},
);
});
test("objects lists one prefix flat and drops folder entries", async () => {
await serving(
{
"/v1/s3/buckets/media/objects": jsonRoute({
objects: [
{ key: "calm.mp3", isDir: false, size: 10 },
{ key: "old/", isDir: true, size: 0 },
],
}),
},
async (base, seen) => {
await withEnv({ HANZO_BASE_URL: base, HANZO_API_KEY: "k", HANZO_ORG: "acme" }, async () => {
const out = await api.objects("bgm/");
assert.deepEqual(
out.map((o) => o.key),
["calm.mp3"],
);
assert.match(seen[0].url, /prefix=bgm%2F/);
assert.match(seen[0].url, /recursive=true/);
assert.equal(seen[0].headers["x-org-id"], "acme");
});
},
);
});
test("link returns the signed URL for one key", async () => {
await serving(
{
"/v1/s3/buckets/media/objects/": jsonRoute({ url: "https://signed.example/x", method: "GET" }),
},
async (base, seen) => {
await withEnv({ HANZO_BASE_URL: base, HANZO_API_KEY: "k", HANZO_ORG: "acme" }, async () => {
assert.equal(await api.link("bgm/calm track.mp3"), "https://signed.example/x");
assert.equal(seen[0].url, "/v1/s3/buckets/media/objects/bgm/calm%20track.mp3");
});
},
);
});
test("save writes a URL to disk and reports the byte count", async () => {
const dir = mkdtempSync(join(tmpdir(), "mu-api-"));
try {
await serving({ "/blob": (res) => res.end("twelve bytes") }, async (base) => {
const dest = join(dir, "nested", "out.bin");
assert.equal(await api.save(`${base}/blob`, dest), 12);
assert.equal(readFileSync(dest, "utf8"), "twelve bytes");
});
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
skills/media-use/scripts/lib/bgm-provider.mjs
+7 -15
View File
@@ -1,20 +1,12 @@
import { heygenSearch } from "./heygen-search.mjs";
import { asResult, compose, find } from "./catalog.mjs";
export const bgmProvider = {
async search(intent) {
const results = heygenSearch("audio sounds list", intent, { type: "music" });
if (!results) return null;
const best = results[0];
return {
url: best.audio_url,
source: "search",
// ext derived from audio_url by resolve.mjs — catalog tracks are .mp3 or .wav
metadata: {
description: best.description || intent,
duration: best.duration || null,
provider: "heygen.audio.sounds",
provenance: { track_id: best.id, score: best.score, query: intent },
},
};
const hit = await find("bgm", intent);
if (!hit || hit.empty) return null;
return asResult(hit, intent, "bgm");
},
async generate(intent, ctx) {
return compose(intent, { kind: "bgm", seconds: ctx?.seconds });
},
};
skills/media-use/scripts/lib/candidates.test.mjs
+2 -2
View File
@@ -6,7 +6,7 @@ import { tmpdir } from "node:os";
import { listCandidates, formatCandidates, CANDIDATE_CAP } from "./candidates.mjs";
import { findGlobalBySha } from "./cache.mjs";
// candidates + findGlobalBySha are offline (no heygen), so we can override HOME
// candidates + findGlobalBySha are offline (no network), so we can override HOME
// to a temp dir and seed a fake global ~/.media manifest deterministically.
function sandbox() {
const root = mkdtempSync(join(tmpdir(), "mu-cand-"));
@@ -31,7 +31,7 @@ function glob(id, type, description, prompt, sha) {
reusable: true,
cached_path: `/x/${sha}/${id}.wav`,
description,
provenance: { prompt, provider: "heygen.audio.sounds" },
provenance: { prompt, provider: "catalog.bgm" },
};
}
skills/media-use/scripts/lib/catalog.mjs
+94
View File
@@ -0,0 +1,94 @@
// catalog.mjs — the shared media catalog: files in our own object store
// (hanzoai/s3), browsed through api.hanzo.ai/v1/s3 and matched by name.
//
// A catalog entry is just an object under `media/<type>/`, e.g.
// `media/bgm/calm-cinematic-underscore.mp3`. The key IS the description: words
// in the key are what an intent matches against, so name files in plain words.
// There is no index to keep in step with the bytes and no second service to
// run — the listing is the index.
//
// An empty prefix is a fact, not an error: `find` returns
// `{ empty: true, prefix }` so callers can say "the catalog holds no bgm yet"
// and move on to generation rather than failing the render.
import { PREFIX, foley, link, music, objects, ready, write } from "./api.mjs";
import { tmpdir } from "node:os";
import { join } from "node:path";
const STOP = new Set(["a", "an", "the", "and", "of", "for", "with", "some", "sound", "music"]);
const terms = (text) =>
String(text)
.toLowerCase()
.split(/[^a-z0-9]+/)
.filter((t) => t && !STOP.has(t));
/** How well an object key answers an intent: shared words, longest match first. */
export function score(intent, key) {
const want = terms(intent);
if (!want.length) return 0;
const have = new Set(terms(key));
const hits = want.filter((t) => have.has(t)).length;
if (!hits) return 0;
// Every word matched scores above a partial match of the same size.
return hits === want.length ? 100 + hits : hits;
}
/** Rank a listing against an intent. Exported so the ranking is testable alone. */
export function rank(intent, entries) {
return entries
.map((o) => ({ ...o, score: score(intent, o.key) }))
.filter((o) => o.score > 0)
.sort((a, b) => b.score - a.score || a.key.localeCompare(b.key));
}
/**
* Best catalog entry for an intent.
* { key, url, score, prefix } a hit
* { empty: true, prefix } the prefix holds nothing yet
* null the catalog has files, none of them match
*/
export async function find(type, intent) {
const prefix = PREFIX[type];
if (!prefix) throw new Error(`no catalog prefix for type: ${type}`);
if (!ready()) return null;
const entries = await objects(prefix);
if (!entries.length) return { empty: true, prefix };
const best = rank(intent, entries)[0];
if (!best) return null;
return { key: best.key, url: await link(prefix + best.key), score: best.score, prefix };
}
/** A catalog hit shaped as a provider result. */
export function asResult(hit, intent, kind) {
return {
url: hit.url,
source: "catalog",
metadata: {
description: hit.key.replace(/\.\w+$/, "").replace(/[-_/]+/g, " "),
provider: `catalog.${kind}`,
provenance: { key: hit.prefix + hit.key, query: intent, score: hit.score },
},
};
}
/**
* Generate an audio bed when the catalog misses. `kind` picks music or foley.
* Lands in a temp file and reports its path — the same `{ localPath }` shape
* every other provider returns, so resolve has one contract to honor.
*/
export async function compose(intent, { kind = "bgm", seconds } = {}) {
const make = kind === "sfx" ? foley : music;
const localPath = join(tmpdir(), `media-use-${kind}-${process.pid}-${Date.now()}.wav`);
write(await make({ prompt: intent, seconds }), localPath);
return {
localPath,
ext: ".wav",
source: "generated",
metadata: {
description: intent,
provider: kind === "sfx" ? "hanzo.foley" : "hanzo.music",
provenance: { prompt: intent, ...(Number.isFinite(seconds) ? { seconds } : {}) },
},
};
}
skills/media-use/scripts/lib/catalog.test.mjs
+109
View File
@@ -0,0 +1,109 @@
import { strict as assert } from "node:assert";
import { test } from "node:test";
import { createServer } from "node:http";
import { asResult, find, rank, score } from "./catalog.mjs";
// The catalog is a listing, not an index: an object key IS its description.
// These pin the matching rules and, above all, the empty-shelf answer — an
// empty prefix must read as "nothing has been put here yet", never as a miss
// that looks like a bad query.
test("a key that answers every word of the intent outranks a partial match", () => {
assert.ok(score("calm cinematic", "calm-cinematic-underscore.mp3") > score("calm cinematic", "calm-piano.mp3"));
assert.equal(score("calm", "loud-drums.mp3"), 0);
});
test("filler words alone never make a match", () => {
assert.equal(score("some music for the", "the-music.mp3"), 0);
});
test("rank drops non-matches and breaks ties by key", () => {
const ranked = rank("whoosh", [
{ key: "impact.mp3" },
{ key: "whoosh-long.mp3" },
{ key: "whoosh-b.mp3" },
{ key: "whoosh-a.mp3" },
]);
assert.deepEqual(
ranked.map((r) => r.key),
["whoosh-a.mp3", "whoosh-b.mp3", "whoosh-long.mp3"],
);
});
async function serving(handler, fn) {
const server = createServer(handler);
await new Promise((r) => server.listen(0, "127.0.0.1", r));
const saved = { ...process.env };
process.env.HANZO_BASE_URL = `http://127.0.0.1:${server.address().port}`;
process.env.HANZO_API_KEY = "k";
process.env.HANZO_ORG = "acme";
try {
return await fn();
} finally {
for (const k of Object.keys(process.env)) if (!(k in saved)) delete process.env[k];
Object.assign(process.env, saved);
await new Promise((r) => server.close(r));
}
}
const reply = (res, payload) => {
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify(payload));
};
test("an empty prefix reports itself as empty, not as a miss", async () => {
await serving(
(req, res) => reply(res, { objects: [] }),
async () => {
const hit = await find("bgm", "anything at all");
assert.deepEqual(hit, { empty: true, prefix: "bgm/" });
},
);
});
test("a stocked prefix with no match is a plain miss", async () => {
await serving(
(req, res) => reply(res, { objects: [{ key: "drums.mp3", isDir: false }] }),
async () => {
assert.equal(await find("bgm", "harpsichord"), null);
},
);
});
test("a hit carries the signed link and the key it came from", async () => {
await serving(
(req, res) => {
if (req.url.startsWith("/v1/s3/buckets/media/objects?")) {
reply(res, { objects: [{ key: "calm-underscore.mp3", isDir: false }] });
return;
}
reply(res, { url: "https://signed.example/calm", method: "GET" });
},
async () => {
const hit = await find("bgm", "calm underscore");
assert.equal(hit.key, "calm-underscore.mp3");
assert.equal(hit.url, "https://signed.example/calm");
assert.equal(hit.prefix, "bgm/");
const record = asResult(hit, "calm underscore", "bgm");
assert.equal(record.source, "catalog");
assert.equal(record.metadata.provider, "catalog.bgm");
assert.equal(record.metadata.description, "calm underscore");
assert.equal(record.metadata.provenance.key, "bgm/calm-underscore.mp3");
},
);
});
test("no credential means no catalog call at all", async () => {
const saved = process.env.HANZO_API_KEY;
delete process.env.HANZO_API_KEY;
try {
assert.equal(await find("bgm", "calm"), null);
} finally {
if (saved !== undefined) process.env.HANZO_API_KEY = saved;
}
});
test("an unknown type is a programming error, not a miss", async () => {
await assert.rejects(find("nope", "x"), /no catalog prefix for type: nope/);
});
skills/media-use/scripts/lib/codex-provider.mjs
-161
View File
@@ -1,161 +0,0 @@
import { execFileSync } from "node:child_process";
import { copyFileSync, existsSync, readdirSync, statSync, unlinkSync } from "node:fs";
import { homedir, tmpdir } from "node:os";
import { join } from "node:path";
// Image generation via the OpenAI Codex CLI's built-in image tool (gpt-image-2)
// on the user's ChatGPT subscription: the codex CLI owns auth, media-use holds
// no key (CLI-only). The image UPSELL behind local mflux; skipped by --local-only.
//
// Retrieval mirrors illo-skill rather than trusting the model to save a file:
// `--enable image_generation` makes the built-in tool drop the rendered artifact
// into $CODEX_HOME/generated_images/, and we fetch the freshest file that
// postdates this run. The save-to-path instruction is only a best-effort
// verify-first.
//
// The flag was called `imagegenext` until Codex CLI ~0.145, which renamed it to
// `image_generation` and dropped the old row from `codex features list`. Both
// the gate below and the --enable arg track the new name, with a fallback to
// the old one so older CLIs keep working.
const TIMEOUT_MS = 600000; // codex exec round-trips the sub; first-run tool spin-up is slow
const MTIME_SKEW_MS = 2000; // tolerate mtime granularity / clock skew (illo uses 2s)
function codexGeneratedDir() {
// Codex relocates CODEX_HOME on some hosts, so resolve it at run time.
return join(process.env.CODEX_HOME || join(homedir(), ".codex"), "generated_images");
}
// Newest artifact that postdates `sinceMs` (minus skew), so a stale prior render
// or a concurrent session's file can't be mistaken for this run's output.
function freshestGeneratedImage(sinceMs) {
const dir = codexGeneratedDir();
if (!existsSync(dir)) return null;
const floor = sinceMs - MTIME_SKEW_MS;
let best = null;
for (const name of readdirSync(dir)) {
let st;
try {
st = statSync(join(dir, name));
} catch {
continue;
}
if (!st.isFile() || st.mtimeMs < floor) continue;
if (!best || st.mtimeMs > best.mtimeMs) best = { path: join(dir, name), mtimeMs: st.mtimeMs };
}
return best?.path ?? null;
}
// Short `codex` subcommand → combined stdout+stderr, or null if it can't run.
function codexRun(args) {
try {
return execFileSync("codex", args, {
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
timeout: 10000,
});
} catch (err) {
return `${err.stdout?.toString() ?? ""}${err.stderr?.toString() ?? ""}` || null;
}
}
// Name of the feature flag this CLI uses for the extended image tool, or null if
// it exposes neither. Newer CLIs list only `image_generation`; older ones list
// `imagegenext` alongside it and reject the new name on --enable.
export function imageGenFlag(feats) {
if (/\bimagegenext\b/.test(feats)) return "imagegenext";
if (/\bimage_generation\b/.test(feats)) return "image_generation";
return null;
}
// Fail-fast host check (mirrors illo): don't burn a minutes-long exec when Codex
// isn't usable. Returns { flag } when ready, else { reason }. The flag ships
// default-disabled, so we check the ROW is present (the capability signal) — the
// exec enables it per-render with --enable.
function codexUnavailableReason() {
try {
const which = process.platform === "win32" ? "where" : "which";
execFileSync(which, ["codex"], { stdio: ["ignore", "ignore", "ignore"], timeout: 5000 });
} catch {
// A shell alias (e.g. `codex → /Applications/Codex.app/...`) is NOT enough:
// aliases live only in the interactive shell, so a spawned subprocess's PATH
// lookup can't see them. Symlink the real binary onto PATH.
return {
reason:
'codex CLI not reachable on PATH (a shell alias won\'t work — spawned processes can\'t see aliases; symlink the real binary onto PATH, e.g. ln -s "$(readlink -f "$(command -v codex)")" ~/.local/bin/codex)',
};
}
// Auth marker: presence of the credentials file, NOT `codex login status`.
// That command prints "Logged in using ChatGPT" only to a human stream
// (stderr / TTY) and exits 0, so its piped stdout — how media-use spawns it —
// is empty, and the gate falsely reported "not logged in", blocking codex
// image gen in every headless / CI / agent run even when fully authed.
// auth.json is the durable, TTY-independent signal; token validity is proven
// by the exec itself, which fails cleanly if the login is stale.
const authPath = join(process.env.CODEX_HOME || join(homedir(), ".codex"), "auth.json");
if (!existsSync(authPath)) return { reason: "codex not logged in (run: codex login)" };
const feats = codexRun(["features", "list"]);
if (feats == null) return { reason: "could not read `codex features list`" };
const flag = imageGenFlag(feats);
if (!flag) return { reason: "codex image generation feature unavailable (upgrade Codex CLI)" };
return { flag };
}
export async function codexImageGenerate(intent) {
const { reason, flag } = codexUnavailableReason();
if (reason) {
console.error(`media-use: codex image upsell unavailable: ${reason}`);
return null;
}
const outPath = join(tmpdir(), `media-use-codex-${process.pid}-${Date.now()}.png`);
const prompt =
`${intent}\n\n` +
`Use your built-in image generation tool to render this, then save the image ` +
`to ${outPath} (overwrite if it exists). Do not ask for confirmation. ` +
`If you have no built-in image tool, do nothing (no PIL/matplotlib/SVG substitute).`;
try {
unlinkSync(outPath); // clear any prior file so verify-first can't accept a stale render
} catch {
/* no prior file */
}
const started = Date.now();
try {
execFileSync(
"codex",
[
"exec",
"--cd",
tmpdir(),
"-s",
"workspace-write",
"--skip-git-repo-check",
"--enable",
flag,
"-",
],
{ input: prompt, encoding: "utf8", timeout: TIMEOUT_MS, stdio: ["pipe", "pipe", "pipe"] },
);
} catch (err) {
console.error(
`media-use: \`codex exec\` image generation failed: ${err.stderr?.toString().trim().slice(-200) || err.message}`,
);
return null;
}
// Verify-first (save-to-path may have worked), else fetch the image-tool artifact.
const produced =
existsSync(outPath) && statSync(outPath).size > 0 ? outPath : freshestGeneratedImage(started);
if (!produced) return null;
if (produced !== outPath) {
try {
copyFileSync(produced, outPath);
} catch {
return null;
}
}
return {
localPath: outPath,
ext: ".png",
source: "generated",
metadata: { description: intent, provider: "codex.image_gen", provenance: { prompt: intent } },
};
}
skills/media-use/scripts/lib/codex-provider.test.mjs
-34
View File
@@ -1,34 +0,0 @@
import test from "node:test";
import assert from "node:assert";
import { imageGenFlag } from "./codex-provider.mjs";
// Real `codex features list` output is three space-padded columns:
// <name> <stage> <enabled>
const OLD_CLI = [
"image_detail_original removed false",
"image_generation stable true",
"imagegenext under development false",
].join("\n");
const NEW_CLI = [
"image_detail_original removed false",
"image_generation stable true",
].join("\n");
test("older CLIs keep the old flag — they reject --enable image_generation", () => {
assert.equal(imageGenFlag(OLD_CLI), "imagegenext");
});
test("Codex CLI >=0.145 dropped the imagegenext row; fall back to its new name", () => {
// The regression this guards: gating on `imagegenext` alone made the provider
// report "unavailable" on every up-to-date CLI, without attempting a render.
assert.equal(imageGenFlag(NEW_CLI), "image_generation");
});
test("no image feature at all is unavailable, not a silent default", () => {
assert.equal(imageGenFlag("apps stable true\nhooks stable true"), null);
});
test("substring lookalikes do not count as the feature row", () => {
assert.equal(imageGenFlag("image_generation_v2 stable true"), null);
});
skills/media-use/scripts/lib/coverage.test.mjs
+15 -33
View File
@@ -4,7 +4,6 @@ import { existsSync } from "node:fs";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";
import { listTypes, getProviders } from "./registry.mjs";
import { CAPABILITIES, listModels } from "./local-models.mjs";
// Capstone: media-use must actually OWN each frames media weakness. This
// test enforces the weakness→owner matrix in references/meta.md so a claim can't rot — if
@@ -48,14 +47,10 @@ test("weakness: no transcript-driven cutting → cut compiler entrypoints exist"
assert.equal(typeof cutlist.compileCutList, "function");
});
test("weakness: whisper.cpp is weak → better local ASR (Parakeet) entrypoint exists", async () => {
test("weakness: no transcript → one transcription entrypoint on the one client", async () => {
assert.ok(existsSync(join(SKILL, "scripts", "transcribe.mjs")), "transcribe.mjs missing");
const pw = await import("./parakeet-words.mjs");
assert.equal(typeof pw.mergeTokensToWords, "function", "token->word merge missing");
const lm = await import("./local-models.mjs");
const asr = lm.listModels("asr");
const parakeet = asr.find((m) => m.id === "parakeet-mlx");
assert.ok(parakeet && parakeet.rank === 0, "Parakeet must be the rank-0 preferred ASR");
const api = await import("./api.mjs");
assert.equal(typeof api.transcript, "function", "transcription call missing");
});
test("weakness: no auto-duck/loudness → duck compiler and recipes exist", async () => {
@@ -76,38 +71,25 @@ test("weakness: no cross-project memory → global cache + ingest entrypoints ex
assert.equal(typeof freeze.isDirectMediaUrl, "function", "ingest URL guard missing");
});
// Wenbo (06-29): heygen free-usage is the default; local models are the opt-out
// fallback ("if user no, then local"). We still assert the fallback table is
// populated so the opt-out path stays real.
test("weakness: weak local defaults → local models exist as the opt-out fallback (tts/asr/upscale)", () => {
for (const cap of ["tts", "asr", "upscale"]) {
assert.ok(CAPABILITIES.includes(cap), `capability ${cap} missing`);
assert.ok(listModels(cap).length > 0, `no local models for ${cap}`);
}
});
test("weakness: no image generation → local mflux (RAM-graded) + codex upsell", async () => {
test("weakness: no image generation → catalog search then generation", () => {
const ps = getProviders("image");
assert.ok(
ps.some((p) => p.name === "mflux.local" && typeof p.generate === "function"),
"local image gen missing",
ps.some((p) => p.name === "catalog.image" && typeof p.search === "function"),
"image catalog search missing",
);
assert.ok(
ps.some((p) => p.name === "codex.image_gen" && typeof p.generate === "function"),
"codex image upsell missing",
ps.some((p) => p.name === "hanzo.image" && typeof p.generate === "function"),
"image generation missing",
);
const lm = await import("./local-models.mjs");
assert.ok(lm.CAPABILITIES.includes("imagegen"), "imagegen capability missing");
assert.ok(lm.listModels("imagegen").length >= 3, "imagegen RAM ladder too small");
assert.equal(typeof lm.describeModelLadder, "function", "agent-facing ladder missing");
});
test("weakness: no video generation → local videogen ladder + heygen avatar upsell", async () => {
const lm = await import("./local-models.mjs");
assert.ok(lm.CAPABILITIES.includes("videogen"), "videogen capability missing");
assert.ok(lm.listModels("videogen").length >= 2, "videogen ladder too small");
const ops = existsSync(join(SKILL, "references", "operations.md"));
assert.ok(ops, "operations.md (avatar-upsell recipe) missing");
test("weakness: no video generation → the video type generates", () => {
const ps = getProviders("video");
assert.ok(
ps.some((p) => p.name === "hanzo.video" && typeof p.generate === "function"),
"video generation missing",
);
assert.ok(existsSync(join(SKILL, "references", "operations.md")), "operations.md missing");
});
test("every resolve type has at least one enabled provider", () => {
skills/media-use/scripts/lib/heygen-cli.mjs
-191
View File
@@ -1,191 +0,0 @@
import { execFileSync } from "node:child_process";
import { track } from "./telemetry.mjs";
// v0.3.0 is the first CLI that can use an OAuth session; v0.1.x/0.2.x reject it
// ("heygen-cli can't use OAuth yet"), and OAuth is what the free-usage path
// needs — so anything below this can't authenticate for free usage at all.
export const HEYGEN_MIN_VERSION = "0.3.0";
// Free-usage path is OAuth (`--oauth` → subscription/free credits); `--api-key`
// bills API credits, so the onboarding steers to OAuth. Keep pipe-to-shell
// installer text out of the runtime module; the docs are the safer source of
// truth for platform-specific setup.
export const HEYGEN_INSTALL_COMMAND =
"Install the CLI from https://developers.heygen.com/cli, then run: heygen auth login --oauth";
export const HEYGEN_AUTH_COMMAND = "heygen auth login --oauth";
export const HEYGEN_UPDATE_COMMAND = "heygen update";
export const HEYGEN_CLIENT_SOURCE_ARGV = ["--headers", "X-HeyGen-Client-Source: media-use"];
export const HEYGEN_NOT_FOUND_MESSAGE = `media-use: heygen CLI not found — it's the free path for bgm/image/voice/avatar-video. ${HEYGEN_INSTALL_COMMAND}`;
export const HEYGEN_NOT_AUTHENTICATED_MESSAGE = `media-use: heygen CLI not authenticated (free usage) — run: ${HEYGEN_AUTH_COMMAND}`;
export const HEYGEN_OUTDATED_MESSAGE = `media-use: heygen CLI is outdated — run: ${HEYGEN_UPDATE_COMMAND} (need >= v${HEYGEN_MIN_VERSION})`;
const ACTIONABLE_MESSAGES = new Set([
HEYGEN_NOT_FOUND_MESSAGE,
HEYGEN_NOT_AUTHENTICATED_MESSAGE,
HEYGEN_OUTDATED_MESSAGE,
]);
export function classifyHeygenError(err) {
return classifyHeygenErrorResult(err).message;
}
export function classifyHeygenErrorCode(err) {
return classifyHeygenErrorResult(err).code;
}
function classifyHeygenErrorResult(err) {
const detail = heygenErrorDetail(err);
const text = [err?.stderr, err?.stdout, err?.message, detail]
.map((value) => textOf(value))
.filter(Boolean)
.join("\n");
const lower = text.toLowerCase();
// Only ENOENT (spawn of a missing binary) or a shell's "command not found"
// mean the CLI itself is absent. A bare "not found" would misfire on the CLI's
// own resource errors (e.g. a stale voiceId → "voice not found"), whose message
// embeds the `heygen ...` command line — sending users to reinstall a CLI they
// just ran successfully. Keep this narrow.
if (err?.code === "ENOENT" || lower.includes("command not found")) {
return { code: "not_found", message: HEYGEN_NOT_FOUND_MESSAGE };
}
if (
lower.includes("unauthorized") ||
lower.includes("unauthenticated") ||
// \b401\b, not a bare "401" substring — otherwise request IDs (req-401abc),
// URLs, and retry-after headers would misclassify as an auth failure.
/\b401\b/.test(lower) ||
lower.includes("not logged in") ||
lower.includes("no api key") ||
lower.includes("missing api key") ||
lower.includes("invalid api key") ||
lower.includes("login required") ||
lower.includes("auth required") ||
lower.includes("authentication required")
) {
return { code: "not_authenticated", message: HEYGEN_NOT_AUTHENTICATED_MESSAGE };
}
const version = firstSemver(text);
if (version && versionLessThan(version, HEYGEN_MIN_VERSION)) {
return { code: "outdated", message: HEYGEN_OUTDATED_MESSAGE };
}
if (
lower.includes("rate limit") ||
lower.includes("quota") ||
lower.includes("insufficient credit") ||
lower.includes("too many requests") ||
lower.includes("throttled") ||
/\b429\b/.test(lower)
) {
return { code: "rate_limited", message: detail };
}
return { code: "other", message: detail };
}
// reportHeygenFailure's callers (voice-provider.mjs, heygen-search.mjs) are
// synchronous and several layers below the CLI's process.exit() calls, so
// they can't await this tracking call themselves. Stash each attempt's
// promise here so a caller closer to exit (resolve.mjs) can join it first —
// same "awaited so a short-lived run flushes it" discipline telemetry.mjs's
// track() already documents, just reachable from a sync call site.
const pendingFailureTracking = new Set();
// resolve.mjs is a single-shot CLI (one resolve per process), so one shared
// consume-once slot is sufficient. If resolve becomes an in-process/concurrent
// API, move this state into a per-resolve context before reusing that path.
let pendingRemediation = null;
export function consumeHeygenRemediation() {
const remediation = pendingRemediation;
pendingRemediation = null;
return remediation;
}
export function reportHeygenFailure(err, context, trackEvent = track) {
const { code, message } = classifyHeygenErrorResult(err);
if (code === "not_found" || code === "outdated") {
pendingRemediation = { code, message };
}
if (ACTIONABLE_MESSAGES.has(message)) {
console.error(message);
} else {
console.error(`media-use: \`${context}\` failed: ${message}`);
}
try {
const tracked = Promise.resolve(
trackEvent("media_use_provider_error", { provider: "heygen", reason: code }),
).catch(() => {});
pendingFailureTracking.add(tracked);
void tracked.finally(() => pendingFailureTracking.delete(tracked));
return tracked;
} catch {
// Telemetry must never affect the provider failure path.
return Promise.resolve();
}
}
// Awaits every provider-error track fired since the last flush, so a caller
// about to process.exit() doesn't orphan one mid-request (both are separate,
// non-keepalive HTTP connections with no ordering guarantee otherwise).
// Never rejects: each tracked promise already swallows its own failure.
export async function flushHeygenFailureTracking() {
if (pendingFailureTracking.size === 0) return;
await Promise.all(pendingFailureTracking);
}
// Shared discovery/generation helper for the CLI-shelling providers (voice,
// avatar-video): run a heygen JSON subcommand, report+classify on failure,
// and hand the caller the classified reason via onError (used by callers that
// need to distinguish e.g. not_authenticated from other failures).
export function runHeygenJson(bin, argv, label, onError) {
let out;
try {
out = execFileSync(bin, argv, {
encoding: "utf8",
timeout: 120000,
stdio: ["pipe", "pipe", "pipe"],
});
} catch (err) {
reportHeygenFailure(err, `${bin} ${label}`);
onError?.(classifyHeygenErrorCode(err));
return null;
}
try {
return JSON.parse(out);
} catch {
console.error(`media-use: \`${bin} ${label}\` returned non-JSON output`);
return null;
}
}
export function firstSemver(text) {
const match = String(text || "").match(/\bv?(\d+)\.(\d+)\.(\d+)\b/);
return match ? `${match[1]}.${match[2]}.${match[3]}` : null;
}
export function versionLessThan(version, minimum) {
const left = versionParts(version);
const right = versionParts(minimum);
if (!left || !right) return false;
for (let i = 0; i < 3; i++) {
if (left[i] < right[i]) return true;
if (left[i] > right[i]) return false;
}
return false;
}
function heygenErrorDetail(err) {
return textOf(err?.stderr) || textOf(err?.stdout) || err?.message || String(err);
}
function textOf(value) {
return value == null ? "" : String(value).trim();
}
function versionParts(version) {
const match = String(version || "").match(/^v?(\d+)\.(\d+)\.(\d+)$/);
return match ? match.slice(1).map((part) => Number.parseInt(part, 10)) : null;
}
skills/media-use/scripts/lib/heygen-cli.test.mjs
-301
View File
@@ -1,301 +0,0 @@
import { strict as assert } from "node:assert";
import { spawnSync } from "node:child_process";
import { test } from "node:test";
import {
classifyHeygenError,
classifyHeygenErrorCode,
consumeHeygenRemediation,
flushHeygenFailureTracking,
HEYGEN_NOT_AUTHENTICATED_MESSAGE,
HEYGEN_NOT_FOUND_MESSAGE,
HEYGEN_OUTDATED_MESSAGE,
reportHeygenFailure,
} from "./heygen-cli.mjs";
function captureFailureReport(err, context, trackEvent) {
const originalError = console.error;
const stderrCalls = [];
console.error = (...args) => stderrCalls.push(args);
try {
if (trackEvent) {
reportHeygenFailure(err, context, trackEvent);
} else {
reportHeygenFailure(err, context);
}
} finally {
console.error = originalError;
}
return stderrCalls;
}
test("classifies ENOENT-style missing heygen errors with install instructions", () => {
const message = classifyHeygenError({ code: "ENOENT", message: "spawn heygen ENOENT" });
assert.equal(message, HEYGEN_NOT_FOUND_MESSAGE);
});
test("classifies auth failures with login instructions", () => {
const message = classifyHeygenError({ stderr: Buffer.from("Error: not logged in") });
assert.equal(message, HEYGEN_NOT_AUTHENTICATED_MESSAGE);
});
test("classifies a real 401 as auth, but not a bare 401 substring in prose", () => {
assert.equal(
classifyHeygenError({ stderr: Buffer.from("HTTP 401 Unauthorized") }),
HEYGEN_NOT_AUTHENTICATED_MESSAGE,
);
// A request id that merely contains "401" must NOT read as an auth failure.
const noise = classifyHeygenError({ stderr: Buffer.from("upload failed (request req-401abc)") });
assert.notEqual(noise, HEYGEN_NOT_AUTHENTICATED_MESSAGE);
});
test("classifies old heygen versions with update instructions", () => {
const message = classifyHeygenError({
stderr: Buffer.from("heygen v0.1.5 does not support --headers"),
});
assert.equal(message, HEYGEN_OUTDATED_MESSAGE);
});
test("does not misclassify a resource 'not found' error as a missing CLI", () => {
// A stale voiceId makes `heygen voice speech create` fail with "voice not
// found"; the error message embeds the `heygen ...` command line. This must
// pass through as detail, not send the user to reinstall a working CLI.
const message = classifyHeygenError({
stderr: Buffer.from("Error: voice not found (id: stale-123)"),
message: "Command failed: heygen voice speech create --voice stale-123",
});
assert.notEqual(message, HEYGEN_NOT_FOUND_MESSAGE);
assert.equal(message, "Error: voice not found (id: stale-123)");
});
test("classifies a shell 'command not found' as a missing CLI", () => {
const message = classifyHeygenError({ stderr: Buffer.from("bash: heygen: command not found") });
assert.equal(message, HEYGEN_NOT_FOUND_MESSAGE);
});
test("passes through unrelated errors", () => {
const message = classifyHeygenError({
stderr: Buffer.from("rate limit exceeded"),
message: "Command failed",
});
assert.equal(message, "rate limit exceeded");
});
test("classifies existing HeyGen failures with stable reason codes", () => {
assert.equal(classifyHeygenErrorCode({ code: "ENOENT" }), "not_found");
assert.equal(
classifyHeygenErrorCode({ stderr: Buffer.from("HTTP 401 Unauthorized") }),
"not_authenticated",
);
assert.equal(
classifyHeygenErrorCode({ stderr: Buffer.from("heygen v0.1.5 is unsupported") }),
"outdated",
);
assert.equal(classifyHeygenErrorCode({ stderr: Buffer.from("provider unavailable") }), "other");
});
test("classifies rate-limit text case-insensitively", () => {
assert.equal(
classifyHeygenErrorCode({ stderr: Buffer.from("RATE LIMIT exceeded") }),
"rate_limited",
);
});
test("classifies quota and insufficient-credit errors as rate limited", () => {
for (const detail of ["Quota exhausted", "INSUFFICIENT CREDIT remaining"]) {
assert.equal(classifyHeygenErrorCode({ stderr: Buffer.from(detail) }), "rate_limited");
}
});
test("classifies the literal 429 reason phrase and throttling language as rate limited", () => {
for (const detail of ["Too Many Requests", "Error: throttled by upstream, retry later"]) {
assert.equal(classifyHeygenErrorCode({ stderr: Buffer.from(detail) }), "rate_limited");
}
});
test("does not misclassify unrelated errors that share a word with the new phrasing", () => {
// Shares "too many" with "too many requests" but is a distinct failure (fd
// exhaustion, not a rate limit) — the match must require the full phrase.
assert.equal(
classifyHeygenErrorCode({ stderr: Buffer.from("Too many open file descriptors") }),
"other",
);
});
test("classifies a bare 429 as rate limited without matching request IDs", () => {
assert.equal(
classifyHeygenErrorCode({ stderr: Buffer.from("HTTP 429 Too Many Requests") }),
"rate_limited",
);
assert.equal(
classifyHeygenErrorCode({ stderr: Buffer.from("request req-429abc failed") }),
"other",
);
});
test("tracks not-found failures without changing actionable output", () => {
const trackingCalls = [];
const stderrCalls = captureFailureReport({ code: "ENOENT" }, "heygen asset search", (...args) =>
trackingCalls.push(args),
);
assert.deepEqual(stderrCalls, [[HEYGEN_NOT_FOUND_MESSAGE]]);
assert.deepEqual(trackingCalls, [
["media_use_provider_error", { provider: "heygen", reason: "not_found" }],
]);
});
test("records missing and outdated CLI remediation once", () => {
consumeHeygenRemediation();
captureFailureReport({ code: "ENOENT" }, "heygen audio sounds list", () => {});
assert.deepEqual(consumeHeygenRemediation(), {
code: "not_found",
message: HEYGEN_NOT_FOUND_MESSAGE,
});
assert.equal(consumeHeygenRemediation(), null);
captureFailureReport(
{ stderr: Buffer.from("heygen v0.1.5 does not support --headers") },
"heygen audio sounds list",
() => {},
);
assert.deepEqual(consumeHeygenRemediation(), {
code: "outdated",
message: HEYGEN_OUTDATED_MESSAGE,
});
assert.equal(consumeHeygenRemediation(), null);
});
test("does not record non-install remediation", () => {
consumeHeygenRemediation();
for (const error of [
{ stderr: Buffer.from("HTTP 401 Unauthorized") },
{ stderr: Buffer.from("quota exhausted") },
{ stderr: Buffer.from("provider unavailable") },
]) {
captureFailureReport(error, "heygen audio sounds list", () => {});
assert.equal(consumeHeygenRemediation(), null);
}
});
test("tracks generic failures without including raw detail", () => {
const trackingCalls = [];
const stderrCalls = captureFailureReport(
{ stderr: Buffer.from("private provider detail") },
"heygen asset search",
(...args) => trackingCalls.push(args),
);
assert.deepEqual(stderrCalls, [
["media-use: `heygen asset search` failed: private provider detail"],
]);
assert.deepEqual(trackingCalls, [
["media_use_provider_error", { provider: "heygen", reason: "other" }],
]);
});
test("keeps failure output observable when telemetry is opted out", () => {
const previousOptOut = process.env.FRAMES_NO_TELEMETRY;
process.env.FRAMES_NO_TELEMETRY = "1";
try {
const stderrCalls = captureFailureReport({ code: "ENOENT" }, "heygen asset search");
assert.deepEqual(stderrCalls, [[HEYGEN_NOT_FOUND_MESSAGE]]);
} finally {
if (previousOptOut === undefined) {
delete process.env.FRAMES_NO_TELEMETRY;
} else {
process.env.FRAMES_NO_TELEMETRY = previousOptOut;
}
}
});
test("keeps failure output observable when tracking throws synchronously", () => {
const originalError = console.error;
const stderrCalls = [];
let thrown;
console.error = (...args) => stderrCalls.push(args);
try {
try {
reportHeygenFailure({ code: "ENOENT" }, "heygen asset search", () => {
throw new Error("tracking failed");
});
} catch (err) {
thrown = err;
}
} finally {
console.error = originalError;
}
assert.deepEqual(stderrCalls, [[HEYGEN_NOT_FOUND_MESSAGE]]);
assert.equal(thrown, undefined);
});
test("does not leave rejected tracking promises unhandled", () => {
const moduleUrl = new URL("./heygen-cli.mjs", import.meta.url).href;
const script = `
import { reportHeygenFailure } from ${JSON.stringify(moduleUrl)};
reportHeygenFailure(
{ stderr: "provider unavailable" },
"heygen asset search",
() => Promise.reject(new Error("tracking failed")),
);
await new Promise((resolve) => setImmediate(resolve));
`;
const child = spawnSync(
process.execPath,
["--unhandled-rejections=strict", "--input-type=module", "--eval", script],
{ encoding: "utf8", timeout: 5000 },
);
assert.equal(child.error, undefined);
assert.equal(child.signal, null);
assert.equal(child.status, 0, child.stderr);
assert.equal(child.stderr, "media-use: `heygen asset search` failed: provider unavailable\n");
});
test("flushHeygenFailureTracking waits for a pending report before resolving", async () => {
const events = [];
let releaseTrack;
const gate = new Promise((resolve) => {
releaseTrack = resolve;
});
// Mirrors the real call sites (voice-provider.mjs, heygen-search.mjs):
// fire-and-forget, the return value is never awaited by the caller.
reportHeygenFailure({ code: "ENOENT" }, "heygen voice speech", () =>
gate.then(() => {
events.push("track-settled");
}),
);
const flushed = flushHeygenFailureTracking().then(() => {
events.push("flush-resolved");
});
// Let several pending microtasks drain before releasing the gate, so this
// proves flush is genuinely still waiting on the tracked promise -- not
// merely that it hasn't had a tick yet.
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
assert.deepEqual(events, [], "flush must not resolve while the tracked promise is still pending");
releaseTrack();
await flushed;
assert.deepEqual(
events,
["track-settled", "flush-resolved"],
"flush must resolve only after the pending track settles, in that order",
);
});
test("flushHeygenFailureTracking resolves immediately when nothing is pending", async () => {
await flushHeygenFailureTracking();
});
skills/media-use/scripts/lib/heygen-search.mjs
-45
View File
@@ -1,45 +0,0 @@
import { execFileSync } from "node:child_process";
import { HEYGEN_CLIENT_SOURCE_ARGV, reportHeygenFailure } from "./heygen-cli.mjs";
export function heygenSearch(subcommand, query, { type, limit = 5, minScore } = {}) {
// execFileSync with an argv array (no shell), so query/type/etc. are passed as
// literal arguments — no quoting tricks, no command injection. subcommand is a
// hardcoded multi-word string (e.g. "audio sounds list"), split into tokens.
// Tag the caller via the CLI's allowlisted attribution header (heygen >= v0.3.0).
const args = [...HEYGEN_CLIENT_SOURCE_ARGV, ...subcommand.split(" "), "--query", query];
if (type) args.push("--type", type);
args.push("--limit", String(limit));
// Server-side score floor. Honored by `audio sounds list`; the `asset search`
// backend rejects it, so only audio providers pass minScore (see image-provider).
if (minScore != null) args.push("--min-score", String(minScore));
let out;
try {
out = execFileSync("heygen", args, {
encoding: "utf8",
timeout: 15000,
stdio: ["pipe", "pipe", "pipe"],
});
} catch (err) {
// Don't swallow a broken command / auth failure as "no results" — that turns
// a typo or expired key into a silent dead end. Surface it, then give up.
reportHeygenFailure(err, `heygen ${subcommand}`);
return null;
}
let parsed;
try {
parsed = JSON.parse(out);
} catch {
console.error(`media-use: \`heygen ${subcommand}\` returned non-JSON output`);
return null;
}
if (parsed?.error) {
const e = parsed.error;
console.error(`media-use: \`heygen ${subcommand}\` error: ${e.message ?? JSON.stringify(e)}`);
return null;
}
const data = parsed?.data;
return Array.isArray(data) && data.length > 0 ? data : null;
}
skills/media-use/scripts/lib/heygen-search.test.mjs
-49
View File
@@ -1,49 +0,0 @@
import { strict as assert } from "node:assert";
import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { test } from "node:test";
import { HEYGEN_CLIENT_SOURCE_HEADERS } from "../../audio/scripts/lib/heygen.mjs";
import { HEYGEN_CLIENT_SOURCE_ARGV } from "./heygen-cli.mjs";
import { heygenSearch } from "./heygen-search.mjs";
test("tags HeyGen searches with the shared media-use client source", () => {
const dir = mkdtempSync(join(tmpdir(), "media-use-heygen-search-"));
const capturePath = join(dir, "argv.log");
const heygenPath = join(dir, "heygen");
const previousPath = process.env.PATH;
const previousCapturePath = process.env.HEYGEN_CAPTURE_PATH;
writeFileSync(
heygenPath,
`#!/bin/sh
printf '%s\\n' "$*" >> "$HEYGEN_CAPTURE_PATH"
printf '%s\\n' '{"data":[{"id":"x"}]}'
`,
);
chmodSync(heygenPath, 0o755);
process.env.PATH = `${dir}:${previousPath ?? ""}`;
process.env.HEYGEN_CAPTURE_PATH = capturePath;
try {
const result = heygenSearch("audio sounds list", "ocean", { limit: 1 });
const argv = readFileSync(capturePath, "utf8").trim();
assert.deepEqual(result, [{ id: "x" }]);
assert.match(argv, /X-HeyGen-Client-Source: media-use/);
} finally {
if (previousPath === undefined) delete process.env.PATH;
else process.env.PATH = previousPath;
if (previousCapturePath === undefined) delete process.env.HEYGEN_CAPTURE_PATH;
else process.env.HEYGEN_CAPTURE_PATH = previousCapturePath;
rmSync(dir, { recursive: true, force: true });
}
});
test("keeps CLI and REST media-use client source headers in lockstep", () => {
const [entry] = Object.entries(HEYGEN_CLIENT_SOURCE_HEADERS);
assert.ok(entry);
const [key, value] = entry;
assert.equal(HEYGEN_CLIENT_SOURCE_ARGV[1], `${key}: ${value}`);
});
skills/media-use/scripts/lib/heygen-video-provider.mjs
-127
View File
@@ -1,127 +0,0 @@
import { execFileSync } from "node:child_process";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { freezeUrl } from "./freeze.mjs";
import {
classifyHeygenErrorCode,
HEYGEN_AUTH_COMMAND,
HEYGEN_CLIENT_SOURCE_ARGV,
reportHeygenFailure,
runHeygenJson,
} from "./heygen-cli.mjs";
export const AVATAR_VIDEO_SIGNIN_MESSAGE = `media-use: avatar video is free for new API users — sign in: ${HEYGEN_AUTH_COMMAND}`;
// Cache only a truthy id -- a transient discovery failure must not poison the
// cache with `null` and permanently disable heygen.video for the rest of the
// process. `onError` lets the caller distinguish not_authenticated (and nudge
// onboarding) from any other discovery failure.
let cachedAvatarId;
function defaultAvatarId(onError) {
if (cachedAvatarId) return cachedAvatarId;
const j = runHeygenJson(
"heygen",
["avatar", "list", "--ownership", "public", "--limit", "1"],
"avatar list",
onError,
);
cachedAvatarId = j?.data?.[0]?.avatar_id || null;
return cachedAvatarId;
}
let cachedStarfishVoiceId;
function defaultStarfishVoiceId(onError) {
if (cachedStarfishVoiceId) return cachedStarfishVoiceId;
const j = runHeygenJson(
"heygen",
["voice", "list", "--engine", "starfish", "--limit", "1"],
"voice list",
onError,
);
cachedStarfishVoiceId = j?.data?.[0]?.voice_id || null;
return cachedStarfishVoiceId;
}
export async function heygenVideoGenerate(intent, ctx) {
let discoveryFailureReason = null;
const captureReason = (reason) => {
discoveryFailureReason ??= reason;
};
// Short-circuit: once one discovery call fails, the result is null either
// way, so don't attempt the second -- that would double-fire the onboarding
// message and the provider-error telemetry ping for what's really one failure.
const avatarId = ctx?.avatarId || defaultAvatarId(captureReason);
if (!avatarId) {
if (discoveryFailureReason === "not_authenticated") console.error(AVATAR_VIDEO_SIGNIN_MESSAGE);
return null;
}
const voiceId = ctx?.voiceId || defaultStarfishVoiceId(captureReason);
if (!voiceId) {
if (discoveryFailureReason === "not_authenticated") console.error(AVATAR_VIDEO_SIGNIN_MESSAGE);
return null;
}
let out;
try {
out = execFileSync(
"heygen",
[
...HEYGEN_CLIENT_SOURCE_ARGV,
"video",
"create",
"--wait",
"-d",
JSON.stringify({
type: "avatar",
avatar_id: avatarId,
script: intent,
voice_id: voiceId,
}),
],
{
encoding: "utf8",
timeout: 300000,
stdio: ["pipe", "pipe", "pipe"],
},
);
} catch (err) {
if (classifyHeygenErrorCode(err) === "not_authenticated") {
console.error(AVATAR_VIDEO_SIGNIN_MESSAGE);
}
reportHeygenFailure(err, "heygen video create");
return null;
}
let parsed;
try {
parsed = JSON.parse(out);
} catch {
console.error("media-use: `heygen video create` returned invalid JSON");
return null;
}
const videoUrl = parsed?.data?.video_url;
if (typeof videoUrl !== "string" || !videoUrl) {
console.error("media-use: `heygen video create` returned no video URL");
return null;
}
const tmpPath = join(tmpdir(), `media-use-heygen-video-${process.pid}-${Date.now()}.mp4`);
try {
await freezeUrl(videoUrl, tmpPath);
} catch (err) {
console.error(`media-use: heygen video download failed: ${err.message}`);
return null;
}
return {
localPath: tmpPath,
ext: ".mp4",
source: "generated",
metadata: {
description: intent,
provider: "heygen.video",
provenance: { prompt: intent },
},
};
}
skills/media-use/scripts/lib/heygen-video-provider.test.mjs
-344
View File
@@ -1,344 +0,0 @@
import { strict as assert } from "node:assert";
import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import http from "node:http";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { test } from "node:test";
import { HEYGEN_NOT_AUTHENTICATED_MESSAGE } from "./heygen-cli.mjs";
import { AVATAR_VIDEO_SIGNIN_MESSAGE } from "./heygen-video-provider.mjs";
const VIDEO_FIXTURE = Buffer.from("tiny heygen video fixture");
let importCount = 0;
async function freshGenerate() {
importCount += 1;
const module = await import(`./heygen-video-provider.mjs?test=${importCount}`);
return module.heygenVideoGenerate;
}
async function listenVideoServer() {
const server = http.createServer((req, res) => {
if (req.url !== "/video.mp4") {
res.writeHead(404).end();
return;
}
res.writeHead(200, {
"content-length": VIDEO_FIXTURE.length,
"content-type": "video/mp4",
});
res.end(VIDEO_FIXTURE);
});
await new Promise((resolve) => server.listen(0, resolve));
const address = server.address();
assert.ok(address && typeof address !== "string");
return {
server,
url: `http://127.0.0.1:${address.port}/video.mp4`,
};
}
async function listenFailingVideoServer() {
const server = http.createServer((req, res) => {
res.writeHead(500).end();
});
await new Promise((resolve) => server.listen(0, resolve));
const address = server.address();
assert.ok(address && typeof address !== "string");
return {
server,
url: `http://127.0.0.1:${address.port}/video.mp4`,
};
}
function closeServer(server) {
return new Promise((resolve, reject) => {
server.close((err) => (err ? reject(err) : resolve()));
});
}
async function withFakeHeygen(options, run) {
const dir = mkdtempSync(join(tmpdir(), "media-use-heygen-video-provider-"));
const capturePath = join(dir, "argv.log");
const heygenPath = join(dir, "heygen");
const previousEnv = {
PATH: process.env.PATH,
HEYGEN_CAPTURE_PATH: process.env.HEYGEN_CAPTURE_PATH,
HEYGEN_VIDEO_MODE: process.env.HEYGEN_VIDEO_MODE,
HEYGEN_VIDEO_RESPONSE: process.env.HEYGEN_VIDEO_RESPONSE,
HEYGEN_DISCOVERY_MODE: process.env.HEYGEN_DISCOVERY_MODE,
FRAMES_NO_TELEMETRY: process.env.FRAMES_NO_TELEMETRY,
};
writeFileSync(
heygenPath,
`#!/bin/sh
printf '%s\\n' "$*" >> "$HEYGEN_CAPTURE_PATH"
case "$*" in
*"avatar list"*)
case "$HEYGEN_DISCOVERY_MODE" in
auth) printf '%s\\n' 'HTTP 401 Unauthorized' >&2; exit 1 ;;
*) printf '%s\\n' '{"data":[{"avatar_id":"avatar-public-1"}]}' ;;
esac
;;
*"voice list"*)
case "$HEYGEN_DISCOVERY_MODE" in
auth) printf '%s\\n' 'HTTP 401 Unauthorized' >&2; exit 1 ;;
*) printf '%s\\n' '{"data":[{"voice_id":"voice-starfish-1"}]}' ;;
esac
;;
*"video create"*)
case "$HEYGEN_VIDEO_MODE" in
auth) printf '%s\\n' 'HTTP 401 Unauthorized' >&2; exit 1 ;;
other) printf '%s\\n' 'provider unavailable' >&2; exit 1 ;;
*) printf '%s\\n' "$HEYGEN_VIDEO_RESPONSE" ;;
esac
;;
esac
`,
);
chmodSync(heygenPath, 0o755);
process.env.PATH = `${dir}:${previousEnv.PATH ?? ""}`;
process.env.HEYGEN_CAPTURE_PATH = capturePath;
process.env.HEYGEN_VIDEO_MODE = options.mode ?? "success";
process.env.HEYGEN_VIDEO_RESPONSE = options.response ?? "";
process.env.HEYGEN_DISCOVERY_MODE = options.discoveryMode ?? "";
process.env.FRAMES_NO_TELEMETRY = "1";
try {
return await run({
invocations: () => readFileSync(capturePath, "utf8").trim().split("\n"),
});
} finally {
for (const [key, value] of Object.entries(previousEnv)) {
if (value === undefined) delete process.env[key];
else process.env[key] = value;
}
rmSync(dir, { recursive: true, force: true });
}
}
function bodyFromInvocation(invocation) {
const marker = " -d ";
const start = invocation.indexOf(marker);
assert.notEqual(start, -1);
return JSON.parse(invocation.slice(start + marker.length));
}
test("downloads a generated avatar video and returns the generated MP4 result", async () => {
const { server, url } = await listenVideoServer();
let localPath;
try {
await withFakeHeygen(
{ response: JSON.stringify({ data: { video_url: url } }) },
async ({ invocations }) => {
const heygenVideoGenerate = await freshGenerate();
const intent = "Welcome to the Frames launch";
const result = await heygenVideoGenerate(intent, {});
localPath = result?.localPath;
const calls = invocations();
const create = calls.find((call) => call.includes("video create"));
assert.ok(create);
assert.match(create, /--headers X-HeyGen-Client-Source: media-use/);
assert.deepEqual(bodyFromInvocation(create), {
type: "avatar",
avatar_id: "avatar-public-1",
script: intent,
voice_id: "voice-starfish-1",
});
assert.ok(result);
assert.equal(join(tmpdir(), result.localPath.slice(tmpdir().length + 1)), result.localPath);
assert.match(result.localPath, /media-use-heygen-video-\d+-\d+\.mp4$/);
assert.deepEqual(result, {
localPath: result.localPath,
ext: ".mp4",
source: "generated",
metadata: {
description: intent,
provider: "heygen.video",
provenance: { prompt: intent },
},
});
assert.deepEqual(readFileSync(result.localPath), VIDEO_FIXTURE);
},
);
} finally {
if (localPath) rmSync(localPath, { force: true });
await closeServer(server);
}
});
test("tags video creation but not avatar or voice discovery", async () => {
const { server, url } = await listenVideoServer();
let localPath;
try {
await withFakeHeygen(
{ response: JSON.stringify({ data: { video_url: url } }) },
async ({ invocations }) => {
const heygenVideoGenerate = await freshGenerate();
const result = await heygenVideoGenerate("Header regression guard", {});
localPath = result?.localPath;
const calls = invocations();
assert.equal(calls.length, 3);
assert.match(calls[0], /^avatar list --ownership public --limit 1$/);
assert.doesNotMatch(calls[0], /X-HeyGen-Client-Source/);
assert.match(calls[1], /^voice list --engine starfish --limit 1$/);
assert.doesNotMatch(calls[1], /X-HeyGen-Client-Source/);
assert.match(calls[2], /video create/);
assert.match(calls[2], /X-HeyGen-Client-Source: media-use/);
},
);
} finally {
if (localPath) rmSync(localPath, { force: true });
await closeServer(server);
}
});
test("uses explicit avatar and voice overrides without discovery", async () => {
const { server, url } = await listenVideoServer();
let localPath;
try {
await withFakeHeygen(
{ response: JSON.stringify({ data: { video_url: url } }) },
async ({ invocations }) => {
const heygenVideoGenerate = await freshGenerate();
const result = await heygenVideoGenerate("Use my presenter", {
avatarId: "avatar-override",
voiceId: "voice-override",
});
localPath = result?.localPath;
const calls = invocations();
assert.equal(calls.length, 1);
assert.match(calls[0], /video create/);
assert.deepEqual(bodyFromInvocation(calls[0]), {
type: "avatar",
avatar_id: "avatar-override",
script: "Use my presenter",
voice_id: "voice-override",
});
},
);
} finally {
if (localPath) rmSync(localPath, { force: true });
await closeServer(server);
}
});
test("caches discovered avatar and voice IDs for the process", async () => {
const { server, url } = await listenVideoServer();
const localPaths = new Set();
try {
await withFakeHeygen(
{ response: JSON.stringify({ data: { video_url: url } }) },
async ({ invocations }) => {
const heygenVideoGenerate = await freshGenerate();
for (const intent of ["First avatar video", "Second avatar video"]) {
const result = await heygenVideoGenerate(intent, {});
if (result) localPaths.add(result.localPath);
}
const calls = invocations();
assert.equal(calls.filter((call) => call.startsWith("avatar list ")).length, 1);
assert.equal(calls.filter((call) => call.startsWith("voice list ")).length, 1);
assert.equal(calls.filter((call) => call.includes("video create")).length, 2);
},
);
} finally {
for (const localPath of localPaths) rmSync(localPath, { force: true });
await closeServer(server);
}
});
test("prints auth onboarding and reports an unauthenticated create failure", async (t) => {
const errors = [];
t.mock.method(console, "error", (message) => errors.push(message));
await withFakeHeygen({ mode: "auth" }, async () => {
const heygenVideoGenerate = await freshGenerate();
const result = await heygenVideoGenerate("Sign-in failure", {
avatarId: "avatar-override",
voiceId: "voice-override",
});
assert.equal(result, null);
assert.ok(errors.includes(AVATAR_VIDEO_SIGNIN_MESSAGE));
assert.ok(errors.includes(HEYGEN_NOT_AUTHENTICATED_MESSAGE));
});
});
test("reports other create failures without auth onboarding", async (t) => {
const errors = [];
t.mock.method(console, "error", (message) => errors.push(message));
await withFakeHeygen({ mode: "other" }, async () => {
const heygenVideoGenerate = await freshGenerate();
const result = await heygenVideoGenerate("Provider failure", {
avatarId: "avatar-override",
voiceId: "voice-override",
});
assert.equal(result, null);
assert.ok(!errors.includes(AVATAR_VIDEO_SIGNIN_MESSAGE));
assert.ok(errors.includes("media-use: `heygen video create` failed: provider unavailable"));
});
});
test("falls through on non-JSON and error responses", async (t) => {
t.mock.method(console, "error", () => {});
for (const response of ["not JSON", '{"error":{"message":"render failed"}}']) {
await withFakeHeygen({ response }, async () => {
const heygenVideoGenerate = await freshGenerate();
const result = await heygenVideoGenerate("Unusable response", {
avatarId: "avatar-override",
voiceId: "voice-override",
});
assert.equal(result, null);
});
}
});
test("onboards and returns null when avatar/voice discovery itself is unauthenticated", async (t) => {
const errors = [];
t.mock.method(console, "error", (message) => errors.push(message));
// Both avatar list AND voice list would fail unauthenticated (discoveryMode
// "auth" applies to both in the fake CLI) -- the short-circuit after the
// first failure must mean only one is ever attempted, so the onboarding
// message and the provider-error telemetry ping each fire exactly once
// instead of double-firing for what's really one auth failure.
await withFakeHeygen({ discoveryMode: "auth" }, async ({ invocations }) => {
const heygenVideoGenerate = await freshGenerate();
const result = await heygenVideoGenerate("Discovery auth failure", {});
assert.equal(result, null);
assert.equal(
errors.filter((message) => message === AVATAR_VIDEO_SIGNIN_MESSAGE).length,
1,
"onboarding message must fire exactly once, not once per failed discovery call",
);
const calls = invocations();
assert.equal(calls.length, 1, "must short-circuit after the first discovery failure");
assert.match(calls[0], /^avatar list /);
});
});
test("download failure after a successful create returns null and logs a diagnostic", async () => {
const { server, url } = await listenFailingVideoServer();
try {
await withFakeHeygen({ response: JSON.stringify({ data: { video_url: url } }) }, async () => {
const heygenVideoGenerate = await freshGenerate();
const result = await heygenVideoGenerate("Download failure", {
avatarId: "avatar-override",
voiceId: "voice-override",
});
assert.equal(result, null);
});
} finally {
await closeServer(server);
}
});
skills/media-use/scripts/lib/image-provider.mjs
+20 -34
View File
@@ -1,44 +1,30 @@
import { heygenSearch } from "./heygen-search.mjs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { image, write } from "./api.mjs";
import { asResult, find } from "./catalog.mjs";
const catalogSearch = (kind) => async (intent) => {
const hit = await find(kind, intent);
if (!hit || hit.empty) return null;
return asResult(hit, intent, kind);
};
export const imageProvider = {
async search(intent) {
const results = heygenSearch("asset search", intent, { type: "image" });
if (!results) return null;
const best = results[0];
search: catalogSearch("image"),
async generate(intent, ctx) {
const localPath = join(tmpdir(), `media-use-image-${process.pid}-${Date.now()}.png`);
write(await image({ prompt: intent, size: ctx?.size }), localPath);
return {
url: best.url,
source: "search",
// ext derived from the asset URL by resolve.mjs (.jpg/.png/.webp)
localPath,
ext: ".png",
source: "generated",
metadata: {
description: intent,
width: best.width || null,
height: best.height || null,
transparent: best.is_transparent || false,
provider: "heygen.asset.search",
provenance: { asset_id: best.id, score: best.score },
provider: "hanzo.image",
provenance: { prompt: intent, ...(ctx?.size ? { size: ctx.size } : {}) },
},
};
},
};
export const iconProvider = {
async search(intent) {
// No minScore: the `asset search` backend rejects --min-score and returns no score field.
const results = heygenSearch("asset search", intent, { type: "icon" });
if (!results) return null;
const best = results[0];
return {
url: best.url,
source: "search",
// ext derived from the asset URL by resolve.mjs — catalog icons are .png, not .svg
metadata: {
description: intent,
width: best.width || null,
height: best.height || null,
transparent: best.is_transparent ?? true,
provider: "heygen.asset.search",
provenance: { asset_id: best.id, score: best.score, type: "icon" },
},
};
},
};
export const iconProvider = { search: catalogSearch("icon") };
skills/media-use/scripts/lib/local-models.mjs
-290
View File
@@ -1,290 +0,0 @@
// Declarative table of USER-INSTALLED local models, for the spec-gated fallback.
//
// These models run on the user's own machine for their own use; media-use
// recommends, spec-checks, and assists install; it does not bundle, redistribute,
// or sell them. Because nothing is redistributed, selection is purely by
// quality / size / spec-fit / word-timestamp support (there is deliberately NO
// license field gating availability).
//
// Tiers (`small`|`medium`|`large`|`xlarge`) are human labels; `needs.ramMB` is
// what selection actually gates on. selectModel() returns the best model that
// fits the machine's AVAILABLE RAM, best-first: by explicit `rank` when set
// (quality that is NOT size, e.g. ASR), else by RAM footprint (the quality
// proxy for generation). No fit -> recommend the CLI/cloud path.
//
// Picks reflect the 2026 research pass, verified live where noted.
export const CAPABILITIES = ["tts", "asr", "upscale", "videogen", "imagegen"];
const MODELS = {
tts: [
{
id: "kokoro",
tier: "medium",
sizeMB: 330,
needs: { ramMB: 2048, gpu: false },
wordTimestamps: "native",
install: "pip install kokoro",
invoke: "python -m kokoro --text {text} --voice {voice} --out {out}",
notes: "CPU, faster-than-realtime, native per-word timestamps. Default floor.",
},
{
id: "fish-speech",
tier: "large",
sizeMB: 1100,
needs: { ramMB: 16000, gpu: true, vramMB: 12000 },
wordTimestamps: "whisperx", // needs forced alignment (run ASR over output)
install: "pip install fish-speech",
invoke: "fish-speech synth --text {text} --ref {ref} --out {out}",
notes: "Expressive zero-shot voice cloning; meeting pick. WhisperX for word timing.",
},
],
asr: [
// Parakeet is BETTER than Whisper yet SMALLER (0.6B vs 1.5B), so quality is
// not size here: `rank` pins it ahead of whisper regardless of footprint.
// Open ASR Leaderboard avg WER: Parakeet ~6.05% vs whisper-large-v3 7.44%
// (~19% better); on NOISY test-other 4.73% vs 5.96%, and whisper-v3
// hallucinated to 308% WER on meetings where Parakeet held. 5-10x faster.
//
// Cohere Transcribe 2B tops the leaderboard (5.42%) and is nominally the most
// accurate, but its mlx-audio community MLX quants (4bit AND 8bit, with and
// without --language en) produced multilingual token-soup garbage AND ran
// 40-70x slower than Parakeet on a 24GB Mac (live-tested 2026-07). Excluded
// until the mlx-audio Cohere decoder stabilizes; Parakeet is the default.
{
id: "parakeet-mlx",
tier: "small",
rank: 0,
sizeMB: 2400,
needs: { ramMB: 4000, gpu: true },
wordTimestamps: "tokens", // sub-word tokens; merged to words by parakeet-words.mjs
repo: "mlx-community/parakeet-tdt-0.6b-v3",
install:
"uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx",
invoke:
"parakeet-mlx {audio} --model mlx-community/parakeet-tdt-0.6b-v3 --output-format json --output-dir {outdir}",
notes:
"NVIDIA Parakeet-TDT 0.6B via parakeet-mlx. VERIFIED on 24GB: accurate transcript, ~3s (cached model) for 8s audio, word timestamps drive transcript-cut. English + 25 European languages. Beats whisper.cpp on accuracy (6.05% vs 7.44% WER) AND speed (5-10x).",
},
{
id: "whisperx",
tier: "medium",
rank: 1,
sizeMB: 1500,
needs: { ramMB: 4096, gpu: false },
wordTimestamps: "native", // faster-whisper + wav2vec2 forced alignment
install: "pip install whisperx",
invoke: "whisperx {audio} --output_format json --out {out}",
notes:
"CPU-only fallback (no GPU): faster-whisper + wav2vec2 forced alignment, native word timestamps. The packaged `frames transcribe` (whisper.cpp) is the zero-setup baseline below this.",
},
],
upscale: [
{
id: "real-esrgan",
tier: "medium",
sizeMB: 70,
needs: { ramMB: 2048, gpu: false },
wordTimestamps: false,
install: "brew install real-esrgan-ncnn-vulkan # or download the ncnn binary",
invoke: "realesrgan-ncnn-vulkan -i {in} -o {out} -s 4",
notes: "ncnn-vulkan binary, CPU-capable. GFPGAN for faces.",
},
{
id: "seedvr2",
tier: "large",
sizeMB: 6000,
needs: { ramMB: 24000, gpu: true, vramMB: 16000 },
wordTimestamps: false,
install: "pip install seedvr2",
invoke: "seedvr2 upscale --in {in} --out {out}",
notes: "Diffusion upscaler, GPU-only. Video2X for video.",
},
],
videogen: [
// 2026-07 X research pass + live verification on a 24GB M-series Mac.
// The Mac-local video story is LTX 2.3 on MLX via dgrauet/ltx-2-mlx (the
// pipeline these weights were converted for; also powers Phosphene).
// Wan 2.x MLX exists only as A14B conversions (too large for consumer
// unified memory); revisit when a 5B Wan MLX conversion lands.
// IMPORTANT: download the weights with a targeted include list first;
// pointing tools at the repo blind snapshot-downloads all 60 GB:
// hf download dgrauet/ltx-2.3-mlx-q4 --include \
// transformer-distilled-1.1.safetensors connector.safetensors \
// "vae_*.safetensors" audio_vae.safetensors vocoder.safetensors "*.json"
{
id: "ltx-2.3-mlx-q4",
tier: "medium",
sizeMB: 20000, // distilled subset; gemma-3-12b-4bit text encoder adds ~7GB
needs: { ramMB: 16384, gpu: true },
wordTimestamps: false,
install:
"git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras",
invoke:
"ltx-2-mlx generate --prompt {prompt} --distilled --low-ram --model dgrauet/ltx-2.3-mlx-q4 --width {w} --height {h} --frames {frames} --frame-rate 24 --output {out}",
notes:
"LTX 2.3 int4 on MLX. Verified on 24GB unified: 512x320 x 33 frames in ~19 min cold (incl. text-encoder download), t2v with audio. Dims must be multiples of 64. i2v, retake/extend, keyframe interpolation supported.",
},
{
id: "ltx-2.3-mlx-bf16",
tier: "large",
sizeMB: 45000,
needs: { ramMB: 32768, gpu: true },
wordTimestamps: false,
install:
"git clone https://github.com/dgrauet/ltx-2-mlx && cd ltx-2-mlx && uv sync --all-extras",
invoke:
"ltx-2-mlx generate --prompt {prompt} --two-stage --model dgrauet/ltx-2.3-mlx-bf16 --width {w} --height {h} --frames {frames} --frame-rate 24 --output {out}",
notes:
"Full-precision two-stage pipeline (upstream production default). 32GB with --low-ram block streaming; 64-128GB Macs for long/HD runs (the 25s multi-scene spots seen in the wild).",
},
],
imagegen: [
// 2026-07 X research + live verification on a 24GB M-series Mac. mflux
// (FLUX-on-MLX) is the Mac-native runner; FLUX is the quality leader. Two
// hard-won findings baked into `needs.ramMB`:
// 1. The OFFICIAL FLUX repos are HF-gated (license wall). Point --path at a
// non-gated community 4-bit re-upload (self-contained, incl. VAE).
// 2. Without --low-ram, FLUX's T5-XXL text encoder + transformer blow past
// 24GB into swap: a 768x512 run took 90 MINUTES. With --low-ram (streams
// components from disk) the SAME machine did 512x512 in ~20s at 7.6GB
// free. So the medium tier's needs.ramMB is the streamed floor, not the
// resident footprint; the large tiers are the no-streaming thresholds.
// The runner resolves `repo` to a local snapshot (hf download) before --path;
// a bare repo id in --path breaks mlx unflatten.
{
id: "flux-schnell-mflux-q4",
tier: "medium",
sizeMB: 8700,
needs: { ramMB: 8000, gpu: true },
repo: "dhairyashil/FLUX.1-schnell-mflux-4bit",
wordTimestamps: false,
install: "uv venv ~/.venvs/mflux && VIRTUAL_ENV=~/.venvs/mflux uv pip install mflux==0.9.6",
invoke:
"mflux-generate --model schnell --path {model_path} --low-ram --steps 4 --prompt {prompt} --width {w} --height {h} --seed {seed} --output {out}",
notes:
"FLUX.1 schnell int4. VERIFIED on 24GB (7.6GB free): --low-ram 512x512 in ~20s, photoreal. --low-ram is MANDATORY at this tier (streams to avoid swap). Few-step, fast.",
},
{
id: "flux2-klein-mflux-q4",
tier: "large",
sizeMB: 12000,
needs: { ramMB: 32000, gpu: true },
repo: "Runpod/FLUX.2-klein-4B-mflux-4bit",
wordTimestamps: false,
install: "uv venv ~/.venvs/mflux && VIRTUAL_ENV=~/.venvs/mflux uv pip install mflux",
invoke:
"mflux-generate --base-model flux2-klein-4b --path {model_path} --steps 8 --prompt {prompt} --width {w} --height {h} --seed {seed} --output {out}",
notes:
"FLUX.2 Klein 4B int4 (most-downloaded mflux community repo). Newer, higher quality than schnell; full-resident (no streaming) so needs 32GB+ to stay fast. Needs mflux >= 0.18 for the flux2-klein base model.",
},
{
id: "qwen-image-mflux",
tier: "xlarge",
sizeMB: 40000,
needs: { ramMB: 64000, gpu: true },
repo: "Qwen/Qwen-Image",
wordTimestamps: false,
install: "uv venv ~/.venvs/mflux && VIRTUAL_ENV=~/.venvs/mflux uv pip install mflux",
invoke:
"mflux-generate --base-model qwen --steps 20 --prompt {prompt} --width {w} --height {h} --seed {seed} --output {out}",
notes:
"Qwen-Image, top-tier quality. Heavy: 'several minutes' even on 128GB M4 Max, 'almost fried' a 32GB M4 Pro. 64GB+ only. Below that, the cloud upsell (codex) is faster and better.",
},
],
};
function tableFor(capability) {
const t = MODELS[capability];
if (!t) throw new Error(`unknown local-model capability: ${capability}`);
return t;
}
/** All local models for a capability. */
export function listModels(capability) {
return tableFor(capability).slice();
}
// Tokenize an `invoke` template on whitespace first, then substitute each
// token, so a `{prompt}`/`{model_path}` value with spaces stays a single argv
// entry. Shared by every local-model provider (mflux, LTX) that builds argv
// from a MODELS[...].invoke template.
export function buildArgv(template, vars) {
return template
.trim()
.split(/\s+/)
.map((tok) => tok.replace(/\{(\w+)\}/g, (_, k) => (k in vars ? String(vars[k]) : `{${k}}`)));
}
/** Does this machine meet a model's needs? Apple Silicon unified memory counts as VRAM. */
export function meetsSpecs(model, specs) {
const n = model.needs || {};
// Gate on AVAILABLE RAM when the probe reported it (the real budget with the
// OS + open apps resident); fall back to total RAM otherwise. Older specs
// objects (and unit fixtures) that only set ramMB keep working unchanged.
const budget = specs.availableRamMB ?? specs.ramMB;
if (n.ramMB && budget < n.ramMB) return false;
if (n.gpu && !specs.gpu?.present) return false;
if (n.vramMB) {
const vram = specs.gpu?.vramMB ?? 0;
if (vram < n.vramMB) return false;
}
return true;
}
// "Best model the machine can run" == best-first among those that fit. Ordering:
// 1. explicit `rank` (lower = better) when a model declares it. Needed where
// quality is NOT size: Parakeet-0.6B beats Whisper-large-1.5B at ASR, so
// footprint would pick the wrong one.
// 2. otherwise RAM footprint descending, the quality proxy for generation
// (a 40GB image model out-renders a 12GB one).
function rankedByPreference(table) {
return [...table].sort((a, b) => {
const ra = a.rank ?? Infinity;
const rb = b.rank ?? Infinity;
if (ra !== rb) return ra - rb;
return (b.needs?.ramMB ?? 0) - (a.needs?.ramMB ?? 0);
});
}
/**
* Pick the best local model the machine can run for a capability: the
* highest-footprint model that fits the available-RAM budget (and GPU/VRAM).
* `preferTier` pins the search to one tier (e.g. force a smaller/faster model).
* Returns `{ model, tier }`, or `{ recommend: "cli", reason }` when nothing fits.
*/
export function selectModel(capability, specs, { preferTier } = {}) {
const table = tableFor(capability);
const pool = preferTier ? table.filter((m) => m.tier === preferTier) : table;
for (const model of rankedByPreference(pool)) {
if (meetsSpecs(model, specs)) return { model, tier: model.tier };
}
const smallest = table.reduce((a, b) => (a.sizeMB <= b.sizeMB ? a : b));
return {
recommend: "cli",
reason: `machine does not meet specs for any local ${capability} model (smallest needs ~${smallest.needs.ramMB}MB RAM${smallest.needs.gpu ? " + GPU" : ""}); use the CLI path instead`,
};
}
/**
* Agent-facing ladder: every model for a capability, best-first, each flagged
* with whether it fits this machine and why. Lets the agent see the RAM-graded
* options and choose (e.g. trade the auto-picked best for a smaller/faster one,
* or step up to a cloud upsell) rather than only getting one auto-selection.
*/
export function describeModelLadder(capability, specs) {
const budget = specs.availableRamMB ?? specs.ramMB;
return rankedByPreference(tableFor(capability)).map((model) => {
const fits = meetsSpecs(model, specs);
return {
id: model.id,
tier: model.tier,
needsRamMB: model.needs?.ramMB ?? 0,
fits,
reason: fits
? `fits (needs ~${model.needs?.ramMB}MB, ${budget}MB available)`
: `too big (needs ~${model.needs?.ramMB}MB${model.needs?.gpu ? " + GPU" : ""}, ${budget}MB available)`,
notes: model.notes,
};
});
}
skills/media-use/scripts/lib/local-models.test.mjs
-154
View File
@@ -1,154 +0,0 @@
import { strict as assert } from "node:assert";
import { test } from "node:test";
import {
listModels,
meetsSpecs,
selectModel,
describeModelLadder,
CAPABILITIES,
} from "./local-models.mjs";
const TIERS = ["small", "medium", "large", "xlarge"];
const strongGpu = {
ramMB: 64000,
gpu: { present: true, kind: "nvidia", vramMB: 24000 },
appleSilicon: false,
};
const cpuOnly = { ramMB: 16000, gpu: { present: false, vramMB: 0 }, appleSilicon: false };
const tiny = { ramMB: 1024, gpu: { present: false, vramMB: 0 }, appleSilicon: false };
test("every capability table is non-empty and well-formed", () => {
for (const cap of CAPABILITIES) {
const models = listModels(cap);
assert.ok(models.length > 0, `no models for ${cap}`);
for (const m of models) {
assert.ok(m.id && m.tier && m.needs, `${cap}/${m.id} missing fields`);
assert.ok(TIERS.includes(m.tier), `${cap}/${m.id} bad tier: ${m.tier}`);
assert.equal(typeof m.install, "string", `${cap}/${m.id} needs an install command`);
assert.equal(typeof m.invoke, "string", `${cap}/${m.id} needs an invoke command`);
// user-installed, local-use-only: there is NO license gate on selection
assert.equal("license" in m, false, `${cap}/${m.id} must not carry a license gate`);
}
}
});
test("meetsSpecs enforces RAM, GPU presence, and VRAM", () => {
const gpuModel = { needs: { ramMB: 8000, gpu: true, vramMB: 12000 } };
assert.equal(meetsSpecs(gpuModel, strongGpu), true);
assert.equal(meetsSpecs(gpuModel, cpuOnly), false, "no GPU -> fails a GPU model");
const cpuModel = { needs: { ramMB: 2000, gpu: false } };
assert.equal(meetsSpecs(cpuModel, cpuOnly), true);
assert.equal(meetsSpecs(cpuModel, tiny), false, "too little RAM");
});
test("Apple Silicon unified memory counts as VRAM", () => {
const apple = {
ramMB: 24000,
appleSilicon: true,
gpu: { present: true, kind: "apple", vramMB: 24000 },
};
const gpuModel = { needs: { ramMB: 8000, gpu: true, vramMB: 16000 } };
assert.equal(meetsSpecs(gpuModel, apple), true);
});
test("selectModel picks the large tier on a strong machine", () => {
const r = selectModel("tts", strongGpu);
assert.equal(r.tier, "large");
assert.ok(r.model.id);
});
test("selectModel falls back to medium on a CPU-only machine", () => {
const r = selectModel("tts", cpuOnly);
assert.equal(r.tier, "medium");
assert.equal(r.model.id, "kokoro", "Kokoro is the CPU/medium default (native word timestamps)");
});
test("selectModel recommends the CLI path when no tier fits", () => {
const r = selectModel("tts", tiny);
assert.equal(r.recommend, "cli");
assert.ok(r.reason && /spec/i.test(r.reason));
assert.equal(r.model, undefined);
});
test("preferTier:'medium' avoids the large model even on a strong machine", () => {
const r = selectModel("tts", strongGpu, { preferTier: "medium" });
assert.equal(r.tier, "medium");
});
test("selectModel gates on AVAILABLE RAM, not total, when both are present", () => {
// 64GB total but only 6GB free right now -> the large tier must not be chosen.
const busy = {
ramMB: 64000,
availableRamMB: 6000,
appleSilicon: true,
gpu: { present: true, kind: "apple", vramMB: 64000 },
};
const r = selectModel("tts", busy);
assert.equal(r.tier, "medium", "available RAM (6GB) rules out the 16GB large tier");
});
test("imagegen is a RAM-graduated ladder; agent picks the best that fits", () => {
const ladder = describeModelLadder("imagegen", {
ramMB: 24000,
availableRamMB: 12000,
appleSilicon: true,
gpu: { present: true, kind: "apple", vramMB: 24000 },
});
// best-first order, each flagged with fit
assert.ok(ladder.length >= 3, "imagegen offers multiple RAM tiers");
assert.ok(
ladder[0].needsRamMB >= ladder[ladder.length - 1].needsRamMB,
"ladder is ordered best (biggest) first",
);
// on 24GB / 12GB-free the schnell --low-ram tier fits, the 32GB+ tiers do not
const fitting = ladder.filter((m) => m.fits);
assert.ok(fitting.length >= 1, "at least the low-ram tier fits a 24GB Mac");
assert.ok(
fitting.every((m) => m.needsRamMB <= 12000),
"only sub-budget models flagged as fitting",
);
const pick = selectModel("imagegen", {
ramMB: 24000,
availableRamMB: 12000,
gpu: { present: true, vramMB: 24000 },
});
assert.equal(
pick.model.id,
"flux-schnell-mflux-q4",
"best fit on 24GB is the low-ram schnell tier",
);
});
test("imagegen on a 64GB Mac steps up to the higher-quality tier", () => {
const pick = selectModel("imagegen", {
ramMB: 96000,
availableRamMB: 80000,
gpu: { present: true, vramMB: 96000 },
});
assert.equal(pick.tier, "xlarge", "80GB free unlocks the top-quality model");
});
test("ASR prefers Parakeet by rank even though it is smaller than whisper", () => {
// quality != size for ASR: Parakeet 0.6B beats whisper-1.5B, so `rank` wins
// over footprint. On a capable machine both fit; Parakeet must be chosen.
const capable = {
ramMB: 24000,
availableRamMB: 12000,
appleSilicon: true,
gpu: { present: true, kind: "apple", vramMB: 24000 },
};
const pick = selectModel("asr", capable);
assert.equal(pick.model.id, "parakeet-mlx", "Parakeet is the rank-0 preferred ASR");
// whisperx (rank 1, CPU-only) is the fallback when no GPU
const cpu = { ramMB: 16000, availableRamMB: 12000, gpu: { present: false, vramMB: 0 } };
assert.equal(selectModel("asr", cpu).model.id, "whisperx", "CPU-only falls back to whisperx");
});
test("ASR offers word-timestamp-capable models (better than plain whisper)", () => {
const asr = listModels("asr");
assert.ok(
asr.every((m) => m.wordTimestamps),
"every ASR model must support word timestamps",
);
});
skills/media-use/scripts/lib/local-run.mjs
-64
View File
@@ -1,64 +0,0 @@
import { execFileSync } from "node:child_process";
import { selectModel } from "./local-models.mjs";
import { probeSpecs } from "./specs.mjs";
// Run a USER-INSTALLED local model for a capability (tts/asr/upscale).
// Picks the best tier the machine supports (selectModel), checks the tool is on
// PATH, fills the model's invoke template, and runs it. Returns:
// { model, tier, out } on success
// { recommend:"install", model, command, reason } when the tool isn't installed
// { recommend:"cli", reason } when no tier fits the machine
// `exec` / `which` are injectable for tests.
//
// ponytail: "installed" = the invoke's first token is on PATH (e.g. `whisperx`,
// `realesrgan-ncnn-vulkan`). For `python -m kokoro` this only proves python
// exists; good enough to gate — the recommend.command names the real package.
// Upgrade to a per-tool probe if a "python present but package missing" run ever
// produces a confusing error instead of a clean recommend.
function defaultWhich(bin) {
execFileSync("command", ["-v", bin], { stdio: "ignore", shell: true });
}
function defaultExec(cmd) {
execFileSync(cmd, { stdio: ["ignore", "pipe", "pipe"], shell: true, timeout: 600000 });
}
const fill = (tpl, vars) =>
tpl.replace(/\{(\w+)\}/g, (_, k) => (vars[k] != null ? String(vars[k]) : ""));
export function runLocalModel(capability, opts = {}) {
const {
specs = probeSpecs(),
exec = defaultExec,
which = defaultWhich,
vars = {},
preferTier,
} = opts;
const sel = selectModel(capability, specs, { preferTier });
if (sel.recommend) return sel; // no tier fits -> recommend the CLI path
const { model } = sel;
const bin = model.invoke.split(/\s+/)[0];
try {
which(bin);
} catch {
return {
recommend: "install",
model: model.id,
command: model.install,
reason: `${model.id} not installed`,
};
}
try {
exec(fill(model.invoke, vars));
} catch (e) {
return {
recommend: "install",
model: model.id,
command: model.install,
reason: e.message || String(e),
};
}
return { model: model.id, tier: sel.tier, out: vars.out };
}
skills/media-use/scripts/lib/local-run.test.mjs
-54
View File
@@ -1,54 +0,0 @@
import { strict as assert } from "node:assert";
import { test } from "node:test";
import { runLocalModel } from "./local-run.mjs";
const strongCpu = { ramMB: 16000, gpu: { present: false, vramMB: 0 }, appleSilicon: false };
const tiny = { ramMB: 512, gpu: { present: false, vramMB: 0 }, appleSilicon: false };
const ok = () => {}; // which/exec that succeed
test("recommends the CLI path when no local tier fits the machine", () => {
const r = runLocalModel("tts", { specs: tiny, which: ok, exec: ok });
assert.equal(r.recommend, "cli");
});
test("recommends install when the tool is not on PATH", () => {
const r = runLocalModel("tts", {
specs: strongCpu,
which: () => {
throw new Error("not found");
},
exec: ok,
vars: { text: "hi", out: "/tmp/v.wav" },
});
assert.equal(r.recommend, "install");
assert.equal(r.model, "kokoro");
assert.match(r.command, /pip install kokoro/);
});
test("runs the model and returns the output path when installed", () => {
let ran = "";
const r = runLocalModel("tts", {
specs: strongCpu,
which: ok,
exec: (cmd) => {
ran = cmd;
},
vars: { text: "hello world", voice: "af_heart", out: "/tmp/v.wav" },
});
assert.equal(r.model, "kokoro");
assert.equal(r.out, "/tmp/v.wav");
assert.match(ran, /hello world/, "invoke template filled with vars");
assert.match(ran, /\/tmp\/v\.wav/);
});
test("a failing run degrades to an install recommendation, never throws", () => {
const r = runLocalModel("upscale", {
specs: strongCpu,
which: ok,
exec: () => {
throw new Error("boom");
},
vars: { in: "a.png", out: "b.png" },
});
assert.equal(r.recommend, "install");
});
skills/media-use/scripts/lib/logo-provider.mjs
+5 -5
View File
@@ -16,10 +16,10 @@
// 4. domain favicon — small-raster last resort (DuckDuckGo ip3). Responses
// under ~500B are DDG's globe placeholder, not a hit.
//
// HeyGen asset search is deliberately absent: for brand queries it returns
// generic look-alike icons (0/3 in testing) — worse than a miss. A total miss
// falls through to resolve's normal failure path (`no provider could resolve
// logo`, exit 1).
// A general image search is deliberately absent: for brand queries it returns
// generic look-alike icons, which is worse than a miss. A total miss falls
// through to resolve's normal failure path (`no provider could resolve logo`,
// exit 1).
import { mkdtempSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
@@ -55,7 +55,7 @@ const GITHUB_ORGS = {
vercel: "vercel",
nextjs: "vercel",
alibaba: "alibaba",
heygen: "heygen-com",
hanzo: "hanzoai",
};
// Favicon domains that aren't `<entity>.com`.
skills/media-use/scripts/lib/logo-provider.test.mjs
+1 -1
View File
@@ -41,7 +41,7 @@ test("simple-icons slugs cover the renamed entries", () => {
test("github avatar tier never guesses an org", () => {
assert.equal(githubOrgFor("slack"), "slackhq");
assert.equal(githubOrgFor("heygen"), "heygen-com");
assert.equal(githubOrgFor("hanzo"), "hanzoai");
assert.equal(githubOrgFor("some-random-startup"), null);
});
skills/media-use/scripts/lib/ltx-video-provider.mjs
-70
View File
@@ -1,70 +0,0 @@
import { execFileSync } from "node:child_process";
import { existsSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { probeSpecs } from "./specs.mjs";
import { buildArgv, selectModel } from "./local-models.mjs";
export async function ltxVideoGenerate(
intent,
ctx,
execFn = execFileSync,
pathExists = existsSync,
) {
const specs = ctx?.specs || probeSpecs();
const sel = selectModel("videogen", specs, { preferTier: ctx?.preferTier });
if (sel.recommend) {
console.error(
`media-use: local video gen not enabled (${sel.reason}). Enable a fitting free on-device LTX model to use this provider.`,
);
return null;
}
const { model } = sel;
const bin = model.invoke.trim().split(/\s+/)[0];
try {
execFn("which", [bin], { stdio: ["ignore", "ignore", "ignore"] });
} catch {
console.error(
`media-use: local video gen not enabled (\`${bin}\` not on PATH). Install for free on-device LTX: ${model.install}`,
);
return null;
}
const outPath = join(tmpdir(), `media-use-ltx-${process.pid}-${Date.now()}.mp4`);
const width = ctx?.width || 512;
const height = ctx?.height || 320;
const frames = ctx?.frames || 33;
const argv = buildArgv(model.invoke, {
prompt: intent,
w: width,
h: height,
frames,
out: outPath,
});
argv.shift();
try {
execFn(bin, argv, {
encoding: "utf8",
timeout: 1_800_000,
stdio: ["ignore", "pipe", "pipe"],
});
} catch (err) {
console.error(
`media-use: local video gen (${model.id}) failed: ${err.stderr?.toString().trim().slice(-200) || err.message}`,
);
return null;
}
if (!pathExists(outPath)) return null;
return {
localPath: outPath,
ext: ".mp4",
source: "generated",
metadata: {
description: intent,
provider: "ltx.local",
provenance: { prompt: intent },
},
};
}
skills/media-use/scripts/lib/ltx-video-provider.test.mjs
-136
View File
@@ -1,136 +0,0 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { dirname } from "node:path";
import { tmpdir } from "node:os";
import { ltxVideoGenerate } from "./ltx-video-provider.mjs";
const fittingSpecs = { availableRamMB: 20000, gpu: { present: true } };
test("no fitting local model: falls through without checking for a binary", async (t) => {
t.mock.method(console, "error", () => {});
const calls = [];
const result = await ltxVideoGenerate(
"a calm ocean wave at sunset",
{ specs: { availableRamMB: 100, gpu: { present: true } } },
(...call) => calls.push(call),
() => true,
);
assert.equal(result, null);
assert.deepEqual(calls, []);
});
test("binary missing from PATH: prints the model install hint and falls through", async (t) => {
const errors = [];
t.mock.method(console, "error", (message) => errors.push(message));
const calls = [];
const fakeExec = (...call) => {
calls.push(call);
throw new Error("not found");
};
const result = await ltxVideoGenerate(
"a calm ocean wave at sunset",
{ specs: fittingSpecs },
fakeExec,
);
assert.equal(result, null);
assert.equal(calls.length, 1);
assert.deepEqual(calls[0].slice(0, 2), ["which", ["ltx-2-mlx"]]);
assert.equal(errors.length, 1);
assert.match(errors[0], /git clone https:\/\/github\.com\/dgrauet\/ltx-2-mlx/);
});
test("generate argv substitutes a spaced prompt after tokenizing and uses verified defaults", async () => {
const calls = [];
const checkedPaths = [];
const fakeExec = (...call) => calls.push(call);
const pathExists = (path) => {
checkedPaths.push(path);
return false;
};
const intent = "a calm ocean wave at sunset";
const result = await ltxVideoGenerate(intent, { specs: fittingSpecs }, fakeExec, pathExists);
assert.equal(result, null);
assert.equal(calls.length, 2);
const [bin, argv, opts] = calls[1];
assert.equal(bin, "ltx-2-mlx");
assert.equal(opts.timeout, 1_800_000);
const expectedPairs = [
["--prompt", intent],
["--width", "512"],
["--height", "320"],
["--frames", "33"],
["--output", checkedPaths[0]],
];
let previousIndex = -1;
for (const [flag, value] of expectedPairs) {
const index = argv.indexOf(flag);
assert.ok(index > previousIndex, `${flag} should follow the previous required option`);
assert.equal(argv[index + 1], value);
previousIndex = index;
}
assert.equal(argv.filter((arg) => arg === intent).length, 1);
});
test("successful generation returns the generated MP4 result", async () => {
const calls = [];
const fakeExec = (...call) => calls.push(call);
const intent = "a calm ocean wave at sunset";
const result = await ltxVideoGenerate(intent, { specs: fittingSpecs }, fakeExec, () => true);
assert.ok(result);
assert.equal(calls.length, 2);
assert.equal(dirname(result.localPath), tmpdir());
assert.match(result.localPath, /media-use-ltx-\d+-\d+\.mp4$/);
assert.deepEqual(result, {
localPath: result.localPath,
ext: ".mp4",
source: "generated",
metadata: {
description: intent,
provider: "ltx.local",
provenance: { prompt: intent },
},
});
});
test("generate failure returns null instead of throwing", async (t) => {
t.mock.method(console, "error", () => {});
let calls = 0;
const fakeExec = () => {
calls += 1;
if (calls === 2) {
const error = new Error("generation failed");
error.stderr = "LTX failed";
throw error;
}
};
const result = await ltxVideoGenerate(
"storm clouds",
{ specs: fittingSpecs },
fakeExec,
() => true,
);
assert.equal(result, null);
assert.equal(calls, 2);
});
test("missing generated output returns null", async () => {
const result = await ltxVideoGenerate(
"storm clouds",
{ specs: fittingSpecs },
() => {},
() => false,
);
assert.equal(result, null);
});
skills/media-use/scripts/lib/lut-preset-provider.test.mjs
+8 -11
View File
@@ -47,19 +47,19 @@ test("complete-filter intent aliases resolve deterministically", () => {
assert.equal(matchColorLook("creator video").preset, "creator-camcorder");
});
test("library look freezes a validated cube from params offline (--local-only)", async () => {
test("library look freezes a validated cube from params, online or off", async () => {
const projectDir = mkdtempSync(join(tmpdir(), "mu-lut-provider-"));
try {
const match = matchColorLook("teal orange blockbuster");
assert.equal(match.kind, "library");
// localOnly forces the deterministic params path (no network); online, the
// same look downloads its .cube from the CDN url (via "url").
// Bundled looks are params-only: the same bytes on every machine, with or
// without a network. `url` remains for a look someone hosts themselves.
const frozen = await freezeLibraryLut(match, { projectDir, type: "grade", localOnly: true });
assert.match(frozen.localPath, /^\.media\/luts\/grade_001\.cube$/);
assert.ok(existsSync(join(projectDir, frozen.localPath)));
assert.equal(validateCubeFile(join(projectDir, frozen.localPath)).ok, true);
assert.equal(frozen.lut.src, frozen.localPath);
assert.equal(frozen.metadata.provenance.via, "params-fallback");
assert.equal(frozen.metadata.provenance.via, "params");
} finally {
rmSync(projectDir, { recursive: true, force: true });
}
@@ -82,16 +82,13 @@ test("zero-overlap intent returns no preset or library match", () => {
assert.equal(matchColorLook("zqxv imaginary neutron look"), null);
});
test("bundled LUT index entries resolve from params or url", () => {
test("every bundled LUT index entry builds from params, and none needs a host", () => {
for (const entry of readBundledLutIndex()) {
assert.ok(entry.id);
assert.ok(entry.description);
assert.ok(entry.params || entry.url, `${entry.id} should define params or url`);
if (entry.params) {
assert.equal(typeof entry.params, "object");
assert.equal(validateCube(buildCube(entry.params)).ok, true, `${entry.id} params validate`);
}
if (entry.url) assert.equal(typeof entry.url, "string");
assert.equal(typeof entry.params, "object", `${entry.id} should build from params`);
assert.equal(validateCube(buildCube(entry.params)).ok, true, `${entry.id} params validate`);
assert.equal("url" in entry, false, `${entry.id} should not depend on a hosted cube`);
}
});
skills/media-use/scripts/lib/manifest.test.mjs
+1 -1
View File
@@ -52,7 +52,7 @@ function makeRecord(overrides = {}) {
source: "search",
description: "soft minimal ambient",
duration: 11,
provenance: { provider: "heygen.audio.sounds", prompt: "subtle tech" },
provenance: { provider: "catalog.bgm", prompt: "subtle tech" },
...overrides,
};
}
skills/media-use/scripts/lib/mflux-provider.mjs
-86
View File
@@ -1,86 +0,0 @@
import { execFileSync } from "node:child_process";
import { existsSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { probeSpecs } from "./specs.mjs";
import { buildArgv, selectModel } from "./local-models.mjs";
// Local image generation via mflux (FLUX-on-MLX), the Mac-native runner.
// Spec-gated: selectModel("imagegen", specs) returns the best FLUX-class model
// the machine's AVAILABLE RAM can actually run (medium FLUX-schnell --low-ram on
// ~24GB, up to Qwen-Image on 64GB+). When nothing local fits, this returns null
// so the registry falls through to the codex image upsell.
//
// The official FLUX repos are HF-gated, so the model entries point --path at
// non-gated community 4-bit re-uploads; the repo is resolved to a local snapshot
// (hf download, idempotent) because a bare repo id breaks mlx unflatten.
// Resolve an HF repo to its local snapshot dir. `hf download` is idempotent and
// prints the snapshot path as its last line.
function resolveSnapshot(repo) {
const out = execFileSync("hf", ["download", repo], {
encoding: "utf8",
timeout: 1_800_000,
stdio: ["ignore", "pipe", "pipe"],
});
const path = out.trim().split(/\r?\n/).pop()?.trim();
return path && existsSync(path) ? path : null;
}
export async function mfluxImageGenerate(intent, ctx) {
const specs = ctx?.specs || probeSpecs();
const sel = selectModel("imagegen", specs, { preferTier: ctx?.preferTier });
if (sel.recommend) return null; // no local model fits -> codex upsell/fallback
const { model } = sel;
const bin = model.invoke.trim().split(/\s+/)[0];
// Not installed? Surface the exact enable-command FIRST (before the model
// download) so the agent learns the free local path is available instead of
// silently taking the codex upsell.
try {
execFileSync("which", [bin], { stdio: ["ignore", "ignore", "ignore"] });
} catch {
console.error(
`media-use: local image gen not enabled (\`${bin}\` not on PATH). Install for free on-device FLUX: ${model.install}`,
);
return null;
}
const outPath = join(tmpdir(), `media-use-mflux-${process.pid}-${Date.now()}.png`);
const width = ctx?.width || 512;
const height = ctx?.height || 512;
const seed = ctx?.seed ?? 42;
const vars = { prompt: intent, w: width, h: height, seed, out: outPath };
if (model.repo && model.invoke.includes("{model_path}")) {
const snap = model.repo ? resolveSnapshot(model.repo) : null;
if (!snap) return null;
vars.model_path = snap;
}
const argv = buildArgv(model.invoke, vars);
argv.shift(); // drop the bin (already validated)
try {
execFileSync(bin, argv, {
encoding: "utf8",
timeout: 1_800_000,
stdio: ["ignore", "pipe", "pipe"],
});
} catch (err) {
console.error(
`media-use: local image gen (${model.id}) failed: ${err.stderr?.toString().trim().slice(-200) || err.message}`,
);
return null;
}
if (!existsSync(outPath)) return null;
return {
localPath: outPath,
ext: ".png",
source: "generated",
metadata: {
description: intent,
provider: `mflux.${model.id}`,
provenance: { model: model.id, tier: model.tier, prompt: intent },
},
};
}
skills/media-use/scripts/lib/npx-sync.mjs
-31
View File
@@ -1,31 +0,0 @@
import { existsSync } from "node:fs";
import { resolveSpawnCommand } from "../../audio/scripts/lib/tts.mjs";
// Sync-spawn analog of the audio engine's spawnP, for execFileSync call sites
// that must hard-fail (rather than fall through to another provider) when npx
// cannot be resolved. On Windows a bare "npx" is npx.cmd, which
// execFileSync/spawnSync cannot exec (spawnSync npx ENOENT) —
// resolveSpawnCommand reroutes it through node + npx-cli.js, no shell:true.
//
// `platform`/`env`/`pathExists` params (defaulting to the real values) exist
// so tests can exercise the win32 branch without mocking node:child_process
// (its ESM exports are non-configurable) — same idiom as spawnP and
// localTtsGenerate.
export function resolveNpxInvocation(
argv,
opts,
platform = process.platform,
env = process.env,
pathExists = existsSync,
) {
const resolved = resolveSpawnCommand("npx", argv, opts, platform, env, pathExists);
if (!resolved) {
// npx-on-win32 with no resolvable npx-cli.js — same terminal condition
// spawnP warns about, surfaced as a throw for callers with no fallback.
throw new Error(
"cannot run npx on Windows: npm's npx-cli.js was not found " +
"(install npm with Node, or run via npx/npm run so npm_execpath is set)",
);
}
return resolved;
}
skills/media-use/scripts/lib/npx-sync.test.mjs
-50
View File
@@ -1,50 +0,0 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { resolveNpxInvocation } from "./npx-sync.mjs";
// Coverage parity with tts-local-provider.test.mjs, for the whisper.cpp
// fallback's call-site shape (transcribe.mjs runWhisper): the same three
// branches, but with the hard-fail contract — no fallback provider exists,
// so an unresolvable npx must throw actionably instead of returning null.
const envWithNpxCli = {
npm_execpath: "C:/Program Files/nodejs/node_modules/npm/bin/npm-cli.js",
npm_node_execpath: "C:/Program Files/nodejs/node.exe",
};
const npxCliPath = "C:/Program Files/nodejs/node_modules/npm/bin/npx-cli.js";
const pathExists = (path) => path === npxCliPath;
const WHISPER_ARGV = ["frames", "transcribe", "C:\\media\\input.wav", "--dir", "C:\\work"];
const WHISPER_OPTS = { stdio: ["ignore", "pipe", "pipe"], timeout: 1_800_000 };
test("win32: routes the whisper fallback through node + npx-cli, never bare npx", () => {
const resolved = resolveNpxInvocation(
WHISPER_ARGV,
WHISPER_OPTS,
"win32",
envWithNpxCli,
pathExists,
);
assert.equal(resolved.cmd, envWithNpxCli.npm_node_execpath);
assert.equal(resolved.args[0], npxCliPath);
assert.deepEqual(resolved.args.slice(1, 3), ["frames", "transcribe"]);
// execFileSync options survive the rerouting (the timeout bounds the
// whisper build/model download; the pipes surface its errors).
assert.deepEqual(resolved.opts.stdio, ["ignore", "pipe", "pipe"]);
assert.equal(resolved.opts.timeout, 1_800_000);
});
test("win32 without a resolvable npx-cli: throws the actionable install hint", () => {
assert.throws(
() => resolveNpxInvocation(WHISPER_ARGV, WHISPER_OPTS, "win32", {}, () => false),
/npx-cli\.js was not found/,
);
});
test("non-win32: spawns plain npx unchanged", () => {
const resolved = resolveNpxInvocation(WHISPER_ARGV, WHISPER_OPTS, "darwin", {}, () => false);
assert.equal(resolved.cmd, "npx");
assert.deepEqual(resolved.args, WHISPER_ARGV);
});
skills/media-use/scripts/lib/parakeet-words.mjs
-27
View File
@@ -1,27 +0,0 @@
// Merge Parakeet-MLX token timestamps into word timestamps.
//
// parakeet-mlx JSON emits SUB-WORD tokens (" H", "ello", ...) with per-token
// start/end. Captions + transcript-cut need WORD timestamps, so join tokens
// into words on the space boundary: a token whose text starts with a space
// (or the very first token) begins a new word; the rest append. Output matches
// the { words: [{ text, start, end }] } shape the rest of media-use consumes
// (see words.mjs / cutlist.mjs).
export function mergeTokensToWords(parakeet) {
const sentences = Array.isArray(parakeet?.sentences) ? parakeet.sentences : [];
const words = [];
for (const s of sentences) {
for (const t of s.tokens ?? []) {
const raw = typeof t.text === "string" ? t.text : "";
const startsWord = raw.startsWith(" ") || words.length === 0;
if (startsWord) {
words.push({ text: raw.trim(), start: t.start, end: t.end });
} else {
const w = words[words.length - 1];
w.text += raw;
w.end = t.end;
}
}
}
return { text: (parakeet?.text ?? "").trim(), words: words.filter((w) => w.text.length > 0) };
}
skills/media-use/scripts/lib/parakeet-words.test.mjs
-44
View File
@@ -1,44 +0,0 @@
import { strict as assert } from "node:assert";
import { test } from "node:test";
import { mergeTokensToWords } from "./parakeet-words.mjs";
test("mergeTokensToWords joins sub-word tokens on the space boundary", () => {
const parakeet = {
text: "Hello everyone. Um,",
sentences: [
{
tokens: [
{ text: " H", start: 0.0, end: 0.24 },
{ text: "ello", start: 0.24, end: 0.48 },
{ text: " everyone.", start: 0.48, end: 1.28 },
{ text: " Um,", start: 1.28, end: 1.92 },
],
},
],
};
const { words } = mergeTokensToWords(parakeet);
assert.deepEqual(words, [
{ text: "Hello", start: 0.0, end: 0.48 },
{ text: "everyone.", start: 0.48, end: 1.28 },
{ text: "Um,", start: 1.28, end: 1.92 },
]);
});
test("mergeTokensToWords spans multiple sentences and drops empties", () => {
const parakeet = {
text: "Hi there",
sentences: [
{ tokens: [{ text: "Hi", start: 0, end: 0.2 }] },
{ tokens: [{ text: " there", start: 0.5, end: 0.9 }] },
],
};
const { words } = mergeTokensToWords(parakeet);
assert.equal(words.length, 2);
assert.equal(words[1].text, "there");
assert.equal(words[1].start, 0.5);
});
test("mergeTokensToWords tolerates missing sentences/tokens", () => {
assert.deepEqual(mergeTokensToWords({}).words, []);
assert.deepEqual(mergeTokensToWords({ sentences: [{}] }).words, []);
});
skills/media-use/scripts/lib/providers.mjs
+3 -4
View File
@@ -1,5 +1,4 @@
// Back-compat surface for the v1 provider API. The ordered, capability-based
// registry now lives in registry.mjs; this re-exports the v1 helpers so existing
// callers keep working. New code should import from registry.mjs directly
// (getProviders / runCapability).
// The v1 provider surface. The ordered, capability-based registry lives in
// registry.mjs; this names the two functions a caller needs to ask "what serves
// this type?" without reaching into the cascade.
export { getProvider, listTypes } from "./registry.mjs";
skills/media-use/scripts/lib/registry.mjs
+43 -70
View File
@@ -3,22 +3,21 @@
// Each media type maps to an ORDERED list of provider entries. Providers are
// tried in order; the first to return a non-null result wins, which keeps
// resolution deterministic (same request -> same provider -> same file ->
// reproducible renders). heygen-CLI is always first for the types it serves.
// reproducible renders).
//
// An entry exposes any of three capability methods — search / generate /
// process — plus { name }. media-use holds no keys; each external tool owns its
// own auth. Providers, by type:
// - heygen CLI: catalog + TTS, first for every type it serves (OAuth free
// allowance first, then the user's HeyGen billing path)
// - mflux: local FLUX-class image gen, spec-selected to the machine's RAM
// (free, private, offline once cached)
// - codex CLI: image gen on the user's ChatGPT sub — the better-quality upsell
// and the fallback when no local model fits
// - Kokoro (via the frames CLI): local voiceover, free/private fallback
// when HeyGen credentials are absent or --local-only is requested
// process — plus { name }. Everything remote goes through api.hanzo.ai on one
// credential ($HANZO_API_KEY, from Hanzo KMS), so a type's cascade is: look in
// the shared catalog first, make it if the catalog cannot answer.
//
// Generation is local-first, cloud-upsell. `ctx.provider` forces one provider
// (e.g. "make an image with codex").
// catalog.* files under media/<type>/ in our object store — free, instant,
// and byte-identical run to run
// hanzo.* the generating routes: /v1/audio/{speech,music,foley},
// /v1/images/generations, /v1/videos/generations
// bundled.sfx the 21 sound effects shipped inside this skill
// svgl … favicon public brand marks, for the `logo` type only
//
// `ctx.provider` forces one provider (e.g. "resolve this bgm from the catalog").
import { bgmProvider } from "./bgm-provider.mjs";
import { sfxProvider } from "./sfx-provider.mjs";
@@ -31,68 +30,47 @@ import {
githubAvatarSearch,
faviconSearch,
} from "./logo-provider.mjs";
import { heygenTtsGenerate } from "./voice-provider.mjs";
import { heygenVideoGenerate } from "./heygen-video-provider.mjs";
import { ltxVideoGenerate } from "./ltx-video-provider.mjs";
import { localTtsGenerate } from "./tts-local-provider.mjs";
import { codexImageGenerate } from "./codex-provider.mjs";
import { mfluxImageGenerate } from "./mflux-provider.mjs";
import { ttsGenerate } from "./voice-provider.mjs";
import { videoGenerate } from "./video-provider.mjs";
// Provider markers: `network` = hits a remote service (skipped by --local-only).
// `paid` = may cost wallet credits after any OAuth/web-plan free allowance
// (documentation for the agent's cost judgment, X4: agent-initiated paid should
// confirm). HeyGen catalog SEARCH is free; HeyGen TTS is free for eligible
// OAuth CLI users up to the monthly allowance, then follows the user's billing.
const A = (name, caps) => ({ name, ...caps }); // local, free
// Provider markers: `network` = reaches api.hanzo.ai (skipped by --local-only).
// `paid` = metered against the org's balance, which is documentation for the
// agent's cost judgment (X4: agent-initiated paid should confirm). Catalog reads
// are metered per storage operation; generation is metered per model call.
const A = (name, caps) => ({ name, ...caps }); // on this machine, free
const N = (name, caps) => ({ name, network: true, ...caps }); // remote, free
const P = (name, caps) => ({ name, network: true, paid: true, ...caps }); // remote, paid
const P = (name, caps) => ({ name, network: true, paid: true, ...caps }); // remote, metered
// heygen-CLI first. All remote providers are skipped by --local-only.
// Catalog before generation, everywhere it applies. All remote providers are
// skipped by --local-only.
const REGISTRY = {
bgm: [N("heygen.audio.sounds", { search: bgmProvider.search })],
bgm: [
N("catalog.bgm", { search: bgmProvider.search }),
P("hanzo.music", { generate: bgmProvider.generate }),
],
sfx: [
N("heygen.audio.sounds", { search: sfxProvider.search }),
A("bundled.sfx", { search: bundledSfxProvider.search }),
N("catalog.sfx", { search: sfxProvider.search }),
P("hanzo.foley", { generate: sfxProvider.generate }),
],
image: [
N("heygen.asset.search", { search: imageProvider.search }),
// Catalog miss -> generate. Local first (best FLUX-class model the machine's
// RAM can run, spec-selected; free, private, kept under --local-only), then
// the codex CLI on the user's ChatGPT sub as the better-quality upsell and
// the fallback when no local model fits.
A("mflux.local", { generate: mfluxImageGenerate }),
N("codex.image_gen", { generate: codexImageGenerate }),
N("catalog.image", { search: imageProvider.search }),
P("hanzo.image", { generate: imageProvider.generate }),
],
icon: [N("heygen.asset.search", { search: iconProvider.search })],
icon: [N("catalog.icon", { search: iconProvider.search })],
logo: [
// Official brand marks. Tiers verified by a 54-brand stress test (100%
// cascade hit); HeyGen asset search is deliberately absent — it returns
// generic look-alike icons for brand queries. All free, all network →
// Official brand marks, from the sources that publish them. Tiers verified
// by a 54-brand stress test (100% cascade hit). All free, all network, so
// --local-only leaves only the cache rungs.
N("svgl", { search: svglSearch }),
N("simple-icons", { search: simpleIconsSearch }),
N("github.avatar", { search: githubAvatarSearch }),
N("favicon.ddg", { search: faviconSearch }),
],
voice: [
// HeyGen TTS first when credentialed so CLI/OAuth users consume the free
// web-plan allowance (10 min/month) before any paid path. --local-only skips
// it and keeps Kokoro as the private/offline fallback.
// Deliberately kept `paid` (X4 confirm-before-call) even though the first
// 10 min/month are free: the client can't know the remaining allowance, so
// confirming is safer than risking a silent charge once it's spent. (A
// tri-state "quota-first, paid after" would need backend quota state.)
P("heygen.tts", { generate: heygenTtsGenerate }),
A("kokoro.local", { generate: localTtsGenerate }),
],
video: [
// HeyGen avatar video first when credentialed; --local-only skips it and
// keeps LTX as the local fallback.
P("heygen.video", { generate: heygenVideoGenerate }),
A("ltx.local", { generate: ltxVideoGenerate }),
],
voice: [P("hanzo.voice", { generate: ttsGenerate })],
video: [P("hanzo.video", { generate: videoGenerate })],
brand: [
// Local design spec, not heygen — reads frame.md / design.md tokens.
// Local design spec — reads frame.md / design.md tokens.
A("design_spec", { search: brandProvider.search }),
],
grade: [
@@ -128,19 +106,15 @@ export function providerNamesFor(type) {
}
/**
* Does an override token (full name like "codex.image_gen" or a prefix like
* "codex") match any provider declared for the type? Same match rule as
* Does an override token (a full name like "catalog.bgm" or a prefix like
* "catalog") match any provider declared for the type? Same match rule as
* runProviders, so validation and dispatch never disagree.
*/
export function providerMatches(type, want) {
return providerNamesFor(type).some((n) => n === want || n.startsWith(`${want}.`));
}
/**
* Back-compat shim for the v1 single-provider API. Returns the first declared
* provider for the type (tagged with `type`); throws for an unknown type.
* Kept for v1 callers only — new code should use getProviders/runCapability.
*/
/** The first declared provider for a type, tagged with `type`. */
export function getProvider(type) {
const first = listFor(type)[0] || {};
return { ...first, type };
@@ -156,10 +130,9 @@ export function getProvider(type) {
* safety flag: it must never make a network call. Forcing a network provider
* while offline yields a clean miss (the caller explains the conflict), never a
* silent network request.
* Provider override: `ctx.provider` (a full name like "codex.image_gen" or a
* prefix like "codex") pins resolution to matching providers only — this is how
* a user "make an image WITH codex" forces the upsell instead of taking the
* free-first default.
* Provider override: `ctx.provider` (a full name like "hanzo.image" or a prefix
* like "catalog") pins resolution to matching providers only — this is how a
* user "generate this image, do not search" skips the catalog rung.
*/
export async function runProviders(providers, capability, intent, ctx) {
const want = ctx?.provider;
@@ -174,7 +147,7 @@ export async function runProviders(providers, capability, intent, ctx) {
return null;
}
/** Run a capability over the providers for a type (deterministic, heygen-first). */
/** Run a capability over the providers for a type (deterministic, catalog-first). */
export async function runCapability(type, capability, intent, ctx) {
return runProviders(getProviders(type), capability, intent, ctx);
}
skills/media-use/scripts/lib/registry.test.mjs
+50 -63
View File
@@ -30,94 +30,81 @@ test("listTypes exposes the v2 media types", () => {
}
});
test("heygen provider is first for every type it serves", () => {
for (const t of ["bgm", "sfx", "image", "icon"]) {
const first = getProviders(t)[0];
assert.ok(first, `no enabled provider for ${t}`);
assert.match(first.name, /^heygen/, `${t} first provider is ${first.name}`);
test("the catalog rung comes before the generating rung", () => {
for (const t of ["bgm", "image"]) {
const names = providerNamesFor(t);
const cat = names.findIndex((n) => n.startsWith("catalog."));
const gen = names.findIndex((n) => n.startsWith("hanzo."));
assert.ok(cat >= 0, `${t} has no catalog rung`);
assert.ok(gen < 0 || cat < gen, `${t} generates before it searches`);
}
});
test("sanctioned providers only: heygen, local mflux/kokoro/ltx, codex, design spec, logo tiers", () => {
test("sanctioned providers only: catalog, hanzo, bundled sfx, design spec, logo tiers", () => {
const allowed =
/^heygen|^bundled\.sfx$|^mflux\.local$|^kokoro\.local$|^ltx\.local$|^codex\.image_gen$|^design_spec$|^svgl$|^simple-icons$|^github\.avatar$|^favicon\.ddg$|^color_grade\.local$|^cube_lut\.local$/;
/^catalog\.|^hanzo\.|^bundled\.sfx$|^design_spec$|^svgl$|^simple-icons$|^github\.avatar$|^favicon\.ddg$|^color_grade\.local$|^cube_lut\.local$/;
for (const t of listTypes()) {
for (const p of getProviders(t)) {
assert.ok(allowed.test(p.name), `${t} lists unsanctioned provider: ${p.name}`);
for (const name of providerNamesFor(t)) {
assert.match(name, allowed, `${t} declares an unsanctioned provider: ${name}`);
}
}
});
test("image cascade: heygen catalog, then local mflux, then the codex upsell", () => {
test("image cascade: the shared catalog, then generation", () => {
assert.deepEqual(providerNamesFor("image"), ["catalog.image", "hanzo.image"]);
const ps = getProviders("image");
assert.match(ps[0].name, /^heygen/, "heygen catalog first");
const names = ps.map((p) => p.name);
const mflux = ps.find((p) => p.name === "mflux.local");
const codex = ps.find((p) => p.name === "codex.image_gen");
assert.ok(mflux && typeof mflux.generate === "function", "local mflux registered");
assert.ok(codex && typeof codex.generate === "function", "codex upsell registered");
assert.ok(names.indexOf("mflux.local") < names.indexOf("codex.image_gen"), "local before codex");
assert.ok(!mflux.network, "local mflux is kept under --local-only");
assert.ok(codex.network, "codex is network (skipped under --local-only)");
assert.ok(ps.every((p) => p.network), "every image provider reaches the API");
assert.ok(!ps[0].paid, "a catalog read is not a model call");
assert.ok(ps[1].paid, "generation is metered");
});
test("voice cascade: HeyGen TTS first, Kokoro remains the local fallback", () => {
const ps = getProviders("voice");
assert.equal(ps[0].name, "heygen.tts", "HeyGen TTS is first when credentials exist");
assert.ok(ps[0].network, "HeyGen TTS is network (skipped under --local-only)");
assert.ok(ps[0].paid, "HeyGen TTS may bill after the OAuth free allowance");
assert.equal(ps[1].name, "kokoro.local", "local Kokoro is the offline fallback");
assert.ok(!ps[1].network, "local Kokoro kept under --local-only");
assert.ok(!ps[1].paid, "local Kokoro is free");
test("voice is one route, not a cascade", () => {
assert.deepEqual(providerNamesFor("voice"), ["hanzo.voice"]);
const [voice] = getProviders("voice");
assert.ok(voice.network, "voice is network (skipped under --local-only)");
assert.ok(voice.paid, "voice generation is metered");
assert.equal(typeof voice.generate, "function");
});
test("video cascade: HeyGen first, LTX local fallback, generate-only", async () => {
assert.deepEqual(providerNamesFor("video"), ["heygen.video", "ltx.local"]);
assert.equal(providerMatches("video", "ltx.local"), true);
const ps = getProviders("video");
assert.ok(ps[0].network, "HeyGen video is network (skipped under --local-only)");
assert.ok(ps[0].paid, "HeyGen video may bill after the OAuth free allowance");
assert.ok(!ps[1].network, "local LTX is kept under --local-only");
assert.equal(await runCapability("video", "search", "x", {}), null);
test("video is one route, generate-only", () => {
assert.deepEqual(providerNamesFor("video"), ["hanzo.video"]);
const [video] = getProviders("video");
assert.equal(typeof video.search, "undefined", "video has no search capability");
assert.equal(typeof video.generate, "function");
});
test("sfx cascade: HeyGen catalog first, bundled library remains the local fallback", () => {
test("sfx cascade: bundled library first, then the catalog, then foley", () => {
assert.deepEqual(providerNamesFor("sfx"), ["bundled.sfx", "catalog.sfx", "hanzo.foley"]);
const ps = getProviders("sfx");
assert.equal(ps[0].name, "heygen.audio.sounds");
assert.ok(ps[0].network, "HeyGen SFX catalog is network-only");
assert.equal(ps[1].name, "bundled.sfx");
assert.equal(typeof ps[1].search, "function");
assert.ok(!ps[1].network, "bundled SFX remain available offline");
assert.ok(!ps[0].network, "the bundled library is kept under --local-only");
assert.ok(ps[1].network && ps[2].network, "catalog and foley are network");
});
test("ctx.provider forces one generator (e.g. 'make an image WITH codex')", async () => {
test("ctx.provider pins one provider", async () => {
const providers = [
{ name: "heygen.asset.search", network: true, search: async () => null },
{ name: "mflux.local", generate: async () => ({ hit: "local" }) },
{ name: "codex.image_gen", network: true, generate: async () => ({ hit: "codex" }) },
{ name: "catalog.image", network: true, search: async () => null },
{ name: "bundled.image", generate: async () => ({ hit: "bundled" }) },
{ name: "hanzo.image", network: true, generate: async () => ({ hit: "generated" }) },
];
// no override: local wins (first generate to return non-null)
assert.deepEqual(await runProviders(providers, "generate", "x", {}), { hit: "local" });
// override to codex: skip local, use codex even though local would have worked
assert.deepEqual(await runProviders(providers, "generate", "x", { provider: "codex" }), {
hit: "codex",
// no override: the first generate to return non-null wins
assert.deepEqual(await runProviders(providers, "generate", "x", {}), { hit: "bundled" });
// pinned by prefix, and by full name
assert.deepEqual(await runProviders(providers, "generate", "x", { provider: "hanzo" }), {
hit: "generated",
});
// override matches the full name too
assert.deepEqual(
await runProviders(providers, "generate", "x", { provider: "codex.image_gen" }),
{ hit: "codex" },
);
// --local-only wins even over a forced network provider: no network call,
// clean miss (the caller surfaces the conflict). A forced LOCAL provider under
assert.deepEqual(await runProviders(providers, "generate", "x", { provider: "hanzo.image" }), {
hit: "generated",
});
// --local-only wins even over a pinned network provider: no network call,
// clean miss (the caller surfaces the conflict). A pinned LOCAL provider under
// --local-only still runs.
assert.equal(
await runProviders(providers, "generate", "x", { provider: "codex", localOnly: true }),
await runProviders(providers, "generate", "x", { provider: "hanzo", localOnly: true }),
null,
);
assert.deepEqual(
await runProviders(providers, "generate", "x", { provider: "mflux", localOnly: true }),
{ hit: "local" },
await runProviders(providers, "generate", "x", { provider: "bundled", localOnly: true }),
{ hit: "bundled" },
);
});
@@ -185,11 +172,11 @@ test("runCapability('bgm','process') is null — process slot is graceful when u
assert.equal(await runCapability("bgm", "process", "x", {}), null);
});
test("--local-only skips every network provider (even free remote ones)", async () => {
test("--local-only skips every network provider", async () => {
let remoteRan = false;
const providers = [
{
name: "heygen",
name: "remote",
network: true,
search: async () => {
remoteRan = true;
skills/media-use/scripts/lib/sfx-provider.mjs
+7 -18
View File
@@ -1,23 +1,12 @@
import { heygenSearch } from "./heygen-search.mjs";
import { asResult, compose, find } from "./catalog.mjs";
export const sfxProvider = {
async search(intent) {
const results = heygenSearch("audio sounds list", intent, {
type: "sound_effects",
minScore: 0.4,
});
if (!results) return null;
const best = results[0];
return {
url: best.audio_url,
source: "search",
// ext derived from audio_url by resolve.mjs — catalog SFX are .mp3 or .wav
metadata: {
description: best.description || best.name || intent,
duration: best.duration || null,
provider: "heygen.audio.sounds",
provenance: { track_id: best.id, score: best.score, query: intent },
},
};
const hit = await find("sfx", intent);
if (!hit || hit.empty) return null;
return asResult(hit, intent, "sfx");
},
async generate(intent, ctx) {
return compose(intent, { kind: "sfx", seconds: ctx?.seconds });
},
};
skills/media-use/scripts/lib/specs.mjs
-81
View File
@@ -1,81 +0,0 @@
// Machine-capability probe for the spec-gated local-model fallback.
//
// Local models are USER-INSTALLED and local-use-only — media-use recommends,
// spec-checks, and assists install, but never bundles or runs them as a service.
// This probe answers "what tier can this machine actually run?" so selection can
// offer a medium/large local model, or fall back to recommending the CLI path.
//
// `osMod` and `exec` are injectable for tests. `exec(cmd)` returns the command's
// stdout as a string, or throws / returns null on failure.
import os from "node:os";
import { execSync } from "node:child_process";
function defaultExec(cmd) {
return execSync(cmd, { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 4000 });
}
// Available (not just total) RAM — the real budget for "will this model fit
// alongside the OS + open apps". On unified-memory Macs the model shares system
// RAM, so a big model on a busy machine OOMs/thrashes even if total RAM looks
// ample. macOS: vm_stat free + inactive + speculative + purgeable pages (all
// reclaimable). Linux: /proc/meminfo MemAvailable. Fallback: total (optimistic
// but stable when the probe is unavailable, e.g. in tests).
function availableRamMB(platform, exec, totalMB) {
try {
if (platform === "darwin") {
const out = String(exec("vm_stat"));
const pageSize = parseInt((out.match(/page size of (\d+)/) || [])[1] || "16384", 10);
const pages = (name) => {
const m = out.match(new RegExp(`${name}:\\s+(\\d+)`));
return m ? parseInt(m[1], 10) : 0;
};
const reclaimable =
pages("Pages free") +
pages("Pages inactive") +
pages("Pages speculative") +
pages("Pages purgeable");
const mb = Math.round((reclaimable * pageSize) / (1024 * 1024));
if (mb > 0) return mb;
} else if (platform === "linux") {
const out = String(exec("cat /proc/meminfo"));
const m = out.match(/MemAvailable:\s+(\d+)\s+kB/);
if (m) return Math.round(parseInt(m[1], 10) / 1024);
}
} catch {
// probe unavailable — fall through to the total-RAM estimate
}
return totalMB;
}
function detectGpu(platform, arch, ramMB, exec) {
// Apple Silicon: Metal GPU with unified memory — VRAM tracks system RAM.
if (platform === "darwin" && arch === "arm64") {
return { present: true, kind: "apple", vramMB: ramMB };
}
// NVIDIA: query total VRAM. Any failure (no driver, no GPU) -> no GPU.
try {
const out = exec("nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits");
const mb = parseInt(String(out).trim().split(/\r?\n/)[0], 10);
if (Number.isFinite(mb) && mb > 0) return { present: true, kind: "nvidia", vramMB: mb };
} catch {
// fall through — no usable GPU
}
return { present: false, kind: null, vramMB: 0 };
}
export function probeSpecs({ osMod = os, exec = defaultExec } = {}) {
const platform = osMod.platform();
const arch = osMod.arch();
const cpuCores = osMod.cpus().length;
const ramMB = Math.round(osMod.totalmem() / (1024 * 1024));
return {
platform,
arch,
cpuCores,
ramMB,
availableRamMB: availableRamMB(platform, exec, ramMB),
appleSilicon: platform === "darwin" && arch === "arm64",
gpu: detectGpu(platform, arch, ramMB, exec),
};
}
skills/media-use/scripts/lib/specs.test.mjs
-75
View File
@@ -1,75 +0,0 @@
import { strict as assert } from "node:assert";
import { test } from "node:test";
import { probeSpecs } from "./specs.mjs";
// Fake os module + exec so the probe is deterministic across CI machines.
const fakeOs = (over = {}) => ({
platform: () => over.platform ?? "linux",
arch: () => over.arch ?? "x64",
cpus: () => Array.from({ length: over.cores ?? 8 }),
totalmem: () => (over.ramMB ?? 16384) * 1024 * 1024,
});
test("probeSpecs reports structured caps", () => {
const s = probeSpecs({ osMod: fakeOs({ cores: 12, ramMB: 32768 }), exec: () => null });
assert.equal(s.cpuCores, 12);
assert.equal(s.ramMB, 32768);
assert.equal(s.platform, "linux");
assert.equal(s.gpu.present, false);
// probe unavailable (exec returns null) -> availableRamMB falls back to total
assert.equal(s.availableRamMB, 32768);
});
test("availableRamMB is read from /proc/meminfo on Linux", () => {
const exec = (cmd) =>
cmd.includes("meminfo") ? "MemTotal: 33554432 kB\nMemAvailable: 8388608 kB\n" : null;
const s = probeSpecs({ osMod: fakeOs({ platform: "linux", ramMB: 32768 }), exec });
assert.equal(s.availableRamMB, 8192, "8388608 kB -> 8192 MB");
});
test("availableRamMB is summed from reclaimable vm_stat pages on macOS", () => {
const vmStat =
"Mach Virtual Memory Statistics: (page size of 16384 bytes)\n" +
"Pages free: 100000.\n" +
"Pages inactive: 200000.\n" +
"Pages speculative: 50000.\n" +
"Pages purgeable: 10000.\n";
const exec = (cmd) => (cmd.includes("vm_stat") ? vmStat : null);
const s = probeSpecs({
osMod: fakeOs({ platform: "darwin", arch: "arm64", ramMB: 24576 }),
exec,
});
// (100000+200000+50000+10000) pages * 16384 B / 1MiB = 5625 MB
assert.equal(s.availableRamMB, 5625);
});
test("Apple Silicon is detected as a unified-memory GPU", () => {
const s = probeSpecs({
osMod: fakeOs({ platform: "darwin", arch: "arm64", ramMB: 24576 }),
exec: () => null,
});
assert.equal(s.appleSilicon, true);
assert.equal(s.gpu.present, true);
assert.equal(s.gpu.kind, "apple");
// unified memory: VRAM tracks system RAM
assert.equal(s.gpu.vramMB, 24576);
});
test("NVIDIA GPU is detected via nvidia-smi VRAM query", () => {
const exec = (cmd) => (cmd.includes("nvidia-smi") ? "24564" : null);
const s = probeSpecs({ osMod: fakeOs({ platform: "linux" }), exec });
assert.equal(s.gpu.present, true);
assert.equal(s.gpu.kind, "nvidia");
assert.equal(s.gpu.vramMB, 24564);
});
test("no GPU when nvidia-smi is absent / fails", () => {
const s = probeSpecs({
osMod: fakeOs({ platform: "linux" }),
exec: () => {
throw new Error("command not found");
},
});
assert.equal(s.gpu.present, false);
assert.equal(s.gpu.vramMB, 0);
});
skills/media-use/scripts/lib/stats.test.mjs
+3 -3
View File
@@ -42,7 +42,7 @@ test("buildStats aggregates resolves, misses, providers, sources, and reuse", ()
type: "bgm",
source: "search",
ts: isoDaysAgo(0),
provenance: { provider: "heygen.audio.sounds", prompt: "upbeat", via: "url" },
provenance: { provider: "catalog.bgm", prompt: "upbeat", via: "url" },
},
{
id: "image_001",
@@ -67,7 +67,7 @@ test("buildStats aggregates resolves, misses, providers, sources, and reuse", ()
type: "bgm",
reusable: true,
cached_path: cachedFile,
provenance: { provider: "heygen.audio.sounds", reused_by: "agent" },
provenance: { provider: "catalog.bgm", reused_by: "agent" },
},
]);
writeFileSync(
@@ -84,7 +84,7 @@ test("buildStats aggregates resolves, misses, providers, sources, and reuse", ()
assert.equal(stats.total_resolves, 3);
assert.deepEqual(stats.by_type, { bgm: 2, image: 1 });
assert.deepEqual(stats.by_source, { search: 1, "reused-explicit": 1, generated: 1 });
assert.equal(stats.by_provider["heygen.audio.sounds"], 1);
assert.equal(stats.by_provider["catalog.bgm"], 1);
assert.equal(stats.by_via.url, 1);
assert.equal(stats.misses, 1);
assert.equal(stats.hit_rate, 0.75);
skills/media-use/scripts/lib/telemetry.mjs
+8 -41
View File
@@ -1,10 +1,9 @@
// Opt-out usage tracking for media-use, sharing the frames CLI/studio
// identity (packages/cli/src/telemetry): the same install id from
// ~/.frames/config.json, plus a $identify to the HeyGen account on sign-in,
// so a person is one PostHog profile across surfaces — not a fresh id per tool.
// Not fully anonymous by design (it must dedupe): pseudonymous before sign-in,
// account-linked after. Event PROPERTIES stay coarse — media TYPE, resolution
// SOURCE, winning PROVIDER — never the intent text, file names, or paths.
// Opt-out usage tracking for media-use, sharing the frames CLI/studio install
// id (packages/cli/src/telemetry, ~/.frames/config.json) so a person is one
// profile across surfaces rather than a fresh id per tool. Pseudonymous: the
// install id and nothing else — no account, no credential, no email. Event
// PROPERTIES stay coarse — media TYPE, resolution SOURCE, winning PROVIDER —
// never the intent text, file names, or paths.
//
// Same public PostHog project key as the CLI (a write-only ingestion key, safe
// to ship), same opt-outs (DO_NOT_TRACK / FRAMES_NO_TELEMETRY / CI / dev),
@@ -19,7 +18,6 @@ import { join } from "node:path";
const POSTHOG_API_KEY = "phc_zjjbX0PnWxERXrMHhkEJWj9A9BhGVLRReICgsfTMmpx";
const POSTHOG_HOST = "https://us.i.posthog.com";
const TIMEOUT_MS = 1500;
let identifiedAccount = false;
let warnedNonDefaultHost = false;
// Same CI/test signals the test suite itself sets (resolve.test.mjs's U7 test
@@ -128,26 +126,6 @@ function anonymousId() {
}
}
function heygenAccountDistinctId() {
const file = join(process.env.HEYGEN_CONFIG_DIR || join(homedir(), ".heygen"), "credentials");
try {
if (!existsSync(file)) return null;
const raw = readFileSync(file, "utf8").trim();
if (!raw.startsWith("{")) return null;
const parsed = JSON.parse(raw);
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
const user = parsed.user;
if (!user || typeof user !== "object" || Array.isArray(user)) return null;
const id = typeof user.email === "string" && user.email.trim() ? user.email : user.username;
// Lowercased so this joins with the CLI's own identify call regardless of
// the account's stored email casing — two different-case distinct ids
// would otherwise split one person across two PostHog profiles.
return typeof id === "string" && id.trim() ? id.trim().toLowerCase() : null;
} catch {
return null;
}
}
function showTelemetryNotice() {
if (optedOut()) return;
try {
@@ -158,7 +136,7 @@ function showTelemetryNotice() {
console.error(
[
"media-use sends usage telemetry: media type, resolution source, and provider; never intent text, file names, or paths.",
"If you sign in to HeyGen, usage links to your account email or username. Opt out with FRAMES_NO_TELEMETRY=1 or DO_NOT_TRACK=1.",
"Opt out with FRAMES_NO_TELEMETRY=1 or DO_NOT_TRACK=1.",
].join("\n"),
);
writeSharedConfig({ ...config, telemetryNoticeShown: true });
@@ -191,14 +169,6 @@ async function postEvent(event, properties, distinctId) {
]);
}
async function identifyAccount(anonId) {
if (optedOut() || identifiedAccount) return;
const distinctId = heygenAccountDistinctId();
if (!distinctId) return;
identifiedAccount = true;
await postEvent("$identify", { $anon_distinct_id: anonId }, distinctId);
}
/**
* Fire-and-forget a single event to PostHog. Best-effort: awaited with a short
* timeout so a short-lived script flushes before exit, but any failure (offline,
@@ -207,9 +177,7 @@ async function identifyAccount(anonId) {
export async function track(event, properties = {}) {
if (optedOut()) return;
showTelemetryNotice();
const anonId = anonymousId();
await identifyAccount(anonId);
await postEvent(event, properties, anonId);
await postEvent(event, properties, anonymousId());
}
export function __anonymousIdForTest() {
@@ -217,6 +185,5 @@ export function __anonymousIdForTest() {
}
export function __resetTelemetryForTest() {
identifiedAccount = false;
warnedNonDefaultHost = false;
}
skills/media-use/scripts/lib/telemetry.test.mjs
+1 -79
View File
@@ -143,85 +143,7 @@ test("anonymous id adopts a legacy ~/.media/anon-id on upgrade (persona continui
}
});
test("track identifies a signed-in HeyGen account once and still sends events", async () => {
const savedEnv = { ...process.env };
const originalFetch = globalThis.fetch;
const { root, home } = sandbox();
const calls = [];
globalThis.fetch = async (url, options) => {
calls.push({ url, options });
return { ok: true };
};
try {
withoutTelemetryOptOut();
mkdirSync(join(home, ".frames"), { recursive: true });
writeFileSync(
join(home, ".frames/config.json"),
JSON.stringify({ anonymousId: "anon-1" }),
);
mkdirSync(join(home, ".heygen"), { recursive: true });
writeFileSync(
join(home, ".heygen/credentials"),
JSON.stringify({ user: { email: "[email protected]", username: "alice" } }),
);
await track("media_use_resolve", { type: "bgm", source: "search" });
await track("media_use_resolve", { type: "image", source: "generated" });
const batch = parseFetchBodies(calls);
const identify = batch.filter((item) => item.event === "$identify");
assert.equal(identify.length, 1);
assert.equal(identify[0].distinct_id, "[email protected]");
assert.equal(identify[0].properties.$anon_distinct_id, "anon-1");
assert.equal(batch.filter((item) => item.event === "media_use_resolve").length, 2);
} finally {
globalThis.fetch = originalFetch;
restoreEnv(savedEnv);
rmSync(root, { recursive: true, force: true });
__resetTelemetryForTest();
}
});
test("track identifies with a lowercased email regardless of stored casing", async () => {
const savedEnv = { ...process.env };
const originalFetch = globalThis.fetch;
const { root, home } = sandbox();
const calls = [];
globalThis.fetch = async (url, options) => {
calls.push({ url, options });
return { ok: true };
};
try {
withoutTelemetryOptOut();
mkdirSync(join(home, ".frames"), { recursive: true });
writeFileSync(
join(home, ".frames/config.json"),
JSON.stringify({ anonymousId: "anon-3" }),
);
mkdirSync(join(home, ".heygen"), { recursive: true });
writeFileSync(
join(home, ".heygen/credentials"),
JSON.stringify({ user: { email: "[email protected]", username: "alice" } }),
);
await track("media_use_resolve", { type: "bgm", source: "search" });
const batch = parseFetchBodies(calls);
const identify = batch.filter((item) => item.event === "$identify");
assert.equal(identify.length, 1);
// Lowercased so this joins with heygen-cli's own identify call regardless
// of the account's stored email casing -- otherwise the same person could
// split into two PostHog profiles by email case alone.
assert.equal(identify[0].distinct_id, "[email protected]");
} finally {
globalThis.fetch = originalFetch;
restoreEnv(savedEnv);
rmSync(root, { recursive: true, force: true });
__resetTelemetryForTest();
}
});
test("track does not identify when signed out", async () => {
test("every event carries the install id and never an account", async () => {
const savedEnv = { ...process.env };
const originalFetch = globalThis.fetch;
const { root, home } = sandbox();
skills/media-use/scripts/lib/tts-local-provider.mjs
-92
View File
@@ -1,92 +0,0 @@
import { execFileSync } from "node:child_process";
import { existsSync, statSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { resolveSpawnCommand } from "../../audio/scripts/lib/tts.mjs";
// Local voiceover via the packaged Kokoro-82M TTS (the `frames tts` CLI),
// the free/private default now that HeyGen TTS costs wallet credits. Kokoro runs
// on-device (CPU, faster-than-realtime, bundled voices, native word timestamps),
// so no key and no per-call charge. When Kokoro is not set up, this returns null
// and the registry falls through to the HeyGen TTS upsell.
//
// Delegated to the frames CLI (same as transcribe / remove-background), not
// re-implemented here. ffprobe reads the duration back for the ledger.
function probeDurationSeconds(file) {
try {
const out = execFileSync(
"ffprobe",
["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", file],
{ encoding: "utf8", timeout: 15000 },
);
const d = parseFloat(String(out).trim());
return Number.isFinite(d) ? d : undefined;
} catch {
return undefined;
}
}
// `platform`/`execFn`/`env`/`pathExists` params (defaulting to the real
// values) exist so tests can exercise the win32 branch without mocking
// node:child_process (its ESM exports are non-configurable) — same idiom as
// spawnP in ../../audio/scripts/lib/tts.mjs.
export async function localTtsGenerate(
intent,
ctx,
platform = process.platform,
execFn = execFileSync,
env = process.env,
pathExists = existsSync,
) {
const outPath = join(tmpdir(), `media-use-kokoro-${process.pid}-${Date.now()}.wav`);
const argv = ["frames", "tts", intent, "--output", outPath];
if (ctx?.voice) argv.push("--voice", ctx.voice);
if (ctx?.lang && ctx.lang !== "en") argv.push("--lang", ctx.lang);
// On Windows a bare "npx" is npx.cmd, which execFileSync cannot exec
// (spawnSync npx ENOENT) — resolveSpawnCommand reroutes it through
// node + npx-cli.js, same as the audio engine's TTS spawns.
const resolved = resolveSpawnCommand(
"npx",
argv,
{ encoding: "utf8", timeout: 300000, stdio: ["ignore", "pipe", "pipe"] },
platform,
env,
pathExists,
);
if (!resolved) {
// npx-on-win32 with no resolvable npx-cli.js — same terminal condition
// spawnP warns about. Fall through to the next provider rather than crash.
console.error(
"media-use: local voice not enabled (kokoro). Cannot run npx on Windows: " +
"npm's npx-cli.js was not found (install npm with Node, or run via npx/npm run so npm_execpath is set).",
);
return null;
}
try {
execFn(resolved.cmd, resolved.args, resolved.opts);
} catch (err) {
// `frames tts` prints its "kokoro-onnx not installed" hint to stdout
// (clack UI), so read both streams and surface the actionable enable-command
// rather than a bare "Command failed": otherwise resolve silently falls
// through to the PAID HeyGen TTS upsell when free local voice was one pip away.
const out = `${err.stdout?.toString() ?? ""}${err.stderr?.toString() ?? ""}`.trim();
const hint = /not installed|pip install kokoro/i.test(out)
? "install for free on-device voice: pip install kokoro-onnx soundfile (or set FRAMES_PYTHON to a venv that has it)"
: out.slice(-200) || err.message;
console.error(`media-use: local voice not enabled (kokoro). ${hint}`);
return null;
}
if (!existsSync(outPath) || statSync(outPath).size === 0) return null;
return {
localPath: outPath,
ext: ".wav",
source: "generated",
metadata: {
description: intent,
provider: "kokoro.local",
duration: probeDurationSeconds(outPath),
provenance: { engine: "kokoro-82m", prompt: intent },
},
};
}
skills/media-use/scripts/lib/tts-local-provider.test.mjs
-72
View File
@@ -1,72 +0,0 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { localTtsGenerate } from "./tts-local-provider.mjs";
// Regression: on Windows, a bare "npx" is npx.cmd, which execFileSync cannot
// exec — the Kokoro delegation failed with `spawnSync npx ENOENT` instead of
// synthesizing (or cleanly falling through to the next provider). The spawn
// must route through node + npx-cli.js on win32, same as the audio engine's
// TTS spawns (see ../../audio/scripts/lib/tts.spawn.test.mjs).
const envWithNpxCli = {
npm_execpath: "C:/Program Files/nodejs/node_modules/npm/bin/npm-cli.js",
npm_node_execpath: "C:/Program Files/nodejs/node.exe",
};
const npxCliPath = "C:/Program Files/nodejs/node_modules/npm/bin/npx-cli.js";
const pathExists = (path) => path === npxCliPath;
test("win32: routes the frames tts call through node + npx-cli, never bare npx", async () => {
const captured = [];
const fakeExec = (cmd, args, opts) => {
captured.push({ cmd, args, opts });
// Synthesize nothing — the provider then returns null via the
// missing-output check, which is fine: we only assert the spawn shape.
};
await localTtsGenerate(
"hello there",
{ voice: "am_michael" },
"win32",
fakeExec,
envWithNpxCli,
pathExists,
);
assert.equal(captured.length, 1);
assert.equal(captured[0].cmd, envWithNpxCli.npm_node_execpath);
assert.equal(captured[0].args[0], npxCliPath);
assert.deepEqual(captured[0].args.slice(1, 4), ["frames", "tts", "hello there"]);
assert.ok(captured[0].args.includes("--voice"));
// execFileSync options survive the rerouting (pipes are what let the caller
// read the "kokoro-onnx not installed" hint back out).
assert.deepEqual(captured[0].opts.stdio, ["ignore", "pipe", "pipe"]);
});
test("win32 without a resolvable npx-cli: falls through to the next provider (null), no spawn", async () => {
const captured = [];
const fakeExec = (...call) => captured.push(call);
const result = await localTtsGenerate(
"hello",
{},
"win32",
fakeExec,
{}, // no npm_execpath, and pathExists finds nothing
() => false,
);
assert.equal(result, null);
assert.equal(captured.length, 0);
});
test("non-win32: spawns plain npx unchanged", async () => {
const captured = [];
const fakeExec = (cmd, args) => captured.push({ cmd, args });
await localTtsGenerate("hola", { lang: "es" }, "darwin", fakeExec, {}, () => false);
assert.equal(captured.length, 1);
assert.equal(captured[0].cmd, "npx");
assert.deepEqual(captured[0].args.slice(0, 3), ["frames", "tts", "hola"]);
assert.ok(captured[0].args.includes("--lang"));
});
skills/media-use/scripts/lib/video-provider.mjs
+19
View File
@@ -0,0 +1,19 @@
import { tmpdir } from "node:os";
import { join } from "node:path";
import { save, video } from "./api.mjs";
export async function videoGenerate(intent, ctx) {
const url = await video({ prompt: intent, seconds: ctx?.seconds });
const out = join(tmpdir(), `media-use-video-${process.pid}-${Date.now()}.mp4`);
await save(url, out);
return {
localPath: out,
ext: ".mp4",
source: "generated",
metadata: {
description: intent,
provider: "hanzo.video",
provenance: { prompt: intent, ...(Number.isFinite(ctx?.seconds) ? { seconds: ctx.seconds } : {}) },
},
};
}
skills/media-use/scripts/lib/voice-provider.mjs
+17 -49
View File
@@ -1,57 +1,25 @@
import { HEYGEN_CLIENT_SOURCE_ARGV, runHeygenJson } from "./heygen-cli.mjs";
import { MAX_SPEECH_INPUT, speech, write } from "./api.mjs";
import { tmpdir } from "node:os";
import { join } from "node:path";
// Voice / TTS generation via the HeyGen CLI — the only external CLI media-use
// shells (CLI-only invariant: media-use holds no keys; the CLI owns auth).
// Flags verified against `heygen voice speech create --help` (v0.3.0).
function result(url, duration, provider, intent) {
if (!url) return null;
// Voiceover through the one speech route. The service owns its voice catalog;
// pass ctx.voice to choose one, omit it for the service default.
export async function ttsGenerate(intent, ctx) {
if (Buffer.byteLength(intent, "utf8") > MAX_SPEECH_INPUT) {
console.error(`media-use: script exceeds the ${MAX_SPEECH_INPUT}-byte speech limit — split it`);
return null;
}
const out = join(tmpdir(), `media-use-voice-${process.pid}-${Date.now()}.mp3`);
const buf = await speech({ text: intent, voice: ctx?.voice, lang: ctx?.lang, format: "mp3" });
write(buf, out);
return {
url,
localPath: out,
ext: ".mp3",
source: "generated",
metadata: {
description: intent,
provider,
...(duration != null && { duration }),
provenance: { prompt: intent },
provider: "hanzo.voice",
provenance: { prompt: intent, ...(ctx?.voice ? { voice: ctx.voice } : {}) },
},
};
}
// HeyGen TTS requires a starfish-engine voice. Default to the first one the
// catalog returns (deterministic order); pass ctx.voiceId to override.
// ponytail: listed once per process; the resolved asset is frozen + cached after
// first use, so the network list only happens on a cache miss. Cache only a
// truthy id -- a transient list failure must not poison the cache with `null`
// and permanently disable TTS for the rest of the process.
let cachedVoiceId;
function defaultVoiceId() {
if (cachedVoiceId) return cachedVoiceId;
const j = runHeygenJson(
"heygen",
["voice", "list", "--engine", "starfish", "--limit", "1"],
"voice list",
);
cachedVoiceId = j?.data?.[0]?.voice_id || null;
return cachedVoiceId;
}
export async function heygenTtsGenerate(intent, ctx) {
const voiceId = ctx?.voiceId || defaultVoiceId();
if (!voiceId) return null;
const p = runHeygenJson(
"heygen",
[
...HEYGEN_CLIENT_SOURCE_ARGV,
"voice",
"speech",
"create",
"--text",
intent,
"--voice-id",
voiceId,
],
"tts",
);
return result(p?.data?.audio_url, p?.data?.duration, "heygen.tts", intent);
}
skills/media-use/scripts/lib/voice-provider.test.mjs
-46
View File
@@ -1,46 +0,0 @@
import { strict as assert } from "node:assert";
import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { test } from "node:test";
import { heygenTtsGenerate } from "./voice-provider.mjs";
test("tags TTS generation but not voice discovery with the media-use client source", async () => {
const dir = mkdtempSync(join(tmpdir(), "media-use-voice-provider-"));
const capturePath = join(dir, "argv.log");
const heygenPath = join(dir, "heygen");
const previousPath = process.env.PATH;
const previousCapturePath = process.env.HEYGEN_CAPTURE_PATH;
writeFileSync(
heygenPath,
`#!/bin/sh
printf '%s\\n' "$*" >> "$HEYGEN_CAPTURE_PATH"
case "$*" in
*"voice list"*) printf '%s\\n' '{"data":[{"voice_id":"voice-123"}]}' ;;
*"voice speech create"*) printf '%s\\n' '{"data":{"audio_url":"https://example.com/voice.mp3","duration":1.5}}' ;;
esac
`,
);
chmodSync(heygenPath, 0o755);
process.env.PATH = `${dir}:${previousPath ?? ""}`;
process.env.HEYGEN_CAPTURE_PATH = capturePath;
try {
const result = await heygenTtsGenerate("Hello from media-use", {});
const invocations = readFileSync(capturePath, "utf8").trim().split("\n");
assert.equal(invocations.length, 2);
assert.match(invocations[0], /^voice list /);
assert.doesNotMatch(invocations[0], /X-HeyGen-Client-Source/);
assert.match(invocations[1], /voice speech create/);
assert.match(invocations[1], /X-HeyGen-Client-Source: media-use/);
assert.equal(result?.url, "https://example.com/voice.mp3");
} finally {
if (previousPath === undefined) delete process.env.PATH;
else process.env.PATH = previousPath;
if (previousCapturePath === undefined) delete process.env.HEYGEN_CAPTURE_PATH;
else process.env.HEYGEN_CAPTURE_PATH = previousCapturePath;
rmSync(dir, { recursive: true, force: true });
}
});
skills/media-use/scripts/resolve.mjs
+35 -178
View File
@@ -23,7 +23,7 @@ import { buildStats } from "./lib/stats.mjs";
import { typesMatch } from "./lib/match.mjs";
import { listCandidates, formatCandidates, CANDIDATE_CAP } from "./lib/candidates.mjs";
import { findGlobalBySha } from "./lib/cache.mjs";
import { heygenAuthMethod } from "../audio/scripts/lib/heygen.mjs";
import { base as apiBase, ready } from "./lib/api.mjs";
import { buildCube, paramsFromIntent } from "./lib/cube-build.mjs";
import { validateCubeFile } from "./lib/cube-validate.mjs";
import { analyzeMediaGrade, formatMeasuredNote } from "./lib/grade-analyzer.mjs";
@@ -32,16 +32,6 @@ import {
isLibraryLutOfflineMiss,
matchColorLook,
} from "./lib/lut-preset-provider.mjs";
import {
HEYGEN_AUTH_COMMAND,
HEYGEN_INSTALL_COMMAND,
HEYGEN_MIN_VERSION,
HEYGEN_UPDATE_COMMAND,
consumeHeygenRemediation,
firstSemver,
flushHeygenFailureTracking,
versionLessThan,
} from "./lib/heygen-cli.mjs";
import { BundledSfxAssetsError, inspectBundledSfxAssets } from "./lib/bundled-sfx-provider.mjs";
const INGEST_TYPES = listTypes();
@@ -82,7 +72,6 @@ const { values: args } = parseArgs({
analyze: { type: "boolean", default: false },
"local-only": { type: "boolean", default: false },
provider: { type: "string" },
"avatar-id": { type: "string" },
"voice-id": { type: "string" },
json: { type: "boolean", default: false },
help: { type: "boolean", short: "h", default: false },
@@ -118,9 +107,8 @@ Options:
suggestions (grade only)
--analyze Return --for grade evidence without recording a candidate
--local-only Offline: skip every network provider
--provider Force one generator (e.g. codex, mflux, kokoro, heygen)
--avatar-id Override the default avatar for heygen.video generation
--voice-id Override the default voice for voice/heygen.video generation
--provider Pin one provider (e.g. catalog, hanzo.image)
--voice-id Voice id for voice generation (omit for the service default)
--json Output JSON instead of one-line result
--help, -h Show this help`);
process.exit(0);
@@ -289,15 +277,6 @@ function recordAvailable(projectDir, record) {
return record.type === "grade" && record.grading;
}
// Sparse `{ authMethod }` for a heygen-family provider name (e.g. "heygen.tts"),
// else `{}` — keeps auth_method telemetry absent for every non-heygen resolve
// instead of implying an auth method that doesn't apply.
function heygenAuthMethodFor(provider) {
if (!provider || !provider.startsWith("heygen.")) return {};
const authMethod = heygenAuthMethod();
return authMethod ? { authMethod } : {};
}
function localizeImportedRecord(record, localPath) {
if (record?.type === "grade" && record.grading?.lut) {
record.grading = {
@@ -383,7 +362,7 @@ async function run() {
}
}
// Offline guard: --local-only skips every remote provider (HeyGen catalog),
// Offline guard: --local-only skips every remote provider (the shared catalog),
// leaving the project + global cache and any local provider.
const localOnly = args["local-only"];
const ctx = {
@@ -391,8 +370,7 @@ async function run() {
projectDir,
localOnly,
provider: args.provider,
avatarId: args["avatar-id"],
voiceId: args["voice-id"],
voice: args["voice-id"],
};
// Adherence nudge (offline, no auto-reuse): the exact-cache floor missed and
@@ -415,7 +393,7 @@ async function run() {
return resolveColor(type, intent, { projectDir });
}
// 3. provider search — registry tries providers in order (heygen-CLI first)
// 3. provider search — registry tries providers in order (catalog first)
let searchResult = null;
let providerFailure = null;
try {
@@ -435,14 +413,6 @@ async function run() {
}
}
// A search/generate attempt against heygen may have fired a fire-and-forget
// media_use_provider_error track (reportHeygenFailure — heygen-search.mjs /
// voice-provider.mjs are sync call sites several layers below here and can't
// await it themselves). Join it now, before any process.exit() below can
// race it: both it and the miss/success telemetry below are separate,
// non-keepalive HTTP connections with no ordering guarantee otherwise.
await flushHeygenFailureTracking();
if (!searchResult) {
await track("media_use_resolve_miss", {
type,
@@ -518,25 +488,10 @@ async function run() {
provenance: {
provider: searchResult.metadata?.provider || "unknown",
prompt: intent,
// heygenAuthMethodFor spreads first so an explicit authMethod on a
// future provider's own metadata.provenance can still override it below
// -- safe today (no provider sets authMethod itself), but keep this
// ordering if that ever changes.
...heygenAuthMethodFor(searchResult.metadata?.provider),
...searchResult.metadata?.provenance,
},
};
const heygenRemediation = consumeHeygenRemediation();
if (
searchResult.metadata?.provider === "bundled.sfx" &&
!localOnly &&
!args.provider &&
heygenRemediation
) {
record.advisory = heygenRemediation;
}
appendRecord(projectDir, record);
regenerateIndex(projectDir);
// Auto-promote: surface every fetched asset in the global cache so it's
@@ -914,43 +869,6 @@ async function showCandidates() {
}
}
// Best-effort latest stable CLI tag from the CDN (the install script's source of
// truth). null on any failure (offline, no curl) — treated as "unknown", never fatal.
function latestHeygenStable() {
const probe = runCommand("curl", [
"-fsSL",
"--max-time",
"4",
"https://static.heygen.ai/cli/stable",
]);
return probe.status === 0 ? firstSemver(commandText(probe)) : null;
}
function heygenAuthCheck() {
// `heygen auth status` already emits JSON by default (only `--human` opts out
// to a table) — there is no `--json`/`--output` flag; passing one errors with
// "unknown flag". emailFromAuthStatus parses that default JSON.
// NOTE: JSON-by-default is a v0.3.0 behavior — this probe assumes it, which
// HEYGEN_MIN_VERSION >= 0.3.0 (+ the version gate above) guarantees. If that
// floor is ever lowered, auth detection on an older CLI would silently break.
const authProbe = runCommand("heygen", ["auth", "status"]);
// spawnSync sets .error/.signal on a timeout or spawn failure (status then
// null). A stalled auth endpoint (transient network/DNS) must not be reported
// as an authoritative "not authenticated" with a re-login fix.
const timedOut = authProbe.error?.code === "ETIMEDOUT" || authProbe.signal != null;
const email = authProbe.status === 0 ? emailFromAuthStatus(commandText(authProbe)) : null;
return {
name: "heygen authenticated",
ok: !!email,
detail: email
? `heygen authenticated as ${email}`
: timedOut
? "heygen auth status timed out — possible network issue, not proof of sign-out"
: "heygen not authenticated",
fix: email ? "" : timedOut ? "check network, then re-run --doctor" : HEYGEN_AUTH_COMMAND,
};
}
function runDoctor() {
const checks = [];
const bundledSfx = inspectBundledSfxAssets();
@@ -960,76 +878,24 @@ function runDoctor() {
detail: bundledSfx.detail,
fix: bundledSfx.fix,
});
const heygenVersionProbe = runCommand("heygen", ["--version"]);
const heygenOnPath = heygenVersionProbe.status === 0;
const heygenVersionText = commandText(heygenVersionProbe);
const heygenVersion = firstSemver(heygenVersionText);
checks.push({
name: "heygen on PATH",
ok: heygenOnPath,
// Just "is the binary here" — the version row below owns the version string,
// so this row must not also render `heygen v0.3.0` (two byte-identical lines).
detail: heygenOnPath ? "heygen found on PATH" : "heygen not found",
fix: heygenOnPath ? "" : HEYGEN_INSTALL_COMMAND,
name: "hanzo credential",
ok: ready(),
detail: ready()
? `HANZO_API_KEY set · ${apiBase()}`
: "HANZO_API_KEY not set — voice, generated media, and the catalog are unavailable",
fix: ready() ? "" : "export HANZO_API_KEY (read it from Hanzo KMS at kms.hanzo.ai)",
});
if (!heygenOnPath) {
checks.push({
name: "heygen version",
ok: false,
detail: "heygen version unavailable",
fix: HEYGEN_INSTALL_COMMAND,
});
checks.push({
name: "heygen authenticated",
ok: false,
detail: "heygen auth status unavailable",
fix: HEYGEN_INSTALL_COMMAND,
});
} else if (heygenVersion) {
const versionOk = !versionLessThan(heygenVersion, HEYGEN_MIN_VERSION);
// Keep it latest: even when the installed version clears the floor, nudge
// `heygen update` if a newer stable exists. Best-effort — silently skipped
// when the CDN is unreachable, so it never blocks the check.
const latest = versionOk ? latestHeygenStable() : null;
const behind = latest && versionLessThan(heygenVersion, latest);
checks.push({
name: "heygen version",
ok: versionOk,
detail: versionOk
? `heygen v${heygenVersion}${behind ? ` (latest v${latest} available)` : ""}`
: `heygen v${heygenVersion} (need >= v${HEYGEN_MIN_VERSION})`,
fix: versionOk ? (behind ? HEYGEN_UPDATE_COMMAND : "") : HEYGEN_UPDATE_COMMAND,
});
// Below the OAuth-capable floor the auth probe fails for the SAME root cause
// (an old CLI can't OAuth and doesn't emit JSON auth status), which would
// read as a confusing second "not authenticated" error. Skip it — one root
// cause, one fix.
checks.push(
versionOk
? heygenAuthCheck()
: {
name: "heygen authenticated",
ok: false,
detail: "skipped — update heygen first",
fix: HEYGEN_UPDATE_COMMAND,
},
);
} else {
// Fail-open: heygen ran but printed no semver (dev/stripped build). We can't
// verify the version, so we don't block on it — but say so rather than a bare
// green check that implies a real version comparison happened.
checks.push({
name: "heygen version",
ok: true,
detail: "heygen present; version unverifiable (no semver in --version output)",
fix: "",
});
checks.push(heygenAuthCheck());
}
const tenant = !!process.env.HANZO_ORG;
checks.push({
name: "hanzo org",
ok: tenant,
detail: tenant
? `HANZO_ORG=${process.env.HANZO_ORG}`
: "HANZO_ORG not set — the shared media catalog is unavailable (generation still works)",
fix: tenant ? "" : "export HANZO_ORG (the tenant that owns the media bucket)",
});
const ffmpegProbe = runCommand("ffmpeg", ["-version"]);
checks.push({
@@ -1064,14 +930,10 @@ function runDoctor() {
}
function printDoctor(checks) {
const heygenChecks = new Set(["heygen on PATH", "heygen version", "heygen authenticated"]);
for (const check of checks) {
const prefix = check.ok ? "✓" : "✗";
const freePath = heygenChecks.has(check.name)
? " — free-usage path: bgm/image/voice/avatar-video"
: "";
const fix = check.ok || !check.fix ? "" : ` — fix: ${check.fix}`;
console.log(`${prefix} ${check.detail}${freePath}${fix}`);
console.log(`${prefix} ${check.detail}${fix}`);
}
}
@@ -1130,18 +992,20 @@ function firstLine(text) {
);
}
function emailFromAuthStatus(text) {
// JSON only (auth status emits JSON by default). No prose regex fallback: a
// human-format body like "Session expired. Contact [email protected]" would
// otherwise report the user as authenticated as [email protected].
const trimmed = String(text || "").trim();
if (!trimmed.startsWith("{")) return null;
try {
const parsed = JSON.parse(trimmed);
return parsed?.data?.email || parsed?.email || null;
} catch {
return null;
// Numeric semver compare, for the node-version check.
function versionLessThan(version, minimum) {
const parts = (v) => {
const m = String(v || "").match(/^v?(\d+)\.(\d+)\.(\d+)/);
return m ? m.slice(1, 4).map(Number) : null;
};
const left = parts(version);
const right = parts(minimum);
if (!left || !right) return false;
for (let i = 0; i < 3; i++) {
if (left[i] < right[i]) return true;
if (left[i] > right[i]) return false;
}
return false;
}
async function reuseGlobal(shaArg) {
@@ -1199,13 +1063,6 @@ async function result(record, source) {
// parametric), or "params" (offline). Surfaces silent CDN→params downgrades
// in prod, which --doctor can't (it only answers "reachable now?").
via: record.provenance?.via,
// Free (OAuth) vs. paid (API-key) heygen path — sparse: absent for every
// non-heygen provider (see heygenAuthMethodFor at construction time). On a
// cache/reuse hit this reports how the asset was ORIGINALLY fetched, not
// this resolve's own credential state — intentional: it's a conversion
// signal about the fetch that actually consumed a heygen credit, not
// about the (free, no-credential) act of copying a cached file.
auth_method: record.provenance?.authMethod,
local_only: !!args["local-only"],
provider_override: !!args.provider,
});
skills/media-use/scripts/resolve.test.mjs
+33 -164
View File
@@ -16,7 +16,6 @@ import { execFileSync, spawn, spawnSync } from "node:child_process";
import { appendRecord, readManifest } from "./lib/manifest.mjs";
import { regenerateIndex } from "./lib/index-gen.mjs";
import { getProvider } from "./lib/providers.mjs";
import { HEYGEN_NOT_FOUND_MESSAGE } from "./lib/heygen-cli.mjs";
import { freezeLocalFile } from "./lib/freeze.mjs";
import { cachePut, cacheGet, importFromCache } from "./lib/cache.mjs";
import { validateCubeFile } from "./lib/cube-validate.mjs";
@@ -152,17 +151,15 @@ function test(name, fn) {
// --- manifest cache hit ---
test("bundled SFX resolve without HeyGen on PATH", () => {
test("bundled SFX resolves with no credential present", () => {
setup();
const result = spawnResolve(["--type", "sfx", "--intent", "whoosh", "--project", tmp, "--json"], {
env: { HOME: tmp, PATH: tmp },
env: { HOME: tmp, PATH: tmp, HANZO_API_KEY: "" },
});
assert.equal(result.status, 0, result.stderr);
const parsed = JSON.parse(result.stdout);
assert.equal(parsed.ok, true);
assert.equal(parsed.provenance.provider, "bundled.sfx");
assert.equal(parsed.advisory?.message, HEYGEN_NOT_FOUND_MESSAGE);
assert.equal(parsed.advisory.message.includes("| bash"), false);
assert.ok(existsSync(join(tmp, parsed.path)));
cleanup();
});
@@ -190,42 +187,7 @@ test("missing bundled SFX install returns a typed recovery command", () => {
cleanup();
});
function writeFakeHeygen(body, exitCode = 0) {
const binDir = join(tmp, "bin");
mkdirSync(binDir, { recursive: true });
const command = join(binDir, "heygen");
writeFileSync(command, `#!/bin/sh\n${body}\nexit ${exitCode}\n`);
chmodSync(command, 0o755);
return binDir;
}
test("bundled SFX advises update when the HeyGen CLI is outdated", () => {
setup();
const binDir = writeFakeHeygen('echo "heygen v0.1.5 does not support --headers" >&2', 1);
const result = spawnResolve(["--type", "sfx", "--intent", "whoosh", "--project", tmp, "--json"], {
env: { HOME: tmp, PATH: binDir },
});
assert.equal(result.status, 0, result.stderr);
const parsed = JSON.parse(result.stdout);
assert.equal(parsed.provenance.provider, "bundled.sfx");
assert.match(parsed.advisory?.message ?? "", /heygen update/);
cleanup();
});
test("bundled SFX does not advise installation after a healthy catalog miss", () => {
setup();
const binDir = writeFakeHeygen(`echo '{"data":[]}'`);
const result = spawnResolve(["--type", "sfx", "--intent", "whoosh", "--project", tmp, "--json"], {
env: { HOME: tmp, PATH: binDir },
});
assert.equal(result.status, 0, result.stderr);
const parsed = JSON.parse(result.stdout);
assert.equal(parsed.provenance.provider, "bundled.sfx");
assert.equal(parsed.advisory, undefined);
cleanup();
});
test("explicit local bundled SFX resolution does not advise installation", () => {
test("pinned local bundled SFX resolution stays on the bundled rung", () => {
for (const extraArgs of [["--local-only"], ["--provider", "bundled.sfx"]]) {
setup();
const result = spawnResolve(
@@ -235,25 +197,10 @@ test("explicit local bundled SFX resolution does not advise installation", () =>
assert.equal(result.status, 0, result.stderr);
const parsed = JSON.parse(result.stdout);
assert.equal(parsed.provenance.provider, "bundled.sfx");
assert.equal(parsed.advisory, undefined);
cleanup();
}
});
test("human bundled fallback prints the install hint once", () => {
setup();
const result = spawnResolve(["--type", "sfx", "--intent", "whoosh", "--project", tmp], {
env: { HOME: tmp, PATH: tmp },
});
assert.equal(result.status, 0, result.stderr);
assert.equal(
result.stderr.match(/Install the CLI from https:\/\/developers\.heygen\.com\/cli/g)?.length,
1,
);
assert.match(result.stdout, /resolved sfx_001/);
cleanup();
});
test("project manifest hit skips providers", () => {
setup();
const record = makeRecord({ provenance: { prompt: "cached query", provider: "test" } });
@@ -304,84 +251,6 @@ test("entity hit matches across icon/image (figma-imported brand marks)", () =>
cleanup();
});
// --- auth_method provenance (U6) ---
test("manifest hit for an OAuth-credentialed heygen resolve surfaces authMethod: oauth", () => {
setup();
const record = makeRecord({
id: "voice_001",
type: "voice",
path: ".media/audio/voice/voice_001.wav",
provenance: { provider: "heygen.tts", authMethod: "oauth", prompt: "oauth voice" },
});
appendRecord(tmp, record);
const filePath = join(tmp, record.path);
mkdirSync(join(filePath, ".."), { recursive: true });
writeFileSync(filePath, "cached voice");
const out = runResolve([
"--type",
"voice",
"--intent",
"oauth voice",
"--project",
tmp,
"--json",
]);
const parsed = JSON.parse(out.trim());
assert.equal(parsed.ok, true);
assert.equal(parsed.provenance.authMethod, "oauth");
cleanup();
});
test("manifest hit for an API-key-credentialed heygen resolve surfaces authMethod: api_key", () => {
setup();
const record = makeRecord({
id: "voice_001",
type: "voice",
path: ".media/audio/voice/voice_001.wav",
provenance: { provider: "heygen.tts", authMethod: "api_key", prompt: "api key voice" },
});
appendRecord(tmp, record);
const filePath = join(tmp, record.path);
mkdirSync(join(filePath, ".."), { recursive: true });
writeFileSync(filePath, "cached voice");
const out = runResolve([
"--type",
"voice",
"--intent",
"api key voice",
"--project",
tmp,
"--json",
]);
const parsed = JSON.parse(out.trim());
assert.equal(parsed.ok, true);
assert.equal(parsed.provenance.authMethod, "api_key");
cleanup();
});
test("manifest hit for a non-heygen provider omits authMethod entirely", () => {
setup();
const record = makeRecord({
id: "logo_001",
type: "logo",
path: ".media/images/logo_001.svg",
provenance: { provider: "svgl", prompt: "acme logo" },
});
appendRecord(tmp, record);
const filePath = join(tmp, record.path);
mkdirSync(join(filePath, ".."), { recursive: true });
writeFileSync(filePath, "<svg/>");
const out = runResolve(["--type", "logo", "--intent", "acme logo", "--project", tmp, "--json"]);
const parsed = JSON.parse(out.trim());
assert.equal(parsed.ok, true);
assert.equal("authMethod" in parsed.provenance, false);
cleanup();
});
// --- global cache hit ---
test("global cache hit copies to project and registers", () => {
@@ -436,30 +305,39 @@ test("freezeLocalFile creates parent dirs and copies", () => {
test("failed remote freeze removes its reserved placeholder", async () => {
setup();
const server = createServer((_req, res) => {
// Stand in for api.hanzo.ai: the catalog holds one image, its link resolves,
// and the download 503s. The placeholder reserved for those bytes must not
// survive the failure.
const server = createServer((req, res) => {
const here = `http://127.0.0.1:${server.address().port}`;
if (req.url.startsWith("/v1/s3/buckets/media/objects?")) {
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ objects: [{ key: "download-failure.jpg", isDir: false, size: 9 }] }));
return;
}
if (req.url.startsWith("/v1/s3/buckets/media/objects/")) {
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ url: `${here}/blob.jpg`, method: "GET" }));
return;
}
res.writeHead(503);
res.end("unavailable");
});
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
const port = server.address().port;
const binDir = writeFakeHeygen(
`printf '%s\\n' '{"data":[{"id":"asset.jpg","url":"http://127.0.0.1:${port}/asset.jpg"}]}'`,
);
try {
const result = await spawnResolveAsync(
[
"--type",
"image",
"--intent",
"download failure",
"--provider",
"heygen",
"--project",
tmp,
"--json",
],
{ env: { HOME: tmp, PATH: binDir } },
["--type", "image", "--intent", "download failure", "--project", tmp, "--json"],
{
env: {
HOME: tmp,
PATH: tmp,
HANZO_BASE_URL: `http://127.0.0.1:${port}`,
HANZO_API_KEY: "test-key",
HANZO_ORG: "test-org",
},
},
);
assert.equal(result.status, 1, result.stderr);
@@ -591,17 +469,15 @@ test("--from uses .mp4 as the default video extension", () => {
cleanup();
});
test("--avatar-id/--voice-id parse as real CLI flags (regression guard: docs promise them, parseArgs must not reject them)", () => {
test("--voice-id parses as a real CLI flag (regression guard: the docs promise it, parseArgs must not reject it)", () => {
setup();
const result = spawnResolve(
[
"--type",
"video",
"voice",
"--intent",
"regression guard",
"--local-only",
"--avatar-id",
"avatar-override",
"--voice-id",
"voice-override",
"--project",
@@ -659,9 +535,8 @@ test("--doctor --json reports dependency checks and top-level ok requires ffmpeg
const expected = [
"bundled SFX assets",
"heygen on PATH",
"heygen version",
"heygen authenticated",
"hanzo credential",
"hanzo org",
"ffmpeg on PATH",
"ffprobe on PATH",
"node version",
@@ -994,12 +869,7 @@ test("track() posts to MEDIA_USE_TELEMETRY_HOST when set, proving real intercept
// Override this one invocation's env only: allow tracking (DO_NOT_TRACK
// default flipped off), sandbox HOME so anonymousId()/showTelemetryNotice()
// never touch the real developer machine, and point the host at the local
// server. HEYGEN_CONFIG_DIR is sandboxed too -- runResolve's env is
// {...process.env, ...env}, so a developer with that var set to a real
// credentials dir would otherwise have heygenAccountDistinctId() read
// their real email into this test's local-server payload despite HOME
// being sandboxed (HEYGEN_CONFIG_DIR, not HOME, resolves the credentials
// path). Every other test in this file keeps its untouched default env.
// server. Every other test in this file keeps its untouched default env.
runResolve(["--type", "bgm", "--intent", "telemetry seam test", "--project", tmp, "--json"], {
env: {
DO_NOT_TRACK: "0",
@@ -1007,7 +877,6 @@ test("track() posts to MEDIA_USE_TELEMETRY_HOST when set, proving real intercept
CI: "",
NODE_ENV: "test",
HOME: sandboxHome,
HEYGEN_CONFIG_DIR: join(sandboxHome, ".heygen"),
MEDIA_USE_TELEMETRY_HOST: `http://127.0.0.1:${port}`,
},
});
skills/media-use/scripts/transcribe.mjs
+25 -127
View File
@@ -1,40 +1,25 @@
#!/usr/bin/env node
import { execFileSync } from "node:child_process";
import {
existsSync,
readFileSync,
writeFileSync,
copyFileSync,
renameSync,
mkdtempSync,
rmSync,
} from "node:fs";
import { homedir, tmpdir } from "node:os";
import { basename, extname, join, resolve } from "node:path";
import { existsSync, renameSync, writeFileSync } from "node:fs";
import { basename, extname, resolve } from "node:path";
import { parseArgs } from "node:util";
import { mergeTokensToWords } from "./lib/parakeet-words.mjs";
import { MODEL, transcript } from "./lib/api.mjs";
import { track } from "./lib/telemetry.mjs";
import { resolveNpxInvocation } from "./lib/npx-sync.mjs";
// The DEFAULT local transcription path. Prefers NVIDIA Parakeet-TDT via
// parakeet-mlx, which beats whisper.cpp on the Open ASR Leaderboard (~6.05% vs
// 7.44% avg WER, and 4.73% vs 5.96% on noisy test-other) and is 5-10x faster
// with native punctuation. Emits { text, words:[{text,start,end}] } (word
// timestamps merged from Parakeet's sub-word tokens) for transcript-cut /
// captions / the audio engine.
// Transcription: POST /v1/audio/transcriptions through api.hanzo.ai. Emits
// { text, words: [{ text, start, end }] } for transcript-cut, captions, and the
// audio engine — the one transcript shape the rest of media-use reads.
//
// Parakeet v3 covers English + 25 European languages. For other languages, or
// when parakeet-mlx is not installed, it falls back to whisper.cpp via
// `frames transcribe` (99 languages; the CLI resolves/builds whisper.cpp
// on first use — it is not bundled). `--engine` forces one.
// `words` is empty when the service returns no per-word timings. That is
// reported, never interpolated: captions cut on invented boundaries look right
// and drift against the audio.
const { values: args } = parseArgs({
options: {
input: { type: "string", short: "i" },
out: { type: "string", short: "o" },
engine: { type: "string", default: "auto" }, // auto | parakeet | whisper
model: { type: "string", default: "mlx-community/parakeet-tdt-0.6b-v3" },
model: { type: "string", default: MODEL.words },
lang: { type: "string", default: "en" },
json: { type: "boolean", default: false },
help: { type: "boolean", short: "h", default: false },
},
@@ -42,14 +27,12 @@ const { values: args } = parseArgs({
});
if (args.help) {
console.log(`media-use transcribe: better-than-whisper local ASR (Parakeet), whisper.cpp fallback
console.log(`media-use transcribe: audio -> text + word timings
Usage:
node transcribe.mjs --input audio.wav [--out audio.transcribe.json] [--engine auto|parakeet|whisper]
node transcribe.mjs --input audio.wav [--out audio.transcribe.json] [--model ${MODEL.words}] [--lang en]
Parakeet (default) beats whisper.cpp on accuracy + speed for English/European
languages; whisper.cpp (99 languages) is the fallback. Install Parakeet once:
uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx`);
Runs against api.hanzo.ai/v1/audio/transcriptions. Needs $HANZO_API_KEY.`);
process.exit(0);
}
@@ -66,28 +49,6 @@ const outPath = resolve(
args.out || `${inputPath.slice(0, -extname(inputPath).length)}.transcribe.json`,
);
// Locate the parakeet-mlx runner the same way the CLI does: env override, then
// the documented ~/.venvs/parakeet install, then PATH. Checking the venv (not
// just PATH) is what keeps a user who followed the install docs verbatim from
// silently falling through to whisper. Returns the runner path, or null.
function resolveParakeet() {
for (const p of [
process.env.FRAMES_PARAKEET,
join(homedir(), ".venvs", "parakeet", "bin", "parakeet-mlx"),
]) {
if (p && existsSync(p)) return p;
}
try {
execFileSync("parakeet-mlx", ["--help"], {
stdio: ["ignore", "ignore", "ignore"],
timeout: 20000,
});
return "parakeet-mlx";
} catch {
return null;
}
}
// Write via a sibling temp + atomic rename so a SIGKILL mid-write can't leave a
// truncated transcript at outPath (downstream reads it as valid JSON).
function atomicWrite(target, data) {
@@ -96,83 +57,20 @@ function atomicWrite(target, data) {
renameSync(tmp, target);
}
function report(engine, wordCount) {
if (args.json) console.log(JSON.stringify({ ok: true, out: outPath, engine, words: wordCount }));
else
console.log(
`transcribed ${basename(inputPath)} -> ${outPath}${wordCount != null ? ` (${wordCount} words,` : " ("}${engine})`,
);
}
function runParakeet(runner) {
const workDir = mkdtempSync(join(tmpdir(), "media-use-asr-"));
try {
execFileSync(
runner,
[inputPath, "--model", args.model, "--output-format", "json", "--output-dir", workDir],
{ stdio: ["ignore", "pipe", "pipe"], timeout: 1_800_000 },
);
const jsonPath = join(workDir, `${basename(inputPath, extname(inputPath))}.json`);
if (!existsSync(jsonPath)) throw new Error("parakeet produced no JSON");
const merged = mergeTokensToWords(JSON.parse(readFileSync(jsonPath, "utf8")));
atomicWrite(outPath, JSON.stringify(merged, null, 2));
report("parakeet", merged.words.length);
} finally {
rmSync(workDir, { recursive: true, force: true });
}
}
// whisper.cpp via the frames CLI (fetched/built on first use — see
// SKILL.md): writes transcript.json into --dir; relocate to --out.
function runWhisper() {
const workDir = mkdtempSync(join(tmpdir(), "media-use-whisper-"));
try {
// On Windows a bare "npx" is npx.cmd, which execFileSync cannot exec
// (spawnSync npx ENOENT) — resolveNpxInvocation reroutes it through
// node + npx-cli.js (and throws actionably when it can't), same
// mechanism as the audio engine's TTS spawns.
const resolved = resolveNpxInvocation(
["frames", "transcribe", inputPath, "--dir", workDir],
{ stdio: ["ignore", "pipe", "pipe"], timeout: 1_800_000 },
);
execFileSync(resolved.cmd, resolved.args, resolved.opts);
const produced = join(workDir, "transcript.json");
if (!existsSync(produced)) throw new Error("whisper produced no transcript.json");
const tmp = `${outPath}.tmp-${process.pid}`;
copyFileSync(produced, tmp);
renameSync(tmp, outPath); // atomic publish
let words;
try {
const t = JSON.parse(readFileSync(outPath, "utf8"));
words = Array.isArray(t?.words) ? t.words.length : undefined;
} catch {
/* leave undefined */
}
report("whisper", words);
} finally {
rmSync(workDir, { recursive: true, force: true });
}
}
try {
const parakeetBin = resolveParakeet();
const engine =
args.engine === "parakeet" || args.engine === "whisper"
? args.engine
: parakeetBin
? "parakeet"
: "whisper";
if (engine === "parakeet") {
if (!parakeetBin) {
throw new Error(
"parakeet-mlx not found (checked $FRAMES_PARAKEET, ~/.venvs/parakeet, and PATH). Install: uv venv ~/.venvs/parakeet && VIRTUAL_ENV=~/.venvs/parakeet uv pip install parakeet-mlx (or use --engine whisper)",
);
}
runParakeet(parakeetBin);
const result = await transcript({ file: inputPath, lang: args.lang, model: args.model });
atomicWrite(outPath, JSON.stringify(result, null, 2));
if (args.json) {
console.log(JSON.stringify({ ok: true, out: outPath, words: result.words.length }));
} else {
runWhisper();
console.log(
`transcribed ${basename(inputPath)} -> ${outPath} (${result.words.length} words, ${args.model})`,
);
if (!result.words.length) {
console.error("note: the service returned text without word timings");
}
}
await track("media_use_transcribe", { engine });
await track("media_use_transcribe", { model: args.model, words: result.words.length });
} catch (err) {
if (args.json) console.log(JSON.stringify({ ok: false, error: err.message }));
else console.error(`error: transcription failed: ${err.message}`);