Add HyperFrames skills
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011h6NB8JQtfPc6eRCefEzBB
This commit is contained in:
964 files changed
+142321
No files matched your search
@@ -0,0 +1,35 @@
|
||||
# SFX Credits
|
||||
|
||||
All sound effects in this directory are sourced from [Pixabay](https://pixabay.com/sound-effects/) and used under the [Pixabay Content License](https://pixabay.com/service/license-summary/).
|
||||
|
||||
The Pixabay license allows free use for commercial and non-commercial purposes without attribution, but attribution is appreciated and given here for transparency.
|
||||
|
||||
## Files
|
||||
|
||||
The following `.mp3` files are bundled with this skill:
|
||||
|
||||
- `chime.mp3`
|
||||
- `click.mp3` / `click-soft.mp3`
|
||||
- `error.mp3`
|
||||
- `glitch-1.mp3` / `glitch-2.mp3` / `glitch-3.mp3`
|
||||
- `impact-bass-1.mp3` / `impact-bass-2.mp3`
|
||||
- `key-press.mp3`
|
||||
- `notification.mp3`
|
||||
- `ping.mp3`
|
||||
- `pop.mp3`
|
||||
- `riser.mp3`
|
||||
- `sparkle.mp3`
|
||||
- `typing.mp3`
|
||||
- `whoosh.mp3` / `whoosh-short.mp3` / `whoosh-cinematic.mp3`
|
||||
|
||||
See `manifest.json` for per-file metadata (duration, energy character, recommended use).
|
||||
|
||||
## License
|
||||
|
||||
All files are distributed under the [Pixabay Content License](https://pixabay.com/service/license-summary/), which permits:
|
||||
|
||||
- Commercial and non-commercial use
|
||||
- Modification and remixing
|
||||
- Redistribution as part of derivative works (such as videos rendered with HyperFrames)
|
||||
|
||||
without any attribution requirement.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,97 @@
|
||||
{
|
||||
"chime": {
|
||||
"file": "chime.mp3",
|
||||
"duration": 2.5,
|
||||
"description": "Soft melodic chime — gentle positive beat: success/confirmation or a lighthearted transition. Sync to the visual moment."
|
||||
},
|
||||
"click-soft": {
|
||||
"file": "click-soft.mp3",
|
||||
"duration": 0.37,
|
||||
"description": "Quiet short click — low-key UI tap / soft selection. Short accent, sync exactly to the on-screen action."
|
||||
},
|
||||
"click": {
|
||||
"file": "click.mp3",
|
||||
"duration": 0.37,
|
||||
"description": "Crisp UI click — button press, toggle, selection. Short accent, sync exactly to the on-screen action."
|
||||
},
|
||||
"error": {
|
||||
"file": "error.mp3",
|
||||
"duration": 1.62,
|
||||
"description": "Negative / error tone — failure state, a 'wrong' beat, or a glitchy interruption."
|
||||
},
|
||||
"glitch-1": {
|
||||
"file": "glitch-1.mp3",
|
||||
"duration": 2.64,
|
||||
"description": "Punchy digital glitch — hard-cut accent or sudden reveal. Trigger on the hit; let the decay bleed into the next shot (J-cut)."
|
||||
},
|
||||
"glitch-2": {
|
||||
"file": "glitch-2.mp3",
|
||||
"duration": 3.5,
|
||||
"description": "Harsh, longer glitch — chaotic / jarring transition or a distorted reveal."
|
||||
},
|
||||
"glitch-3": {
|
||||
"file": "glitch-3.mp3",
|
||||
"duration": 3.1,
|
||||
"description": "Low-key glitch texture — subtle digital shift, minimal transition that sits under other audio."
|
||||
},
|
||||
"impact-bass-1": {
|
||||
"file": "impact-bass-1.mp3",
|
||||
"duration": 2.12,
|
||||
"description": "Bass impact hit — logo/hero snap, headline slam. Trigger on the visual landing; decay carries into the next shot (J-cut)."
|
||||
},
|
||||
"impact-bass-2": {
|
||||
"file": "impact-bass-2.mp3",
|
||||
"duration": 2.59,
|
||||
"description": "Bass impact with a short swell — brief anticipation then a deep hit. Place so the peak lands on the reveal."
|
||||
},
|
||||
"key-press": {
|
||||
"file": "key-press.mp3",
|
||||
"duration": 0.4,
|
||||
"description": "Single key press — one keystroke / terminal-input beat. Short accent, sync to the typed character."
|
||||
},
|
||||
"notification": {
|
||||
"file": "notification.mp3",
|
||||
"duration": 2.46,
|
||||
"description": "Notification chime — alert, message-in, toast/badge appears. Sync to the element entering."
|
||||
},
|
||||
"ping": {
|
||||
"file": "ping.mp3",
|
||||
"duration": 1.32,
|
||||
"description": "Sharp electronic ping — punchy accent on a key reveal or data point. Sync to the beat."
|
||||
},
|
||||
"pop": {
|
||||
"file": "pop.mp3",
|
||||
"duration": 0.72,
|
||||
"description": "Quick pop — element appear/spawn, chip/tag/badge in. Small precise accent, sync to the pop-in."
|
||||
},
|
||||
"riser": {
|
||||
"file": "riser.mp3",
|
||||
"duration": 10.03,
|
||||
"description": "Long cinematic riser (~10s build, peak at the end). Trigger at (climax_time − 10.03s) so it crests exactly on the reveal."
|
||||
},
|
||||
"sparkle": {
|
||||
"file": "sparkle.mp3",
|
||||
"duration": 1.8,
|
||||
"description": "Bright sparkle / shimmer — magical reveal or 'shine' highlight on a hero element. Sync to the highlight."
|
||||
},
|
||||
"typing": {
|
||||
"file": "typing.mp3",
|
||||
"duration": 1.5,
|
||||
"description": "Typing burst (~1.5s of keys) — keyboard / code typing reveal, text-being-typed beat. Start as the text begins typing."
|
||||
},
|
||||
"whoosh-cinematic": {
|
||||
"file": "whoosh-cinematic.mp3",
|
||||
"duration": 5.54,
|
||||
"description": "Cinematic whoosh build (~5.5s) — sweeping scene transition. Align so the swell peaks on the cut."
|
||||
},
|
||||
"whoosh-short": {
|
||||
"file": "whoosh-short.mp3",
|
||||
"duration": 0.57,
|
||||
"description": "Short whoosh — quick swipe/slide accent, fast element move, snappy transition. Sync to the motion."
|
||||
},
|
||||
"whoosh": {
|
||||
"file": "whoosh.mp3",
|
||||
"duration": 0.57,
|
||||
"description": "Punchy whoosh/impact — fast reveal or hard transition accent. Sync to the motion."
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,76 @@
|
||||
# Background music (BGM)
|
||||
|
||||
One music bed per composition, produced by the shared audio engine (`scripts/audio.mjs` → `scripts/lib/bgm.mjs`). Two routes, chosen by the engine's one switch — whether a HeyGen credential is present:
|
||||
|
||||
- **HeyGen retrieval — the default when credentialed.** Search HeyGen's music catalog by mood, download the top track. No generation; same `~/.heygen` / `$HEYGEN_API_KEY` credential as TTS.
|
||||
- **Local generation (Lyria → MusicGen) — the fallback when there is no credential** (or when asked for explicitly). Generate a WAV from a mood prompt. There is **no `npx hyperframes bgm` command**; the engine spawns `scripts/lyria-recipe.py` or an inline MusicGen script directly.
|
||||
|
||||
> **Run the Preflight first — no credential is not a green light to silently generate locally.** Before generating, complete the sign-in **Preflight** (see `../../SKILL.md` → Preflight): run `npx hyperframes auth status`, recommend signing in, and **STOP for the user's choice** (sign in for HeyGen's music library, or continue offline with local generation). This applies to a one-off "generate a BGM" request just as much as inside a full workflow.
|
||||
|
||||
## Driving it from the request
|
||||
|
||||
`audio_request.json` → `bgm: { mode?, query?, prompt? }`:
|
||||
|
||||
- **`mode`** — `retrieve | generate | none`. Omit for **auto** (retrieve when credentialed, else generate). An **explicit** `retrieve` is strict: no credential ⇒ skip, never a detached generate (so a caller with no `wait-bgm` step, e.g. product-launch, can't get a pending job it won't await).
|
||||
- **`query`** — the mood, used for retrieval and as a fallback prompt seed (e.g. a storyboard's `music:` field, falling back to `message` → `arc` → `"calm cinematic underscore"`).
|
||||
- **`prompt`** — an explicit full prompt for generation; omit and the engine infers one (see Mood inference). Optional `blob` / `archetype` / `arc` feed that inference.
|
||||
|
||||
Both routes keep a file of yours already at the output name: the engine writes the next free name (`track-2.mp3`), reports it as an anomaly, and `bgm.path` carries the real path.
|
||||
|
||||
## HeyGen retrieval (default)
|
||||
|
||||
`searchSounds(query, "music", { limit: 5 })` → `GET /audio/sounds?query=<mood>&type=music&limit=5`. Take the top result (ranked by `score`), download its presigned `audio_url` → `assets/bgm/track.mp3`. Synchronous. No match → skip (BGM is optional; never fail the render over it). Cue written to `audio_meta.json`:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"path": "assets/bgm/track.mp3",
|
||||
"volume": 0.12,
|
||||
"mode": "retrieve",
|
||||
"query": "calm cinematic underscore",
|
||||
"duration_s": 42.0,
|
||||
}
|
||||
```
|
||||
|
||||
`volume` comes from the engine's `bgmDefaultVolume()`: `BGM_BED_VOLUME` (currently `0.12` ≈ -18 dB — a bed under the voice) under narration, `BGM_SILENT_VOLUME` (currently `0.9`) for a silent film (no voice). Tune those constants in `scripts/lib/bgm.mjs`, not call sites. An explicit `volume` in `audio_meta.json` always overrides this default. `bgm_pending` is `false` — the file is on disk when the engine returns.
|
||||
|
||||
For short launch videos, do not assume the beginning of the retrieved file is the best edit point. Check the opening against later five-second sections. If the track starts with a quiet build but a later section has a stronger, clean musical entrance, trim from that section and apply a short fade-in and longer fade-out. Repeat this check whenever the composition duration changes; the final music file must cover the full cut without a silent tail.
|
||||
|
||||
## Local generation (fallback) — Lyria → MusicGen
|
||||
|
||||
Spawned **detached** so voice work isn't blocked; `audio_meta.bgm_pending: true` and `bgm_pid` / `bgm_log` are set until it finishes. **Run `scripts/wait-bgm.mjs` before assembling** — it polls the output file / process / log, detects crashes, and writes `bgm_status.json` (`status: ready | failed | timeout | disabled`). A failed/absent track is simply omitted; it never blocks voice/SFX.
|
||||
|
||||
| Order | Provider | Env / deps | Speed | Quality |
|
||||
| ----- | ------------------------------------ | ------------------------------------------------------------------------------------- | --------------------------------------- | --------------------------- |
|
||||
| 1 | Google Lyria RealTime | `$GEMINI_API_KEY` or `$GOOGLE_API_KEY` + `google-genai` (auto-installed on demand) | Real-time stream (≈ requested duration) | Production-grade |
|
||||
| 2 | MusicGen (`facebook/musicgen-small`) | Python `transformers + torch + soundfile + numpy` (~300 MB first run; auto-installed) | Slow on CPU; fast on Apple MPS / CUDA | Decent; prompt-only control |
|
||||
|
||||
Output → `assets/bgm/track.wav`, target = total voice duration. MusicGen generates **one** seed clip (≤28–30s, under the decoder's positional limit) then crossfade-loops it up to the target (or trims down if shorter), avoiding per-segment seams. Backend selection is by what can actually **run**: Lyria only when `import google.genai` succeeds, else MusicGen; if neither can be made to run, BGM is skipped (voice + SFX still render).
|
||||
|
||||
## Mood inference (the generate prompt)
|
||||
|
||||
`inferBgmPrompt()` in `scripts/lib/bgm.mjs`: an explicit `prompt` wins; otherwise industry-keyword **base** → narrative-**archetype** shape → emotional-**arc** tiebreaker.
|
||||
|
||||
| Match in `blob` / `query` | Base prompt | BPM |
|
||||
| ------------------------------------------------------ | --------------------------------------------------------------------------- | --- |
|
||||
| `crypto / nft / web3 / defi / token / blockchain` | atmospheric electronic, deep bass, futuristic synths, restrained percussion | 100 |
|
||||
| `finance / fintech / bank / payment / invest / wealth` | calm cinematic, soft strings, subtle piano, restrained percussion | 92 |
|
||||
| `creative / agency / design / studio / art / brand` | playful electronic, warm pads, light percussion | 115 |
|
||||
| _(default: SaaS / tech / platform)_ | uplifting corporate tech, bright modern piano with synth pads | 108 |
|
||||
|
||||
Archetype then reshapes the arc — PAS → "MINOR to MAJOR" build; BAB / future-pacing → aspirational rising; feature-cascade → +10 BPM driving; demo-loop → −8 BPM minimal. The emotional arc breaks remaining ties (tension→relief, excitement, trust/reassurance).
|
||||
|
||||
## Lyria knobs (direct recipe use)
|
||||
|
||||
The engine bakes BPM / scale into the **prompt text** (via the inference above) and passes only `--output` / `--duration` / `--prompt` to the recipe. If you invoke `scripts/lyria-recipe.py` directly you can also set: `--bpm` (90–110 calm, 110–130 energetic), `--brightness` (0–1, ≥0.7 promotional), `--density` (0–1, higher = fuller), `--scale` (`MAJOR` / `MINOR` / `PENTATONIC` / …), `--negative-prompt` (styles to exclude). MusicGen ignores all of these — put the mood in the prompt.
|
||||
|
||||
## Failure modes
|
||||
|
||||
| Failure | Behavior |
|
||||
| --------------------------------------------- | ---------------------------------------------------------------------------------------- |
|
||||
| No music match (retrieve) | `bgm: null`, anomaly logged. Render proceeds without BGM. |
|
||||
| Explicit `retrieve`, no credential | Skipped (no silent generate fallback). Use `mode: generate` or omit `mode` for auto. |
|
||||
| Neither Lyria nor MusicGen can run (generate) | `bgm` disabled with a `pip install …` hint. Voice + SFX still render. |
|
||||
| Generate still rendering at assemble time | `bgm_pending: true`; `wait-bgm.mjs` waits/checks and writes `bgm_status.json` first. |
|
||||
| Generate crashed | `wait-bgm.mjs` → `bgm_status.json { status: "failed" }`; the `<audio>` track is omitted. |
|
||||
|
||||
BGM failure never blocks a render.
|
||||
@@ -0,0 +1,163 @@
|
||||
# Captions
|
||||
|
||||
<!-- registry-items: allow=max-width,data-composition-src,hyperframes-registry,blend-mode,caption-style,font-family -->
|
||||
|
||||
**The live search is the source of truth for what the registry has.** The table(s) below are a hand-maintained sample and under-cover by design: run `npx hyperframes catalog --query "<what you want>" --json` — it needs nothing installed — before concluding the registry lacks something. Item names here are checked against `registry/registry.json` by `bun run lint:skills`.
|
||||
|
||||
Before authoring: confirm the transcript came from the right Whisper model. CLI default `small.en` silently translates non-English audio — see [`../transcribe.md`](../transcribe.md) → "Language Rule" and [`transcript-handling.md`](transcript-handling.md) for the mandatory quality check.
|
||||
|
||||
Analyze spoken content to determine caption style. If user specifies a style, use that. Otherwise, detect tone from the transcript.
|
||||
|
||||
## Transcript Source
|
||||
|
||||
```json
|
||||
[
|
||||
{ "id": "w0", "text": "Hello", "start": 0.0, "end": 0.5 },
|
||||
{ "id": "w1", "text": "world.", "start": 0.6, "end": 1.2 }
|
||||
]
|
||||
```
|
||||
|
||||
`id` (`w0`, `w1`, …) is the stable reference for per-word overrides and is added by `hyperframes transcribe`. It's optional for backwards compatibility with hand-authored transcripts. See [`../transcribe.md`](../transcribe.md) → "Output Shape" for how this is produced, and [`transcript-handling.md`](transcript-handling.md) for cleanup before consumption.
|
||||
|
||||
## Style Detection (When No Style Specified)
|
||||
|
||||
Read the full transcript before choosing. Four dimensions:
|
||||
|
||||
**1. Visual feel** — corporate→clean; energetic→bold; storytelling→elegant; technical→precise; social→playful.
|
||||
|
||||
**2. Color palette** — dark+bright for energy; muted for professional; high contrast for clarity; one accent color.
|
||||
|
||||
**3. Font mood** — heavy/condensed for impact; clean sans for modern; rounded for friendly; serif for elegance.
|
||||
|
||||
**4. Animation character** — scale-pop for punchy; gentle fade for calm; word-by-word for emphasis; typewriter for technical.
|
||||
|
||||
## Per-Word Styling
|
||||
|
||||
Scan for words deserving distinct treatment:
|
||||
|
||||
- **Brand/product names** — larger size, unique color
|
||||
- **ALL CAPS** — scale boost, flash, accent color
|
||||
- **Numbers/statistics** — bold weight, accent color
|
||||
- **Emotional keywords** — exaggerated animation (overshoot, bounce)
|
||||
- **Call-to-action** — highlight, underline, color pop
|
||||
- **Marker highlight** — for beyond-color emphasis (highlight sweep, circle, burst, scribble, sketchout), see `hyperframes-animation/rules/css-marker-patterns.md`.
|
||||
|
||||
## Script-to-Style Mapping
|
||||
|
||||
| Tone | Font mood | Animation | Color | Size |
|
||||
| ------------ | ------------------------ | ---------------------------------- | --------------------------- | ------- |
|
||||
| Hype/launch | Heavy condensed, 800-900 | Scale-pop, back.out(1.7), 0.1-0.2s | Bright on dark | 72-96px |
|
||||
| Corporate | Clean sans, 600-700 | Fade+slide, power3.out, 0.3s | White/neutral, muted accent | 56-72px |
|
||||
| Tutorial | Mono/clean sans, 500-600 | Typewriter/fade, 0.4-0.5s | High contrast, minimal | 48-64px |
|
||||
| Storytelling | Serif/elegant, 400-500 | Slow fade, power2.out, 0.5-0.6s | Warm muted tones | 44-56px |
|
||||
| Social | Rounded sans, 700-800 | Bounce, elastic.out, word-by-word | Playful, colored pills | 56-80px |
|
||||
|
||||
## Word Grouping
|
||||
|
||||
- **High energy:** 2-3 words. Quick turnover.
|
||||
- **Conversational:** 3-5 words. Natural phrases.
|
||||
- **Measured/calm:** 4-6 words. Longer groups.
|
||||
|
||||
Break on sentence boundaries, 150ms+ pauses, or max word count.
|
||||
|
||||
## Positioning
|
||||
|
||||
- **Landscape (1920x1080):** Bottom 80-120px, centered
|
||||
- **Portrait (1080x1920):** Lower middle ~600-700px from bottom, centered
|
||||
- Never cover the subject's face
|
||||
- `position: absolute` — never relative
|
||||
- One caption group visible at a time
|
||||
|
||||
## Text Overflow Prevention
|
||||
|
||||
Use `window.__hyperframes.fitTextFontSize()`:
|
||||
|
||||
```js
|
||||
var result = window.__hyperframes.fitTextFontSize(group.text.toUpperCase(), {
|
||||
fontFamily: "Outfit",
|
||||
fontWeight: 900,
|
||||
maxWidth: 1600,
|
||||
});
|
||||
el.style.fontSize = result.fontSize + "px";
|
||||
```
|
||||
|
||||
Options: `maxWidth` (1600 landscape, 900 portrait), `baseFontSize` (78), `minFontSize` (42), `fontWeight`, `fontFamily`, `step` (2).
|
||||
|
||||
CSS safety nets: `max-width` on container, `overflow: visible` (**not** `hidden` — hidden clips scaled emphasis words and glow effects), `position: absolute`, explicit `height`. When per-word styling uses `scale > 1.0`, compute `maxWidth = safeWidth / maxScale` to leave headroom.
|
||||
|
||||
**Container pattern:** Full-width absolute container, centered. Do **not** use `left: 50%; transform: translateX(-50%)` — causes clipping at composition edges.
|
||||
|
||||
## Caption Exit Guarantee
|
||||
|
||||
Every group **must** have a hard kill after exit animation:
|
||||
|
||||
```js
|
||||
tl.to(groupEl, { opacity: 0, scale: 0.95, duration: 0.12, ease: "power2.in" }, group.end - 0.12);
|
||||
// `tl.set` is an instant flip, not a tween — safe to set `visibility` here (core's "no animating
|
||||
// visibility" rule applies to tweens, which can't smoothly interpolate non-numeric values anyway).
|
||||
tl.set(groupEl, { opacity: 0, visibility: "hidden" }, group.end);
|
||||
```
|
||||
|
||||
Self-lint after building timeline — place **before** `window.__timelines[id] = tl` so it runs at composition init:
|
||||
|
||||
```js
|
||||
GROUPS.forEach(function (group, gi) {
|
||||
var el = document.getElementById("cg-" + gi);
|
||||
if (!el) return;
|
||||
tl.seek(group.end + 0.01);
|
||||
var computed = window.getComputedStyle(el);
|
||||
if (computed.opacity !== "0" && computed.visibility !== "hidden") {
|
||||
console.warn(
|
||||
"[caption-lint] group " + gi + " still visible at t=" + (group.end + 0.01).toFixed(2) + "s",
|
||||
);
|
||||
}
|
||||
});
|
||||
tl.seek(0);
|
||||
```
|
||||
|
||||
## Pre-Built Caption Components
|
||||
|
||||
Before building caption styles from scratch, check the registry — 15 ready-to-use caption components cover the most common styles. Install with `npx hyperframes add <name>` and wire as a sub-composition via `data-composition-src` (see `hyperframes-registry`).
|
||||
|
||||
```bash
|
||||
npx hyperframes catalog --tag caption-style # list all caption components
|
||||
npx hyperframes add caption-highlight # install a specific one
|
||||
```
|
||||
|
||||
| Style | Component | Best for |
|
||||
| ------------------------- | ---------------------------- | ---------------------------- |
|
||||
| TikTok-style highlight | `caption-highlight` | Social, high-energy |
|
||||
| Karaoke pill | `caption-pill-karaoke` | Music, lyric videos |
|
||||
| Cinematic editorial | `caption-editorial-emphasis` | Documentary, storytelling |
|
||||
| Glitch / cyber | `caption-glitch-rgb` | Tech, gaming |
|
||||
| Full-screen slam | `caption-kinetic-slam` | Hype, announcements |
|
||||
| Neon glow | `caption-neon-glow` | Night, club, neon aesthetics |
|
||||
| Neon accent (multi-color) | `caption-neon-accent` | Colorful, playful |
|
||||
| Wipe reveal | `caption-clip-wipe` | Clean, modern |
|
||||
| Gradient fill | `caption-gradient-fill` | Vibrant, eye-catching |
|
||||
| Matrix decode | `caption-matrix-decode` | Sci-fi, tech reveals |
|
||||
| Emoji pop | `caption-emoji-pop` | Social, casual |
|
||||
| Parallax layers | `caption-parallax-layers` | Depth, cinematic |
|
||||
| Particle burst | `caption-particle-burst` | Celebration, impact keywords |
|
||||
| Lava texture | `caption-texture` | Bold, dramatic |
|
||||
| Weight shift | `caption-weight-shift` | Elegant, typographic |
|
||||
|
||||
Related: `caption-blend-difference` (tagged `text` / `blend-mode`, not `caption-style`, so it won't appear under the filter above) auto-inverts text against any background via `mix-blend-mode: difference` — useful when the background is busy or unpredictable.
|
||||
|
||||
Browse all with previews: [hyperframes.heygen.com/catalog](https://hyperframes.heygen.com/catalog)
|
||||
|
||||
Caption components ship with transparent backgrounds — they're pure overlays. If the underlying video is bright or busy, add a contrast layer (e.g. a semi-transparent dark div) in the host composition beneath the caption sub-composition, not inside the component itself.
|
||||
|
||||
## Further References
|
||||
|
||||
- [`motion.md`](motion.md) — karaoke, marker effects, audio-reactive modulation, scatter exits.
|
||||
- [`transcript-handling.md`](transcript-handling.md) — input formats, quality checks, cleaning, external API fallback.
|
||||
- `hyperframes-animation/rules/css-marker-patterns.md` — marker highlighting (deterministic, fully seekable).
|
||||
|
||||
## Constraints
|
||||
|
||||
- Deterministic. No `Math.random()`, no `Date.now()`.
|
||||
- Sync to transcript timestamps.
|
||||
- One group visible at a time.
|
||||
- Every group must have a hard `tl.set` kill at `group.end`.
|
||||
- Fonts: the compiler auto-embeds only its **built-in mapped set** (Inter, Roboto, Montserrat, …) — for those, just declare `font-family` in CSS. Any **other** font (a brand/custom font like `TT Norms Pro`, or a non-Latin CJK/Devanagari family) is **not** auto-supplied: it needs an `@font-face` pointing at a real `.woff2` shipped with the project, or the text silently falls back to a generic font in the render. Don't assume a `font-family` you can see locally will render — the render machine is a clean headless Chrome with no installed fonts.
|
||||
@@ -0,0 +1,87 @@
|
||||
# Dynamic Caption Techniques
|
||||
|
||||
You are here because SKILL.md told you to read this file before writing animation code. Pick your technique combination from the table below based on the energy level you detected from the transcript, then implement using standard GSAP patterns.
|
||||
|
||||
## Technique Selection by Energy
|
||||
|
||||
| Energy level | Highlight | Exit | Cycle pattern |
|
||||
| ------------ | ------------------------------------- | ------------------- | ----------------------------------------- |
|
||||
| High | Karaoke with accent glow + scale pop | Scatter or drop | Alternate highlight styles every 2 groups |
|
||||
| Medium-high | Karaoke with color pop | Scatter or collapse | Alternate every 3 groups |
|
||||
| Medium | Karaoke (subtle, white only) | Fade + slide | Alternate every 3 groups |
|
||||
| Medium-low | Karaoke (minimal scale change) | Fade | Single style, vary ease per group |
|
||||
| Low | Karaoke (warm tones, slow transition) | Collapse | Alternate every 4 groups |
|
||||
|
||||
**All energy levels use karaoke highlight as the baseline.** The difference is intensity — high energy gets accent color + glow + 15% scale pop on active words, low energy gets a gentle white shift with 3% scale.
|
||||
|
||||
**Emphasis words always break the pattern.** When a word is flagged as emphasis (emotional keyword, ALL CAPS, brand name), give it a stronger animation than surrounding words (larger scale, accent color, overshoot ease). This creates contrast.
|
||||
|
||||
**Marker highlight modes add a visual layer on top of karaoke.** For emphasis words that need more than color/scale, add a marker-style effect: highlight sweep, circle, burst, scribble, or sketchout. See `hyperframes-animation/rules/css-marker-patterns.md` for implementation details. Match mode to energy: burst for hype, circle for key terms, highlight for standard, scribble for subtle.
|
||||
|
||||
## Audio-Reactive Captions (Mandatory for Music)
|
||||
|
||||
**If the source audio is music (vocals over instrumentation, beats, any musical content), you MUST extract audio data and add audio-reactive animations.** This is not optional — music without audio reactivity looks disconnected. Even low-energy ballads get subtle bass pulse and treble glow.
|
||||
|
||||
No special wiring is needed. The group loop already iterates over every caption group to build entrance, karaoke, and exit tweens. At that point, read the audio data for each group's time range and use it to modulate the group's animation intensity with regular GSAP tweens.
|
||||
|
||||
```js
|
||||
// Load audio data inline (same pattern as TRANSCRIPT)
|
||||
var AUDIO = JSON.parse(audioDataJson); // { fps, totalFrames, frames: [{ bands: [...] }] }
|
||||
|
||||
GROUPS.forEach(function (group, gi) {
|
||||
var groupEl = document.getElementById("cg-" + gi);
|
||||
if (!groupEl) return;
|
||||
|
||||
// Read peak energy for this group's time range
|
||||
var startFrame = Math.floor(group.start * AUDIO.fps);
|
||||
var endFrame = Math.min(Math.floor(group.end * AUDIO.fps), AUDIO.totalFrames - 1);
|
||||
var peakBass = 0;
|
||||
var peakTreble = 0;
|
||||
for (var f = startFrame; f <= endFrame; f++) {
|
||||
var frame = AUDIO.frames[f];
|
||||
if (!frame) continue;
|
||||
peakBass = Math.max(peakBass, frame.bands[0] || 0, frame.bands[1] || 0);
|
||||
peakTreble = Math.max(peakTreble, frame.bands[6] || 0, frame.bands[7] || 0);
|
||||
}
|
||||
|
||||
// Modulate entrance — louder groups enter bigger and glowier
|
||||
tl.to(
|
||||
groupEl,
|
||||
{
|
||||
scale: 1 + peakBass * 0.06,
|
||||
textShadow:
|
||||
"0 0 " + Math.round(peakTreble * 12) + "px rgba(255,255,255," + peakTreble * 0.4 + ")",
|
||||
duration: 0.3,
|
||||
ease: "power2.out",
|
||||
},
|
||||
group.start,
|
||||
);
|
||||
|
||||
// Reset at exit so audio-driven values don't persist
|
||||
tl.set(groupEl, { scale: 1, textShadow: "none" }, group.end - 0.15);
|
||||
});
|
||||
```
|
||||
|
||||
This shapes the animation at build time, not playback time — no per-frame callbacks, no `tl.call()` loops, no async fetch timing issues. Loud groups come in with more weight and glow; quiet groups come in soft. The audio data modulates _how much_, the content determines _what_.
|
||||
|
||||
Keep audio reactivity subtle — 3-6% scale variation and soft glow. Heavy pulsing makes text unreadable.
|
||||
|
||||
To generate the audio data file:
|
||||
|
||||
```bash
|
||||
python3 skills/hyperframes-creative/scripts/extract-audio-data.py audio.mp3 --fps 30 --bands 8 -o audio-data.json
|
||||
```
|
||||
|
||||
## Combining Techniques
|
||||
|
||||
Don't use the same highlight animation on every group — cycle through styles using the group index. Don't combine multiple competing animations on the same word at the same timestamp. Vary techniques across groups to match the content's pace changes.
|
||||
|
||||
**Marker highlight effects** layer well with karaoke — use karaoke for the word-by-word reveal, then add a marker effect on emphasis words only. For example: karaoke highlights each word in white, but brand names get a yellow highlight sweep and stats get a red circle. Cycle marker modes across groups for visual variety.
|
||||
|
||||
## Runtime Tools
|
||||
|
||||
Caption motion uses standard HyperFrames runtime APIs. Use the canonical sources:
|
||||
|
||||
- **GSAP timeline + tween syntax** — `hyperframes-animation/adapters/gsap.md` (eases, position parameter, performance)
|
||||
- **`window.__hyperframes.fitTextFontSize` / `pretext`** — `hyperframes-core/references/determinism-rules.md` → Layout Contract (overflow prevention, per-frame text measurement)
|
||||
- **Audio data extraction** — generate via `python3 skills/hyperframes-creative/scripts/extract-audio-data.py audio.mp3 --fps 30 --bands 8 -o audio-data.json`, then load inline as shown in "Audio-Reactive Captions" above
|
||||
@@ -0,0 +1,97 @@
|
||||
# Transcript Guide
|
||||
|
||||
For the `transcribe` CLI invocation, the `.en`-translates-non-English rule, and whisper model selection, see [`../transcribe.md`](../transcribe.md). This file covers what to do with the resulting transcript when authoring captions: input formats, mandatory quality checks, cleaning code, external-API fallbacks.
|
||||
|
||||
## Supported Input Formats
|
||||
|
||||
The CLI auto-detects and normalizes these formats:
|
||||
|
||||
| Format | Extension | Source | Word-level? |
|
||||
| --------------------- | --------- | --------------------------------------------------------------------------- | ----------------- |
|
||||
| whisper.cpp JSON | `.json` | `hyperframes init --video`, `hyperframes transcribe` | Yes |
|
||||
| OpenAI Whisper API | `.json` | `openai.audio.transcriptions.create({ timestamp_granularities: ["word"] })` | Yes |
|
||||
| SRT subtitles | `.srt` | Video editors, subtitle tools, YouTube | No (phrase-level) |
|
||||
| VTT subtitles | `.vtt` | Web players, YouTube, transcription services | No (phrase-level) |
|
||||
| Normalized word array | `.json` | Pre-processed by any tool | Yes |
|
||||
|
||||
**Word-level timestamps produce better captions.** SRT/VTT give phrase-level timing, which works but can't do per-word animation effects.
|
||||
|
||||
## Transcript Quality Check (Mandatory)
|
||||
|
||||
After every transcription, **read the transcript and check for quality issues before proceeding.** Bad transcripts produce nonsensical captions. Never skip this step.
|
||||
|
||||
### What to look for
|
||||
|
||||
| Signal | Example | Cause |
|
||||
| ---------------------------- | -------------------------------------- | ---------------------------------------------------------------------------- |
|
||||
| Music note tokens (`♪`, `�`) | `{ "text": "♪" }` or `{ "text": "�" }` | Whisper detected music, not speech |
|
||||
| Garbled / nonsense words | "Do a chin", "Get so gay", "huh" | Model misheard lyrics or background noise |
|
||||
| Long gaps with no words | 20+ seconds of only `♪` tokens | Instrumental section — expected, but high ratio means speech is being missed |
|
||||
| Repeated filler | Many "huh", "uh", "oh" entries | Model is hallucinating on music |
|
||||
| Very short word spans | Words with `end - start < 0.05` | Unreliable timestamp alignment |
|
||||
|
||||
### Automatic retry rules
|
||||
|
||||
**If more than 20% of entries are `♪`/`�` tokens, or the transcript contains obvious nonsense words, the transcription failed.** Do not proceed with the bad transcript. Instead:
|
||||
|
||||
1. **Retry with `medium.en`** if the original used `small.en` or smaller:
|
||||
```bash
|
||||
npx hyperframes transcribe audio.mp3 --model medium.en
|
||||
```
|
||||
2. **If `medium.en` also fails** (still >20% music tokens or garbled), tell the user the audio is too noisy for local transcription and suggest:
|
||||
- Providing lyrics manually as an SRT/VTT file
|
||||
- Using an external API (OpenAI or Groq Whisper — see below)
|
||||
3. **Always clean the transcript** before building captions — filter out `♪`/`�` tokens and entries where `text` is a single non-word character. Only real words should reach the caption composition.
|
||||
|
||||
### Cleaning a transcript
|
||||
|
||||
After transcription (even with a good model), strip non-word entries:
|
||||
|
||||
```js
|
||||
var raw = JSON.parse(transcriptJson);
|
||||
var words = raw.filter(function (w) {
|
||||
if (!w.text || w.text.trim().length === 0) return false;
|
||||
if (/^[♪�\u266a\u266b\u266c\u266d\u266e\u266f]+$/.test(w.text)) return false;
|
||||
if (/^(huh|uh|um|ah|oh)$/i.test(w.text) && w.end - w.start < 0.1) return false;
|
||||
return true;
|
||||
});
|
||||
```
|
||||
|
||||
For model-selection guidance by content type, see [`../transcribe.md`](../transcribe.md) → "Picking a model by content type".
|
||||
|
||||
## Using External Transcription APIs
|
||||
|
||||
For the best accuracy, use an external API and import the result:
|
||||
|
||||
**OpenAI Whisper API** (recommended for quality):
|
||||
|
||||
```bash
|
||||
# Generate with word timestamps, then import
|
||||
curl https://api.openai.com/v1/audio/transcriptions \
|
||||
-H "Authorization: Bearer $OPENAI_API_KEY" \
|
||||
-F file=@audio.mp3 -F model=whisper-1 \
|
||||
-F response_format=verbose_json \
|
||||
-F "timestamp_granularities[]=word" \
|
||||
-o transcript-openai.json
|
||||
|
||||
npx hyperframes transcribe transcript-openai.json
|
||||
```
|
||||
|
||||
**Groq Whisper API** (fast, free tier available):
|
||||
|
||||
```bash
|
||||
curl https://api.groq.com/openai/v1/audio/transcriptions \
|
||||
-H "Authorization: Bearer $GROQ_API_KEY" \
|
||||
-F file=@audio.mp3 -F model=whisper-large-v3 \
|
||||
-F response_format=verbose_json \
|
||||
-F "timestamp_granularities[]=word" \
|
||||
-o transcript-groq.json
|
||||
|
||||
npx hyperframes transcribe transcript-groq.json
|
||||
```
|
||||
|
||||
## If No Transcript Exists
|
||||
|
||||
1. Check the project root for `transcript.json`, `.srt`, or `.vtt` files.
|
||||
2. If none found, run [`../transcribe.md`](../transcribe.md) — pick the starting model from "Picking a model by content type" there.
|
||||
3. Run the quality check above. If it fails, retry with a larger model or fall back to manual lyrics / external API.
|
||||
@@ -0,0 +1,143 @@
|
||||
# Background Removal
|
||||
|
||||
Make a transparent overlay (typical: a talking head over an arbitrary scene). Uses `u2net_human_seg` (Apache-2.0).
|
||||
|
||||
```bash
|
||||
npx hyperframes remove-background subject.mp4 -o transparent.webm # default: VP9 + alpha
|
||||
npx hyperframes remove-background subject.mp4 -o transparent.mov # ProRes 4444 (editing)
|
||||
npx hyperframes remove-background portrait.jpg -o cutout.png # single-image cutout
|
||||
npx hyperframes remove-background subject.mp4 -o subject.webm \
|
||||
--background-output plate.webm # both layers, one pass
|
||||
npx hyperframes remove-background subject.mp4 -o transparent.webm --device cpu
|
||||
npx hyperframes remove-background --info # detected providers
|
||||
```
|
||||
|
||||
## Output Format
|
||||
|
||||
- **`.webm` (VP9 alpha)** — default. Plug straight into `<video>` for Chrome-native transparent playback (~1 MB / 4s @ 1080p).
|
||||
- **`.mov` (ProRes 4444)** — round-trip in editors (Premiere / Resolve / DaVinci). ~50 MB / 4s.
|
||||
- **`.png`** — single-image cutout.
|
||||
|
||||
## Quality (`--quality`)
|
||||
|
||||
Controls VP9 encoder CRF only — segmentation quality is fixed. Higher quality keeps the cutout's RGB closer to the source MP4 (important when overlaying the cutout on its own source).
|
||||
|
||||
| Preset | CRF | When |
|
||||
| ---------- | --- | --------------------------------------------- |
|
||||
| `fast` | 30 | Iterating, smaller files, looser color match |
|
||||
| `balanced` | 18 | **Default**; visually identical for most uses |
|
||||
| `best` | 12 | Master / final delivery, tightest color match |
|
||||
|
||||
## Device (`--device`)
|
||||
|
||||
`auto` (default) picks CoreML on Apple Silicon, CUDA when available, otherwise CPU. Force with `--device cpu | coreml | cuda`. CUDA requires `HYPERFRAMES_CUDA=1` plus a GPU-enabled `onnxruntime-node` build. Use `--info` to inspect detected providers without rendering.
|
||||
|
||||
## Compositing patterns — pick the right one
|
||||
|
||||
The cutout WebM is a **re-encoded copy** of the source MP4's RGB. What sits behind it matters.
|
||||
|
||||
| Pattern | Behind the cutout | Result |
|
||||
| -------------------------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------------------------------------------- |
|
||||
| **Cutout over a different scene** (most common) | Static image, gradient, unrelated video | Looks great. Single RGB source for the subject. |
|
||||
| **Cutout over its own source mp4** (text-behind-subject) | Same mp4 the cutout came from | At `balanced` doubling is barely visible; at `fast` you'll see color shift / edge halo. Use `best` for masters. |
|
||||
| **Cutout over a different take of the same person** | Footage of the same subject | **Two overlapping people. Don't do this.** |
|
||||
|
||||
## Text-behind-subject pattern (two non-obvious rules)
|
||||
|
||||
Putting a headline behind a presenter cutout:
|
||||
|
||||
```html
|
||||
<video
|
||||
src="presenter.mp4"
|
||||
id="bg"
|
||||
data-start="0"
|
||||
data-duration="6"
|
||||
data-track-index="0"
|
||||
muted
|
||||
playsinline
|
||||
></video>
|
||||
|
||||
<h1 id="headline" style="z-index:2; ...">MAKE IT IN HYPERFRAMES</h1>
|
||||
|
||||
<div class="cutout-wrap" style="position:absolute; inset:0; z-index:3; opacity:0">
|
||||
<video
|
||||
src="presenter.webm"
|
||||
data-start="0"
|
||||
data-duration="6"
|
||||
data-track-index="1"
|
||||
muted
|
||||
playsinline
|
||||
></video>
|
||||
</div>
|
||||
```
|
||||
|
||||
```js
|
||||
// Flip the wrapper's opacity at the cut, NOT the video's
|
||||
tl.set(".cutout-wrap", { opacity: 1 }, 3.3);
|
||||
```
|
||||
|
||||
Two rules that are easy to miss:
|
||||
|
||||
1. **Wrap the cutout `<video>` in a non-timed `<div>` and animate the wrapper's opacity, not the video element's.** The framework forces `opacity: 1` on active clips (any element with `data-start` / `data-duration`), so animating the video's opacity directly is silently overridden. The wrapper has no `data-*` attributes, so it's owned by your CSS / GSAP.
|
||||
2. **Both videos use `data-start="0"` and `data-media-start="0"`** so the framework decodes them in sync from t=0. Late-mounting the cutout (`data-start=3.3`) introduces a seek + warm-up that lands a frame off the base mp4 — visible as one frame of misalignment at the cut.
|
||||
|
||||
## Layer separation (`--background-output`)
|
||||
|
||||
Emits a **second** transparent video alongside the cutout: same source RGB, alpha is `255 - mask` instead of `mask`. The cutout has the subject opaque; the plate has the surroundings opaque (with a transparent hole where the subject was). Use it when text / graphics need to live **between** the two layers.
|
||||
|
||||
| File | Alpha is… | Use it for |
|
||||
| -------------------------------- | ------------------------------------------------------- | ---------------------------------------------------------------- |
|
||||
| `-o subject.webm` | mask — subject opaque, background transparent | Foreground layer (top) |
|
||||
| `--background-output plate.webm` | inverse mask — surroundings opaque, subject transparent | Bottom layer; place text / graphics between this and the subject |
|
||||
|
||||
Both share the same `--quality` and run from a single inference pass — only encode cost roughly doubles. Only valid for video inputs with `.webm` / `.mov` outputs.
|
||||
|
||||
**Hole-cut, not inpainted.** The subject region in `plate.webm` is fully transparent — composite something opaque under it to fill the hole.
|
||||
|
||||
**Single test for whether `--background-output` is the right tool:** _will anything ever be visible through the subject's silhouette where the subject used to be?_ If no, you don't need the plate — `subject.webm` alone over a different background is enough.
|
||||
|
||||
### Use case → right tool
|
||||
|
||||
| Use case | Right tool |
|
||||
| ----------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- |
|
||||
| Text/graphics between the cutout and the plate (this command's reason for existing) | **Hole-cut** (`--background-output`) |
|
||||
| Subject onto an unrelated scene | Just `subject.webm`; ignore the plate |
|
||||
| Show the room _without_ the person, alone over no other content | **Clean plate** — needs an inpainter (LaMa, ProPainter, E2FGVI). Not this command. |
|
||||
| Replace the subject with a different subject | **Clean plate** — same as above |
|
||||
|
||||
### Canonical 3-layer template (plate + content + cutout)
|
||||
|
||||
Ship just the two transparent layers and let arbitrary content live between them — no original mp4 needed:
|
||||
|
||||
```html
|
||||
<!-- z=1 plate: surroundings opaque, subject silhouette transparent -->
|
||||
<video
|
||||
src="plate.webm"
|
||||
data-start="0"
|
||||
data-duration="6"
|
||||
data-track-index="0"
|
||||
muted
|
||||
playsinline
|
||||
></video>
|
||||
|
||||
<!-- z=2 your content lives between the layers -->
|
||||
<h1 id="headline" style="z-index:2; ...">MAKE IT IN HYPERFRAMES</h1>
|
||||
|
||||
<!-- z=3 cutout floats the subject back on top -->
|
||||
<div class="cutout-wrap" style="position:absolute; inset:0; z-index:3">
|
||||
<video
|
||||
src="subject.webm"
|
||||
data-start="0"
|
||||
data-duration="6"
|
||||
data-track-index="1"
|
||||
muted
|
||||
playsinline
|
||||
></video>
|
||||
</div>
|
||||
```
|
||||
|
||||
Functionally equivalent to the text-behind-subject pattern above, but doesn't require shipping the original mp4 — the plate replaces it. Use this when delivering just the two transparent layers as a reusable asset.
|
||||
|
||||
## When `remove-background` is NOT the right tool
|
||||
|
||||
If a user asks for "the room **without** the person, displayed standalone" (no subject anywhere, no compositing on top), `--background-output` is wrong — its plate has a transparent hole, not a filled-in clean plate. They need an **inpainter**: LaMa, ProPainter, or E2FGVI. Tell them this command can't do it.
|
||||
@@ -0,0 +1,29 @@
|
||||
# Requirements & Caches
|
||||
|
||||
## Credential & key priority
|
||||
|
||||
Run `npx hyperframes auth status` to see what's configured and which engines a workflow will use (see the skill's **Preflight** section). Keys resolve in this order — **first match wins**:
|
||||
|
||||
| Provider | Resolution order (first non-empty wins) | Local deps when used |
|
||||
| ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ |
|
||||
| **HeyGen** (TTS + BGM/SFX retrieval) | `$HEYGEN_API_KEY` → `$HYPERFRAMES_API_KEY` → `~/.heygen/credentials` (shared with heygen-cli; `$HEYGEN_CONFIG_DIR` overrides the dir; written by `hyperframes auth login`) | none (REST) |
|
||||
| **ElevenLabs** (TTS fallback) | `$ELEVENLABS_API_KEY` | `pip install elevenlabs` |
|
||||
| **Lyria** (BGM fallback) | `$GEMINI_API_KEY` → `$GOOGLE_API_KEY` | `pip install google-genai` |
|
||||
| **Kokoro** (TTS, no key) | always — final voice fallback | `pip install kokoro-onnx soundfile` |
|
||||
| **MusicGen** (BGM, no key) | always — final music fallback | `pip install transformers torch soundfile numpy` |
|
||||
|
||||
`hyperframes auth login` (browser OAuth) is the recommended setup: one sign-in, every project, no per-repo `.env`. An OAuth login is sent as `Authorization: Bearer`; an API key as `X-Api-Key`; both are tagged with `X-HeyGen-Source: cli`. OAuth CLI users can consume the web-plan free allowance for HeyGen TTS (10 min/month); API keys follow the normal API billing path. With no HeyGen credential, voice/BGM run fully locally (Kokoro / MusicGen) — `hyperframes auth status` and `hyperframes doctor` both report whether those local deps are installed.
|
||||
|
||||
## Model caches & system dependencies
|
||||
|
||||
Each command downloads its own model on first run and caches it under `~/.cache/hyperframes/`:
|
||||
|
||||
- **TTS (HeyGen)** — no local deps; needs a HeyGen credential + `ffmpeg` on PATH (to transcode the mp3 response to `.wav`). Credential resolves like the CLI: `$HEYGEN_API_KEY` → `$HYPERFRAMES_API_KEY` → `~/.heygen/credentials` (shared with heygen-cli; run `npx hyperframes auth login`). An OAuth login is sent as `Authorization: Bearer`; an API key as `X-Api-Key`; both include `X-HeyGen-Source: cli` so the backend can apply CLI OAuth free usage.
|
||||
- **TTS (ElevenLabs)** — same as HeyGen: API key + `ffmpeg`.
|
||||
- **TTS (Kokoro)** — Kokoro-82M (~311 MB) + voices (~27 MB) in `tts/`. Requires Python 3.8+ with `kokoro-onnx` and `soundfile` (`pip install kokoro-onnx soundfile`). Non-English text also needs `espeak-ng` system-wide.
|
||||
- **BGM (Lyria)** — needs `$GEMINI_API_KEY` or `$GOOGLE_API_KEY` + `pip install google-genai`. No local model cache.
|
||||
- **BGM (MusicGen)** — `pip install transformers torch soundfile`. `facebook/musicgen-small` (~300 MB) cached under `~/.cache/huggingface/` on first run.
|
||||
- **Transcribe** — Whisper model size depending on choice (75 MB – 3.1 GB) in `whisper/`, downloaded from HuggingFace on first use. `whisper.cpp` itself is NOT bundled: the CLI resolves it from PATH, installs via Homebrew (macOS), or builds it from source with git+cmake on first use (`$HYPERFRAMES_WHISPER_PATH` overrides).
|
||||
- **Remove-background** — `u2net_human_seg` (~168 MB ONNX) in `background-removal/models/`. Peak inference RAM ~1.5 GB.
|
||||
|
||||
Run `npx hyperframes doctor` if a command fails because of a missing dependency.
|
||||
@@ -0,0 +1,42 @@
|
||||
# Sound effects (SFX)
|
||||
|
||||
Named sound effects, produced by the shared audio engine (`scripts/audio.mjs` → `scripts/lib/sfx.mjs`). **Provider-gated** by the engine's one switch — whether a HeyGen credential is present, decided once (not per cue):
|
||||
|
||||
- **HeyGen credential present → retrieve every cue** from HeyGen's audio library (`/v3/audio/sounds`, `type=sound_effects`, `min_score=0.4`). Search-and-download, **not** generation. The bundled library is NOT consulted.
|
||||
- **No credential → the bundled 21-file library** (`assets/sfx/` + `manifest.json`): match each cue name, copy the matched file into the project. Offline, deterministic, free.
|
||||
|
||||
There is no `npx hyperframes sfx` command. SFX is never generated — it is retrieved (online) or taken from the bundled library (offline).
|
||||
|
||||
## Cues — request → meta
|
||||
|
||||
Each line names the effects it wants: `lines[].sfx: ["whoosh", "ui click"]`. The engine flattens these into cues, resolves them per the switch, dedupes identical `(id, name)` pairs (the same effect named twice downloads/copies once), and writes `audio_meta.sfx[]`:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"id": "3", // joins the cue to the caller's model (frame / scene / segment)
|
||||
"name": "whoosh",
|
||||
"file": "assets/sfx/whoosh.mp3", // downloaded or copied, relative to project root
|
||||
"source": "heygen" | "local", // which route resolved it
|
||||
"offset_s": 0, // delay from the line's start
|
||||
"duration_s": 0.57,
|
||||
"volume": 0.35 // SFX sit UNDER voice + BGM
|
||||
}
|
||||
```
|
||||
|
||||
A cue that matches nothing is **skipped** (recorded as an anomaly); SFX never blocks a render. Neither route replaces a file of yours already at the output name: the cue gets the next free name (`whoosh-2.mp3`), reported as an anomaly, and `file` carries the real path.
|
||||
|
||||
## HeyGen retrieval (credentialed)
|
||||
|
||||
`searchSounds(name, "sound_effects", { limit: 3, minScore: 0.4 })` → top hit → `assets/sfx/<slug>.mp3`. Results are ranked by `score` (each carries a presigned `audio_url`, `duration`, `description`). The floor is **0.4** because good SFX hits score ~0.5–0.67 — below the API's default `0.7`, which would silently drop most named cues (only whoosh/swoosh-family clears 0.7). `duration_s` comes from the result (else 1.0). Name effects concretely (`glass shatter`, not `dramatic sound`); a vague query returns a poor match.
|
||||
|
||||
## Bundled library (no credential)
|
||||
|
||||
21 curated files in `assets/sfx/`, indexed by `manifest.json` — `{ file, duration, description }` per key (e.g. `whoosh`, `pop`, `click`, `chime`, `riser`, `impact-bass-1`, `glitch-1`, `typing`, …). A cue name resolves by **manifest key, file basename, or slug**, so `whoosh`, `whoosh.mp3`, or `"ui click"` (→ slug) all match. Matched files are copied into the project's `assets/sfx/`; `duration_s` comes from the manifest, so timing is known **offline** — e.g. `riser` is 10.03s, so trigger it at `climax − 10.03s`. The manifest's `description` field carries placement hints per effect; read `assets/sfx/manifest.json` for the full set and usage.
|
||||
|
||||
## Rules
|
||||
|
||||
- **Volume ~0.35.** SFX must sit under narration and BGM, not fight them.
|
||||
- **No match → skip, don't fail.** A missing effect logs an anomaly and moves on; never a render blocker.
|
||||
- **Retrieval (credentialed) or bundled library (offline) — never generation.** You search HeyGen by text, or match a name against the 21-file manifest.
|
||||
- **One asset per distinct name.** Reuse across lines is deduped to a single download/copy, many cues.
|
||||
- **The switch is global, not per cue.** With a credential, retrieval handles even the long tail (effects not in the 21); without one, only the 21 bundled names resolve.
|
||||
@@ -0,0 +1,52 @@
|
||||
# Transcription
|
||||
|
||||
Create normalized word-level timestamps. **Always specify `--model` explicitly** — the CLI default is `small.en`, which silently translates non-English audio into English.
|
||||
|
||||
```bash
|
||||
npx hyperframes transcribe audio.mp3 --model small.en # known English
|
||||
npx hyperframes transcribe video.mp4 --model small --language es # known Spanish
|
||||
npx hyperframes transcribe audio.mp3 --model small # unknown language (auto-detect)
|
||||
npx hyperframes transcribe subtitles.srt # import existing
|
||||
npx hyperframes transcribe subtitles.vtt
|
||||
npx hyperframes transcribe openai-response.json
|
||||
```
|
||||
|
||||
## Language Rule (Non-Negotiable)
|
||||
|
||||
`.en` models (`tiny.en` / `base.en` / `small.en` / `medium.en`) **translate** non-English audio into English. This silently destroys the original language.
|
||||
|
||||
1. **Known English** → `--model small.en` (or `medium.en` for music / noisy audio)
|
||||
2. **Known non-English** → `--model small --language <iso-code>` (no `.en` suffix)
|
||||
3. **Unknown language** → `--model small` (whisper auto-detects)
|
||||
|
||||
**CLI default is `small.en`** — do not rely on it; always pass `--model` to make the choice explicit. `--language` also filters out non-target-language segments from mixed-language audio.
|
||||
|
||||
## Model Sizes
|
||||
|
||||
| Model | Size | Speed | When |
|
||||
| ---------- | ------ | -------- | ------------------------------------- |
|
||||
| `tiny` | 75 MB | Fastest | Quick previews, smoke tests |
|
||||
| `base` | 142 MB | Fast | Short clips, clear audio |
|
||||
| `small` | 466 MB | Moderate | Default for most multilingual content |
|
||||
| `medium` | 1.5 GB | Slow | Music with vocals, noisy audio |
|
||||
| `large-v3` | 3.1 GB | Slowest | Production quality |
|
||||
|
||||
### Picking a model by content type
|
||||
|
||||
1. Speech over silence / light background → `small.en`
|
||||
2. Speech over music, or music with vocals → start with `medium.en`
|
||||
3. Produced music track (vocals + full instrumentation) → start with `medium.en`; expect to need manual lyrics or an external API ([`captions/transcript-handling.md`](captions/transcript-handling.md) → "Using External Transcription APIs")
|
||||
4. Multilingual → `medium` or `large-v3` (no `.en` suffix), pair with `--language`
|
||||
|
||||
## Output Shape
|
||||
|
||||
Compositions consume a flat array of word objects. The `id` (`w0`, `w1`, …) is added during normalization for stable references in caption overrides; optional for backwards compatibility.
|
||||
|
||||
```json
|
||||
[
|
||||
{ "id": "w0", "text": "Hello", "start": 0.0, "end": 0.5 },
|
||||
{ "id": "w1", "text": "world.", "start": 0.6, "end": 1.2 }
|
||||
]
|
||||
```
|
||||
|
||||
For mandatory caption-quality checks, retry rules, and the OpenAI/Groq Whisper API import path, see `captions/transcript-handling.md`.
|
||||
@@ -0,0 +1,35 @@
|
||||
# TTS → Captions
|
||||
|
||||
When no recorded voiceover exists, generate one and obtain word-level caption timing. Two paths depending on which TTS provider is in use:
|
||||
|
||||
## Path A — HeyGen (single call, no Whisper)
|
||||
|
||||
HeyGen returns word timestamps in the same response as the audio. Use the
|
||||
bundled REST helper (the `hyperframes tts` command is Kokoro-only):
|
||||
|
||||
```bash
|
||||
node skills/media-use/audio/scripts/heygen-tts.mjs \
|
||||
script.txt --output narration.wav --words narration.words.json
|
||||
```
|
||||
|
||||
`narration.words.json` is already in the `[{ id, text, start, end }]` shape the captions pipeline consumes — no separate transcribe pass.
|
||||
|
||||
## Path B — Gemini / ElevenLabs / Kokoro (TTS → transcription)
|
||||
|
||||
These adapters supply audio without word data. The shared audio engine runs
|
||||
transcription automatically when timings are absent. For Gemini, use the
|
||||
request in [Text to speech](tts.md#gemini-narration), then consume
|
||||
`audio_meta.json` → `voices[].words`.
|
||||
|
||||
For a standalone local Kokoro generation, generate the audio, then transcribe:
|
||||
|
||||
```bash
|
||||
npx hyperframes tts script.txt --voice af_heart --output narration.wav
|
||||
npx hyperframes transcribe narration.wav --model small.en # voice af_heart is American English
|
||||
```
|
||||
|
||||
Whisper extracts precise word boundaries from the generated audio, so caption timing matches delivery without hand-tuning. Match `--model` to the voice's language (use `small.en` for `a`/`b` prefixes, `small --language <code>` otherwise). Then consume `transcript.json` via the caption references in `captions/`.
|
||||
|
||||
For Gemini, verify that transcription preserved the script, especially names,
|
||||
numbers, and delivery pauses. If `words` is empty, resolve the transcription
|
||||
failure before captioning. Generate and align again after changing the read.
|
||||
@@ -0,0 +1,260 @@
|
||||
# Text To Speech
|
||||
|
||||
`npx hyperframes tts` synthesizes locally with Kokoro. It does not accept a
|
||||
`--provider` or `--words` flag. For HeyGen audio plus word timestamps, use the
|
||||
bundled `heygen-tts.mjs` script below.
|
||||
|
||||
> **Run the Preflight first — no credential is not a green light to silently use the local voice.** Before generating a voiceover, complete the sign-in **Preflight** (see `../../SKILL.md` → Preflight): run `npx hyperframes auth status`, recommend signing in, and **STOP for the user's choice** (sign in for HeyGen voices, or continue offline with local Kokoro). This applies to a one-off "generate a voiceover" request just as much as inside a full workflow.
|
||||
|
||||
## Narrating a HyperFrames docs video
|
||||
|
||||
Anything that will sit on hyperframes.heygen.com uses one narrator, so the site
|
||||
does not sound like two products.
|
||||
|
||||
| | |
|
||||
| -------------- | ---------------------------------------------- |
|
||||
| Voice | **River** — "Relaxed, Neutral, Informative" |
|
||||
| Provider | ElevenLabs |
|
||||
| `voice_id` | `SAz9YHcvj6GT2YYXdXww` |
|
||||
| Model | `eleven_multilingual_v2` |
|
||||
| Pace | 145–155 words per minute, with room to breathe |
|
||||
| Music under it | about −31 LUFS, never masking the voice |
|
||||
|
||||
```bash
|
||||
curl -s -X POST "https://api.elevenlabs.io/v1/text-to-speech/SAz9YHcvj6GT2YYXdXww" \
|
||||
-H "xi-api-key: $ELEVENLABS_API_KEY" -H "Content-Type: application/json" \
|
||||
-d '{"text":"...","model_id":"eleven_multilingual_v2"}' -o take.mp3
|
||||
```
|
||||
|
||||
This is the voice every user-journey film on the docs site already uses. Falling
|
||||
back to local Kokoro because a key was not to hand produces a film that sounds
|
||||
wrong beside the others — three docs videos were built that way and had to be
|
||||
re-voiced. If you cannot reach ElevenLabs, say so and stop rather than
|
||||
substituting a different voice.
|
||||
|
||||
Use another voice only for a documented reason, and write the reason down.
|
||||
|
||||
## Available routes
|
||||
|
||||
Gemini is an explicit alternative to the automatic provider order below. A
|
||||
request to use Gemini already chooses the provider; do not redirect that user
|
||||
to HeyGen sign-in. Read the Gemini section for its credential requirement.
|
||||
|
||||
| Order | Provider | Env trigger | Voice IDs | Word timestamps | Audio format |
|
||||
| ----- | ----------------- | -------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | ----------------------------------------- | -------------------- |
|
||||
| 1 | HeyGen (Starfish) | `$HEYGEN_ACCESS_TOKEN` / `$HEYGEN_API_KEY` / `~/.heygen/credentials` | UUIDs from `GET /v3/voices?engine=starfish`, or your cloned voice ids (`heygen-voice.mjs`) | **Yes** (`word_timestamps[]` in response) | mp3 → wav via ffmpeg |
|
||||
| 2 | ElevenLabs | `$ELEVENLABS_API_KEY` | UUIDs from elevenlabs.io dashboard | No | mp3 → wav via ffmpeg |
|
||||
| 3 | Kokoro-82M | always (local fallback) | `am_michael`, `af_heart`, … (54 voices) | No | wav direct |
|
||||
|
||||
```bash
|
||||
# Local Kokoro CLI
|
||||
npx hyperframes tts "Welcome to HyperFrames" -o narration.wav
|
||||
```
|
||||
|
||||
## Self-contained HeyGen (no CLI) — `scripts/heygen-tts.mjs`
|
||||
|
||||
The published `hyperframes tts` CLI synthesizes locally with Kokoro only. When you
|
||||
want HeyGen specifically — best quality **plus** word timestamps in one call — use
|
||||
the skill's bundled script, which calls the HeyGen v3 REST API directly and needs
|
||||
no CLI provider plumbing:
|
||||
|
||||
The script resolves a HeyGen credential the same way the CLI does — first source
|
||||
wins: a host-injected OAuth `$HEYGEN_ACCESS_TOKEN` → `$HEYGEN_API_KEY` →
|
||||
`$HYPERFRAMES_API_KEY` → a project `.env` (auto-loaded,
|
||||
walks up ≤5 dirs) → `~/.heygen/credentials` (shared with heygen-cli;
|
||||
`$HEYGEN_CONFIG_DIR` overrides the dir). An OAuth login is sent as
|
||||
`Authorization: Bearer`; an API key as `X-Api-Key`; both include
|
||||
`X-HeyGen-Source: cli`. OAuth CLI users can consume the web-plan free allowance
|
||||
(10 min/month) before paid usage; API keys follow normal API billing. If the
|
||||
only credential is an expired OAuth token it stops with a hint to run
|
||||
`npx hyperframes auth refresh`.
|
||||
|
||||
```bash
|
||||
# Only needed if you haven't run `npx hyperframes auth login`:
|
||||
export HEYGEN_API_KEY=... # or put it in a project .env
|
||||
|
||||
# Synthesize + capture word timestamps in one call (skips a Whisper pass)
|
||||
node skills/media-use/audio/scripts/heygen-tts.mjs \
|
||||
"Welcome to HyperFrames." -o narration.wav --words narration.words.json
|
||||
|
||||
node skills/media-use/audio/scripts/heygen-tts.mjs ./script.txt -o narration.wav
|
||||
node skills/media-use/audio/scripts/heygen-tts.mjs --list # public starfish voices
|
||||
```
|
||||
|
||||
- **Voice:** `--voice <id>` must be a **starfish** voice_id (`--list`, or `GET /v3/voices?engine=starfish`) or one of your cloned voice ids (below). v2-catalog ids are rejected with HTTP 400. Omit `--voice` (English) and it defaults to **Marcia** (`05f19352e8f74b0392a8f411eba40de1`, a fixed default so the choice is deterministic). Non-English with no `--voice` falls back to the first matching catalog voice.
|
||||
- **Output:** `.wav` → transcoded to 44.1k mono via ffmpeg; `.mp3` → raw bytes (no ffmpeg needed).
|
||||
- **Words:** `--words <path>` writes the flat `[{id,text,start,end}]` shape below, drop-in for the captions pipeline. HeyGen's `<start>`/`<end>` boundary sentinels are filtered out and ids are re-contiguous.
|
||||
- **Non-English:** `--lang <code>` (anything but `en`) is sent as the request `language`.
|
||||
|
||||
## Cloned voices — `scripts/heygen-voice.mjs`
|
||||
|
||||
Clone a voice from an mp3 or wav recording, then pass the printed id to
|
||||
`heygen-tts.mjs --voice`. Same credential resolution as above. Any API refusal
|
||||
(clone limit reached, plan upgrade required, voice not found) prints HeyGen's own
|
||||
message to stderr and exits 1.
|
||||
|
||||
```bash
|
||||
node skills/media-use/audio/scripts/heygen-voice.mjs clone take.mp3 --name "My voice" # → {"voice_id":"..."}
|
||||
node skills/media-use/audio/scripts/heygen-voice.mjs list --prefix "My" # → [{"voice_id","name","created_at"}]
|
||||
node skills/media-use/audio/scripts/heygen-voice.mjs delete <voice_id>
|
||||
```
|
||||
|
||||
`clone` waits up to 120 s for the clone to finish. `list` reads
|
||||
`GET /v3/voices?type=private`.
|
||||
|
||||
## When to use which provider
|
||||
|
||||
| Goal | Use |
|
||||
| --------------------------------------------------------- | --------------------------------------------------------- |
|
||||
| Best voice quality + word timestamps in one call | **HeyGen** |
|
||||
| Drop-in cloud TTS, big voice catalog | **ElevenLabs** |
|
||||
| Offline, no API key, fast iteration | **Kokoro** |
|
||||
| Directed delivery with Gemini prebuilt or custom voices | **Gemini** (explicit selection; transcription for timing) |
|
||||
| Non-English multilingual with deterministic phonemization | **Kokoro** (`ef_dora`, `jf_alpha`, `zf_xiaobei`, …) |
|
||||
|
||||
## Gemini narration
|
||||
|
||||
Use the shared audio engine, not `hyperframes tts`. Authenticate with either:
|
||||
|
||||
- `GEMINI_API_KEY` or `GOOGLE_API_KEY` (first set key wins).
|
||||
- A service-account JSON file at `GOOGLE_APPLICATION_CREDENTIALS`, or injected
|
||||
JSON in `GCS_CREDS`. Install `google-auth requests` in the Python 3 environment
|
||||
used by the helper. A configured file takes precedence over injected JSON.
|
||||
|
||||
API keys take precedence over service accounts: unset both key variables to use
|
||||
OAuth. Never put credentials in a request file or composition. The helper
|
||||
obtains a fresh OAuth token for each generation with the
|
||||
`generative-language.retriever` scope. The quota project resolves from
|
||||
`GOOGLE_CLOUD_PROJECT`, then `GCLOUD_PROJECT_ID`, then the service-account JSON.
|
||||
User ADC files and metadata-server authentication are not supported.
|
||||
|
||||
Both routes call the Gemini Developer Interactions API, not Cloud TTS or Vertex
|
||||
AI. Those APIs have separate model catalogs and access requirements.
|
||||
|
||||
Save this as `audio_request.json` in the project:
|
||||
|
||||
```json
|
||||
{
|
||||
"provider": "gemini",
|
||||
"tts_model": "gemini-3.8-flash-tts",
|
||||
"voice": "Kore",
|
||||
"lang": "en",
|
||||
"style": "Warm, clear, conversational. Leave a short pause between sentences.",
|
||||
"lines": [
|
||||
{ "id": "intro", "text": "Every word has a moment. Let the picture follow the voice." }
|
||||
],
|
||||
"bgm": { "mode": "none" }
|
||||
}
|
||||
```
|
||||
|
||||
```bash
|
||||
node <SKILL_DIR>/audio/scripts/audio.mjs \
|
||||
--request ./audio_request.json --hyperframes . --out ./audio_meta.json --only tts
|
||||
```
|
||||
|
||||
The engine saves `assets/voice/intro.wav` (or `intro-2.wav` when a file of yours already has that name; `voices[].path` says which), measures its duration, and transcribes
|
||||
it into `voices[].words` in `audio_meta.json`. Check that every requested line
|
||||
has audio and nonempty word timings before building a captioned video. Review
|
||||
the timings against the actual audio; transcription is estimated alignment,
|
||||
not native TTS timestamps. Do not distribute words evenly across a clip.
|
||||
|
||||
- **Models:** `gemini-3.8-flash-tts` (default), `gemini-3.8-flash-lite-tts`,
|
||||
`gemini-3.1-flash-tts-preview`, `gemini-2.5-pro-preview-tts`, and
|
||||
`gemini-2.5-flash-preview-tts`. Use these exact Developer API IDs; Cloud TTS
|
||||
aliases such as `gemini-2.5-flash-tts` are not accepted.
|
||||
- **Voice:** `Kore` by default; pass another prebuilt voice or an existing custom
|
||||
voice ID for 3.8. Older models require prebuilt voices. Creating or replicating
|
||||
voices is outside this helper.
|
||||
- **Delivery:** Put directions in `style`, not in spoken `text`. Each line can
|
||||
override `style`. 3.8 uses structured annotations; older models receive a
|
||||
delivery prompt before the transcript. Check that directions were not spoken.
|
||||
Use style for pacing; numeric `speed` must be omitted or 1.
|
||||
- **Audio:** 3.8 returns a complete WAV. Older models return mono 16-bit PCM,
|
||||
which the helper wraps as WAV at the returned sample rate without resampling.
|
||||
No generation calls run during playback or rendering.
|
||||
- **Timing:** This adapter requests no native word timestamps. It uses the
|
||||
existing transcription pass. `lang` selects transcription language; Gemini
|
||||
infers speech language from the text.
|
||||
- **Workflow adapters:** Product-launch, faceless-explainer, and PR-video audio
|
||||
scripts accept `--provider gemini --voice Kore --tts-model gemini-3.8-flash-tts
|
||||
--style "Warm and clear"`.
|
||||
|
||||
An API error is reported as a failed line; an explicitly chosen Gemini voice
|
||||
never silently falls back to another provider. Check the engine's anomalies
|
||||
and output metadata, not only its exit code.
|
||||
|
||||
API contract: [Google's speech generation guide](https://ai.google.dev/gemini-api/docs/speech-generation).
|
||||
|
||||
## ffmpeg requirement
|
||||
|
||||
HeyGen + ElevenLabs return mp3. The bundled HeyGen helper transcodes to wav
|
||||
when `--output` ends in `.wav` (the default and what downstream `ffprobe` +
|
||||
Whisper expect). If you'd rather skip the transcode, pass `-o file.mp3`.
|
||||
Without `ffmpeg` on PATH, wav output from cloud providers fails; the local
|
||||
Kokoro CLI writes wav directly.
|
||||
|
||||
## Voice selection (Kokoro)
|
||||
|
||||
Default `af_heart`. Curated picks:
|
||||
|
||||
| Content type | Voice |
|
||||
| ----------------- | ---------------------- |
|
||||
| Product demo | `af_heart`, `af_nova` |
|
||||
| Tutorial / how-to | `am_adam`, `bf_emma` |
|
||||
| Marketing / promo | `af_sky`, `am_michael` |
|
||||
| Documentation | `bf_emma`, `bm_george` |
|
||||
| Casual / social | `af_heart`, `af_sky` |
|
||||
|
||||
Run `npx hyperframes tts --list` for the bundled set.
|
||||
|
||||
## Multilingual (Kokoro voice prefix → language)
|
||||
|
||||
The first letter of a Kokoro voice ID picks the phonemizer language; `--lang` overrides auto-detection.
|
||||
|
||||
| Prefix | Language |
|
||||
| ------ | -------------------- |
|
||||
| `a` | American English |
|
||||
| `b` | British English |
|
||||
| `e` | Spanish |
|
||||
| `f` | French |
|
||||
| `h` | Hindi |
|
||||
| `i` | Italian |
|
||||
| `j` | Japanese |
|
||||
| `p` | Brazilian Portuguese |
|
||||
| `z` | Mandarin |
|
||||
|
||||
```bash
|
||||
npx hyperframes tts "La reunión empieza a las nueve" --voice ef_dora
|
||||
npx hyperframes tts "Today is a nice day" --voice af_heart
|
||||
```
|
||||
|
||||
Valid `--lang` codes (only needed to override the voice's auto-detected language): `en-us`, `en-gb`, `es`, `fr-fr`, `hi`, `it`, `pt-br`, `ja`, `zh`.
|
||||
|
||||
Non-English phonemization requires `espeak-ng` system-wide (`brew install espeak-ng` / `apt-get install espeak-ng`).
|
||||
|
||||
## Speed
|
||||
|
||||
- `0.7-0.8` — tutorial, complex content, accessibility
|
||||
- `1.0` — natural pace (default)
|
||||
- `1.1-1.2` — intros, transitions, upbeat content
|
||||
- `1.5+` — rarely appropriate, test carefully
|
||||
|
||||
The `hyperframes tts` command honors `--speed` for Kokoro. Provider-specific
|
||||
helpers document their own pacing controls.
|
||||
|
||||
## Long scripts
|
||||
|
||||
Past a few paragraphs, write the text to a `.txt` file and pass the path. Inputs over ~5 minutes of speech may benefit from splitting into segments.
|
||||
|
||||
## HeyGen word-timestamp shape
|
||||
|
||||
When `--words <path>` is passed to a HeyGen call, the file is written in the same flat shape `transcribe` produces — drop-in compatible with the captions pipeline:
|
||||
|
||||
```json
|
||||
[
|
||||
{ "id": "w0", "text": "Hi", "start": 0.0, "end": 0.21 },
|
||||
{ "id": "w1", "text": "there", "start": 0.22, "end": 0.55 }
|
||||
]
|
||||
```
|
||||
|
||||
For ElevenLabs / Kokoro, run `npx hyperframes transcribe narration.wav --model small.en` to get the same shape.
|
||||
@@ -0,0 +1,311 @@
|
||||
#!/usr/bin/env node
|
||||
// audio.mjs — the shared HyperFrames audio engine. ONE implementation of TTS +
|
||||
// BGM + SFX for every video workflow (product-launch, general-video, pr-to-video,
|
||||
// …). Workflows do NOT vendor a copy: they write a neutral `audio_request.json`
|
||||
// (a tiny per-workflow adapter maps their storyboard/scenes into it) and call:
|
||||
//
|
||||
// node <MEDIA_DIR>/scripts/audio.mjs --request ./audio_request.json --hyperframes . --out ./audio_meta.json
|
||||
//
|
||||
// The three capabilities degrade on ONE switch — whether HeyGen is configured
|
||||
// (credential present, NOT the CLI). This mirrors the table in ../SKILL.md:
|
||||
//
|
||||
// TTS : HeyGen REST → ElevenLabs → Kokoro (CLI)
|
||||
// BGM : HeyGen retrieve → (no credential) Lyria/MusicGen generate
|
||||
// SFX : HeyGen retrieve → (no credential) bundled 19-file library
|
||||
//
|
||||
// ── audio_request.json (input) ────────────────────────────────────────────────
|
||||
// {
|
||||
// "provider": "auto", // auto|heygen|elevenlabs|kokoro|gemini (override: --provider)
|
||||
// "lang": "en", "speed": 1.0,
|
||||
// "lines": [ // one TTS unit each; id joins back to the caller's model
|
||||
// { "id": "01", "text": "...", "sfx": ["whoosh", "ui click"] }
|
||||
// ],
|
||||
// "bgm": { "mode": "retrieve", // retrieve|generate|none (override: --bgm-mode / --no-bgm)
|
||||
// "query": "calm cinematic underscore", // mood for retrieval
|
||||
// "prompt": null, // full prompt for generation (else inferred)
|
||||
// "blob": "...", "archetype": "...", "arc": "..." } // optional mood-inference hints
|
||||
// }
|
||||
//
|
||||
// ── audio_meta.json (output, id-keyed) ───────────────────────────────────────
|
||||
// { tts_provider, voice_id,
|
||||
// bgm: { path, volume, mode, query?, duration_s? } | null,
|
||||
// bgm_pending, bgm_provider, bgm_pid, bgm_log, bgm_mode, bgm_target_duration_s, …,
|
||||
// voices: [ { id, path, duration_s, words: [{id,text,start,end}] } ],
|
||||
// sfx: [ { id, name, file, source, offset_s, duration_s, volume } ],
|
||||
// total_duration_s }
|
||||
//
|
||||
// --only tts,bgm,sfx runs a subset and MERGES into an existing --out (so a
|
||||
// workflow can do TTS+BGM early, then SFX later once cues exist). When BGM uses
|
||||
// the generate path it is spawned detached (bgm_pending:true) — run wait-bgm.mjs
|
||||
// before assembling.
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { heygenAuthHeaders, heygenCredential, loadEnvFromDir } from "./lib/heygen.mjs";
|
||||
import {
|
||||
ffprobeDuration,
|
||||
pickProvider,
|
||||
resolveVoiceId,
|
||||
synthesizeOne,
|
||||
transcribeWav,
|
||||
withWordIds,
|
||||
} from "./lib/tts.mjs";
|
||||
import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs";
|
||||
import { resolveSfx } from "./lib/sfx.mjs";
|
||||
import { mapWithConcurrency } from "./lib/concurrency.mjs";
|
||||
import { openAudioMeta } from "./lib/audio-meta.mjs";
|
||||
import { recordInManifest, voicePaths, writtenAssets } from "./lib/media-record.mjs";
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name, def) => {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : def;
|
||||
};
|
||||
const has = (name) => argv.includes(`--${name}`);
|
||||
const die = (m) => {
|
||||
console.error(`✗ audio engine: ${m}`);
|
||||
process.exit(1);
|
||||
};
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
|
||||
// Two independent reports of an unbounded Promise.all over TTS lines
|
||||
// overwhelming a machine: one OOM'd 12/13 concurrent Kokoro TTS +
|
||||
// whisper-transcribe lines on a resource-constrained laptop, the other saw
|
||||
// 7/8 lines fail on first run (concurrent cold-start model loads) and pass on
|
||||
// retry once the model was cached. Kokoro/Whisper each load their own local
|
||||
// model per subprocess, so firing every line at once multiplies that cost by
|
||||
// the line count. mapWithConcurrency caps how many run at once — still
|
||||
// parallel, just bounded.
|
||||
const ttsConcurrency = Math.max(1, Number(process.env.HYPERFRAMES_TTS_CONCURRENCY) || 4);
|
||||
|
||||
const hyperframesDir = resolve(flag("hyperframes", "."));
|
||||
const requestPath = resolve(flag("request", join(hyperframesDir, "audio_request.json")));
|
||||
const outPath = resolve(flag("out", join(hyperframesDir, "audio_meta.json")));
|
||||
const sfxLibDir = resolve(flag("sfx-lib", join(HERE, "..", "assets", "sfx")));
|
||||
const lyriaRecipe = resolve(flag("lyria-recipe", join(HERE, "lyria-recipe.py")));
|
||||
const onlyArg = flag("only", "tts,bgm,sfx");
|
||||
const only = new Set(
|
||||
onlyArg
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean),
|
||||
);
|
||||
const providerOverride = flag("provider", null);
|
||||
const bgmModeOverride = flag("bgm-mode", null);
|
||||
const noBgm = has("no-bgm");
|
||||
const voiceOverride = flag("voice", null);
|
||||
const speedOverride = flag("speed", null);
|
||||
const langOverride = flag("lang", null);
|
||||
const seedSeconds = Number(flag("seed-seconds", "28")) || 28;
|
||||
|
||||
if (!existsSync(requestPath)) die(`audio_request.json not found at ${requestPath}`);
|
||||
let request;
|
||||
try {
|
||||
request = JSON.parse(readFileSync(requestPath, "utf8"));
|
||||
} catch (e) {
|
||||
die(`audio_request.json parse: ${e.message}`);
|
||||
}
|
||||
const lines = Array.isArray(request.lines) ? request.lines : [];
|
||||
const lang = langOverride || request.lang || "en";
|
||||
const speedInput = speedOverride ?? request.speed ?? 1;
|
||||
const speed = Number(speedInput);
|
||||
if (!(speed > 0 && speed <= 3))
|
||||
die(`speed must be above 0 and at most 3, got ${JSON.stringify(speedInput)}`);
|
||||
|
||||
// ── env + HeyGen availability (the single switch) ─────────────────────────────
|
||||
loadEnvFromDir(hyperframesDir);
|
||||
const heygenOK = heygenCredential() !== null;
|
||||
|
||||
// ── merge base: preserve sections not selected by --only ──────────────────────
|
||||
const audioMeta = openAudioMeta(outPath);
|
||||
const prev = audioMeta.value;
|
||||
const anomalies = [];
|
||||
|
||||
// ── TTS ───────────────────────────────────────────────────────────────────────
|
||||
let voices = prev.voices ?? [];
|
||||
let ttsProvider = prev.tts_provider ?? null;
|
||||
let voiceId = prev.voice_id ?? null;
|
||||
if (only.has("tts") && lines.length) {
|
||||
try {
|
||||
ttsProvider = pickProvider(
|
||||
providerOverride || (request.provider === "auto" ? null : request.provider),
|
||||
);
|
||||
} catch (e) {
|
||||
die(e.message);
|
||||
}
|
||||
voiceId = await resolveVoiceId({
|
||||
provider: ttsProvider,
|
||||
userVoice: voiceOverride || request.voice,
|
||||
lang,
|
||||
});
|
||||
console.error(`· tts: ${ttsProvider} · voice ${voiceId} · ${lines.length} line(s)`);
|
||||
const paths = voicePaths(hyperframesDir, lines, anomalies);
|
||||
const synthLine = async (line) => {
|
||||
const id = String(line.id);
|
||||
const text = String(line.text ?? "").trim();
|
||||
if (!text) {
|
||||
anomalies.push(`line ${id}: empty text — skipped`);
|
||||
return null;
|
||||
}
|
||||
const rel = paths.get(id);
|
||||
const abs = join(hyperframesDir, rel);
|
||||
const { ok, words, error } = await synthesizeOne({
|
||||
provider: ttsProvider,
|
||||
text,
|
||||
voiceId,
|
||||
lang,
|
||||
speed,
|
||||
model: flag("tts-model", request.tts_model),
|
||||
style: flag("style", line.style ?? request.style),
|
||||
wavAbs: abs,
|
||||
hyperframesDir,
|
||||
});
|
||||
if (!ok) {
|
||||
anomalies.push(`line ${id}: TTS failed — omitted${error ? ` (${error})` : ""}`);
|
||||
return null;
|
||||
}
|
||||
let wordArr = words; // heygen: native; else transcribe
|
||||
if (!wordArr) wordArr = await transcribeWav({ wavRel: rel, lang, hyperframesDir });
|
||||
const dur = ffprobeDuration(abs);
|
||||
if (!isFinite(dur) || dur <= 0) {
|
||||
anomalies.push(`line ${id}: bad voice duration — omitted`);
|
||||
return null;
|
||||
}
|
||||
return { id, path: rel, duration_s: r3(dur), words: withWordIds(wordArr) };
|
||||
};
|
||||
const results = await mapWithConcurrency(lines, ttsConcurrency, synthLine);
|
||||
voices = results.filter(Boolean);
|
||||
for (const v of voices)
|
||||
console.error(` voice ${v.id}: ${v.path} (${v.duration_s}s, ${v.words.length} words)`);
|
||||
}
|
||||
const hasVoice = voices.length > 0;
|
||||
const totalDuration = r3(voices.reduce((a, v) => a + (v.duration_s || 0), 0));
|
||||
|
||||
// ── BGM ─────────────────────────────────────────────────────────────────────
|
||||
let bgm = prev.bgm ?? null;
|
||||
const bgmFields = {
|
||||
bgm_pending: prev.bgm_pending ?? false,
|
||||
bgm_provider: prev.bgm_provider ?? null,
|
||||
bgm_pid: prev.bgm_pid ?? null,
|
||||
bgm_log: prev.bgm_log ?? null,
|
||||
bgm_mode: prev.bgm_mode ?? null,
|
||||
bgm_target_duration_s: prev.bgm_target_duration_s ?? null,
|
||||
bgm_seed_duration_s: prev.bgm_seed_duration_s ?? null,
|
||||
bgm_loop_count: prev.bgm_loop_count ?? null,
|
||||
};
|
||||
if (only.has("bgm")) {
|
||||
bgm = null;
|
||||
Object.keys(bgmFields).forEach((k) => (bgmFields[k] = k === "bgm_pending" ? false : null));
|
||||
// Mode resolution. An EXPLICIT mode (flag or request.bgm.mode) is strict:
|
||||
// "retrieve" means retrieve-or-nothing — it never silently starts a detached
|
||||
// generate (a caller with no wait-bgm step, e.g. product-launch, must not get
|
||||
// a pending job it can't await). Only the UNSET/auto default picks generate
|
||||
// when HeyGen is absent.
|
||||
const explicitMode = bgmModeOverride || request.bgm?.mode || null;
|
||||
let mode = noBgm ? "none" : explicitMode || (heygenOK ? "retrieve" : "generate");
|
||||
if (mode === "retrieve" && !heygenOK) {
|
||||
anomalies.push(
|
||||
"bgm: retrieve requires a HeyGen credential — skipped (no generate fallback for an explicit retrieve)",
|
||||
);
|
||||
mode = "none";
|
||||
}
|
||||
|
||||
if (mode === "none") {
|
||||
console.error(`· bgm: disabled`);
|
||||
} else if (mode === "retrieve") {
|
||||
try {
|
||||
bgm = await retrieveBgm({
|
||||
query: request.bgm?.query,
|
||||
headers: heygenAuthHeaders(),
|
||||
hyperframesDir,
|
||||
hasVoice,
|
||||
anomalies,
|
||||
});
|
||||
if (bgm) {
|
||||
bgmFields.bgm_provider = "heygen";
|
||||
bgmFields.bgm_mode = "retrieve";
|
||||
console.error(` bgm: ${bgm.path} (retrieve "${bgm.query}")`);
|
||||
} else {
|
||||
anomalies.push(`bgm: no music match for "${request.bgm?.query ?? ""}" — skipped`);
|
||||
}
|
||||
} catch (e) {
|
||||
anomalies.push(`bgm retrieve failed: ${e.message} — skipped`);
|
||||
}
|
||||
} else {
|
||||
// generate
|
||||
const prompt = inferBgmPrompt({
|
||||
userPrompt: request.bgm?.prompt,
|
||||
blob: request.bgm?.blob || request.bgm?.query,
|
||||
archetype: request.bgm?.archetype,
|
||||
arc: request.bgm?.arc,
|
||||
});
|
||||
const gen = generateBgmDetached({
|
||||
prompt,
|
||||
durationS: totalDuration || 30,
|
||||
hyperframesDir,
|
||||
lyriaRecipe: existsSync(lyriaRecipe) ? lyriaRecipe : null,
|
||||
seedSeconds,
|
||||
hasVoice,
|
||||
anomalies,
|
||||
});
|
||||
if (gen.disabled) {
|
||||
anomalies.push(`bgm: ${gen.reason}`);
|
||||
} else {
|
||||
bgm = { path: gen.path, volume: gen.volume, mode: gen.mode, duration_s: null };
|
||||
bgmFields.bgm_pending = true;
|
||||
bgmFields.bgm_provider = gen.provider;
|
||||
bgmFields.bgm_pid = gen.pid;
|
||||
bgmFields.bgm_log = gen.log;
|
||||
bgmFields.bgm_mode = gen.mode;
|
||||
bgmFields.bgm_target_duration_s = gen.target_duration_s ?? null;
|
||||
bgmFields.bgm_seed_duration_s = gen.seed_duration_s ?? null;
|
||||
bgmFields.bgm_loop_count = gen.loop_count ?? null;
|
||||
console.error(` bgm: launched ${gen.provider} (detached, pid ${gen.pid}) → ${gen.path}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── SFX ─────────────────────────────────────────────────────────────────────
|
||||
let sfx = prev.sfx ?? [];
|
||||
if (only.has("sfx")) {
|
||||
const cues = lines.flatMap((l) =>
|
||||
(Array.isArray(l.sfx) ? l.sfx : [])
|
||||
.map((name) => ({ id: String(l.id), name: String(name).trim() }))
|
||||
.filter((c) => c.name),
|
||||
);
|
||||
const headers = heygenOK && cues.length ? heygenAuthHeaders() : null;
|
||||
const res = await resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxLibDir });
|
||||
sfx = res.sfx;
|
||||
anomalies.push(...res.anomalies);
|
||||
console.error(
|
||||
`· sfx: ${sfx.length} cue(s) resolved (${heygenOK ? "heygen retrieval" : "bundled library"})`,
|
||||
);
|
||||
}
|
||||
|
||||
// ── write audio_meta.json ─────────────────────────────────────────────────────
|
||||
const meta = {
|
||||
tts_provider: ttsProvider,
|
||||
voice_id: voiceId,
|
||||
bgm,
|
||||
...bgmFields,
|
||||
voices,
|
||||
sfx,
|
||||
total_duration_s: totalDuration,
|
||||
};
|
||||
mkdirSync(dirname(outPath), { recursive: true });
|
||||
audioMeta.write(meta);
|
||||
const written = writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields, sfx });
|
||||
anomalies.push(...recordInManifest(hyperframesDir, written));
|
||||
|
||||
console.log(`✓ audio engine → ${outPath}`);
|
||||
console.log(` heygen: ${heygenOK ? "yes" : "no"} · ran: ${[...only].join(",")}`);
|
||||
console.log(
|
||||
` voices: ${voices.length} · bgm: ${bgm ? `${bgmFields.bgm_provider}${bgmFields.bgm_pending ? " (pending)" : ""}` : "none"} · sfx: ${sfx.length}`,
|
||||
);
|
||||
console.log(` total voice duration: ${totalDuration}s`);
|
||||
if (anomalies.length) {
|
||||
console.log(`\nanomalies (non-fatal):`);
|
||||
for (const a of anomalies) console.log(` - ${a}`);
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import { test } from "node:test";
|
||||
import { mkdtempSync, mkdirSync, readFileSync, rmSync, existsSync, writeFileSync } from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { join, dirname } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { recordInManifest } from "./lib/media-record.mjs";
|
||||
import { resolveSfx } from "./lib/sfx.mjs";
|
||||
|
||||
// Proves the relocated engine (skills/media-use/audio/) still resolves its
|
||||
// bundled SFX library from the moved location — the path most likely to break
|
||||
// on a subtree move. Offline (heygenOK:false), no network.
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const sfxLibDir = join(HERE, "..", "assets", "sfx"); // same offset the engine uses
|
||||
|
||||
test("bundled SFX library resolves from the relocated path", async () => {
|
||||
assert.ok(existsSync(join(sfxLibDir, "manifest.json")), "moved manifest is present");
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
assert.equal(sfx.length, 1, `expected 1 resolved cue, got anomalies: ${anomalies.join("; ")}`);
|
||||
assert.equal(sfx[0].source, "local");
|
||||
assert.match(sfx[0].file, /assets\/sfx\//);
|
||||
assert.ok(existsSync(join(dir, sfx[0].file)), "matched SFX copied into the project");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("an unknown cue is reported, not fatal", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "1", name: "definitely-not-a-real-sfx" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
assert.equal(sfx.length, 0);
|
||||
assert.ok(anomalies.some((a) => /not in bundled library/.test(a)));
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("a person's own file under a bundled name survives, and the cue gets the next free name", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
const own = join(dir, "assets", "sfx", "whoosh.mp3");
|
||||
mkdirSync(dirname(own), { recursive: true });
|
||||
writeFileSync(own, "the person's own whoosh");
|
||||
const resolve = () =>
|
||||
resolveSfx({
|
||||
cues: [{ id: "1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
|
||||
const first = await resolve();
|
||||
const second = await resolve();
|
||||
|
||||
assert.equal(readFileSync(own, "utf8"), "the person's own whoosh");
|
||||
assert.deepEqual(
|
||||
[first, second].map(({ sfx }) => [sfx[0].file, sfx[0].source]),
|
||||
[
|
||||
["assets/sfx/whoosh-2.mp3", "local"],
|
||||
["assets/sfx/whoosh-2.mp3", "local"],
|
||||
],
|
||||
);
|
||||
assert.deepEqual(
|
||||
readFileSync(join(dir, "assets/sfx/whoosh-2.mp3")),
|
||||
readFileSync(join(sfxLibDir, "whoosh.mp3")),
|
||||
);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("two effects never share a file when one's name is taken by the person, run after run", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
mkdirSync(join(dir, "assets", "sfx"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets", "sfx", "glitch.mp3"), "the person's own glitch");
|
||||
const realFetch = globalThis.fetch;
|
||||
globalThis.fetch = async (url) => {
|
||||
const query = new URL(url).searchParams.get("query");
|
||||
if (query)
|
||||
return Response.json({ data: [{ audio_url: `https://sound.test/${query}`, score: 0.6 }] });
|
||||
return new Response(`bytes of ${new URL(url).pathname}`);
|
||||
};
|
||||
const run = async (names) => {
|
||||
const { sfx } = await resolveSfx({
|
||||
cues: names.map((name, index) => ({ id: String(index), name })),
|
||||
heygenOK: true,
|
||||
headers: {},
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
const files = sfx.map(({ file }) => file);
|
||||
recordInManifest(
|
||||
dir,
|
||||
[...new Set(files)].map((path) => ({ path, type: "sfx", source: "search" })),
|
||||
);
|
||||
return files;
|
||||
};
|
||||
try {
|
||||
assert.deepEqual(await run(["glitch"]), ["assets/sfx/glitch-2.mp3"]);
|
||||
assert.deepEqual(await run(["glitch", "glitch 2", "glitch"]), [
|
||||
"assets/sfx/glitch-2.mp3",
|
||||
"assets/sfx/glitch-2-2.mp3",
|
||||
"assets/sfx/glitch-2.mp3",
|
||||
]);
|
||||
} finally {
|
||||
globalThis.fetch = realFetch;
|
||||
}
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// The engine owns speed parsing; a fallback before the check would turn 0 or junk into 1.
|
||||
function runEngineWith(request, ...args) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "audio-speed-range-"));
|
||||
try {
|
||||
writeFileSync(join(dir, "audio_request.json"), JSON.stringify({ lines: [], ...request }));
|
||||
return spawnSync(process.execPath, [join(HERE, "audio.mjs"), "--hyperframes", dir, ...args], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
for (const [label, request, args, shown] of [
|
||||
["request speed 5", { speed: 5 }, [], "5"],
|
||||
["request speed 0", { speed: 0 }, [], "0"],
|
||||
["request speed -1", { speed: -1 }, [], "-1"],
|
||||
["request speed text", { speed: "fast" }, [], '"fast"'],
|
||||
["--speed 0", {}, ["--speed", "0"], '"0"'],
|
||||
["--speed text", { speed: 1 }, ["--speed", "abc"], '"abc"'],
|
||||
]) {
|
||||
test(`${label} stops the engine before any TTS runs`, () => {
|
||||
const r = runEngineWith(request, ...args);
|
||||
assert.equal(r.status, 1);
|
||||
assert.ok(r.stderr.includes(`speed must be above 0 and at most 3, got ${shown}`), r.stderr);
|
||||
});
|
||||
}
|
||||
|
||||
test("a numeric string speed, as the adapters forward it, passes the check", () => {
|
||||
assert.doesNotMatch(runEngineWith({ speed: "0.8" }, "--only", "tts").stderr, /speed must/);
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { mkdtempSync, writeFileSync, readFileSync, mkdirSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
// Exercise the actual engine and provider together; only the paid API and
|
||||
// external audio tools are fixtures. This is not a live synthesis/render test.
|
||||
for (const expired of [false, true]) {
|
||||
for (const only of ["tts", "tts,bgm,sfx"]) {
|
||||
// With expired credentials the person already has an assets/voice/intro.wav, which the engine must keep.
|
||||
const voicePath = expired ? "assets/voice/intro-2.wav" : "assets/voice/intro.wav";
|
||||
test(`Gemini engine returns caption metadata with ${expired ? "expired" : "absent"} HeyGen credentials (${only})`, (t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-gemini-pipeline-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
if (expired) {
|
||||
mkdirSync(join(dir, "assets/voice"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets/voice/intro.wav"), "the person's own intro");
|
||||
}
|
||||
const config = join(dir, "heygen");
|
||||
mkdirSync(config);
|
||||
if (expired)
|
||||
writeFileSync(
|
||||
join(config, "credentials"),
|
||||
JSON.stringify({
|
||||
oauth: { access_token: "expired-fixture", expires_at: "2000-01-01T00:00:00Z" },
|
||||
}),
|
||||
);
|
||||
const bin = join(dir, "bin");
|
||||
mkdirSync(bin);
|
||||
const preload = join(dir, "fetch.mjs");
|
||||
writeFileSync(
|
||||
preload,
|
||||
`
|
||||
import assert from 'node:assert/strict';
|
||||
globalThis.fetch = async (url, options) => {
|
||||
assert.equal(url, 'https://generativelanguage.googleapis.com/v1beta/interactions');
|
||||
const body = JSON.parse(options.body);
|
||||
assert.equal(body.model, 'gemini-3.8-flash-lite-tts');
|
||||
assert.equal(body.input[0].content[0].annotations[0].style, 'Line direction');
|
||||
const bytes = Buffer.alloc(48);
|
||||
bytes.write('RIFF'); bytes.write('WAVE', 8);
|
||||
return Response.json({status:'completed', steps:[{type:'model_output', content:[
|
||||
{type:'audio', mime_type:'audio/wav', data:bytes.toString('base64')}
|
||||
]}]});
|
||||
};
|
||||
`,
|
||||
);
|
||||
writeFileSync(
|
||||
join(bin, "npx"),
|
||||
`#!${process.execPath}
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const args = process.argv.slice(2);
|
||||
assert.deepEqual(args.slice(0, 3), ['hyperframes', 'transcribe', '${voicePath}']);
|
||||
assert.ok(fs.existsSync(args[2]));
|
||||
assert.equal(args[args.indexOf('--model')+1], 'small.en');
|
||||
fs.writeFileSync(path.join(args[args.indexOf('--dir')+1], 'transcript.json'), JSON.stringify([
|
||||
{text:'Hello',start:0.1,end:0.4}, {text:'there.',start:0.5,end:0.9}
|
||||
]));
|
||||
`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
writeFileSync(join(bin, "ffprobe"), "#!/bin/sh\necho 1.25\n", { mode: 0o755 });
|
||||
writeFileSync(join(dir, ".env"), "");
|
||||
writeFileSync(
|
||||
join(dir, "audio_request.json"),
|
||||
JSON.stringify({
|
||||
provider: "gemini",
|
||||
tts_model: "gemini-3.8-flash-lite-tts",
|
||||
style: "Global direction",
|
||||
lines: [{ id: "intro", text: "Hello there.", style: "Line direction" }],
|
||||
bgm: { mode: "none" },
|
||||
}),
|
||||
);
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
preload,
|
||||
new URL("./audio.mjs", import.meta.url).pathname,
|
||||
"--hyperframes",
|
||||
dir,
|
||||
"--only",
|
||||
only,
|
||||
],
|
||||
{
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
HEYGEN_CONFIG_DIR: config,
|
||||
HEYGEN_API_KEY: "",
|
||||
HYPERFRAMES_API_KEY: "",
|
||||
GEMINI_API_KEY: "fixture-key",
|
||||
PATH: `${bin}:${process.env.PATH}`,
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8"));
|
||||
assert.equal(meta.tts_provider, "gemini");
|
||||
assert.equal(meta.voice_id, "Kore");
|
||||
assert.deepEqual(meta.voices, [
|
||||
{
|
||||
id: "intro",
|
||||
path: voicePath,
|
||||
duration_s: 1.25,
|
||||
words: [
|
||||
{ id: "w0", text: "Hello", start: 0.1, end: 0.4 },
|
||||
{ id: "w1", text: "there.", start: 0.5, end: 0.9 },
|
||||
],
|
||||
},
|
||||
]);
|
||||
assert.equal(meta.total_duration_s, 1.25);
|
||||
const manifest = readFileSync(join(dir, ".media/manifest.jsonl"), "utf8").trim().split("\n");
|
||||
assert.deepEqual(
|
||||
manifest
|
||||
.map((line) => JSON.parse(line))
|
||||
.map(({ path, type, source }) => [path, type, source]),
|
||||
[[voicePath, "voice", "generated"]],
|
||||
);
|
||||
if (expired)
|
||||
assert.equal(
|
||||
readFileSync(join(dir, "assets/voice/intro.wav"), "utf8"),
|
||||
"the person's own intro",
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env node
|
||||
// Self-contained HeyGen TTS — single text in → one wav (+ optional words JSON)
|
||||
// out. A thin CLI over lib/tts.mjs (the same code the audio engine uses), so the
|
||||
// HeyGen REST call, starfish voice pick, mp3→wav transcode, and word-timestamp
|
||||
// filtering live in exactly one place. Bypasses the `hyperframes` CLI, which in
|
||||
// the published build is Kokoro-only.
|
||||
//
|
||||
// Usage:
|
||||
// node heygen-tts.mjs "Text to speak" -o narration.wav [--words narration.words.json]
|
||||
// node heygen-tts.mjs ./script.txt -o narration.wav --words narration.words.json
|
||||
// node heygen-tts.mjs "Bonjour" -o fr.wav --lang fr --voice <id>
|
||||
// node heygen-tts.mjs --list # list starfish voices and exit
|
||||
//
|
||||
// Flags: -o/--output (.wav → ffmpeg transcode; .mp3 → raw bytes), --words,
|
||||
// --voice (starfish id), --speed, --lang, --list.
|
||||
// Requires: $HEYGEN_API_KEY / OAuth ~/.heygen credentials and ffmpeg for .wav output.
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { heygenAuthHeaders, heygenJSON, loadEnvFromDir } from "./lib/heygen.mjs";
|
||||
import { ffprobeDuration, resolveVoiceId, synthesizeOne, withWordIds } from "./lib/tts.mjs";
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
function flag(name, def) {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
if (i < 0) return def;
|
||||
if (i + 1 >= argv.length) return true;
|
||||
const v = argv[i + 1];
|
||||
return v.startsWith("--") ? true : v;
|
||||
}
|
||||
const die = (m) => {
|
||||
console.error(`✗ heygen-tts: ${m}`);
|
||||
process.exit(1);
|
||||
};
|
||||
|
||||
// First arg that isn't a flag or the -o value is the text / .txt path.
|
||||
const positional = (() => {
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a.startsWith("--")) {
|
||||
const next = argv[i + 1];
|
||||
if (next && !next.startsWith("--")) i++;
|
||||
continue;
|
||||
}
|
||||
if (a === "-o") {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
return a;
|
||||
}
|
||||
return null;
|
||||
})();
|
||||
|
||||
const output = resolve(
|
||||
(typeof flag("output") === "string" && flag("output")) ||
|
||||
(argv.includes("-o") && argv[argv.indexOf("-o") + 1]) ||
|
||||
"narration.wav",
|
||||
);
|
||||
const wordsPath = typeof flag("words") === "string" ? resolve(flag("words")) : null;
|
||||
const userVoice = typeof flag("voice") === "string" ? flag("voice") : null;
|
||||
const speedRaw = typeof flag("speed") === "string" ? Number(flag("speed")) : 1.0;
|
||||
const speed = isFinite(speedRaw) && speedRaw > 0 ? speedRaw : 1.0;
|
||||
const lang = typeof flag("lang") === "string" ? flag("lang") : "en";
|
||||
const listOnly = flag("list") === true;
|
||||
|
||||
loadEnvFromDir(process.cwd());
|
||||
let authHeaders;
|
||||
try {
|
||||
authHeaders = heygenAuthHeaders();
|
||||
} catch (e) {
|
||||
die(e.message);
|
||||
}
|
||||
|
||||
// ---------- --list ----------
|
||||
if (listOnly) {
|
||||
const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, {
|
||||
headers: authHeaders,
|
||||
});
|
||||
for (const v of payload.data ?? payload.voices ?? []) {
|
||||
console.log(`${v.voice_id}\t${v.name}\t${v.language ?? ""}`);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// ---------- resolve text + voice ----------
|
||||
if (!positional) die("no text given. Pass a string or a .txt path, or use --list.");
|
||||
const text =
|
||||
positional.endsWith(".txt") && existsSync(resolve(positional))
|
||||
? readFileSync(resolve(positional), "utf8").trim()
|
||||
: positional;
|
||||
if (!text) die("input text is empty");
|
||||
|
||||
const voiceId = await resolveVoiceId({ provider: "heygen", userVoice, lang });
|
||||
if (!userVoice) console.error(`· using voice ${voiceId}`);
|
||||
|
||||
// ---------- synthesize (shared engine code) ----------
|
||||
const { ok, words, error } = await synthesizeOne({
|
||||
provider: "heygen",
|
||||
text,
|
||||
voiceId,
|
||||
lang,
|
||||
speed,
|
||||
wavAbs: output,
|
||||
hyperframesDir: process.cwd(),
|
||||
});
|
||||
// HeyGen's own refusal, verbatim: a caller shows it or acts on it (a deleted voice, a limit).
|
||||
if (!ok) die(error ?? "synthesis failed (HeyGen request/transcode error)");
|
||||
|
||||
let wordCount = 0;
|
||||
if (wordsPath) {
|
||||
if (words && words.length) {
|
||||
mkdirSync(dirname(wordsPath), { recursive: true });
|
||||
writeFileSync(wordsPath, JSON.stringify(withWordIds(words), null, 2));
|
||||
wordCount = words.length;
|
||||
} else {
|
||||
console.error("⚠ no word_timestamps in response — run `hyperframes transcribe` instead");
|
||||
}
|
||||
}
|
||||
|
||||
const dur = ffprobeDuration(output);
|
||||
const durStr = isFinite(dur) ? ` (${dur.toFixed(2)}s)` : "";
|
||||
console.log(`✓ ${output}${durStr}${wordCount ? ` · ${wordsPath} (${wordCount} words)` : ""}`);
|
||||
@@ -0,0 +1,38 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const script = fileURLToPath(new URL("./heygen-tts.mjs", import.meta.url));
|
||||
// Every fetch answers as HeyGen does for a voice that no longer exists.
|
||||
const refusing = `data:text/javascript,globalThis.fetch = async () => new Response(${JSON.stringify(
|
||||
JSON.stringify({ error: { code: "voice_not_found", message: "Voice not found: vc_gone" } }),
|
||||
)}, { status: 404, headers: { "content-type": "application/json" } });`;
|
||||
|
||||
test("a refused speech prints HeyGen's own message, so a caller can act on it", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-tts-"));
|
||||
const run = spawnSync(
|
||||
process.execPath,
|
||||
["--import", refusing, script, "Hello", "--voice", "vc_gone", "-o", join(dir, "out.mp3")],
|
||||
{
|
||||
cwd: dir,
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
HEYGEN_API_KEY: "",
|
||||
HYPERFRAMES_API_KEY: "",
|
||||
HEYGEN_ACCESS_TOKEN: "t",
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(run.status, 1);
|
||||
assert.match(run.stderr, /✗ heygen-tts: Voice not found: vc_gone\n/);
|
||||
assert.doesNotMatch(
|
||||
run.stderr,
|
||||
/HTTP 404|"error"/,
|
||||
"HeyGen's message alone, not the raw response",
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env node
|
||||
// HeyGen voice clones: `clone <audio> --name <n>`, `delete <voice_id>`, `list --prefix <p>` (references/tts.md).
|
||||
// The clone id works as `heygen-tts.mjs --voice <id>`. A refusal prints HeyGen's own message and exits 1.
|
||||
// Auth: same resolver as heygen-tts.mjs (lib/heygen.mjs).
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { extname } from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import { heygenAuthHeaders, heygenJSON, heygenMessage, loadEnvFromDir } from "./lib/heygen.mjs";
|
||||
import { isMainModule } from "./lib/main-module.mjs";
|
||||
|
||||
const MEDIA_TYPES = { ".mp3": "audio/mpeg", ".wav": "audio/wav" };
|
||||
const POLL_MS = 2_000;
|
||||
const DEADLINE_MS = 120_000;
|
||||
|
||||
async function clone(file, name, headers, { sleep, now }) {
|
||||
const mediaType = MEDIA_TYPES[extname(file).toLowerCase()];
|
||||
if (!mediaType) throw new Error(`unsupported audio file ${file}: use .mp3 or .wav`);
|
||||
const data = readFileSync(file).toString("base64");
|
||||
const created = await heygenJSON("/voices/clone", {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: { voice_name: name, audio: { type: "base64", media_type: mediaType, data } },
|
||||
});
|
||||
const id = created.data.voice_clone_id;
|
||||
const deadline = now() + DEADLINE_MS;
|
||||
for (;;) {
|
||||
const { data: voice } = await heygenJSON(`/voices/${id}`, { headers });
|
||||
if (voice.status === "complete") return { voice_id: voice.voice_id ?? id };
|
||||
if (voice.status === "failed") throw new Error(voice.failure_message || "voice clone failed");
|
||||
if (now() >= deadline) throw new Error(`voice clone ${id} not ready after 120 s`);
|
||||
await sleep(POLL_MS);
|
||||
}
|
||||
}
|
||||
|
||||
async function list(prefix, headers) {
|
||||
const mine = [];
|
||||
const seen = new Set();
|
||||
let token;
|
||||
do {
|
||||
if (token && seen.has(token)) throw new Error(`HeyGen returned page token ${token} twice`);
|
||||
if (token) seen.add(token);
|
||||
const query = new URLSearchParams({ type: "private", limit: "100" });
|
||||
if (token) query.set("token", token);
|
||||
const page = await heygenJSON(`/voices?${query}`, { headers });
|
||||
mine.push(...page.data.filter((v) => (v.name ?? "").startsWith(prefix)));
|
||||
token = page.has_more ? page.next_token : null;
|
||||
} while (token);
|
||||
// ponytail: the list response has no created_at, so one GET per match; bounded by the clone limit.
|
||||
return Promise.all(
|
||||
mine.map(async ({ voice_id, name }) => {
|
||||
const { data } = await heygenJSON(`/voices/${voice_id}`, { headers });
|
||||
return { voice_id, name, created_at: data.created_at ?? null };
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function main(
|
||||
argv,
|
||||
{
|
||||
sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
|
||||
now = Date.now,
|
||||
out = (s) => process.stdout.write(`${s}\n`),
|
||||
err = (s) => process.stderr.write(`${s}\n`),
|
||||
} = {},
|
||||
) {
|
||||
try {
|
||||
const { positionals, values } = parseArgs({
|
||||
args: argv,
|
||||
allowPositionals: true,
|
||||
options: { name: { type: "string" }, prefix: { type: "string", default: "" } },
|
||||
});
|
||||
const [command, arg] = positionals;
|
||||
if (command === "clone") {
|
||||
if (!arg || !values.name) throw new Error("usage: clone <audio-file> --name <name>");
|
||||
out(JSON.stringify(await clone(arg, values.name, heygenAuthHeaders(), { sleep, now })));
|
||||
} else if (command === "delete") {
|
||||
if (!arg) throw new Error("usage: delete <voice_id>");
|
||||
await heygenJSON(`/voices/${encodeURIComponent(arg)}`, {
|
||||
method: "DELETE",
|
||||
headers: heygenAuthHeaders(),
|
||||
});
|
||||
} else if (command === "list") {
|
||||
out(JSON.stringify(await list(values.prefix, heygenAuthHeaders())));
|
||||
} else {
|
||||
throw new Error("usage: heygen-voice.mjs clone|delete|list (see the header of this file)");
|
||||
}
|
||||
return 0;
|
||||
} catch (e) {
|
||||
err(heygenMessage(e));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (isMainModule(import.meta.url)) {
|
||||
loadEnvFromDir(process.cwd());
|
||||
process.exitCode = await main(process.argv.slice(2));
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
import { afterEach, beforeEach, test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { main } from "./heygen-voice.mjs";
|
||||
|
||||
const realFetch = globalThis.fetch;
|
||||
const savedEnv = {
|
||||
HEYGEN_ACCESS_TOKEN: process.env.HEYGEN_ACCESS_TOKEN,
|
||||
HEYGEN_API_KEY: process.env.HEYGEN_API_KEY,
|
||||
};
|
||||
let dir;
|
||||
let calls;
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), "heygen-voice-"));
|
||||
delete process.env.HEYGEN_ACCESS_TOKEN;
|
||||
process.env.HEYGEN_API_KEY = "hg_test";
|
||||
calls = [];
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = realFetch;
|
||||
for (const [k, v] of Object.entries(savedEnv)) {
|
||||
if (v === undefined) delete process.env[k];
|
||||
else process.env[k] = v;
|
||||
}
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// routes: "METHOD /path" → array of [status, body] answered in order (last one repeats).
|
||||
function mockFetch(routes) {
|
||||
globalThis.fetch = async (url, opts) => {
|
||||
const key = `${opts.method} ${String(url).replace("https://api.heygen.com/v3", "")}`;
|
||||
calls.push({ key, opts });
|
||||
const queue = routes[key];
|
||||
if (!queue) throw new Error(`unexpected request ${key}`);
|
||||
const [status, body] = queue.length > 1 ? queue.shift() : queue[0];
|
||||
return new Response(body === undefined ? null : JSON.stringify(body), { status });
|
||||
};
|
||||
}
|
||||
|
||||
async function run(argv, { sleep = async () => {}, now = () => 0 } = {}) {
|
||||
const out = [];
|
||||
const err = [];
|
||||
const code = await main(argv, {
|
||||
sleep,
|
||||
now,
|
||||
out: (s) => out.push(s),
|
||||
err: (s) => err.push(s),
|
||||
});
|
||||
return { code, out: out.join("\n"), err: err.join("\n") };
|
||||
}
|
||||
|
||||
function audioFile(name = "sample.mp3") {
|
||||
const path = join(dir, name);
|
||||
writeFileSync(path, Buffer.from("fake-audio"));
|
||||
return path;
|
||||
}
|
||||
|
||||
test("clone uploads base64 audio, polls until complete, prints the voice id", async () => {
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[200, { data: { voice_clone_id: "vc1" } }]],
|
||||
"GET /voices/vc1": [
|
||||
[200, { data: { status: "processing", voice_id: "vc1" } }],
|
||||
[200, { data: { status: "complete", voice_id: "vc1", name: "Me" } }],
|
||||
],
|
||||
});
|
||||
const sleeps = [];
|
||||
const r = await run(["clone", audioFile(), "--name", "Me"], {
|
||||
sleep: async (ms) => sleeps.push(ms),
|
||||
});
|
||||
assert.deepEqual(r, { code: 0, out: '{"voice_id":"vc1"}', err: "" });
|
||||
assert.deepEqual(sleeps, [2000]);
|
||||
const post = calls[0];
|
||||
assert.equal(post.opts.headers["X-Api-Key"], "hg_test");
|
||||
assert.deepEqual(JSON.parse(post.opts.body), {
|
||||
voice_name: "Me",
|
||||
audio: {
|
||||
type: "base64",
|
||||
media_type: "audio/mpeg",
|
||||
data: Buffer.from("fake-audio").toString("base64"),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("clone sends audio/wav for a .wav file", async () => {
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[200, { data: { voice_clone_id: "vc1" } }]],
|
||||
"GET /voices/vc1": [[200, { data: { status: "complete", voice_id: "vc1" } }]],
|
||||
});
|
||||
const r = await run(["clone", audioFile("take.WAV"), "--name", "Me"]);
|
||||
assert.equal(r.code, 0);
|
||||
assert.equal(JSON.parse(calls[0].opts.body).audio.media_type, "audio/wav");
|
||||
});
|
||||
|
||||
test("clone prints HeyGen's refusal verbatim and exits 1", async () => {
|
||||
const message =
|
||||
"Voice clone limit reached (10). Delete unused clones or contact support to increase your limit.";
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[400, { error: { code: "resource_limit_reached", message } }]],
|
||||
});
|
||||
const r = await run(["clone", audioFile(), "--name", "Me"]);
|
||||
assert.deepEqual(r, { code: 1, out: "", err: message });
|
||||
});
|
||||
|
||||
test("clone surfaces a failed clone's failure_message", async () => {
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[200, { data: { voice_clone_id: "vc1" } }]],
|
||||
"GET /voices/vc1": [[200, { data: { status: "failed", failure_message: "Audio too noisy" } }]],
|
||||
});
|
||||
const r = await run(["clone", audioFile(), "--name", "Me"]);
|
||||
assert.deepEqual(r, { code: 1, out: "", err: "Audio too noisy" });
|
||||
});
|
||||
|
||||
test("clone gives up after the 120 s deadline", async () => {
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[200, { data: { voice_clone_id: "vc1" } }]],
|
||||
"GET /voices/vc1": [[200, { data: { status: "processing" } }]],
|
||||
});
|
||||
let clock = 0;
|
||||
const r = await run(["clone", audioFile(), "--name", "Me"], {
|
||||
now: () => clock,
|
||||
sleep: async (ms) => {
|
||||
clock += ms;
|
||||
},
|
||||
});
|
||||
assert.deepEqual(r, { code: 1, out: "", err: "voice clone vc1 not ready after 120 s" });
|
||||
assert.equal(calls.filter((c) => c.key === "GET /voices/vc1").length, 61);
|
||||
});
|
||||
|
||||
test("clone rejects an audio type other than mp3 or wav without calling HeyGen", async () => {
|
||||
mockFetch({});
|
||||
const r = await run(["clone", audioFile("take.ogg"), "--name", "Me"]);
|
||||
assert.equal(r.code, 1);
|
||||
assert.match(r.err, /use \.mp3 or \.wav/);
|
||||
assert.equal(calls.length, 0);
|
||||
});
|
||||
|
||||
test("delete calls DELETE /voices/<id> and exits 0", async () => {
|
||||
mockFetch({ "DELETE /voices/vc1": [[200, { data: {} }]] });
|
||||
const r = await run(["delete", "vc1"]);
|
||||
assert.deepEqual(r, { code: 0, out: "", err: "" });
|
||||
});
|
||||
|
||||
test("delete exits 0 on a 204 with no body, and escapes the id in the path", async () => {
|
||||
mockFetch({ "DELETE /voices/a%2Fb": [[204]] });
|
||||
const r = await run(["delete", "a/b"]);
|
||||
assert.deepEqual(r, { code: 0, out: "", err: "" });
|
||||
});
|
||||
|
||||
test("clone falls back to the clone id when the finished voice has no voice_id", async () => {
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[200, { data: { voice_clone_id: "vc9" } }]],
|
||||
"GET /voices/vc9": [[200, { data: { status: "complete" } }]],
|
||||
});
|
||||
const r = await run(["clone", audioFile(), "--name", "Me"]);
|
||||
assert.deepEqual(r, { code: 0, out: '{"voice_id":"vc9"}', err: "" });
|
||||
});
|
||||
|
||||
test("list skips a nameless voice and stops on a page token HeyGen repeats", async () => {
|
||||
mockFetch({
|
||||
"GET /voices?type=private&limit=100": [
|
||||
[200, { data: [{ voice_id: "n", name: null }], has_more: true, next_token: "t2" }],
|
||||
],
|
||||
"GET /voices?type=private&limit=100&token=t2": [
|
||||
[200, { data: [], has_more: true, next_token: "t2" }],
|
||||
],
|
||||
});
|
||||
// A fake answers at once, so a list that never stops would starve any timeout: cap the calls instead.
|
||||
const answer = globalThis.fetch;
|
||||
globalThis.fetch = (...args) =>
|
||||
calls.length >= 10 ? Promise.reject(new Error("paged forever")) : answer(...args);
|
||||
const r = await run(["list", "--prefix", "desk-"]);
|
||||
assert.deepEqual(r, { code: 1, out: "", err: "HeyGen returned page token t2 twice" });
|
||||
});
|
||||
|
||||
test("delete prints HeyGen's 404 message verbatim and exits 1", async () => {
|
||||
mockFetch({
|
||||
"DELETE /voices/gone": [
|
||||
[404, { error: { code: "voice_not_found", message: "Voice not found: gone" } }],
|
||||
],
|
||||
});
|
||||
const r = await run(["delete", "gone"]);
|
||||
assert.deepEqual(r, { code: 1, out: "", err: "Voice not found: gone" });
|
||||
});
|
||||
|
||||
test("list pages through private voices and returns prefix matches with created_at", async () => {
|
||||
mockFetch({
|
||||
"GET /voices?type=private&limit=100": [
|
||||
[
|
||||
200,
|
||||
{
|
||||
data: [
|
||||
{ voice_id: "a", name: "desk-1" },
|
||||
{ voice_id: "b", name: "other" },
|
||||
],
|
||||
has_more: true,
|
||||
next_token: "t2",
|
||||
},
|
||||
],
|
||||
],
|
||||
"GET /voices?type=private&limit=100&token=t2": [
|
||||
[200, { data: [{ voice_id: "c", name: "desk-2" }], has_more: false }],
|
||||
],
|
||||
"GET /voices/a": [[200, { data: { voice_id: "a", created_at: 1700000000 } }]],
|
||||
"GET /voices/c": [[200, { data: { voice_id: "c", created_at: 1700000100 } }]],
|
||||
});
|
||||
const r = await run(["list", "--prefix", "desk-"]);
|
||||
assert.equal(r.code, 0);
|
||||
assert.deepEqual(JSON.parse(r.out), [
|
||||
{ voice_id: "a", name: "desk-1", created_at: 1700000000 },
|
||||
{ voice_id: "c", name: "desk-2", created_at: 1700000100 },
|
||||
]);
|
||||
});
|
||||
|
||||
test("list prints HeyGen's refusal verbatim and exits 1", async () => {
|
||||
const message = "Invalid or expired API key. Verify your x-api-key header.";
|
||||
mockFetch({
|
||||
"GET /voices?type=private&limit=100": [[401, { error: { code: "unauthorized", message } }]],
|
||||
});
|
||||
const r = await run(["list", "--prefix", "desk-"]);
|
||||
assert.deepEqual(r, { code: 1, out: "", err: message });
|
||||
});
|
||||
|
||||
test("a host-injected HEYGEN_ACCESS_TOKEN is sent as Bearer", async () => {
|
||||
process.env.HEYGEN_ACCESS_TOKEN = "at_host";
|
||||
mockFetch({ "DELETE /voices/vc1": [[200, { data: {} }]] });
|
||||
await run(["delete", "vc1"]);
|
||||
assert.equal(calls[0].opts.headers.Authorization, "Bearer at_host");
|
||||
assert.equal(calls[0].opts.headers["X-Api-Key"], undefined);
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
import { closeSync, ftruncateSync, openSync, readFileSync, writeSync } from "node:fs";
|
||||
|
||||
// Keep the merge base and output on the same file even if its pathname changes
|
||||
// while audio generation runs. The CLI owns this handle until write or exit.
|
||||
export function openAudioMeta(path) {
|
||||
let fd;
|
||||
try {
|
||||
fd = openSync(path, "r+");
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT") throw error;
|
||||
}
|
||||
let value = {};
|
||||
if (fd !== undefined) {
|
||||
try {
|
||||
value = JSON.parse(readFileSync(fd, "utf8"));
|
||||
} catch (error) {
|
||||
closeSync(fd);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return {
|
||||
value,
|
||||
write(meta) {
|
||||
const bytes = Buffer.from(JSON.stringify(meta, null, 2));
|
||||
// Defer new-file creation until generation succeeds. Never overwrite a
|
||||
// file (or follow a link) that appeared since the missing merge base.
|
||||
if (fd === undefined) fd = openSync(path, "wx");
|
||||
try {
|
||||
let offset = 0;
|
||||
while (offset < bytes.length) {
|
||||
offset += writeSync(fd, bytes, offset, bytes.length - offset, offset);
|
||||
}
|
||||
ftruncateSync(fd, bytes.length);
|
||||
} finally {
|
||||
closeSync(fd);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import {
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { openAudioMeta } from "./audio-meta.mjs";
|
||||
|
||||
function fixture(t) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "audio-meta-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
}
|
||||
|
||||
test("updates the read file from byte zero, truncates, and retains its mode", (t) => {
|
||||
const path = join(fixture(t), "meta.json");
|
||||
writeFileSync(path, JSON.stringify({ voices: ["long existing voice metadata"] }), {
|
||||
mode: 0o600,
|
||||
});
|
||||
const originalMode = statSync(path).mode;
|
||||
const handle = openAudioMeta(path);
|
||||
assert.deepEqual(handle.value, { voices: ["long existing voice metadata"] });
|
||||
handle.write({ voices: ["声"] });
|
||||
assert.equal(readFileSync(path, "utf8"), JSON.stringify({ voices: ["声"] }, null, 2));
|
||||
assert.equal(statSync(path).mode, originalMode);
|
||||
});
|
||||
|
||||
test("pathname replacement cannot redirect the existing-file write", (t) => {
|
||||
const dir = fixture(t);
|
||||
const path = join(dir, "meta.json");
|
||||
const moved = join(dir, "original.json");
|
||||
writeFileSync(path, "{}");
|
||||
const handle = openAudioMeta(path);
|
||||
renameSync(path, moved);
|
||||
writeFileSync(path, "replacement must survive");
|
||||
handle.write({ voices: [] });
|
||||
assert.equal(readFileSync(path, "utf8"), "replacement must survive");
|
||||
assert.deepEqual(JSON.parse(readFileSync(moved, "utf8")), { voices: [] });
|
||||
});
|
||||
|
||||
test("existing symlink outputs retain their original target even after retargeting", (t) => {
|
||||
const dir = fixture(t);
|
||||
const path = join(dir, "meta.json");
|
||||
const target = join(dir, "target.json");
|
||||
const victim = join(dir, "victim.json");
|
||||
writeFileSync(target, "{}");
|
||||
writeFileSync(victim, "untouched");
|
||||
symlinkSync(target, path);
|
||||
const handle = openAudioMeta(path);
|
||||
rmSync(path);
|
||||
symlinkSync(victim, path);
|
||||
handle.write({ bgm: null });
|
||||
assert.equal(readFileSync(victim, "utf8"), "untouched");
|
||||
assert.deepEqual(JSON.parse(readFileSync(target, "utf8")), { bgm: null });
|
||||
});
|
||||
|
||||
test("missing output is created only when saved", (t) => {
|
||||
const path = join(fixture(t), "meta.json");
|
||||
const handle = openAudioMeta(path);
|
||||
assert.deepEqual(handle.value, {});
|
||||
assert.throws(() => statSync(path), { code: "ENOENT" });
|
||||
handle.write({ sfx: [] });
|
||||
assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), { sfx: [] });
|
||||
});
|
||||
|
||||
for (const replacement of ["file", "symlink"]) {
|
||||
test(`new output does not overwrite a racing ${replacement}`, (t) => {
|
||||
const dir = fixture(t);
|
||||
const path = join(dir, "meta.json");
|
||||
const victim = join(dir, "victim.json");
|
||||
const handle = openAudioMeta(path);
|
||||
writeFileSync(victim, "untouched");
|
||||
if (replacement === "symlink") symlinkSync(victim, path);
|
||||
else writeFileSync(path, "untouched");
|
||||
assert.throws(() => handle.write({}), { code: "EEXIST" });
|
||||
assert.equal(readFileSync(path, "utf8"), "untouched");
|
||||
assert.equal(readFileSync(victim, "utf8"), "untouched");
|
||||
});
|
||||
}
|
||||
|
||||
test("malformed merge base remains an error without changing bytes", (t) => {
|
||||
const path = join(fixture(t), "meta.json");
|
||||
writeFileSync(path, "bad json");
|
||||
assert.throws(() => openAudioMeta(path), SyntaxError);
|
||||
assert.equal(readFileSync(path, "utf8"), "bad json");
|
||||
});
|
||||
|
||||
test("CLI partial run retains unselected voices, BGM and SFX", (t) => {
|
||||
const dir = fixture(t);
|
||||
const path = join(dir, "meta.json");
|
||||
const request = join(dir, "request.json");
|
||||
const previous = {
|
||||
voices: [{ id: "a", duration_s: 2 }],
|
||||
bgm: { path: "bgm.wav" },
|
||||
sfx: [{ name: "click" }],
|
||||
tts_provider: "kokoro",
|
||||
voice_id: "voice-a",
|
||||
};
|
||||
writeFileSync(path, JSON.stringify(previous));
|
||||
writeFileSync(request, "{}");
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
fileURLToPath(new URL("../audio.mjs", import.meta.url)),
|
||||
"--request",
|
||||
request,
|
||||
"--hyperframes",
|
||||
dir,
|
||||
"--out",
|
||||
path,
|
||||
"--only",
|
||||
"none",
|
||||
],
|
||||
{
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
HEYGEN_CONFIG_DIR: dir,
|
||||
HEYGEN_API_KEY: "",
|
||||
HYPERFRAMES_API_KEY: "",
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const actual = JSON.parse(readFileSync(path, "utf8"));
|
||||
for (const key of Object.keys(previous)) assert.deepEqual(actual[key], previous[key]);
|
||||
});
|
||||
@@ -0,0 +1,4 @@
|
||||
// Music sits under narration; a silent film brings music forward.
|
||||
export const BGM_BED_VOLUME = 0.12;
|
||||
export const BGM_SILENT_VOLUME = 0.9;
|
||||
export const bgmDefaultVolume = (hasVoice) => (hasVoice ? BGM_BED_VOLUME : BGM_SILENT_VOLUME);
|
||||
@@ -0,0 +1,257 @@
|
||||
// bgm.mjs — background music for the media audio engine. Two routes, gated the
|
||||
// same way as TTS/SFX:
|
||||
//
|
||||
// retrieve (default when HeyGen is configured) — search HeyGen's music library
|
||||
// by mood, download the top track. Synchronous. assets/bgm/track.mp3.
|
||||
// generate (the alternative; the automatic choice when HeyGen is absent) —
|
||||
// Lyria (cloud, $GEMINI_API_KEY/$GOOGLE_API_KEY + google-genai) preferred,
|
||||
// else local MusicGen (facebook/musicgen-small via transformers). Spawned
|
||||
// DETACHED so the engine can return while audio renders; the caller marks
|
||||
// bgm_pending and runs wait-bgm.mjs before assembling. assets/bgm/track.wav.
|
||||
//
|
||||
// Missing/failed BGM never blocks a render.
|
||||
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, openSync, closeSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { downloadTo, searchSounds } from "./heygen.mjs";
|
||||
import { agentWritePath } from "./media-record.mjs";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
const lyriaKey = () => process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || "";
|
||||
|
||||
import { bgmDefaultVolume } from "./bgm-volume.mjs";
|
||||
export { BGM_BED_VOLUME, BGM_SILENT_VOLUME, bgmDefaultVolume } from "./bgm-volume.mjs";
|
||||
|
||||
const BGM_PY_DEPS = ["transformers", "torch", "soundfile", "numpy"];
|
||||
const BGM_PY_PROBE =
|
||||
"import transformers, soundfile, torch, numpy; from transformers import MusicgenForConditionalGeneration";
|
||||
const LYRIA_PY_DEPS = ["google-genai", "python-dotenv"];
|
||||
const LYRIA_PY_PROBE = "import google.genai";
|
||||
|
||||
function pyOk(probe) {
|
||||
const { cmd, args } = pythonInvocation(["-c", probe]);
|
||||
return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;
|
||||
}
|
||||
// `python -m pip`, not a bare `pip` binary: a Homebrew/system Python often
|
||||
// exposes only `python3`/`pip3` on PATH, so a plain `pip` spawn silently
|
||||
// no-ops (ENOENT) and the documented "auto-installed on demand" path never
|
||||
// actually installs. `-m pip` also guarantees the packages land in the SAME
|
||||
// interpreter pyOk() probes — a bare `pip`/`pip3` could resolve to a
|
||||
// different Python installation than `python3` if more than one is on PATH.
|
||||
function pipInstall(deps) {
|
||||
const { cmd, args } = pythonInvocation(["-m", "pip", "install", "-q", ...deps]);
|
||||
return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;
|
||||
}
|
||||
|
||||
// ── retrieval (HeyGen music library) ──────────────────────────────────────────
|
||||
export async function retrieveBgm({ query, headers, hyperframesDir, hasVoice, anomalies }) {
|
||||
const q = query || "calm cinematic underscore";
|
||||
const results = await searchSounds(q, "music", headers, { limit: 5 });
|
||||
if (!results.length) return null;
|
||||
const top = results[0];
|
||||
const rel = agentWritePath(hyperframesDir, "assets/bgm/track.mp3", { anomalies });
|
||||
await downloadTo(top.audio_url, join(hyperframesDir, rel));
|
||||
return {
|
||||
path: rel,
|
||||
volume: bgmDefaultVolume(hasVoice),
|
||||
query: q,
|
||||
mode: "retrieve",
|
||||
duration_s: typeof top.duration === "number" ? r3(top.duration) : null,
|
||||
};
|
||||
}
|
||||
|
||||
// ── mood inference (for the generate path's prompt) ──────────────────────────
|
||||
// Industry base → archetype shape → emotional-arc tiebreaker. Exported so a
|
||||
// workflow adapter can build a rich prompt from its own narrative metadata; the
|
||||
// engine also calls it when generate has only a plain mood query.
|
||||
export function inferBgmPrompt({ blob = "", archetype = "", arc = "", userPrompt = "" } = {}) {
|
||||
if (userPrompt) return userPrompt;
|
||||
const b = String(blob).toLowerCase();
|
||||
let base;
|
||||
let bpm;
|
||||
if (/\b(crypto|nft|web3|defi|token|blockchain|exchange|wallet|dao)\b/.test(b)) {
|
||||
base = "atmospheric electronic, deep bass, futuristic synths, restrained percussion";
|
||||
bpm = 100;
|
||||
} else if (/\b(finance|fintech|bank|payment|invest|wealth|insurance|treasury)\b/.test(b)) {
|
||||
base = "calm cinematic, soft strings, subtle piano, restrained percussion";
|
||||
bpm = 92;
|
||||
} else if (/\b(creative|agency|design|studio|art|brand|marketing|content)\b/.test(b)) {
|
||||
base = "playful electronic, warm pads, light percussion";
|
||||
bpm = 115;
|
||||
} else {
|
||||
base = "uplifting corporate tech, bright modern piano with synth pads";
|
||||
bpm = 108;
|
||||
}
|
||||
const at = String(archetype).toLowerCase();
|
||||
const ar = String(arc).toLowerCase();
|
||||
if (/\bpas\b|pain.agitate|pain.+solve/.test(at))
|
||||
return `${base}, starts with subtle tension then builds to resolution, BPM ${bpm}, transitions from MINOR to MAJOR`;
|
||||
if (/\bbab\b|before.after|future.pac|vision/.test(at))
|
||||
return `${base}, cinematic and aspirational, steady build with rising energy, BPM ${bpm}, MAJOR`;
|
||||
if (/cascade|feature.benefit/.test(at))
|
||||
return `${base}, energetic and driving, consistent momentum, BPM ${Math.min(bpm + 10, 128)}, MAJOR`;
|
||||
if (/demo.loop|question.+answer/.test(at))
|
||||
return `${base}, clean and focused, minimal arrangement, BPM ${Math.max(bpm - 8, 88)}`;
|
||||
if (/frustrat|anxiety|overwhelm|tension/.test(ar) && /relief|excite|triumph/.test(ar))
|
||||
return `${base}, builds from understated tension to uplifting resolution, BPM ${bpm}, MINOR to MAJOR`;
|
||||
if (/excit|awe|power|triumph/.test(ar))
|
||||
return `${base}, energetic and confident, BPM ${bpm}, MAJOR`;
|
||||
if (/trust|ease|clarity|reassur/.test(ar))
|
||||
return `${base}, warm and reassuring, BPM ${Math.max(bpm - 5, 85)}`;
|
||||
return `${base}, BPM ${bpm}, MAJOR`;
|
||||
}
|
||||
|
||||
// ── generation (Lyria → MusicGen, detached) ──────────────────────────────────
|
||||
// Returns a bgmMeta the caller folds into audio_meta:
|
||||
// { path, mode, volume, provider, pid, log, target_duration_s, seed_duration_s,
|
||||
// loop_count, pending:true } on success, or { disabled:true, reason }.
|
||||
export function generateBgmDetached({
|
||||
prompt,
|
||||
durationS,
|
||||
hyperframesDir,
|
||||
lyriaRecipe,
|
||||
seedSeconds = 28,
|
||||
hasVoice,
|
||||
anomalies,
|
||||
}) {
|
||||
const rel = agentWritePath(hyperframesDir, "assets/bgm/track.wav", { anomalies });
|
||||
const abs = join(hyperframesDir, rel);
|
||||
mkdirSync(join(hyperframesDir, "assets", "bgm"), { recursive: true });
|
||||
const log = join(hyperframesDir, "assets", "bgm", `bgm-${Date.now()}.log`);
|
||||
const targetS = Math.max(1, durationS);
|
||||
const baseMeta = { path: rel, mode: null, volume: bgmDefaultVolume(hasVoice), pending: true };
|
||||
|
||||
const lyriaConfigured = !!lyriaKey() && !!lyriaRecipe && existsSync(lyriaRecipe);
|
||||
|
||||
// Make a backend runnable: prefer Lyria when configured (install google-genai
|
||||
// on demand), else ensure local MusicGen deps. Installs are synchronous here —
|
||||
// generation itself is detached, so the engine still returns promptly.
|
||||
if (lyriaConfigured && !pyOk(LYRIA_PY_PROBE)) pipInstall(LYRIA_PY_DEPS);
|
||||
const useLyria = lyriaConfigured && pyOk(LYRIA_PY_PROBE);
|
||||
if (!useLyria && !pyOk(BGM_PY_PROBE)) pipInstall(BGM_PY_DEPS);
|
||||
|
||||
const fd = openSync(log, "w");
|
||||
if (useLyria) {
|
||||
const { cmd, args } = pythonInvocation([
|
||||
lyriaRecipe,
|
||||
"--output",
|
||||
abs,
|
||||
"--duration",
|
||||
String(targetS),
|
||||
"--prompt",
|
||||
prompt,
|
||||
]);
|
||||
rmSync(abs, { force: true }); // wait-bgm takes any file here as the finished track
|
||||
const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] });
|
||||
proc.unref();
|
||||
closeSync(fd);
|
||||
return {
|
||||
...baseMeta,
|
||||
mode: "detached-single",
|
||||
provider: "lyria",
|
||||
pid: proc.pid,
|
||||
log,
|
||||
target_duration_s: r3(targetS),
|
||||
};
|
||||
}
|
||||
|
||||
if (pyOk(BGM_PY_PROBE)) {
|
||||
const seedS = Math.min(Math.max(seedSeconds, 10), 30);
|
||||
const loops = targetS > seedS ? Math.ceil(targetS / seedS) : 1;
|
||||
const script = musicgenScript({ prompt, abs, targetS, seedS });
|
||||
const { cmd, args } = pythonInvocation(["-c", script]);
|
||||
rmSync(abs, { force: true });
|
||||
const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] });
|
||||
proc.unref();
|
||||
closeSync(fd);
|
||||
return {
|
||||
...baseMeta,
|
||||
mode: targetS > seedS ? "detached-seed-loop" : "detached-seed-trim",
|
||||
provider: "musicgen",
|
||||
pid: proc.pid,
|
||||
log,
|
||||
target_duration_s: r3(targetS),
|
||||
seed_duration_s: seedS,
|
||||
loop_count: loops,
|
||||
};
|
||||
}
|
||||
|
||||
closeSync(fd);
|
||||
return {
|
||||
disabled: true,
|
||||
reason: lyriaConfigured
|
||||
? `Lyria configured but google-genai uninstallable, and local MusicGen unavailable (pip install ${BGM_PY_DEPS.join(" ")})`
|
||||
: `no Lyria key/recipe and local MusicGen deps unavailable (pip install ${BGM_PY_DEPS.join(" ")})`,
|
||||
};
|
||||
}
|
||||
|
||||
// Inline MusicGen: generate ONE seed clip (≤30s to stay under the decoder's
|
||||
// positional limit), then trim it down or crossfade-loop it up to the target.
|
||||
function musicgenScript({ prompt, abs, targetS, seedS }) {
|
||||
return `
|
||||
import math, os, sys, traceback
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from transformers import MusicgenForConditionalGeneration, AutoProcessor
|
||||
|
||||
prompt = ${JSON.stringify(prompt)}
|
||||
out_path = ${JSON.stringify(abs)}
|
||||
target_s = float(${targetS.toFixed(3)})
|
||||
seed_s = float(${seedS.toFixed(3)})
|
||||
token_rate = 50
|
||||
crossfade_s = 0.3
|
||||
|
||||
def apply_fade(arr, sr, fade_in_s=0.08, fade_out_s=0.5):
|
||||
n_in = min(int(round(fade_in_s * sr)), arr.shape[0] // 2)
|
||||
n_out = min(int(round(fade_out_s * sr)), arr.shape[0] // 2)
|
||||
if n_in > 1: arr[:n_in] *= np.linspace(0.0, 1.0, n_in, dtype="float32")
|
||||
if n_out > 1: arr[-n_out:] *= np.linspace(1.0, 0.0, n_out, dtype="float32")
|
||||
return arr
|
||||
|
||||
def loop_crossfade(seed, target_len, xf):
|
||||
if seed.shape[0] >= target_len: return seed[:target_len]
|
||||
xf = min(xf, seed.shape[0] // 2)
|
||||
if xf < 1:
|
||||
reps = int(math.ceil(target_len / seed.shape[0]))
|
||||
return np.tile(seed, reps)[:target_len]
|
||||
t = np.linspace(0.0, 1.0, xf, dtype="float32")
|
||||
fade_out = np.cos(t * (math.pi / 2)); fade_in = np.sin(t * (math.pi / 2))
|
||||
out = seed.copy()
|
||||
while out.shape[0] < target_len:
|
||||
tail = out[-xf:] * fade_out; head = seed[:xf] * fade_in
|
||||
out = np.concatenate([out[:-xf], tail + head, seed[xf:]])
|
||||
return out[:target_len]
|
||||
|
||||
try:
|
||||
Path(os.path.dirname(out_path)).mkdir(parents=True, exist_ok=True)
|
||||
processor = AutoProcessor.from_pretrained("facebook/musicgen-small")
|
||||
model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small")
|
||||
model.eval()
|
||||
sr = int(model.config.audio_encoder.sampling_rate)
|
||||
gen_s = min(seed_s, target_s)
|
||||
tokens = max(1, int(math.ceil(gen_s * token_rate)))
|
||||
print(f"[musicgen] seed dur={gen_s:.2f}s tokens={tokens}", flush=True)
|
||||
inputs = processor(text=[prompt], padding=True, return_tensors="pt")
|
||||
audio = model.generate(**inputs, max_new_tokens=tokens)
|
||||
seed = audio[0, 0].detach().cpu().numpy().astype("float32")
|
||||
peak = float(np.max(np.abs(seed)))
|
||||
if peak > 1e-6: seed = seed * (0.89 / peak)
|
||||
want = max(1, int(round(target_s * sr)))
|
||||
if seed.shape[0] >= want:
|
||||
final = seed[:want].copy()
|
||||
else:
|
||||
final = loop_crossfade(seed, want, int(round(crossfade_s * sr)))
|
||||
if final.shape[0] < want: final = np.pad(final, (0, want - final.shape[0]))
|
||||
else: final = final[:want]
|
||||
final = apply_fade(final, sr)
|
||||
peak = float(np.max(np.abs(final)))
|
||||
if peak > 1.0: final = final / peak
|
||||
sf.write(out_path, final, sr)
|
||||
print(f"[musicgen] wrote {out_path} samples={final.shape[0]} sr={sr}", flush=True)
|
||||
except Exception:
|
||||
traceback.print_exc(); sys.exit(1)
|
||||
`;
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { appendRecord } from "../../../scripts/lib/manifest.mjs";
|
||||
import {
|
||||
BGM_BED_VOLUME,
|
||||
BGM_SILENT_VOLUME,
|
||||
bgmDefaultVolume,
|
||||
generateBgmDetached,
|
||||
} from "./bgm.mjs";
|
||||
|
||||
// Regression: narrated pipelines used to ship BGM at 0.8 (≈ -2 dB), ~16 dB
|
||||
// hotter than a music bed under a voice should be. The default under narration
|
||||
// must be a proper bed (≈ -18 dB); a silent film keeps the louder default.
|
||||
|
||||
const dbfs = (linear) => 20 * Math.log10(linear);
|
||||
|
||||
test("BGM under narration is a bed near -18 dB", () => {
|
||||
assert.equal(bgmDefaultVolume(true), BGM_BED_VOLUME);
|
||||
assert.equal(BGM_BED_VOLUME, 0.12);
|
||||
const db = dbfs(BGM_BED_VOLUME);
|
||||
assert.ok(db < -17 && db > -19, `bed should be ≈ -18 dB, got ${db.toFixed(1)} dB`);
|
||||
});
|
||||
|
||||
test("a silent film (no voice) keeps BGM forward", () => {
|
||||
assert.equal(bgmDefaultVolume(false), BGM_SILENT_VOLUME);
|
||||
assert.equal(BGM_SILENT_VOLUME, 0.9);
|
||||
});
|
||||
|
||||
test("the narrated default is well below the voice (≈ 0 dBFS)", () => {
|
||||
// Voice sits at data-volume="1" (0 dBFS); the bed must be ~16+ dB under it.
|
||||
const separation = dbfs(1) - dbfs(bgmDefaultVolume(true));
|
||||
assert.ok(
|
||||
separation >= 16,
|
||||
`bed should sit ≥16 dB under the voice, got ${separation.toFixed(1)} dB`,
|
||||
);
|
||||
});
|
||||
|
||||
test(
|
||||
"generating music again clears the engine's old track, so waiting cannot mistake it for the new one",
|
||||
{ skip: process.platform === "win32" && "the fake python is a shell script" },
|
||||
(t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-bgm-"));
|
||||
const path = process.env.PATH;
|
||||
t.after(() => {
|
||||
process.env.PATH = path;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
mkdirSync(join(dir, "bin"));
|
||||
for (const name of ["python3", "python"])
|
||||
writeFileSync(join(dir, "bin", name), "#!/bin/sh\nexit 0\n", { mode: 0o755 });
|
||||
process.env.PATH = `${join(dir, "bin")}:${path}`;
|
||||
mkdirSync(join(dir, "assets/bgm"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets/bgm/track.wav"), "last run's track");
|
||||
appendRecord(dir, {
|
||||
id: "bgm_001",
|
||||
type: "bgm",
|
||||
path: "assets/bgm/track.wav",
|
||||
source: "generated",
|
||||
});
|
||||
|
||||
const gen = generateBgmDetached({
|
||||
prompt: "calm",
|
||||
durationS: 5,
|
||||
hyperframesDir: dir,
|
||||
anomalies: [],
|
||||
});
|
||||
|
||||
assert.equal(gen.path, "assets/bgm/track.wav");
|
||||
assert.equal(existsSync(join(dir, "assets/bgm/track.wav")), false);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,14 @@
|
||||
// mapWithConcurrency — run `fn` over `items` with at most `limit` in flight at
|
||||
// once. Preserves input order in the result array regardless of completion order.
|
||||
export async function mapWithConcurrency(items, limit, fn) {
|
||||
const results = new Array(items.length);
|
||||
let next = 0;
|
||||
async function worker() {
|
||||
while (next < items.length) {
|
||||
const i = next++;
|
||||
results[i] = await fn(items[i], i);
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
||||
return results;
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mapWithConcurrency } from "./concurrency.mjs";
|
||||
|
||||
// Regression: audio.mjs used a bare Promise.all(lines.map(synthLine)) to
|
||||
// synthesize every TTS line at once, spawning one Kokoro/whisper model load
|
||||
// per line concurrently. Two independent reports of this overwhelming a
|
||||
// machine (OOM, and cold-start contention causing spurious failures).
|
||||
// mapWithConcurrency is the extracted cap; test it in isolation since
|
||||
// audio.mjs itself is a script (runs CLI/exit side effects on import).
|
||||
test("processes every item and preserves input order regardless of completion order", async () => {
|
||||
const order = [5, 1, 3, 2, 4];
|
||||
const results = await mapWithConcurrency(order, 2, async (n) => {
|
||||
await new Promise((r) => setTimeout(r, n));
|
||||
return n * 10;
|
||||
});
|
||||
assert.deepEqual(results, [50, 10, 30, 20, 40]);
|
||||
});
|
||||
|
||||
test("never runs more than `limit` at once", async () => {
|
||||
let inFlight = 0;
|
||||
let maxInFlight = 0;
|
||||
const items = Array.from({ length: 10 }, (_, i) => i);
|
||||
await mapWithConcurrency(items, 3, async () => {
|
||||
inFlight++;
|
||||
maxInFlight = Math.max(maxInFlight, inFlight);
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
inFlight--;
|
||||
});
|
||||
assert.equal(maxInFlight, 3);
|
||||
});
|
||||
|
||||
test("limit larger than the item count runs everything without hanging", async () => {
|
||||
const results = await mapWithConcurrency([1, 2], 10, async (n) => n * 2);
|
||||
assert.deepEqual(results, [2, 4]);
|
||||
});
|
||||
|
||||
test("empty input resolves to an empty array", async () => {
|
||||
const results = await mapWithConcurrency([], 4, async (n) => n);
|
||||
assert.deepEqual(results, []);
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
|
||||
export function geminiConfigured() {
|
||||
return Boolean(
|
||||
process.env.GEMINI_API_KEY ||
|
||||
process.env.GOOGLE_API_KEY ||
|
||||
process.env.GOOGLE_APPLICATION_CREDENTIALS ||
|
||||
process.env.GCS_CREDS,
|
||||
);
|
||||
}
|
||||
|
||||
export function geminiAuth({ run = spawnSync } = {}) {
|
||||
const key = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
|
||||
if (key) return { headers: { "x-goog-api-key": key }, secret: key };
|
||||
if (!geminiConfigured()) {
|
||||
throw new Error(
|
||||
"Gemini TTS needs GEMINI_API_KEY or GOOGLE_API_KEY, or service-account credentials (GOOGLE_APPLICATION_CREDENTIALS or GCS_CREDS)",
|
||||
);
|
||||
}
|
||||
const { cmd, args } = pythonInvocation([
|
||||
fileURLToPath(new URL("./gemini-auth.py", import.meta.url)),
|
||||
]);
|
||||
const result = run(cmd, args, { encoding: "utf8", timeout: 60_000, maxBuffer: 1024 * 1024 });
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse(result.stdout);
|
||||
} catch {
|
||||
throw new Error(
|
||||
"Gemini service-account authentication failed; check Python and install google-auth requests",
|
||||
);
|
||||
}
|
||||
if (result.status !== 0 || !data.token) {
|
||||
// Do not echo subprocess output: it could contain credentials.
|
||||
throw new Error(
|
||||
"Gemini service-account authentication failed; check service-account JSON, permissions, and Python packages google-auth requests",
|
||||
);
|
||||
}
|
||||
const headers = { Authorization: `Bearer ${data.token}` };
|
||||
if (data.project) headers["x-goog-user-project"] = data.project;
|
||||
return { headers, secret: data.token };
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Obtain Gemini OAuth headers without persisting credentials or tokens."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Keep the credential subprocess protocol UTF-8 on every platform.
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(_stream, "reconfigure"):
|
||||
_stream.reconfigure(encoding="utf-8", errors=_stream.errors)
|
||||
|
||||
|
||||
def authenticate():
|
||||
try:
|
||||
from google.auth.transport.requests import Request
|
||||
from google.oauth2 import service_account
|
||||
except ImportError:
|
||||
return {"error": "Service-account auth needs Python packages: pip install google-auth requests"}
|
||||
|
||||
try:
|
||||
path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")
|
||||
if path:
|
||||
with open(path, encoding="utf-8") as source:
|
||||
info = json.load(source)
|
||||
else:
|
||||
info = json.loads(os.environ["GCS_CREDS"])
|
||||
if info.get("type") != "service_account":
|
||||
return {"error": "Gemini requires service-account JSON, not user ADC credentials"}
|
||||
# Only exchange credentials with Google's OAuth server.
|
||||
info["token_uri"] = "https://oauth2.googleapis.com/token"
|
||||
credentials = service_account.Credentials.from_service_account_info(
|
||||
info,
|
||||
scopes=[
|
||||
"https://www.googleapis.com/auth/generative-language.retriever",
|
||||
],
|
||||
)
|
||||
credentials.refresh(Request())
|
||||
project = (
|
||||
os.environ.get("GOOGLE_CLOUD_PROJECT")
|
||||
or os.environ.get("GCLOUD_PROJECT_ID")
|
||||
or info.get("project_id")
|
||||
)
|
||||
return {"token": credentials.token, "project": project}
|
||||
except Exception:
|
||||
# Library exceptions can contain credential data. Never forward them.
|
||||
return {"error": "Gemini service-account authentication failed; check credentials, permissions, and network access"}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
result = authenticate()
|
||||
print(json.dumps(result))
|
||||
sys.exit(1 if "error" in result else 0)
|
||||
@@ -0,0 +1,86 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { geminiAuth, geminiConfigured } from "./gemini-auth.mjs";
|
||||
import { pickProvider } from "./tts.mjs";
|
||||
|
||||
function env(t) {
|
||||
const saved = { ...process.env };
|
||||
for (const key of [
|
||||
"GEMINI_API_KEY",
|
||||
"GOOGLE_API_KEY",
|
||||
"GOOGLE_APPLICATION_CREDENTIALS",
|
||||
"GCS_CREDS",
|
||||
])
|
||||
delete process.env[key];
|
||||
t.after(() => {
|
||||
process.env = saved;
|
||||
});
|
||||
}
|
||||
|
||||
test("API keys take precedence over service accounts without invoking Python", (t) => {
|
||||
env(t);
|
||||
process.env.GEMINI_API_KEY = "gemini-key";
|
||||
process.env.GOOGLE_API_KEY = "google-key";
|
||||
process.env.GCS_CREDS = "private-json";
|
||||
const run = () => assert.fail("must not spawn Python");
|
||||
assert.deepEqual(geminiAuth({ run }).headers, { "x-goog-api-key": "gemini-key" });
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
assert.deepEqual(geminiAuth({ run }).headers, { "x-goog-api-key": "google-key" });
|
||||
});
|
||||
|
||||
test("service-account configuration selects Gemini and returns scoped helper headers", (t) => {
|
||||
env(t);
|
||||
assert.equal(geminiConfigured(), false);
|
||||
assert.throws(() => geminiAuth(), /needs GEMINI_API_KEY/);
|
||||
for (const key of ["GOOGLE_APPLICATION_CREDENTIALS", "GCS_CREDS"]) {
|
||||
process.env[key] = "credential-source";
|
||||
assert.equal(pickProvider("gemini"), "gemini");
|
||||
const auth = geminiAuth({
|
||||
run: (_, args, options) => {
|
||||
assert.ok(args.at(-1).endsWith("gemini-auth.py"));
|
||||
assert.ok(!args.includes("credential-source"));
|
||||
assert.equal(options.timeout, 60000);
|
||||
return {
|
||||
status: 0,
|
||||
stdout: JSON.stringify({ token: "access-token", project: "quota-project" }),
|
||||
};
|
||||
},
|
||||
});
|
||||
assert.deepEqual(auth.headers, {
|
||||
Authorization: "Bearer access-token",
|
||||
"x-goog-user-project": "quota-project",
|
||||
});
|
||||
delete process.env[key];
|
||||
}
|
||||
});
|
||||
|
||||
test("failed, malformed, or missing Python output cannot leak secrets or fall back", (t) => {
|
||||
env(t);
|
||||
process.env.GCS_CREDS = "private-json";
|
||||
for (const result of [
|
||||
{ status: 1, stdout: '{"error":"private-json"}', stderr: "private-json" },
|
||||
{ status: null, stdout: "private-json", error: new Error("private-json") },
|
||||
{ status: 0, stdout: "{}" },
|
||||
]) {
|
||||
assert.throws(
|
||||
() => geminiAuth({ run: () => result }),
|
||||
(error) => {
|
||||
assert.match(error.message, /authentication failed/);
|
||||
assert.ok(!error.message.includes("private-json"));
|
||||
return true;
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// Keep the Python credential-boundary suite reachable from the normal Node CI runner.
|
||||
test("Python service-account boundary validates credentials, scope, transport and errors", () => {
|
||||
const { cmd, args } = pythonInvocation([
|
||||
fileURLToPath(new URL("./gemini-auth_test.py", import.meta.url)),
|
||||
]);
|
||||
const result = spawnSync(cmd, args, { encoding: "utf8", timeout: 30_000 });
|
||||
assert.equal(result.status, 0, result.stderr || result.error?.message);
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Credential-boundary tests; stub Google's transport, never use real secrets."""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
|
||||
# Keep the credential subprocess protocol UTF-8 on every platform.
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(_stream, "reconfigure"):
|
||||
_stream.reconfigure(encoding="utf-8", errors=_stream.errors)
|
||||
|
||||
spec = importlib.util.spec_from_file_location("gemini_auth", Path(__file__).with_name("gemini-auth.py"))
|
||||
auth = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(auth)
|
||||
|
||||
|
||||
class AuthenticationTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.info = {"type": "service_account", "project_id": "account-project", "token_uri": "https://untrusted.invalid/token"}
|
||||
self.credentials = Mock(token="fixture-token")
|
||||
self.factory = Mock(return_value=self.credentials)
|
||||
self.request = Mock(return_value=object())
|
||||
transport = types.ModuleType("google.auth.transport.requests")
|
||||
transport.Request = self.request
|
||||
oauth = types.ModuleType("google.oauth2")
|
||||
oauth.service_account = types.SimpleNamespace(Credentials=types.SimpleNamespace(from_service_account_info=self.factory))
|
||||
modules = {name: types.ModuleType(name) for name in ("google", "google.auth", "google.auth.transport")}
|
||||
modules.update({"google.auth.transport.requests": transport, "google.oauth2": oauth})
|
||||
self.addCleanup(patch.stopall)
|
||||
patch.dict("sys.modules", modules).start()
|
||||
patch.dict(os.environ, {"GCS_CREDS": json.dumps(self.info)}, clear=True).start()
|
||||
|
||||
def test_scope_endpoint_and_refresh(self):
|
||||
self.assertEqual(auth.authenticate(), {"token": "fixture-token", "project": "account-project"})
|
||||
info = self.factory.call_args.args[0]
|
||||
self.assertEqual(info["token_uri"], "https://oauth2.googleapis.com/token")
|
||||
self.assertEqual(self.factory.call_args.kwargs["scopes"], ["https://www.googleapis.com/auth/generative-language.retriever"])
|
||||
self.credentials.refresh.assert_called_once_with(self.request.return_value)
|
||||
|
||||
def test_file_takes_precedence_over_json_environment(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "service.json"
|
||||
path.write_text(json.dumps({**self.info, "project_id": "file-project"}), encoding="utf-8")
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = str(path)
|
||||
os.environ["GCS_CREDS"] = "invalid-json"
|
||||
self.assertEqual(auth.authenticate()["project"], "file-project")
|
||||
|
||||
def test_quota_project_precedence(self):
|
||||
os.environ["GCLOUD_PROJECT_ID"] = "alias-project"
|
||||
self.assertEqual(auth.authenticate()["project"], "alias-project")
|
||||
os.environ["GOOGLE_CLOUD_PROJECT"] = "explicit-project"
|
||||
self.assertEqual(auth.authenticate()["project"], "explicit-project")
|
||||
|
||||
def test_user_adc_is_rejected_before_exchange(self):
|
||||
os.environ["GCS_CREDS"] = json.dumps({"type": "authorized_user"})
|
||||
self.assertIn("service-account JSON", auth.authenticate()["error"])
|
||||
self.factory.assert_not_called()
|
||||
|
||||
def test_invalid_json_or_missing_file_does_not_fall_back(self):
|
||||
os.environ["GCS_CREDS"] = "private-invalid-json"
|
||||
self.assertIn("error", auth.authenticate())
|
||||
os.environ["GCS_CREDS"] = json.dumps(self.info)
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/missing/service-account-fixture.json"
|
||||
self.assertIn("error", auth.authenticate())
|
||||
self.factory.assert_not_called()
|
||||
|
||||
def test_library_errors_are_redacted(self):
|
||||
for boundary in (self.factory, self.credentials.refresh):
|
||||
with self.subTest(boundary=boundary):
|
||||
boundary.side_effect = RuntimeError("private-key-and-token")
|
||||
result = auth.authenticate()
|
||||
self.assertIn("error", result)
|
||||
self.assertNotIn("private-key-and-token", json.dumps(result))
|
||||
boundary.side_effect = None
|
||||
|
||||
def test_missing_dependency_has_actionable_error(self):
|
||||
with patch.dict("sys.modules", {"google.auth.transport.requests": None}):
|
||||
self.assertIn("pip install google-auth requests", auth.authenticate()["error"])
|
||||
self.factory.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,122 @@
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { geminiAuth } from "./gemini-auth.mjs";
|
||||
|
||||
export const GEMINI_TTS_MODEL = "gemini-3.8-flash-tts";
|
||||
|
||||
export const GEMINI_TTS_MODELS = [
|
||||
GEMINI_TTS_MODEL,
|
||||
"gemini-3.8-flash-lite-tts",
|
||||
"gemini-3.1-flash-tts-preview",
|
||||
"gemini-2.5-pro-preview-tts",
|
||||
"gemini-2.5-flash-preview-tts",
|
||||
];
|
||||
|
||||
// 3.8 returns WAV; older models return PCM that we wrap without resampling.
|
||||
// The shared engine transcribes the saved audio for word timings.
|
||||
export async function synthesizeGemini(
|
||||
{ text, voiceId = "Kore", model = GEMINI_TTS_MODEL, style, speed = 1, wavAbs },
|
||||
{ fetchImpl = fetch, authenticate = geminiAuth } = {},
|
||||
) {
|
||||
let secret;
|
||||
try {
|
||||
if (!GEMINI_TTS_MODELS.includes(model)) {
|
||||
throw new Error(`Unsupported Gemini TTS model: ${model}`);
|
||||
}
|
||||
if (speed !== 1) throw new Error("Gemini TTS uses style for pacing; omit speed or use 1");
|
||||
const modern = model.startsWith("gemini-3.8-");
|
||||
if (!modern && /^(voice_|voicekey_)/.test(voiceId)) {
|
||||
throw new Error(
|
||||
"Custom Gemini voices require a 3.8 TTS model; use a prebuilt voice with older models",
|
||||
);
|
||||
}
|
||||
const auth = authenticate();
|
||||
secret = auth.secret;
|
||||
const content = {
|
||||
type: "text",
|
||||
text:
|
||||
!modern && style ? `Read the following text with this delivery: ${style}\n\n${text}` : text,
|
||||
};
|
||||
if (modern && style) content.annotations = [{ type: "speech_metadata", style }];
|
||||
const response = await fetchImpl(
|
||||
"https://generativelanguage.googleapis.com/v1beta/interactions",
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", ...auth.headers },
|
||||
signal: AbortSignal.timeout(120_000),
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
input: [{ type: "user_input", content: [content] }],
|
||||
response_format: modern ? { type: "audio", mime_type: "audio/wav" } : { type: "audio" },
|
||||
generation_config: { speech_config: [{ voice: voiceId }] },
|
||||
store: false,
|
||||
}),
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
const detail = await response.text();
|
||||
throw new Error(`Gemini TTS HTTP ${response.status}: ${detail}`);
|
||||
}
|
||||
const payload = await response.json();
|
||||
if (payload.status !== "completed") {
|
||||
throw new Error(`Gemini TTS did not complete (${payload.status ?? "missing status"})`);
|
||||
}
|
||||
const audio = (payload.steps ?? [])
|
||||
.filter((step) => step.type === "model_output")
|
||||
.flatMap((step) => step.content ?? [])
|
||||
.filter((part) => part.type === "audio");
|
||||
if (audio.length !== 1 || !audio[0].data) {
|
||||
throw new Error("Gemini TTS returned no single audio block");
|
||||
}
|
||||
let bytes = Buffer.from(audio[0].data, "base64");
|
||||
if (!modern && /^audio\/l16(?:;|$)/i.test(audio[0].mime_type ?? "")) {
|
||||
bytes = pcmToWav(bytes, audio[0].mime_type);
|
||||
} else if (audio[0].mime_type !== "audio/wav") {
|
||||
throw new Error("Gemini TTS returned an unsupported audio format");
|
||||
}
|
||||
if (
|
||||
bytes.length <= 44 ||
|
||||
bytes.toString("ascii", 0, 4) !== "RIFF" ||
|
||||
bytes.toString("ascii", 8, 12) !== "WAVE"
|
||||
) {
|
||||
throw new Error("Gemini TTS returned invalid WAV audio");
|
||||
}
|
||||
mkdirSync(dirname(wavAbs), { recursive: true });
|
||||
writeFileSync(wavAbs, bytes);
|
||||
return { ok: true, words: null };
|
||||
} catch (error) {
|
||||
// Error responses must never echo the credential into logs or metadata.
|
||||
const message = String(error?.message ?? error);
|
||||
return {
|
||||
ok: false,
|
||||
words: null,
|
||||
error: secret ? message.split(secret).join("[redacted]") : message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function pcmToWav(pcm, mime) {
|
||||
const rate = Number(/(?:^|;)\s*rate=(\d+)(?:;|$)/i.exec(mime)?.[1]);
|
||||
const channels = /(?:^|;)\s*channels=([^;]+)/i.exec(mime)?.[1];
|
||||
const codec = /(?:^|;)\s*codec=([^;]+)/i.exec(mime)?.[1];
|
||||
if ((channels && channels.trim() !== "1") || (codec && codec.trim() !== "pcm")) {
|
||||
throw new Error("Gemini TTS returned unsupported PCM channels or codec");
|
||||
}
|
||||
if (!Number.isInteger(rate) || rate < 8000 || rate > 96000 || !pcm.length || pcm.length % 2) {
|
||||
throw new Error("Gemini TTS returned invalid PCM audio or sample rate");
|
||||
}
|
||||
const header = Buffer.alloc(44);
|
||||
header.write("RIFF");
|
||||
header.writeUInt32LE(36 + pcm.length, 4);
|
||||
header.write("WAVEfmt ", 8);
|
||||
header.writeUInt32LE(16, 16);
|
||||
header.writeUInt16LE(1, 20);
|
||||
header.writeUInt16LE(1, 22);
|
||||
header.writeUInt32LE(rate, 24);
|
||||
header.writeUInt32LE(rate * 2, 28);
|
||||
header.writeUInt16LE(2, 32);
|
||||
header.writeUInt16LE(16, 34);
|
||||
header.write("data", 36);
|
||||
header.writeUInt32LE(pcm.length, 40);
|
||||
return Buffer.concat([header, pcm]);
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, rmSync, existsSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { synthesizeGemini, GEMINI_TTS_MODEL } from "./gemini-tts.mjs";
|
||||
import { pickProvider, resolveVoiceId, synthesizeOne } from "./tts.mjs";
|
||||
|
||||
function fixture(t) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "gemini-tts-"));
|
||||
const saved = { ...process.env };
|
||||
process.env.GEMINI_API_KEY = "test-gemini-key";
|
||||
delete process.env.GOOGLE_API_KEY;
|
||||
delete process.env.GOOGLE_APPLICATION_CREDENTIALS;
|
||||
delete process.env.GCS_CREDS;
|
||||
t.after(() => {
|
||||
process.env = saved;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
const wav = Buffer.alloc(48);
|
||||
wav.write("RIFF");
|
||||
wav.writeUInt32LE(40, 4);
|
||||
wav.write("WAVEfmt ", 8);
|
||||
wav.writeUInt32LE(16, 16);
|
||||
wav.writeUInt16LE(1, 20);
|
||||
wav.writeUInt16LE(1, 22);
|
||||
wav.writeUInt32LE(24000, 24);
|
||||
wav.writeUInt32LE(48000, 28);
|
||||
wav.writeUInt16LE(2, 32);
|
||||
wav.writeUInt16LE(16, 34);
|
||||
wav.write("data", 36);
|
||||
wav.writeUInt32LE(4, 40);
|
||||
const audio = { type: "audio", mime_type: "audio/wav", data: wav.toString("base64") };
|
||||
const payload = { status: "completed", steps: [{ type: "model_output", content: [audio] }] };
|
||||
return {
|
||||
wav,
|
||||
audio,
|
||||
payload,
|
||||
args: { text: "Hello there.", voiceId: "Kore", wavAbs: join(dir, "voice", "one.wav") },
|
||||
};
|
||||
}
|
||||
|
||||
test("Gemini preserves verbatim text, directs style separately, saves WAV and requests transcription", async (t) => {
|
||||
const { wav, payload, args } = fixture(t);
|
||||
t.mock.method(globalThis, "fetch", async (url, options) => {
|
||||
assert.equal(url, "https://generativelanguage.googleapis.com/v1beta/interactions");
|
||||
assert.equal(options.headers["x-goog-api-key"], "test-gemini-key");
|
||||
const body = JSON.parse(options.body);
|
||||
assert.equal(body.model, GEMINI_TTS_MODEL);
|
||||
assert.equal(body.store, false);
|
||||
assert.deepEqual(body.input[0].content, [
|
||||
{
|
||||
type: "text",
|
||||
text: args.text,
|
||||
annotations: [{ type: "speech_metadata", style: "Warm and calm" }],
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(body.generation_config.speech_config, [{ voice: "Kore" }]);
|
||||
assert.equal(body.response_format.mime_type, "audio/wav");
|
||||
return Response.json(payload);
|
||||
});
|
||||
const result = await synthesizeOne({ ...args, provider: "gemini", style: "Warm and calm" });
|
||||
assert.deepEqual(result, { ok: true, words: null });
|
||||
assert.deepEqual(readFileSync(args.wavAbs), wav);
|
||||
});
|
||||
|
||||
test("Flash-Lite and GOOGLE_API_KEY work without a Gemini key", async (t) => {
|
||||
const { payload, args } = fixture(t);
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
process.env.GOOGLE_API_KEY = "test-google-key";
|
||||
assert.equal(pickProvider("gemini"), "gemini");
|
||||
assert.equal(await resolveVoiceId({ provider: "gemini" }), "Kore");
|
||||
const result = await synthesizeGemini(
|
||||
{ ...args, model: "gemini-3.8-flash-lite-tts" },
|
||||
{
|
||||
fetchImpl: async (_, options) => {
|
||||
assert.equal(options.headers["x-goog-api-key"], "test-google-key");
|
||||
const body = JSON.parse(options.body);
|
||||
assert.equal(body.model, "gemini-3.8-flash-lite-tts");
|
||||
assert.equal(body.input[0].content[0].annotations, undefined);
|
||||
return Response.json(payload);
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.ok, true);
|
||||
});
|
||||
|
||||
test("Gemini remains opt-in even with a key and preserves explicitly chosen voices", async (t) => {
|
||||
fixture(t);
|
||||
assert.notEqual(pickProvider(), "gemini");
|
||||
assert.equal(
|
||||
await resolveVoiceId({ provider: "gemini", userVoice: "voice_custom" }),
|
||||
"voice_custom",
|
||||
);
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
assert.throws(() => pickProvider("gemini"), /GEMINI_API_KEY or GOOGLE_API_KEY/);
|
||||
});
|
||||
|
||||
test("invalid config fails before spending a generation request", async (t) => {
|
||||
const { args } = fixture(t);
|
||||
const deps = { fetchImpl: () => assert.fail("must not call the API") };
|
||||
for (const change of [{ model: "gemini-2.5-flash" }, { speed: 1.2 }]) {
|
||||
const result = await synthesizeGemini({ ...args, ...change }, deps);
|
||||
assert.equal(result.ok, false);
|
||||
assert.ok(result.error);
|
||||
}
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
assert.match((await synthesizeGemini(args, deps)).error, /needs GEMINI_API_KEY/);
|
||||
assert.equal(existsSync(args.wavAbs), false);
|
||||
});
|
||||
|
||||
test("HTTP and network failures stay actionable without leaking the key", async (t) => {
|
||||
const { args } = fixture(t);
|
||||
const result = await synthesizeGemini(args, {
|
||||
fetchImpl: async () => new Response("quota exceeded test-gemini-key", { status: 429 }),
|
||||
});
|
||||
assert.equal(result.ok, false);
|
||||
assert.match(result.error, /HTTP 429: quota exceeded/);
|
||||
assert.ok(!result.error.includes("test-gemini-key"));
|
||||
const timeout = await synthesizeGemini(args, {
|
||||
fetchImpl: async () => {
|
||||
throw new Error("timed out");
|
||||
},
|
||||
});
|
||||
assert.match(timeout.error, /timed out/);
|
||||
assert.equal(existsSync(args.wavAbs), false);
|
||||
});
|
||||
|
||||
test("incomplete, missing, raw PCM and malformed audio cannot become a successful WAV", async (t) => {
|
||||
const { args, payload, audio } = fixture(t);
|
||||
const cases = [
|
||||
{ ...payload, status: "incomplete" },
|
||||
{ status: "completed", steps: [] },
|
||||
...[{ ...audio, mime_type: "audio/l16" }, { ...audio, data: "not audio" }, audio].map(
|
||||
(part, i) => ({
|
||||
status: "completed",
|
||||
steps: [{ type: "model_output", content: i === 2 ? [part, part] : [part] }],
|
||||
}),
|
||||
),
|
||||
];
|
||||
for (const body of cases) {
|
||||
const result = await synthesizeGemini(args, { fetchImpl: async () => Response.json(body) });
|
||||
assert.equal(result.ok, false);
|
||||
assert.equal(existsSync(args.wavAbs), false);
|
||||
}
|
||||
});
|
||||
|
||||
for (const model of [
|
||||
"gemini-3.1-flash-tts-preview",
|
||||
"gemini-2.5-pro-preview-tts",
|
||||
"gemini-2.5-flash-preview-tts",
|
||||
]) {
|
||||
test(`${model} uses legacy delivery prompts and wraps PCM as mono WAV`, async (t) => {
|
||||
const { args, wav } = fixture(t);
|
||||
const pcm = wav.subarray(44);
|
||||
const result = await synthesizeGemini(
|
||||
{ ...args, model, style: "Warm and clear" },
|
||||
{
|
||||
fetchImpl: async (_, options) => {
|
||||
const body = JSON.parse(options.body);
|
||||
assert.deepEqual(body.response_format, { type: "audio" });
|
||||
assert.equal(body.input[0].content[0].annotations, undefined);
|
||||
assert.ok(body.input[0].content[0].text.endsWith(args.text));
|
||||
assert.ok(body.input[0].content[0].text.includes("Warm and clear"));
|
||||
return Response.json({
|
||||
status: "completed",
|
||||
steps: [
|
||||
{
|
||||
type: "model_output",
|
||||
content: [
|
||||
{
|
||||
type: "audio",
|
||||
mime_type: "audio/L16;codec=pcm;rate=24000",
|
||||
data: pcm.toString("base64"),
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.ok, true, result.error);
|
||||
assert.deepEqual(readFileSync(args.wavAbs), wav);
|
||||
});
|
||||
}
|
||||
|
||||
test("service-account bearer and quota project reach synthesis, and token errors are redacted", async (t) => {
|
||||
const { args, payload } = fixture(t);
|
||||
const authenticate = () => ({
|
||||
headers: { Authorization: "Bearer secret-token", "x-goog-user-project": "test-project" },
|
||||
secret: "secret-token",
|
||||
});
|
||||
const ok = await synthesizeGemini(args, {
|
||||
authenticate,
|
||||
fetchImpl: async (_, options) => {
|
||||
assert.equal(options.headers.Authorization, "Bearer secret-token");
|
||||
assert.equal(options.headers["x-goog-user-project"], "test-project");
|
||||
assert.equal(options.headers["x-goog-api-key"], undefined);
|
||||
return Response.json(payload);
|
||||
},
|
||||
});
|
||||
assert.equal(ok.ok, true);
|
||||
const failed = await synthesizeGemini(args, {
|
||||
authenticate,
|
||||
fetchImpl: async () => new Response("denied secret-token", { status: 403 }),
|
||||
});
|
||||
assert.match(failed.error, /HTTP 403/);
|
||||
assert.ok(!failed.error.includes("secret-token"));
|
||||
});
|
||||
|
||||
test("older models reject unsupported PCM and custom voices", async (t) => {
|
||||
const { args } = fixture(t);
|
||||
for (const mime of [
|
||||
"audio/l16",
|
||||
"audio/l16;rate=0",
|
||||
"audio/l16;rate=24000;channels=2",
|
||||
"audio/l16;rate=24000;codec=other",
|
||||
]) {
|
||||
const result = await synthesizeGemini(
|
||||
{ ...args, model: "gemini-3.1-flash-tts-preview" },
|
||||
{
|
||||
fetchImpl: async () =>
|
||||
Response.json({
|
||||
status: "completed",
|
||||
steps: [
|
||||
{
|
||||
type: "model_output",
|
||||
content: [{ type: "audio", mime_type: mime, data: "AAAAAA==" }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
);
|
||||
assert.equal(result.ok, false);
|
||||
assert.equal(existsSync(args.wavAbs), false);
|
||||
}
|
||||
const result = await synthesizeGemini(
|
||||
{ ...args, model: "gemini-2.5-pro-preview-tts", voiceId: "voice_custom" },
|
||||
{ fetchImpl: () => assert.fail("must not generate") },
|
||||
);
|
||||
assert.match(result.error, /Custom Gemini voices require/);
|
||||
});
|
||||
@@ -0,0 +1,240 @@
|
||||
import { fetchMedia } from "../../../scripts/lib/media-fetch.mjs";
|
||||
// heygen.mjs — vendored HeyGen REST helpers (auth + transport) for the audio
|
||||
// pipeline. The credential resolver matches the hyperframes CLI auth: first
|
||||
// usable source wins — a host gateway ($HEYGEN_API_BASE with its own
|
||||
// $HEYGEN_API_KEY) → a host-injected OAuth $HEYGEN_ACCESS_TOKEN (Bearer) →
|
||||
// $HEYGEN_API_KEY / $HYPERFRAMES_API_KEY → a nearby .env → ~/.heygen/
|
||||
// credentials (oauth → Bearer, else api_key → X-Api-Key; $HEYGEN_CONFIG_DIR
|
||||
// overrides the dir). $HEYGEN_API_BASE moves every request to that host, as it
|
||||
// does for the heygen CLI; plain HTTP needs $HEYGEN_ALLOW_HTTP=1, as there.
|
||||
// Vendored so the skill ships standalone. Pure node.
|
||||
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
|
||||
export const HEYGEN_BASE = "https://api.heygen.com/v3";
|
||||
|
||||
// The v3 base every request goes to: $HEYGEN_API_BASE when a host app names its own gateway (HyperFrames Desktop
|
||||
// forwards it to HeyGen with the API key saved in its Settings), else HeyGen's public API. Plain HTTP carries the key
|
||||
// in the clear, so it needs $HEYGEN_ALLOW_HTTP=1, the heygen CLI's own rule.
|
||||
export function heygenBase() {
|
||||
const host = process.env.HEYGEN_API_BASE?.trim().replace(/\/+$/, "");
|
||||
if (!host) return HEYGEN_BASE;
|
||||
if (host.startsWith("http://") && process.env.HEYGEN_ALLOW_HTTP !== "1")
|
||||
throw new Error(
|
||||
`HEYGEN_API_BASE (${host}) uses HTTP, which sends the key in plaintext. Set HEYGEN_ALLOW_HTTP=1 to allow it.`,
|
||||
);
|
||||
return `${host}/v3`;
|
||||
}
|
||||
|
||||
// No base override, or one on HeyGen's own hosts (a canary or dev API).
|
||||
function heygenOwnBase() {
|
||||
const host = process.env.HEYGEN_API_BASE?.trim();
|
||||
if (!host) return true;
|
||||
try {
|
||||
const name = new URL(host).hostname;
|
||||
return name === "heygen.com" || name.endsWith(".heygen.com");
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// A host gateway: the host named its own API base and the key that base accepts. It pays for every call, so it wins
|
||||
// over any other credential the environment carries.
|
||||
const hostGatewayKey = () =>
|
||||
process.env.HEYGEN_API_BASE?.trim() && process.env.HEYGEN_API_KEY
|
||||
? process.env.HEYGEN_API_KEY
|
||||
: null;
|
||||
export const HEYGEN_CLI_SOURCE_HEADERS = { "X-HeyGen-Source": "cli" };
|
||||
// Tool-attribution sent on EVERY media-use HeyGen call regardless of auth type, so
|
||||
// the backend can isolate media-use consumption from other free TTS / avatar video.
|
||||
// Unconditional — a paying user's media-use call is still media-use — unlike the
|
||||
// OAuth-only cli-source header above, which also gates the free allowance.
|
||||
export const HEYGEN_CLIENT_SOURCE_HEADERS = { "X-HeyGen-Client-Source": "media-use" };
|
||||
|
||||
// A missing `.env`, or a `.env` folder (some home dirs have one), is no env file: null. Read without checking first,
|
||||
// so the file cannot change between a check and the read.
|
||||
function envFileText(path) {
|
||||
try {
|
||||
return readFileSync(path, "utf8");
|
||||
} catch (error) {
|
||||
if (["ENOENT", "ENOTDIR", "EISDIR"].includes(error.code)) return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// A host app sets these in the environment it spawns, never in a project file: a project's .env naming its own base
|
||||
// would send the person's shell HEYGEN_API_KEY to that host.
|
||||
const HOST_ONLY = new Set(["HEYGEN_API_BASE", "HEYGEN_ALLOW_HTTP"]);
|
||||
|
||||
// Walk up ≤5 dirs from startDir; load the first .env (shell env always wins).
|
||||
export function loadEnvFromDir(startDir) {
|
||||
let dir = resolve(startDir);
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const text = envFileText(join(dir, ".env"));
|
||||
if (text != null) {
|
||||
for (const raw of text.split("\n")) {
|
||||
let line = raw.trim();
|
||||
if (!line || line.startsWith("#")) continue;
|
||||
if (line.startsWith("export ")) line = line.slice(7).trim();
|
||||
const eq = line.indexOf("=");
|
||||
if (eq < 1) continue;
|
||||
const key = line.slice(0, eq).trim();
|
||||
let val = line.slice(eq + 1).trim();
|
||||
if (val.startsWith('"') || val.startsWith("'")) {
|
||||
const q = val[0];
|
||||
const end = val.indexOf(q, 1);
|
||||
val = end > 0 ? val.slice(1, end) : val.slice(1);
|
||||
}
|
||||
if (!HOST_ONLY.has(key) && !(key in process.env)) process.env[key] = val;
|
||||
}
|
||||
return;
|
||||
}
|
||||
const parent = dirname(dir);
|
||||
if (parent === dir) break;
|
||||
dir = parent;
|
||||
}
|
||||
}
|
||||
|
||||
// → { headers } | { expired: true } | null. Never throws.
|
||||
export function heygenCredential() {
|
||||
const cred = resolveCredential();
|
||||
return cred?.unreadable ? null : cred;
|
||||
}
|
||||
|
||||
// heygenCredential's answer, or { unreadable: { file, code } } when the credentials path exists but cannot be read
|
||||
// (a folder, a locked ~/.heygen), so heygenAuthHeaders can say to fix that path: logging in again would fail there too.
|
||||
// Read without checking first, so the file cannot change between a check and the read.
|
||||
function resolveCredential() {
|
||||
const gatewayKey = hostGatewayKey();
|
||||
if (gatewayKey) return { headers: { "X-Api-Key": gatewayKey } };
|
||||
// Every other credential belongs to HeyGen: a base on any other host gets none of them.
|
||||
if (!heygenOwnBase()) return null;
|
||||
const accessToken = process.env.HEYGEN_ACCESS_TOKEN;
|
||||
if (accessToken) return { headers: { Authorization: `Bearer ${accessToken}` } };
|
||||
const envKey = process.env.HEYGEN_API_KEY || process.env.HYPERFRAMES_API_KEY;
|
||||
if (envKey) return { headers: { "X-Api-Key": envKey } };
|
||||
|
||||
const file = join(process.env.HEYGEN_CONFIG_DIR || join(homedir(), ".heygen"), "credentials");
|
||||
let raw;
|
||||
try {
|
||||
raw = readFileSync(file, "utf8").trim();
|
||||
} catch (error) {
|
||||
return error.code === "ENOENT" ? null : { unreadable: { file, code: error.code } };
|
||||
}
|
||||
if (!raw) return null;
|
||||
if (!raw.startsWith("{")) return { headers: { "X-Api-Key": raw } };
|
||||
|
||||
// A malformed credentials file (partial write / wrong shape) must degrade to
|
||||
// "no credential", not crash the engine at startup.
|
||||
let cred;
|
||||
try {
|
||||
cred = JSON.parse(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const oauth = cred.oauth;
|
||||
if (oauth?.access_token) {
|
||||
const expired = oauth.expires_at && new Date(oauth.expires_at).getTime() - 60_000 < Date.now();
|
||||
if (!expired) return { headers: { Authorization: `Bearer ${oauth.access_token}` } };
|
||||
if (!cred.api_key) return { expired: true };
|
||||
}
|
||||
if (cred.api_key) return { headers: { "X-Api-Key": cred.api_key } };
|
||||
return null;
|
||||
}
|
||||
|
||||
// → "oauth" | "api_key" | null. Same oauth-vs-api-key check heygenAuthHeaders()
|
||||
// makes internally, exposed on its own so callers that only need to *tag* the
|
||||
// auth path (telemetry) don't have to parse headers back apart. Never throws:
|
||||
// no credential (or an expired one) is just `null`, same as a fresh resolve
|
||||
// with nothing to tag.
|
||||
export function heygenAuthMethod() {
|
||||
const cred = heygenCredential();
|
||||
if (!cred?.headers) return null;
|
||||
return "Authorization" in cred.headers ? "oauth" : "api_key";
|
||||
}
|
||||
|
||||
// → auth headers object, or throw with a fix hint.
|
||||
export function heygenAuthHeaders() {
|
||||
const cred = resolveCredential();
|
||||
if (cred?.headers) {
|
||||
// Only tag OAuth (Bearer) traffic as cli-source — the backend uses it to
|
||||
// grant the free allowance for OAuth requests and ignores it for API-key
|
||||
// (X-Api-Key) traffic, where it's dead metadata.
|
||||
const isOauth = "Authorization" in cred.headers;
|
||||
return isOauth
|
||||
? { ...cred.headers, ...HEYGEN_CLI_SOURCE_HEADERS, ...HEYGEN_CLIENT_SOURCE_HEADERS }
|
||||
: { ...cred.headers, ...HEYGEN_CLIENT_SOURCE_HEADERS };
|
||||
}
|
||||
if (cred?.unreadable)
|
||||
throw new Error(
|
||||
`HeyGen credentials at ${cred.unreadable.file} can't be read (${cred.unreadable.code}) — fix or remove that path, then run \`npx hyperframes auth login\``,
|
||||
);
|
||||
if (cred?.expired)
|
||||
throw new Error(
|
||||
"HeyGen OAuth token expired — run `npx hyperframes auth refresh` (or `npx hyperframes auth login`)",
|
||||
);
|
||||
throw new Error(
|
||||
"no HeyGen credentials — set $HEYGEN_API_KEY, or run `npx hyperframes auth login` (writes ~/.heygen/credentials)",
|
||||
);
|
||||
}
|
||||
|
||||
// Authed JSON request against the v3 API; throws on a non-OK status.
|
||||
export async function heygenJSON(path, { method = "GET", headers = {}, body } = {}) {
|
||||
const opts = { method, headers: { ...HEYGEN_CLIENT_SOURCE_HEADERS, ...headers } };
|
||||
if (body !== undefined) {
|
||||
opts.headers["Content-Type"] = "application/json";
|
||||
opts.body = JSON.stringify(body);
|
||||
}
|
||||
const res = await fetch(`${heygenBase()}${path}`, opts);
|
||||
if (!res.ok) {
|
||||
const detail = await res.text().catch(() => "");
|
||||
const message = `HeyGen ${method} ${path} → HTTP ${res.status}${detail ? `\n${detail.slice(0, 300)}` : ""}`;
|
||||
throw Object.assign(new Error(message), { status: res.status, body: detail });
|
||||
}
|
||||
// A DELETE may answer 204 with no body.
|
||||
const text = await res.text();
|
||||
return text ? JSON.parse(text) : {};
|
||||
}
|
||||
|
||||
// HeyGen's own words for a failed call: its {"error":{"message"}}, else the raw body, else the error's message.
|
||||
export function heygenMessage(e) {
|
||||
if (!e?.body) return e?.message ? String(e.message) : String(e);
|
||||
try {
|
||||
return JSON.parse(e.body).error?.message ?? e.body;
|
||||
} catch {
|
||||
return e.body;
|
||||
}
|
||||
}
|
||||
|
||||
// Download a (presigned) URL to destPath; returns byte length.
|
||||
export async function downloadTo(url, destPath) {
|
||||
const res = await fetchMedia(url);
|
||||
if (!res.ok) throw new Error(`download HTTP ${res.status}: ${String(url).slice(0, 80)}`);
|
||||
const bytes = Buffer.from(await res.arrayBuffer());
|
||||
mkdirSync(dirname(destPath), { recursive: true });
|
||||
writeFileSync(destPath, bytes);
|
||||
return bytes.length;
|
||||
}
|
||||
|
||||
// Retrieval search over HeyGen's audio catalog (NOT generation). type =
|
||||
// "music" | "sound_effects". Returns the ranked results array (best first); each
|
||||
// item has a presigned `audio_url` (+ `duration`, `description`, `name`, `score`).
|
||||
// `query` is required (≥1 char, empty → HTTP 400) and `limit` is capped at 50.
|
||||
// `minScore`: omit to use the server default (0.7). That default is TOO HIGH for
|
||||
// sound_effects — good SFX hits score ~0.5–0.67, so callers wanting SFX should
|
||||
// pass a lower floor (~0.4); music scores high and is fine at the default.
|
||||
export async function searchSounds(query, type, headers, { limit = 5, minScore } = {}) {
|
||||
const params = new URLSearchParams({ query, type, limit: String(limit) });
|
||||
if (minScore != null) params.set("min_score", String(minScore));
|
||||
const payload = await heygenJSON(`/audio/sounds?${params.toString()}`, { headers });
|
||||
// `data` comes back as a ranked array (best first). Older responses keyed it by
|
||||
// numeric index ("0","1",…); normalize both shapes to an array (empty → []).
|
||||
const data = payload?.data ?? payload;
|
||||
if (Array.isArray(data)) return data;
|
||||
if (data && typeof data === "object") return Object.values(data);
|
||||
throw new Error(
|
||||
`unexpected /audio/sounds shape — top keys: ${Object.keys(payload ?? {}).join(", ")}`,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { createServer } from "node:http";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
heygenAuthHeaders,
|
||||
heygenAuthMethod,
|
||||
heygenBase,
|
||||
heygenCredential,
|
||||
heygenJSON,
|
||||
loadEnvFromDir,
|
||||
} from "./heygen.mjs";
|
||||
|
||||
const HEYGEN_ENV = [
|
||||
"HEYGEN_ACCESS_TOKEN",
|
||||
"HEYGEN_API_KEY",
|
||||
"HYPERFRAMES_API_KEY",
|
||||
"HEYGEN_CONFIG_DIR",
|
||||
"HEYGEN_API_BASE",
|
||||
"HEYGEN_ALLOW_HTTP",
|
||||
];
|
||||
|
||||
// Runs fn with every HeyGen variable unset, then puts them back; an async fn restores once it settles.
|
||||
function withCleanHeygenEnv(fn) {
|
||||
const previous = Object.fromEntries(HEYGEN_ENV.map((name) => [name, process.env[name]]));
|
||||
const restore = () => {
|
||||
for (const [name, value] of Object.entries(previous)) {
|
||||
if (value === undefined) delete process.env[name];
|
||||
else process.env[name] = value;
|
||||
}
|
||||
};
|
||||
for (const name of HEYGEN_ENV) delete process.env[name];
|
||||
let result;
|
||||
try {
|
||||
result = fn();
|
||||
} catch (error) {
|
||||
restore();
|
||||
throw error;
|
||||
}
|
||||
if (result && typeof result.then === "function") return result.finally(restore);
|
||||
restore();
|
||||
return result;
|
||||
}
|
||||
|
||||
test("heygenAuthHeaders does not tag API-key requests as CLI traffic, but still carries the media-use tool tag", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_API_KEY = "hg_test";
|
||||
// API-key requests use normal billing; the backend ignores the cli-source
|
||||
// header for them, so it's not sent. The tool-attribution header IS sent on
|
||||
// every media-use call (any auth type) so the backend can isolate media-use.
|
||||
assert.deepEqual(heygenAuthHeaders(), {
|
||||
"X-Api-Key": "hg_test",
|
||||
"X-HeyGen-Client-Source": "media-use",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthHeaders tags OAuth requests as CLI traffic and with the media-use tool tag", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
writeFileSync(
|
||||
join(dir, "credentials"),
|
||||
JSON.stringify({
|
||||
oauth: {
|
||||
access_token: "at_test",
|
||||
expires_at: "2099-01-01T00:00:00Z",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.deepEqual(heygenAuthHeaders(), {
|
||||
Authorization: "Bearer at_test",
|
||||
"X-HeyGen-Source": "cli",
|
||||
"X-HeyGen-Client-Source": "media-use",
|
||||
});
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("a host-injected HEYGEN_ACCESS_TOKEN wins over an API key and is treated as OAuth", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_ACCESS_TOKEN = "at_host";
|
||||
process.env.HEYGEN_API_KEY = "hg_test";
|
||||
assert.deepEqual(heygenAuthHeaders(), {
|
||||
Authorization: "Bearer at_host",
|
||||
"X-HeyGen-Source": "cli",
|
||||
"X-HeyGen-Client-Source": "media-use",
|
||||
});
|
||||
assert.equal(heygenAuthMethod(), "oauth");
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns api_key for an env API key, without tagging headers", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_API_KEY = "hg_test";
|
||||
assert.equal(heygenAuthMethod(), "api_key");
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns oauth for a live OAuth credential", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
writeFileSync(
|
||||
join(dir, "credentials"),
|
||||
JSON.stringify({
|
||||
oauth: {
|
||||
access_token: "at_test",
|
||||
expires_at: "2099-01-01T00:00:00Z",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.equal(heygenAuthMethod(), "oauth");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns null with no credential at all", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir; // no credentials file written
|
||||
assert.equal(heygenAuthMethod(), null);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("loadEnvFromDir skips a .env folder and loads the .env file above it", () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "heygen-env-"));
|
||||
const project = join(root, "project");
|
||||
mkdirSync(join(project, ".env"), { recursive: true });
|
||||
writeFileSync(join(root, ".env"), "MEDIA_USE_ENV_DIR_TEST=from-parent\n");
|
||||
try {
|
||||
loadEnvFromDir(project);
|
||||
assert.equal(process.env.MEDIA_USE_ENV_DIR_TEST, "from-parent");
|
||||
} finally {
|
||||
delete process.env.MEDIA_USE_ENV_DIR_TEST;
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("a project's .env cannot name the HeyGen base, so a shell key never leaves for its host", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const project = mkdtempSync(join(tmpdir(), "heygen-env-"));
|
||||
writeFileSync(
|
||||
join(project, ".env"),
|
||||
"HEYGEN_API_BASE=https://proxy.example.com\nHEYGEN_ALLOW_HTTP=1\nMEDIA_USE_ENV_BASE_TEST=loaded\n",
|
||||
);
|
||||
try {
|
||||
process.env.HEYGEN_API_KEY = "hg_shell_real";
|
||||
loadEnvFromDir(project);
|
||||
assert.equal(process.env.MEDIA_USE_ENV_BASE_TEST, "loaded");
|
||||
assert.equal(process.env.HEYGEN_API_BASE, undefined);
|
||||
assert.equal(process.env.HEYGEN_ALLOW_HTTP, undefined);
|
||||
assert.equal(heygenBase(), "https://api.heygen.com/v3");
|
||||
} finally {
|
||||
delete process.env.MEDIA_USE_ENV_BASE_TEST;
|
||||
rmSync(project, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns null when the credentials path is a folder", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
mkdirSync(join(dir, "credentials"));
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
assert.equal(heygenAuthMethod(), null);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns null when the credentials path is a symlink loop", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
symlinkSync(join(dir, "credentials"), join(dir, "credentials"));
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
assert.equal(heygenAuthMethod(), null);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthHeaders says to fix an unreadable credentials path, and to log in when there is none", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
assert.throws(() => heygenAuthHeaders(), /no HeyGen credentials/);
|
||||
mkdirSync(join(dir, "credentials"));
|
||||
assert.equal(heygenCredential(), null);
|
||||
assert.throws(
|
||||
() => heygenAuthHeaders(),
|
||||
(error) =>
|
||||
error.message.includes(join(dir, "credentials")) &&
|
||||
/fix or remove that path/.test(error.message),
|
||||
);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenBase is HeyGen's public API unless a host names another", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
assert.equal(heygenBase(), "https://api.heygen.com/v3");
|
||||
process.env.HEYGEN_API_BASE = "https://api-canary.heygen.com/";
|
||||
assert.equal(heygenBase(), "https://api-canary.heygen.com/v3");
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenBase refuses a plain-HTTP host unless HEYGEN_ALLOW_HTTP is set, as the heygen CLI does", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_API_BASE = "http://127.0.0.1:4100";
|
||||
assert.throws(() => heygenBase(), /HEYGEN_ALLOW_HTTP=1/);
|
||||
process.env.HEYGEN_ALLOW_HTTP = "1";
|
||||
assert.equal(heygenBase(), "http://127.0.0.1:4100/v3");
|
||||
});
|
||||
});
|
||||
|
||||
test("a host gateway (HEYGEN_API_BASE with its own HEYGEN_API_KEY) wins over a host OAuth token", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_API_BASE = "http://127.0.0.1:4100";
|
||||
process.env.HEYGEN_ALLOW_HTTP = "1";
|
||||
process.env.HEYGEN_API_KEY = "gateway-token";
|
||||
process.env.HEYGEN_ACCESS_TOKEN = "at_host";
|
||||
assert.deepEqual(heygenAuthHeaders(), {
|
||||
"X-Api-Key": "gateway-token",
|
||||
"X-HeyGen-Client-Source": "media-use",
|
||||
});
|
||||
assert.equal(heygenAuthMethod(), "api_key");
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenJSON sends its request to the host's HEYGEN_API_BASE with the host's key", async () => {
|
||||
/** @type {{ url?: string, key?: string | string[] }} */
|
||||
const seen = {};
|
||||
const server = createServer((req, res) => {
|
||||
seen.url = req.url;
|
||||
seen.key = req.headers["x-api-key"];
|
||||
res.writeHead(200, { "content-type": "application/json" }).end('{"data":[]}');
|
||||
});
|
||||
await new Promise((done) => server.listen(0, "127.0.0.1", done));
|
||||
const { port } = /** @type {import("node:net").AddressInfo} */ (server.address());
|
||||
try {
|
||||
await withCleanHeygenEnv(async () => {
|
||||
process.env.HEYGEN_API_BASE = `http://127.0.0.1:${port}`;
|
||||
process.env.HEYGEN_ALLOW_HTTP = "1";
|
||||
process.env.HEYGEN_API_KEY = "gateway-token";
|
||||
const reply = await heygenJSON("/voices?limit=1", { headers: heygenAuthHeaders() });
|
||||
assert.deepEqual(reply, { data: [] });
|
||||
});
|
||||
assert.equal(seen.url, "/v3/voices?limit=1");
|
||||
assert.equal(seen.key, "gateway-token");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("a base that isn't HeyGen's gets no stored or host OAuth credential, only a key named for it", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
writeFileSync(join(dir, "credentials"), JSON.stringify({ api_key: "hg_stored" }));
|
||||
process.env.HEYGEN_ACCESS_TOKEN = "at_host";
|
||||
process.env.HEYGEN_API_BASE = "https://proxy.example.com";
|
||||
assert.equal(heygenCredential(), null);
|
||||
assert.throws(() => heygenAuthHeaders(), /no HeyGen credentials/);
|
||||
// HeyGen's own hosts keep every credential source.
|
||||
process.env.HEYGEN_API_BASE = "https://api-canary.heygen.com";
|
||||
assert.equal(heygenAuthMethod(), "oauth");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
// Audio the host app supplied survives every pass that rebuilds audio_meta.json from the storyboard: an entry
|
||||
// marked "source": "host" whose file exists. A frame the host scored takes no looked-up cue sound.
|
||||
/** The host's own entries in an earlier audio_meta.json, and why any other host-marked entry was dropped; `exists`
|
||||
* resolves a path from the project root. A frame written as "1" is frame 1. */
|
||||
export function hostAudio(previous, exists) {
|
||||
const marked = [[previous?.bgm, "path"], ...(previous?.sfx ?? []).map((s) => [s, "file"])].filter(
|
||||
([entry]) => entry?.source === "host",
|
||||
);
|
||||
const kept = new Set();
|
||||
const dropped = [];
|
||||
for (const [entry, key] of marked) {
|
||||
const path = entry[key];
|
||||
if (typeof path !== "string") dropped.push(`a host entry has no "${key}"`);
|
||||
else if (!exists(path)) dropped.push(`${path} is not on disk`);
|
||||
else kept.add(entry);
|
||||
}
|
||||
const bgm = kept.has(previous?.bgm) ? previous.bgm : null;
|
||||
const sfx = (previous?.sfx ?? [])
|
||||
.filter((s) => kept.has(s))
|
||||
.map((s) => ({ ...s, frame: Number(s.frame) }));
|
||||
return { bgm, sfx, frames: new Set(sfx.map((s) => s.frame)), dropped };
|
||||
}
|
||||
|
||||
/** `rebuilt` with the host's bed and sounds put back; the host's bed wins over a looked-up one. */
|
||||
export function keepHostAudio(rebuilt, host) {
|
||||
const looked = (rebuilt.sfx ?? []).filter((s) => !host.frames.has(s.frame));
|
||||
return { ...rebuilt, bgm: host.bgm ?? rebuilt.bgm, sfx: [...host.sfx, ...looked] };
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import test from "node:test";
|
||||
import { hostAudio, keepHostAudio } from "./host-audio.mjs";
|
||||
|
||||
const onDisk = new Set([
|
||||
"assets/bgm/host.mp3",
|
||||
"assets/sfx/custom-whoosh.mp3",
|
||||
"assets/sfx/whoosh.mp3",
|
||||
]);
|
||||
const exists = (path) => onDisk.has(path);
|
||||
const hostWhoosh = {
|
||||
frame: 1,
|
||||
file: "assets/sfx/custom-whoosh.mp3",
|
||||
offset_s: 0,
|
||||
duration_s: 2,
|
||||
source: "host",
|
||||
};
|
||||
|
||||
test("a host sound and bed survive a rebuild, and the host's frame takes no looked-up sound", () => {
|
||||
const previous = {
|
||||
bgm: { path: "assets/bgm/host.mp3", source: "host" },
|
||||
sfx: [hostWhoosh, { frame: 2, file: "assets/sfx/whoosh.mp3", offset_s: 0 }],
|
||||
};
|
||||
const rebuilt = {
|
||||
bgm: null,
|
||||
voices: [],
|
||||
sfx: [
|
||||
{ frame: 1, file: "assets/sfx/whoosh.mp3", offset_s: 0 },
|
||||
{ frame: 2, file: "assets/sfx/whoosh.mp3", offset_s: 0 },
|
||||
],
|
||||
};
|
||||
const kept = keepHostAudio(rebuilt, hostAudio(previous, exists));
|
||||
assert.deepEqual(kept.bgm, previous.bgm);
|
||||
assert.deepEqual(kept.sfx, [hostWhoosh, rebuilt.sfx[1]]);
|
||||
});
|
||||
|
||||
test("an unmarked entry or a missing file is not the host's", () => {
|
||||
const previous = {
|
||||
bgm: { path: "assets/bgm/gone.mp3", source: "host" },
|
||||
sfx: [
|
||||
{ frame: 1, file: "assets/sfx/whoosh.mp3", offset_s: 0 },
|
||||
{ frame: 3, file: "assets/sfx/gone.mp3", source: "host" },
|
||||
],
|
||||
};
|
||||
const host = hostAudio(previous, exists);
|
||||
assert.equal(host.bgm, null);
|
||||
assert.deepEqual(host.sfx, []);
|
||||
const rebuilt = { bgm: { path: "assets/bgm/looked.mp3" }, voices: [], sfx: [] };
|
||||
assert.deepEqual(keepHostAudio(rebuilt, host), rebuilt);
|
||||
});
|
||||
|
||||
test("no earlier audio_meta.json keeps the rebuild as it is", () => {
|
||||
const rebuilt = { bgm: null, voices: [], sfx: [{ frame: 1, file: "assets/sfx/whoosh.mp3" }] };
|
||||
assert.deepEqual(keepHostAudio(rebuilt, hostAudio(null, exists)), rebuilt);
|
||||
});
|
||||
|
||||
// The narrated workflows tell the agent the same host-audio contract and the same meaning of silent.
|
||||
const workflowText = (skill, file) =>
|
||||
readFileSync(new URL(`../../../../${skill}/${file}`, import.meta.url), "utf8");
|
||||
|
||||
test("each narrated workflow tells a host app's music and sounds the way in, and when a film is silent", () => {
|
||||
for (const skill of ["product-launch-video", "pr-to-video", "faceless-explainer"]) {
|
||||
const steps = workflowText(skill, "SKILL.md");
|
||||
assert.match(
|
||||
steps,
|
||||
/host app's own tools make the music or a sound effect[^\n]+"source": "host"/,
|
||||
skill,
|
||||
);
|
||||
assert.match(steps, /"duration_s": <its length>[^\n]+Every audio pass keeps these/, skill);
|
||||
assert.match(steps, /no `SCRIPT.md`, \*\*and\*\* no `sfx:` cues or host audio/, skill);
|
||||
assert.match(
|
||||
steps,
|
||||
/marked silent \(`music: none`, no `SCRIPT.md`, no `sfx:` cues or host audio\)/,
|
||||
skill,
|
||||
);
|
||||
assert.match(
|
||||
workflowText(skill, "references/story-design.md"),
|
||||
/no `SCRIPT.md` \+ no `sfx:` cues or host audio\*\* — the canonical/,
|
||||
skill,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("a frame written as a string is the same frame, and a gone host file is reported", () => {
|
||||
const previous = {
|
||||
bgm: { path: "assets/bgm/gone.mp3", source: "host" },
|
||||
sfx: [{ ...hostWhoosh, frame: "1" }],
|
||||
};
|
||||
const host = hostAudio(previous, exists);
|
||||
assert.deepEqual(host.sfx, [hostWhoosh]);
|
||||
assert.ok(host.frames.has(1));
|
||||
assert.deepEqual(host.dropped, ["assets/bgm/gone.mp3 is not on disk"]);
|
||||
const rebuilt = { bgm: null, voices: [], sfx: [{ frame: 1, file: "assets/sfx/whoosh.mp3" }] };
|
||||
assert.deepEqual(keepHostAudio(rebuilt, host).sfx, [hostWhoosh]);
|
||||
});
|
||||
|
||||
test("a host entry with no path is reported, not thrown", () => {
|
||||
const throwsOnNonString = (path) => {
|
||||
if (typeof path !== "string") throw new TypeError("path must be a string");
|
||||
return onDisk.has(path);
|
||||
};
|
||||
const host = hostAudio({ bgm: { src: "x.mp3", source: "host" }, sfx: [] }, throwsOnNonString);
|
||||
assert.equal(host.bgm, null);
|
||||
assert.deepEqual(host.dropped, ['a host entry has no "path"']);
|
||||
});
|
||||
@@ -0,0 +1,13 @@
|
||||
import { realpathSync } from "node:fs";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
// Node resolves symlinks in the main module's import.meta.url but argv[1] keeps the invoked
|
||||
// spelling (a symlinked skills dir, or /tmp vs /private/tmp on macOS), so compare real paths.
|
||||
export function isMainModule(importMetaUrl) {
|
||||
if (!process.argv[1]) return false;
|
||||
try {
|
||||
return pathToFileURL(realpathSync(process.argv[1])).href === importMetaUrl;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { extname, join } from "node:path";
|
||||
import { AGENT_SOURCES, latestRecordFor, recordInPlace } from "../../../scripts/lib/manifest.mjs";
|
||||
import { regenerateIndex } from "../../../scripts/lib/index-gen.mjs";
|
||||
|
||||
/**
|
||||
* Where the engine may write `rel`: there, unless another file of this run has `taken` it or the file there is
|
||||
* the person's (not `reusable` and not recorded as agent-made). Then the first free `name-2.ext`, with an anomaly.
|
||||
*/
|
||||
export function agentWritePath(
|
||||
hyperframesDir,
|
||||
rel,
|
||||
{ anomalies, taken = new Set(), reusable = () => false },
|
||||
) {
|
||||
const free = (path) =>
|
||||
!taken.has(path) &&
|
||||
(!existsSync(join(hyperframesDir, path)) ||
|
||||
reusable(path) ||
|
||||
AGENT_SOURCES.includes(latestRecordFor(hyperframesDir, path)?.source));
|
||||
if (free(rel)) return rel;
|
||||
const ext = extname(rel);
|
||||
const stem = rel.slice(0, rel.length - ext.length);
|
||||
let n = 2;
|
||||
while (!free(`${stem}-${n}${ext}`)) n++;
|
||||
const path = `${stem}-${n}${ext}`;
|
||||
const why = taken.has(rel)
|
||||
? "another file of this run goes there"
|
||||
: "the file there is yours (the media manifest does not record it as made by the engine)";
|
||||
const note = `${rel}: kept, because ${why}; writing ${path} instead (audio_meta.json has the path used)`;
|
||||
if (!anomalies.includes(note)) anomalies.push(note);
|
||||
return path;
|
||||
}
|
||||
|
||||
/** Each spoken line's file, picked before lines synthesize concurrently so two never land on one free name. */
|
||||
export function voicePaths(hyperframesDir, lines, anomalies) {
|
||||
const taken = new Set();
|
||||
const paths = new Map();
|
||||
for (const line of lines.filter((l) => String(l.text ?? "").trim())) {
|
||||
const rel = agentWritePath(hyperframesDir, `assets/voice/${line.id}.wav`, { anomalies, taken });
|
||||
taken.add(rel);
|
||||
paths.set(String(line.id), rel);
|
||||
}
|
||||
return paths;
|
||||
}
|
||||
|
||||
const SFX_SOURCES = { heygen: "search", local: "bundled" };
|
||||
|
||||
// The files one engine run wrote, each with how it was made, for the project's media manifest.
|
||||
export function writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields, sfx }) {
|
||||
const textById = new Map(lines.map((line) => [String(line.id), String(line.text ?? "").trim()]));
|
||||
const assets = [];
|
||||
if (only.has("tts")) {
|
||||
for (const voice of voices) {
|
||||
assets.push({
|
||||
path: voice.path,
|
||||
type: "voice",
|
||||
source: "generated",
|
||||
intent: textById.get(voice.id),
|
||||
duration: voice.duration_s,
|
||||
provider: ttsProvider,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (only.has("bgm") && bgm && !bgmFields.bgm_pending) {
|
||||
assets.push({
|
||||
path: bgm.path,
|
||||
type: "bgm",
|
||||
source: bgmFields.bgm_mode === "retrieve" ? "search" : "generated",
|
||||
intent: bgm.query,
|
||||
duration: bgm.duration_s,
|
||||
provider: bgmFields.bgm_provider,
|
||||
});
|
||||
}
|
||||
if (only.has("sfx")) {
|
||||
for (const cue of new Map(sfx.map((entry) => [entry.file, entry])).values()) {
|
||||
const source = SFX_SOURCES[cue.source];
|
||||
const provider = source === "search" ? "heygen" : "bundled.sfx";
|
||||
assets.push({
|
||||
path: cue.file,
|
||||
type: "sfx",
|
||||
source,
|
||||
intent: cue.name,
|
||||
duration: cue.duration_s,
|
||||
provider,
|
||||
});
|
||||
}
|
||||
}
|
||||
return assets;
|
||||
}
|
||||
|
||||
/** Records each asset where it lies; returns one anomaly per file left unrecorded. */
|
||||
export function recordInManifest(hyperframesDir, assets) {
|
||||
const anomalies = [];
|
||||
for (const { path, type, source, intent, duration, provider } of assets) {
|
||||
try {
|
||||
recordInPlace(hyperframesDir, {
|
||||
type,
|
||||
path,
|
||||
source,
|
||||
description: intent,
|
||||
duration,
|
||||
provenance: { provider: provider || "local", ...(intent && { prompt: intent }) },
|
||||
});
|
||||
} catch (error) {
|
||||
anomalies.push(`${path}: not recorded in the media manifest (${error.message})`);
|
||||
}
|
||||
}
|
||||
try {
|
||||
if (anomalies.length < assets.length) regenerateIndex(hyperframesDir);
|
||||
} catch (error) {
|
||||
anomalies.push(`.media/index.md: not refreshed (${error.message})`);
|
||||
}
|
||||
return anomalies;
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import { test } from "node:test";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { findByPrompt, readManifest } from "../../../scripts/lib/manifest.mjs";
|
||||
import { agentWritePath, recordInManifest, voicePaths, writtenAssets } from "./media-record.mjs";
|
||||
|
||||
const noBgm = { bgm: null, bgmFields: { bgm_pending: false } };
|
||||
|
||||
function project(t) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-record-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
}
|
||||
|
||||
test("a voice run leaves a manifest entry marked generated", (t) => {
|
||||
const dir = project(t);
|
||||
const assets = writtenAssets({
|
||||
only: new Set(["tts"]),
|
||||
lines: [{ id: "01", text: "Welcome to the launch" }],
|
||||
voices: [{ id: "01", path: "assets/voice/01.wav", duration_s: 2.34 }],
|
||||
ttsProvider: "kokoro",
|
||||
sfx: [],
|
||||
...noBgm,
|
||||
});
|
||||
|
||||
assert.deepEqual(recordInManifest(dir, assets), []);
|
||||
|
||||
assert.deepEqual(
|
||||
readManifest(dir).map(({ path, type, source, description, duration, provenance }) => ({
|
||||
path,
|
||||
type,
|
||||
source,
|
||||
description,
|
||||
duration,
|
||||
provenance,
|
||||
})),
|
||||
[
|
||||
{
|
||||
path: "assets/voice/01.wav",
|
||||
type: "voice",
|
||||
source: "generated",
|
||||
description: "Welcome to the launch",
|
||||
duration: 2.3,
|
||||
provenance: { provider: "kokoro", prompt: "Welcome to the launch" },
|
||||
},
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
const voiceLine = (intent, duration) => [
|
||||
{ path: "assets/voice/01.wav", type: "voice", source: "generated", intent, duration },
|
||||
];
|
||||
|
||||
test("a rerun that writes the same take keeps one record, and a new length is a new take", (t) => {
|
||||
const dir = project(t);
|
||||
|
||||
recordInManifest(dir, voiceLine("Hello world", 1.25));
|
||||
recordInManifest(dir, voiceLine("Hello world", 1.25));
|
||||
assert.equal(readManifest(dir).length, 1);
|
||||
|
||||
recordInManifest(dir, voiceLine("Hello world", 2));
|
||||
assert.deepEqual(
|
||||
readManifest(dir).map(({ duration }) => duration),
|
||||
[1.3, 2],
|
||||
);
|
||||
});
|
||||
|
||||
test("a rerun with new text records the new take, and the old text no longer finds the file", (t) => {
|
||||
const dir = project(t);
|
||||
|
||||
recordInManifest(dir, voiceLine("Hello world", 1.25));
|
||||
recordInManifest(dir, voiceLine("Welcome back to the show", 3.5));
|
||||
|
||||
assert.deepEqual(
|
||||
readManifest(dir).map(({ description, duration }) => [description, duration]),
|
||||
[
|
||||
["Hello world", 1.3],
|
||||
["Welcome back to the show", 3.5],
|
||||
],
|
||||
);
|
||||
assert.equal(findByPrompt(dir, "Hello world", "voice"), null);
|
||||
const index = readFileSync(join(dir, ".media/index.md"), "utf8");
|
||||
assert.match(index, /Welcome back to the show/);
|
||||
assert.doesNotMatch(index, /Hello world/);
|
||||
assert.equal(findByPrompt(dir, "Welcome back to the show", "voice")?.path, "assets/voice/01.wav");
|
||||
});
|
||||
|
||||
test("music and sound effects are marked by where they came from", () => {
|
||||
const assets = writtenAssets({
|
||||
only: new Set(["bgm", "sfx"]),
|
||||
lines: [],
|
||||
voices: [{ id: "01", path: "assets/voice/01.wav" }],
|
||||
bgm: { path: "assets/bgm/track.mp3", query: "calm" },
|
||||
bgmFields: { bgm_pending: false, bgm_mode: "retrieve" },
|
||||
sfx: [
|
||||
{ file: "assets/sfx/whoosh.mp3", name: "whoosh", source: "local" },
|
||||
{ file: "assets/sfx/whoosh.mp3", name: "whoosh", source: "local" },
|
||||
{ file: "assets/sfx/glass.mp3", name: "glass", source: "heygen" },
|
||||
],
|
||||
});
|
||||
|
||||
assert.deepEqual(
|
||||
assets.map(({ path, source, provider }) => [path, source, provider]),
|
||||
[
|
||||
["assets/bgm/track.mp3", "search", undefined],
|
||||
["assets/sfx/whoosh.mp3", "bundled", "bundled.sfx"],
|
||||
["assets/sfx/glass.mp3", "search", "heygen"],
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test("music made locally is marked generated once it is ready, not while pending", () => {
|
||||
const written = (bgm_pending) =>
|
||||
writtenAssets({
|
||||
only: new Set(["bgm"]),
|
||||
lines: [],
|
||||
voices: [],
|
||||
sfx: [],
|
||||
bgm: { path: "assets/bgm/track.wav" },
|
||||
bgmFields: { bgm_pending, bgm_mode: "detached-single" },
|
||||
}).map(({ source }) => source);
|
||||
|
||||
assert.deepEqual(written(true), []);
|
||||
assert.deepEqual(written(false), ["generated"]);
|
||||
});
|
||||
|
||||
test("a file that cannot be recorded becomes an anomaly, not a failure", (t) => {
|
||||
const dir = project(t);
|
||||
writeFileSync(join(dir, ".media"), "a file where the media folder should be");
|
||||
|
||||
const anomalies = recordInManifest(dir, [
|
||||
{ path: "assets/voice/01.wav", type: "voice", source: "generated" },
|
||||
]);
|
||||
|
||||
assert.equal(anomalies.length, 1);
|
||||
assert.match(anomalies[0], /^assets\/voice\/01\.wav: not recorded in the media manifest/);
|
||||
});
|
||||
|
||||
test("the engine writes over only its own files, else the next free name", (t) => {
|
||||
const dir = project(t);
|
||||
mkdirSync(join(dir, "assets/sfx"), { recursive: true });
|
||||
const names = ["mine", "made", "adopted", "twice", "twice-2", "kept", "kept-2"];
|
||||
for (const name of names) writeFileSync(join(dir, `assets/sfx/${name}.mp3`), name);
|
||||
for (const path of ["assets/sfx/made.mp3", "assets/sfx/kept-2.mp3"])
|
||||
recordInManifest(dir, [{ path, type: "sfx", source: "search" }]);
|
||||
recordInManifest(dir, [{ path: "assets/sfx/adopted.mp3", type: "sfx", source: "existing" }]);
|
||||
const anomalies = [];
|
||||
const at = (name, reusable) =>
|
||||
agentWritePath(dir, `assets/sfx/${name}.mp3`, { anomalies, reusable });
|
||||
|
||||
assert.equal(at("new"), "assets/sfx/new.mp3");
|
||||
assert.equal(at("made"), "assets/sfx/made.mp3");
|
||||
assert.equal(
|
||||
at("mine", (path) => path === "assets/sfx/mine.mp3"),
|
||||
"assets/sfx/mine.mp3",
|
||||
);
|
||||
assert.deepEqual(anomalies, []);
|
||||
assert.equal(at("mine"), "assets/sfx/mine-2.mp3");
|
||||
assert.equal(at("adopted"), "assets/sfx/adopted-2.mp3");
|
||||
assert.equal(at("twice"), "assets/sfx/twice-3.mp3");
|
||||
assert.equal(at("kept"), "assets/sfx/kept-2.mp3");
|
||||
assert.equal(anomalies.length, 4);
|
||||
assert.match(
|
||||
anomalies[0],
|
||||
/^assets\/sfx\/mine\.mp3: kept, .* writing assets\/sfx\/mine-2\.mp3 instead/,
|
||||
);
|
||||
});
|
||||
|
||||
test("two spoken lines never share a file when one's name is taken by the person", (t) => {
|
||||
const dir = project(t);
|
||||
mkdirSync(join(dir, "assets/voice"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets/voice/hook.wav"), "the person's own hook");
|
||||
const anomalies = [];
|
||||
|
||||
const paths = voicePaths(
|
||||
dir,
|
||||
[
|
||||
{ id: "hook", text: "First" },
|
||||
{ id: "hook-2", text: "Second" },
|
||||
{ id: "blank", text: " " },
|
||||
],
|
||||
anomalies,
|
||||
);
|
||||
|
||||
assert.deepEqual(Object.fromEntries(paths), {
|
||||
hook: "assets/voice/hook-2.wav",
|
||||
"hook-2": "assets/voice/hook-2-2.wav",
|
||||
});
|
||||
assert.equal(anomalies.length, 2);
|
||||
});
|
||||
|
||||
test("a record never takes the id of a download still in flight", (t) => {
|
||||
const dir = project(t);
|
||||
mkdirSync(join(dir, ".media/audio/bgm"), { recursive: true });
|
||||
writeFileSync(join(dir, ".media/audio/bgm/bgm_001.mp3"), "");
|
||||
|
||||
recordInManifest(dir, [{ path: "assets/bgm/track.wav", type: "bgm", source: "generated" }]);
|
||||
|
||||
assert.deepEqual(
|
||||
readManifest(dir).map(({ id }) => id),
|
||||
["bgm_002"],
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
// python.mjs — resolve which Python 3 executable to spawn, per platform.
|
||||
//
|
||||
// The audio engine (tts.mjs, bgm.mjs) shells out to `python3` for ElevenLabs
|
||||
// TTS and the local Lyria/MusicGen BGM paths. `python3` is the right name on
|
||||
// macOS/Linux, but on Windows the python.org installer only creates
|
||||
// `python.exe` plus the `py` launcher — there is no `python3.exe` (only the
|
||||
// Microsoft Store build adds one). So a bare `spawn("python3", …)` ENOENTs on a
|
||||
// standard Windows Python install, silently disabling every Python-backed audio
|
||||
// feature until the user hand-creates a `python3.exe` shim (reported twice).
|
||||
//
|
||||
// Resolve once, per process: probe the platform's candidates in order and take
|
||||
// the first that actually runs. `py` is the launcher, so it needs a `-3` arg to
|
||||
// select Python 3 — hence candidates are argv PREFIXES, not bare names.
|
||||
//
|
||||
// `HYPERFRAMES_PYTHON` (the documented PEP 668 venv setup) wins over the PATH probe, matching
|
||||
// the CLI's `findPython()`; ignoring it made BGM silently disable itself (#4614).
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
// Accept only an interpreter whose `--version` reports Python 3, as the CLI's
|
||||
// `validatePythonOverride()` does. Exit status alone would take a Python 2
|
||||
// `python` (or any executable that exits 0) that `doctor` rejects. Python 2
|
||||
// prints its version to stderr, so read both streams.
|
||||
export function defaultProbe(cmd, args) {
|
||||
try {
|
||||
const result = spawnSync(cmd, args, { encoding: "utf-8", timeout: 5000 });
|
||||
return result.status === 0 && /Python 3/.test(`${result.stdout}${result.stderr}`);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the argv prefix that launches Python 3 on this platform.
|
||||
* Returns e.g. `["python3"]`, `["python"]`, or `["py", "-3"]`.
|
||||
*
|
||||
* Pure except for `probe` (which runs `<cmd> … --version`); `platform`, `probe`
|
||||
* and `env` are injectable so every branch is unit-testable without spawning.
|
||||
* If nothing probes OK, falls back to the canonical name for the platform so
|
||||
* the eventual spawn fails loudly exactly as it did before — never worse.
|
||||
*/
|
||||
export function resolvePythonCommand(
|
||||
platform = process.platform,
|
||||
probe = defaultProbe,
|
||||
env = process.env,
|
||||
) {
|
||||
const override = env.HYPERFRAMES_PYTHON;
|
||||
if (override && probe(override, ["--version"])) return [override];
|
||||
const candidates =
|
||||
platform === "win32" ? [["python3"], ["python"], ["py", "-3"]] : [["python3"], ["python"]];
|
||||
for (const prefix of candidates) {
|
||||
if (probe(prefix[0], [...prefix.slice(1), "--version"])) return prefix;
|
||||
}
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
let cached = null;
|
||||
|
||||
/** Cached `resolvePythonCommand()` — probing spawns, so resolve at most once. */
|
||||
export function pythonCommand() {
|
||||
if (!cached) cached = resolvePythonCommand();
|
||||
return cached;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a `{ cmd, args }` for running Python 3 with `extraArgs`, using the
|
||||
* resolved (or supplied) prefix. Keeps the launcher's `-3` (and any future
|
||||
* prefix args) ahead of the caller's own arguments.
|
||||
*/
|
||||
export function pythonInvocation(extraArgs, prefix = pythonCommand()) {
|
||||
return { cmd: prefix[0], args: [...prefix.slice(1), ...extraArgs] };
|
||||
}
|
||||
|
||||
/** Test-only: clear the cached resolution so a test can re-probe. */
|
||||
export function _resetPythonCommandCacheForTests() {
|
||||
cached = null;
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { defaultProbe, resolvePythonCommand, pythonInvocation } from "./python.mjs";
|
||||
|
||||
// Regression: on Windows a standard python.org install has no `python3.exe`
|
||||
// (only `python.exe` + the `py` launcher), so `spawn("python3", …)` ENOENTs and
|
||||
// every Python-backed audio feature silently no-ops. resolvePythonCommand takes
|
||||
// injectable platform/probe params so all branches are testable without
|
||||
// spawning a real interpreter.
|
||||
|
||||
// probeFor(names): a probe that reports success only for the given argv-0 names.
|
||||
function probeFor(...names) {
|
||||
const ok = new Set(names);
|
||||
return (cmd) => ok.has(cmd);
|
||||
}
|
||||
|
||||
test("non-win32 uses python3 when it runs", () => {
|
||||
assert.deepEqual(resolvePythonCommand("linux", probeFor("python3")), ["python3"]);
|
||||
assert.deepEqual(resolvePythonCommand("darwin", probeFor("python3")), ["python3"]);
|
||||
});
|
||||
|
||||
test("win32 prefers python3 when the Microsoft Store build provides it", () => {
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor("python3", "python", "py")), ["python3"]);
|
||||
});
|
||||
|
||||
test("win32 falls back to python.exe when python3 is absent (python.org install)", () => {
|
||||
// The exact reported scenario: no python3, but `python` exists.
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor("python", "py")), ["python"]);
|
||||
});
|
||||
|
||||
test("win32 falls back to the py launcher with -3 when only py exists", () => {
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor("py")), ["py", "-3"]);
|
||||
});
|
||||
|
||||
test("py launcher is probed as `py -3 --version`, not bare `py`", () => {
|
||||
const seen = [];
|
||||
const probe = (cmd, args) => {
|
||||
seen.push([cmd, ...args]);
|
||||
return cmd === "py";
|
||||
};
|
||||
resolvePythonCommand("win32", probe);
|
||||
assert.deepEqual(seen.at(-1), ["py", "-3", "--version"]);
|
||||
});
|
||||
|
||||
test("falls back to the canonical name (loud failure, unchanged) when nothing runs", () => {
|
||||
// No interpreter anywhere — must not throw, and must return python3 so the
|
||||
// eventual spawn fails exactly as it did before this fix, never worse.
|
||||
assert.deepEqual(
|
||||
resolvePythonCommand("win32", () => false),
|
||||
["python3"],
|
||||
);
|
||||
assert.deepEqual(
|
||||
resolvePythonCommand("linux", () => false),
|
||||
["python3"],
|
||||
);
|
||||
});
|
||||
|
||||
// Regression (#4614): on PEP 668 systems the documented setup is a venv named
|
||||
// by HYPERFRAMES_PYTHON; the audio engine ignored it and probed the bare
|
||||
// system python3, so MusicGen BGM was skipped while `doctor` said installed.
|
||||
const VENV = "/home/u/.venvs/hf/bin/python";
|
||||
|
||||
test("HYPERFRAMES_PYTHON wins over python3 on PATH when it runs", () => {
|
||||
const env = { HYPERFRAMES_PYTHON: VENV };
|
||||
assert.deepEqual(resolvePythonCommand("darwin", probeFor(VENV, "python3"), env), [VENV]);
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor(VENV, "python", "py"), env), [VENV]);
|
||||
});
|
||||
|
||||
test("a HYPERFRAMES_PYTHON that doesn't run falls through to the PATH probe", () => {
|
||||
const env = { HYPERFRAMES_PYTHON: "/nope/python" };
|
||||
assert.deepEqual(resolvePythonCommand("linux", probeFor("python3"), env), ["python3"]);
|
||||
});
|
||||
|
||||
test("an empty HYPERFRAMES_PYTHON is ignored without being probed", () => {
|
||||
const seen = [];
|
||||
const probe = (cmd) => {
|
||||
seen.push(cmd);
|
||||
return cmd === "python3";
|
||||
};
|
||||
assert.deepEqual(resolvePythonCommand("linux", probe, { HYPERFRAMES_PYTHON: "" }), ["python3"]);
|
||||
assert.deepEqual(seen, ["python3"]);
|
||||
});
|
||||
|
||||
// defaultProbe must match the CLI's validatePythonOverride(): exiting 0 is not
|
||||
// enough, `--version` has to report Python 3. Stubs are shell scripts, so
|
||||
// these run on POSIX only.
|
||||
function withStub(script, fn) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-python-probe-"));
|
||||
try {
|
||||
const stub = join(dir, "python");
|
||||
writeFileSync(stub, `#!/bin/sh\n${script}\n`);
|
||||
chmodSync(stub, 0o755);
|
||||
fn(stub);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
const posixOnly = { skip: process.platform === "win32" && "stub scripts need a POSIX shell" };
|
||||
|
||||
test("defaultProbe accepts an interpreter reporting Python 3", posixOnly, () => {
|
||||
withStub('echo "Python 3.12.4"', (stub) => assert.equal(defaultProbe(stub, ["--version"]), true));
|
||||
});
|
||||
|
||||
test("defaultProbe rejects Python 2, which prints its version to stderr", posixOnly, () => {
|
||||
withStub('echo "Python 2.7.18" >&2', (stub) =>
|
||||
assert.equal(defaultProbe(stub, ["--version"]), false),
|
||||
);
|
||||
});
|
||||
|
||||
test(
|
||||
"defaultProbe rejects an executable that exits 0 without a Python 3 version",
|
||||
posixOnly,
|
||||
() => {
|
||||
withStub("exit 0", (stub) => assert.equal(defaultProbe(stub, ["--version"]), false));
|
||||
},
|
||||
);
|
||||
|
||||
test("defaultProbe rejects a command that doesn't exist", () => {
|
||||
assert.equal(defaultProbe("/nonexistent/hf-python", ["--version"]), false);
|
||||
});
|
||||
|
||||
test("pythonInvocation prepends the resolved prefix ahead of caller args", () => {
|
||||
assert.deepEqual(pythonInvocation(["-c", "import x"], ["python"]), {
|
||||
cmd: "python",
|
||||
args: ["-c", "import x"],
|
||||
});
|
||||
// The py launcher's -3 must stay ahead of the caller's own arguments.
|
||||
assert.deepEqual(pythonInvocation(["-c", "import x"], ["py", "-3"]), {
|
||||
cmd: "py",
|
||||
args: ["-3", "-c", "import x"],
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,165 @@
|
||||
// sfx.mjs — sound effects for the media audio engine. Provider-gated (NOT a
|
||||
// per-cue merge): the decision is made once, by whether HeyGen is configured —
|
||||
// mirroring how TTS and BGM degrade.
|
||||
//
|
||||
// HeyGen credential present → retrieve EVERY cue from HeyGen's audio library
|
||||
// (/v3/audio/sounds, type=sound_effects, min_score=0.4). The bundled
|
||||
// library is NOT consulted.
|
||||
// HeyGen credential absent → resolve cues against the bundled 21-file
|
||||
// library (assets/sfx/manifest.json), copying matched files into the
|
||||
// project. Offline, deterministic, free.
|
||||
//
|
||||
// A cue that matches nothing is skipped (recorded as an anomaly); SFX never
|
||||
// blocks a render. Every cue sits at volume ~0.35, under voice + BGM.
|
||||
|
||||
import { copyFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { downloadTo, searchSounds } from "./heygen.mjs";
|
||||
import { latestRecordFor } from "../../../scripts/lib/manifest.mjs";
|
||||
import { agentWritePath } from "./media-record.mjs";
|
||||
|
||||
const SFX_VOLUME = 0.35;
|
||||
const slug = (s) =>
|
||||
s
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.slice(0, 40) || "x";
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
|
||||
// cues: [{ id, name }] (id = the line/frame/scene the cue fires in). Returns
|
||||
// { sfx: [{ id, name, file, source, offset_s, duration_s, volume }], anomalies }.
|
||||
export async function resolveSfx({ cues, heygenOK, headers, hyperframesDir, sfxLibDir }) {
|
||||
const sfx = [];
|
||||
const anomalies = [];
|
||||
const destDir = join(hyperframesDir, "assets", "sfx");
|
||||
// Each effect's file this run: one effect named twice gets one file, and two effects never share one.
|
||||
const fileFor = new Map();
|
||||
|
||||
// Dedupe identical (id,name) cues — the same effect named twice in one line
|
||||
// downloads/copies once.
|
||||
const seen = new Set();
|
||||
const uniq = cues.filter((c) => {
|
||||
const k = `${c.id}:${c.name}`;
|
||||
if (seen.has(k)) return false;
|
||||
seen.add(k);
|
||||
return true;
|
||||
});
|
||||
|
||||
if (heygenOK) {
|
||||
for (const { id, name } of uniq) {
|
||||
try {
|
||||
// SFX hits score low (~0.5–0.67), below the API's default 0.7 which
|
||||
// silently drops most named cues — floor to 0.4. (BGM/music score high
|
||||
// and keep the default.)
|
||||
const results = await searchSounds(name, "sound_effects", headers, {
|
||||
limit: 3,
|
||||
minScore: 0.4,
|
||||
});
|
||||
if (!results.length) {
|
||||
anomalies.push(`sfx "${name}" (id ${id}): no HeyGen match — skipped`);
|
||||
continue;
|
||||
}
|
||||
const top = results[0];
|
||||
const file =
|
||||
fileFor.get(slug(name)) ??
|
||||
agentWritePath(hyperframesDir, `assets/sfx/${slug(name)}.mp3`, {
|
||||
anomalies,
|
||||
taken: new Set(fileFor.values()),
|
||||
});
|
||||
await downloadTo(top.audio_url, join(hyperframesDir, file));
|
||||
fileFor.set(slug(name), file);
|
||||
sfx.push({
|
||||
id,
|
||||
name,
|
||||
file,
|
||||
source: "heygen",
|
||||
offset_s: 0,
|
||||
duration_s: typeof top.duration === "number" ? r3(top.duration) : 1.0,
|
||||
volume: SFX_VOLUME,
|
||||
});
|
||||
} catch (e) {
|
||||
anomalies.push(`sfx "${name}" (id ${id}): retrieval failed — ${e.message}`);
|
||||
}
|
||||
}
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
|
||||
// ── offline: bundled library ──
|
||||
const manifestPath = join(sfxLibDir, "manifest.json");
|
||||
if (!existsSync(manifestPath)) {
|
||||
if (uniq.length)
|
||||
anomalies.push(`no HeyGen credential and no SFX library at ${sfxLibDir} — all cues dropped`);
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
let manifest;
|
||||
try {
|
||||
manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
|
||||
} catch (e) {
|
||||
anomalies.push(`SFX manifest parse failed (${e.message}) — all cues dropped`);
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
// Build lookups: by manifest key, by file basename, and by slug of either, so
|
||||
// a cue can name "whoosh", "whoosh.mp3", or "ui click" (→ slug match).
|
||||
const byKey = new Map();
|
||||
for (const [key, entry] of Object.entries(manifest)) {
|
||||
if (!entry?.file || !isFinite(entry.duration)) continue;
|
||||
const rec = { key, file: entry.file, duration: entry.duration };
|
||||
byKey.set(key, rec);
|
||||
byKey.set(entry.file, rec);
|
||||
byKey.set(slug(key), rec);
|
||||
byKey.set(slug(entry.file.replace(/\.\w+$/, "")), rec);
|
||||
}
|
||||
mkdirSync(destDir, { recursive: true });
|
||||
for (const { id, name } of uniq) {
|
||||
const hit = byKey.get(name) ?? byKey.get(slug(name));
|
||||
if (!hit) {
|
||||
const known = [...new Set([...byKey.values()].map((v) => v.key))].slice(0, 8).join(", ");
|
||||
anomalies.push(
|
||||
`sfx "${name}" (id ${id}): not in bundled library — skipped (have: ${known}…)`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const src = join(sfxLibDir, hit.file);
|
||||
const library = existsSync(src) ? readFileSync(src) : null;
|
||||
const isLibraryCopy = (rel) =>
|
||||
library &&
|
||||
existsSync(join(hyperframesDir, rel)) &&
|
||||
readFileSync(join(hyperframesDir, rel)).equals(library);
|
||||
const destRel =
|
||||
fileFor.get(hit.file) ??
|
||||
agentWritePath(hyperframesDir, `assets/sfx/${hit.file}`, {
|
||||
anomalies,
|
||||
taken: new Set(fileFor.values()),
|
||||
// An unrecorded copy of the library file is one an engine run made before the manifest had it.
|
||||
reusable: (rel) => !latestRecordFor(hyperframesDir, rel) && isLibraryCopy(rel),
|
||||
});
|
||||
const dest = join(hyperframesDir, destRel);
|
||||
// The bundled library may be incomplete: some installs of the skill ship
|
||||
// manifest.json without the actual mp3s. Pushing an sfx entry that points at
|
||||
// a file we never copied produces a dangling reference that silently drops
|
||||
// downstream ("not on disk"). Surface it as a loud anomaly and skip the cue
|
||||
// instead, so the audio_meta never references a missing file.
|
||||
if (library) {
|
||||
if (!isLibraryCopy(destRel)) copyFileSync(src, dest);
|
||||
} else if (!existsSync(dest)) {
|
||||
anomalies.push(
|
||||
`sfx "${name}" (id ${id}): bundled file ${hit.file} missing from the offline ` +
|
||||
`library (${sfxLibDir}) — skipped. Reinstall the media-use skill to ` +
|
||||
`restore assets/sfx/*.mp3, or configure a HeyGen credential for retrieval.`,
|
||||
);
|
||||
continue;
|
||||
} // else the engine's earlier copy at dest stands in for the file this install lacks
|
||||
fileFor.set(hit.file, destRel);
|
||||
sfx.push({
|
||||
id,
|
||||
name,
|
||||
file: destRel,
|
||||
source: "local",
|
||||
offset_s: 0,
|
||||
duration_s: r3(hit.duration),
|
||||
volume: SFX_VOLUME,
|
||||
});
|
||||
}
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, existsSync, readFileSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { appendRecord } from "../../../scripts/lib/manifest.mjs";
|
||||
import { resolveSfx } from "./sfx.mjs";
|
||||
|
||||
// Offline (no HeyGen) SFX resolution: the bundled library may ship manifest.json
|
||||
// without the actual mp3s. The old code copied only when the source existed but
|
||||
// pushed the sfx entry unconditionally — producing a dangling reference that
|
||||
// silently dropped downstream ("not on disk"). These tests lock in the loud
|
||||
// behavior: a present file is copied + referenced; a missing file yields an
|
||||
// anomaly and NO dangling entry.
|
||||
|
||||
async function withDirs(fn) {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-sfx-"));
|
||||
const libDir = join(root, "lib");
|
||||
const projDir = join(root, "proj");
|
||||
mkdirSync(libDir, { recursive: true });
|
||||
mkdirSync(projDir, { recursive: true });
|
||||
try {
|
||||
// `await` is load-bearing: without it the finally cleanup runs before the
|
||||
// async test body resolves, deleting the temp dir mid-assertion.
|
||||
return await fn({ libDir, projDir });
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test("offline: copies and references a present bundled file", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
writeFileSync(
|
||||
join(libDir, "manifest.json"),
|
||||
JSON.stringify({ whoosh: { file: "whoosh.mp3", duration: 0.8 } }),
|
||||
);
|
||||
writeFileSync(join(libDir, "whoosh.mp3"), "ID3-fake-bytes");
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
assert.equal(sfx.length, 1);
|
||||
assert.equal(sfx[0].file, "assets/sfx/whoosh.mp3");
|
||||
assert.equal(sfx[0].source, "local");
|
||||
assert.ok(existsSync(join(projDir, "assets/sfx/whoosh.mp3")), "mp3 copied into project");
|
||||
assert.equal(anomalies.length, 0);
|
||||
});
|
||||
});
|
||||
|
||||
test("offline: a matched-but-missing bundled file yields an anomaly and NO dangling entry", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
// Manifest names whoosh.mp3, but the mp3 was never shipped (the reported bug).
|
||||
writeFileSync(
|
||||
join(libDir, "manifest.json"),
|
||||
JSON.stringify({ whoosh: { file: "whoosh.mp3", duration: 0.8 } }),
|
||||
);
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
assert.equal(sfx.length, 0, "no dangling entry for a file that was never copied");
|
||||
assert.equal(anomalies.length, 1);
|
||||
assert.match(anomalies[0], /missing from the offline library/);
|
||||
assert.ok(!existsSync(join(projDir, "assets/sfx/whoosh.mp3")), "nothing copied");
|
||||
});
|
||||
});
|
||||
|
||||
const whooshLibrary = (libDir, withFile) => {
|
||||
writeFileSync(
|
||||
join(libDir, "manifest.json"),
|
||||
JSON.stringify({ whoosh: { file: "whoosh.mp3", duration: 0.8 } }),
|
||||
);
|
||||
if (withFile) writeFileSync(join(libDir, "whoosh.mp3"), "ID3-library-bytes");
|
||||
};
|
||||
const record = (projDir, source) =>
|
||||
appendRecord(projDir, { id: "sfx_001", type: "sfx", path: "assets/sfx/whoosh.mp3", source });
|
||||
const offline = (libDir, projDir) =>
|
||||
resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
heygenOK: false,
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
|
||||
test("offline: the engine's earlier copy stands in for a library file this install lacks", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
whooshLibrary(libDir, false);
|
||||
mkdirSync(join(projDir, "assets/sfx"), { recursive: true });
|
||||
writeFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "engine copy");
|
||||
record(projDir, "bundled");
|
||||
|
||||
const { sfx, anomalies } = await offline(libDir, projDir);
|
||||
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file }) => file),
|
||||
["assets/sfx/whoosh.mp3"],
|
||||
);
|
||||
assert.deepEqual(anomalies, []);
|
||||
});
|
||||
});
|
||||
|
||||
test("offline: a person's file with the library's bytes stays theirs once recorded as their own", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
whooshLibrary(libDir, true);
|
||||
mkdirSync(join(projDir, "assets/sfx"), { recursive: true });
|
||||
writeFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "ID3-library-bytes");
|
||||
record(projDir, "existing");
|
||||
|
||||
const { sfx } = await offline(libDir, projDir);
|
||||
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file }) => file),
|
||||
["assets/sfx/whoosh-2.mp3"],
|
||||
);
|
||||
assert.equal(readFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "utf8"), "ID3-library-bytes");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,175 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const ttsUrl = new URL("./tts.mjs", import.meta.url).href;
|
||||
const stub = `#!${process.execPath}
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const args = process.argv.slice(2);
|
||||
if (args[0] === "--version") {
|
||||
console.log("Python 3.12.0");
|
||||
process.exit(0);
|
||||
}
|
||||
const input = args[2];
|
||||
const output = args[0] === "-c" ? args[4] : args[args.indexOf("--output") + 1];
|
||||
const text = fs.readFileSync(input, "utf8");
|
||||
fs.writeFileSync(output + ".started", input);
|
||||
const finish = () => {
|
||||
if (text === "slow" && !fs.existsSync(process.env.RELEASE_FILE)) {
|
||||
setTimeout(finish, 5);
|
||||
return;
|
||||
}
|
||||
const consumed = fs.readFileSync(input, "utf8");
|
||||
fs.writeFileSync(output + ".json", JSON.stringify({ input, text, consumed, args }));
|
||||
if (process.env.CASE_MODE === "nonzero") process.exit(7);
|
||||
if (process.env.CASE_MODE !== "no-output") fs.writeFileSync(output, consumed);
|
||||
};
|
||||
setTimeout(finish, 10);
|
||||
`;
|
||||
|
||||
const exercise = `
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import childProcess from "node:child_process";
|
||||
import { syncBuiltinESMExports } from "node:module";
|
||||
import { join, dirname } from "node:path";
|
||||
const { CASE_ROOT: root, CASE_MODE: mode, CASE_PROVIDER: provider } = process.env;
|
||||
const temp = join(root, "tmp");
|
||||
const preexisting = join(temp, "hf-txt-preexisting");
|
||||
fs.mkdirSync(preexisting);
|
||||
fs.writeFileSync(join(preexisting, "marker"), "keep");
|
||||
const injected = Object.assign(new Error("synthetic " + mode), { code: "EACCES" });
|
||||
const originalMkdtemp = fs.mkdtempSync;
|
||||
const originalWrite = fs.writeFileSync;
|
||||
const originalRm = fs.rmSync;
|
||||
if (mode === "allocation") fs.mkdtempSync = (prefix, ...args) => {
|
||||
if (prefix.startsWith(join(temp, "hf-txt-"))) throw injected;
|
||||
return originalMkdtemp(prefix, ...args);
|
||||
};
|
||||
if (mode === "write") fs.writeFileSync = (file, ...args) => {
|
||||
if (String(file).endsWith("line.txt")) {
|
||||
originalWrite(file, "partial");
|
||||
throw injected;
|
||||
}
|
||||
return originalWrite(file, ...args);
|
||||
};
|
||||
if (mode === "spawn-throw") childProcess.spawn = () => { throw injected; };
|
||||
if (mode === "cleanup") fs.rmSync = (dir, ...args) => {
|
||||
if (String(dir).startsWith(join(temp, "hf-txt-"))) throw injected;
|
||||
return originalRm(dir, ...args);
|
||||
};
|
||||
syncBuiltinESMExports();
|
||||
const { synthesizeOne } = await import(${JSON.stringify(ttsUrl)});
|
||||
const project = join(root, "project");
|
||||
const wavAbs = join(project, "line.wav");
|
||||
const request = (chosen = provider, text = "synthetic café", output = wavAbs) =>
|
||||
synthesizeOne({ provider: chosen, text, voiceId: "synthetic-voice", lang: "es",
|
||||
speed: 0.8, wavAbs: output, hyperframesDir: project });
|
||||
const owned = () => fs.readdirSync(temp).filter(name => name !== "hf-txt-preexisting");
|
||||
if (["allocation", "write", "spawn-throw"].includes(mode)) {
|
||||
await assert.rejects(request(), error => error === injected);
|
||||
assert.equal(fs.existsSync(wavAbs), false);
|
||||
} else if (mode === "concurrent") {
|
||||
const slowWav = join(project, "slow.wav");
|
||||
const slow = request("elevenlabs", "slow", slowWav);
|
||||
try {
|
||||
const deadline = Date.now() + 5000;
|
||||
while (!fs.existsSync(slowWav + ".started")) {
|
||||
assert.ok(Date.now() < deadline, "slow subprocess must start");
|
||||
await new Promise(resolve => setTimeout(resolve, 5));
|
||||
}
|
||||
const slowInput = fs.readFileSync(slowWav + ".started", "utf8");
|
||||
assert.deepEqual(await request("kokoro"), { ok: true, words: null });
|
||||
assert.deepEqual(owned(), [dirname(slowInput).split(/[\\\\/]/).pop()]);
|
||||
assert.equal(fs.readFileSync(slowInput, "utf8"), "slow");
|
||||
} finally {
|
||||
fs.writeFileSync(process.env.RELEASE_FILE, "release");
|
||||
await slow;
|
||||
}
|
||||
assert.equal(fs.readFileSync(slowWav, "utf8"), "slow");
|
||||
} else {
|
||||
const result = await request();
|
||||
const label = provider === "elevenlabs" ? "elevenlabs (python)" : "kokoro (npx hyperframes tts)";
|
||||
if (["nonzero", "enoent", "no-output"].includes(mode)) {
|
||||
const error = mode === "no-output" ? label + " produced no wav file" :
|
||||
label + " exited with status " + (mode === "enoent" ? -1 : 7);
|
||||
assert.deepEqual(result, { ok: false, words: null, error });
|
||||
assert.equal(fs.existsSync(wavAbs), false);
|
||||
} else {
|
||||
assert.deepEqual(result, { ok: true, words: null });
|
||||
assert.equal(fs.readFileSync(wavAbs, "utf8"), "synthetic café");
|
||||
}
|
||||
if (mode !== "enoent") {
|
||||
const log = JSON.parse(fs.readFileSync(wavAbs + ".json", "utf8"));
|
||||
assert.equal(log.text, "synthetic café");
|
||||
assert.equal(log.consumed, log.text);
|
||||
if (provider === "kokoro") assert.deepEqual(log.args.slice(-4), ["--lang", "es", "--speed", "0.8"]);
|
||||
}
|
||||
}
|
||||
if (mode === "cleanup") assert.equal(owned().length, 1);
|
||||
else assert.deepEqual(owned(), [], "owned text directories must be removed");
|
||||
assert.equal(fs.readFileSync(join(preexisting, "marker"), "utf8"), "keep");
|
||||
`;
|
||||
|
||||
function runCase(provider, mode) {
|
||||
const root = mkdtempSync(join(tmpdir(), "tts-cleanup-test-"));
|
||||
try {
|
||||
for (const dir of ["home", "tmp", "bin", "project"]) mkdirSync(join(root, dir));
|
||||
if (mode !== "enoent") {
|
||||
for (const name of ["npx", "python3"]) {
|
||||
const executable = join(root, "bin", name);
|
||||
writeFileSync(executable, stub);
|
||||
chmodSync(executable, 0o755);
|
||||
}
|
||||
}
|
||||
const result = spawnSync(process.execPath, ["--input-type=module", "-e", exercise], {
|
||||
encoding: "utf8",
|
||||
timeout: 15000,
|
||||
env: {
|
||||
HOME: join(root, "home"),
|
||||
USERPROFILE: join(root, "home"),
|
||||
TMPDIR: join(root, "tmp"),
|
||||
TMP: join(root, "tmp"),
|
||||
TEMP: join(root, "tmp"),
|
||||
PATH: join(root, "bin"),
|
||||
CASE_ROOT: root,
|
||||
CASE_PROVIDER: provider,
|
||||
CASE_MODE: mode,
|
||||
RELEASE_FILE: join(root, "release"),
|
||||
},
|
||||
});
|
||||
assert.equal(result.error, undefined);
|
||||
assert.equal(result.status, 0, result.stdout + result.stderr);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
for (const provider of ["elevenlabs", "kokoro"]) {
|
||||
for (const mode of [
|
||||
"success",
|
||||
"nonzero",
|
||||
"no-output",
|
||||
"enoent",
|
||||
"spawn-throw",
|
||||
"allocation",
|
||||
"write",
|
||||
"cleanup",
|
||||
]) {
|
||||
test(
|
||||
`synthesizeOne(${provider}) preserves ${mode} behavior and cleans only its text directory`,
|
||||
{ skip: process.platform === "win32" },
|
||||
() => runCase(provider, mode),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
test(
|
||||
"concurrent providers retain each other's text until their subprocess finishes",
|
||||
{ skip: process.platform === "win32" },
|
||||
() => runCase("kokoro", "concurrent"),
|
||||
);
|
||||
@@ -0,0 +1,394 @@
|
||||
import { fetchMedia } from "../../../scripts/lib/media-fetch.mjs";
|
||||
// tts.mjs — multi-provider TTS for the media audio engine. The provider chain,
|
||||
// auto-detected from env, is the one documented in ../SKILL.md:
|
||||
//
|
||||
// 1. HeyGen (Starfish) — $HEYGEN_API_KEY / $HYPERFRAMES_API_KEY / ~/.heygen.
|
||||
// Direct v3 REST (NOT `hyperframes tts`, which in the published build is
|
||||
// Kokoro-only and silently ignores a HeyGen key). Returns word_timestamps
|
||||
// in the same call, so no separate transcribe pass.
|
||||
// 2. ElevenLabs — $ELEVENLABS_API_KEY + `pip install elevenlabs`. No
|
||||
// word timings → caller chains transcribeWav().
|
||||
// 3. Kokoro-82M (local) — always available, via the published `hyperframes tts`
|
||||
// CLI. No word timings → caller chains transcribeWav().
|
||||
//
|
||||
// "HeyGen available" is decided by CREDENTIAL presence (heygenCredential), never
|
||||
// by the CLI — see the note above.
|
||||
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { heygenAuthHeaders, heygenCredential, heygenJSON, heygenMessage } from "./heygen.mjs";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
import { synthesizeGemini } from "./gemini-tts.mjs";
|
||||
import { geminiConfigured } from "./gemini-auth.mjs";
|
||||
|
||||
// ── provider detection ────────────────────────────────────────────────────────
|
||||
export function heygenAvailable() {
|
||||
return heygenCredential() !== null;
|
||||
}
|
||||
export function elevenlabsAvailable() {
|
||||
if (!process.env.ELEVENLABS_API_KEY) return false;
|
||||
const { cmd, args } = pythonInvocation(["-c", "import elevenlabs"]);
|
||||
const r = spawnSync(cmd, args, {
|
||||
stdio: "ignore",
|
||||
});
|
||||
return r.status === 0;
|
||||
}
|
||||
|
||||
// First available provider wins; an explicit choice is honored (and validated).
|
||||
export function pickProvider(userProvider) {
|
||||
if (userProvider) {
|
||||
if (!["heygen", "elevenlabs", "kokoro", "gemini"].includes(userProvider))
|
||||
throw new Error(`invalid provider "${userProvider}" (heygen | elevenlabs | kokoro | gemini)`);
|
||||
if (userProvider === "gemini" && !geminiConfigured())
|
||||
throw new Error(
|
||||
"provider=gemini needs GEMINI_API_KEY or GOOGLE_API_KEY, or service-account credentials (GOOGLE_APPLICATION_CREDENTIALS or GCS_CREDS)",
|
||||
);
|
||||
// heygenAuthHeaders owns the reason and its fix: no credential, or a credentials path that cannot be read.
|
||||
if (userProvider === "heygen" && !heygenAvailable()) heygenAuthHeaders();
|
||||
if (userProvider === "elevenlabs" && !process.env.ELEVENLABS_API_KEY)
|
||||
throw new Error("provider=elevenlabs but $ELEVENLABS_API_KEY is not set");
|
||||
return userProvider;
|
||||
}
|
||||
return heygenAvailable() ? "heygen" : elevenlabsAvailable() ? "elevenlabs" : "kokoro";
|
||||
}
|
||||
|
||||
// ── voice resolution ──────────────────────────────────────────────────────────
|
||||
// HeyGen /v3/voices/speech only accepts STARFISH voice_ids; auto-pick the first
|
||||
// English public starfish voice when none is pinned. ElevenLabs/Kokoro have
|
||||
// their own defaults.
|
||||
export async function resolveVoiceId({ provider, userVoice, lang = "en" }) {
|
||||
if (userVoice) return userVoice;
|
||||
if (provider === "gemini") return "Kore";
|
||||
if (provider === "elevenlabs") return "21m00Tcm4TlvDq8ikWAM"; // Rachel
|
||||
if (provider === "kokoro") {
|
||||
if (lang === "en") return "am_michael";
|
||||
throw new Error("Kokoro non-English needs an explicit --voice (see references/tts.md)");
|
||||
}
|
||||
// heygen — pin a fixed English default so the choice is deterministic. The old
|
||||
// "first English voice the API returns" drifts whenever HeyGen re-sorts the
|
||||
// public catalog. Marcia (mature, low female). Override with --voice / request.voice.
|
||||
if (lang === "en") return "05f19352e8f74b0392a8f411eba40de1"; // Marcia · English · female
|
||||
// Non-English: no fixed default — fall back to the first matching catalog voice.
|
||||
const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, {
|
||||
headers: heygenAuthHeaders(),
|
||||
});
|
||||
const voices = payload.data ?? payload.voices ?? [];
|
||||
const pick = voices.find((v) => v.language === "English") ?? voices[0];
|
||||
if (!pick) throw new Error("no public starfish voice to default to — pass --voice");
|
||||
return pick.voice_id;
|
||||
}
|
||||
|
||||
// ── helpers ─────────────────────────────────────────────────────────────────
|
||||
export function withWordIds(words) {
|
||||
return (words ?? []).map((w, i) => ({
|
||||
id: `w${i}`,
|
||||
text: w.text,
|
||||
start: w.start,
|
||||
end: w.end,
|
||||
}));
|
||||
}
|
||||
|
||||
// `ffmpeg -i <file>` prints a `Duration: HH:MM:SS.ms` line to stderr even
|
||||
// though it exits non-zero with no output requested. Parsing pulled out as
|
||||
// a pure function so the ENOENT fallback below can be tested without
|
||||
// depending on whether ffprobe/ffmpeg are actually installed on the
|
||||
// machine running the tests.
|
||||
export function parseFfmpegDurationBanner(stderrText) {
|
||||
const match = /Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/.exec(stderrText ?? "");
|
||||
if (!match) return NaN;
|
||||
const [, hours, minutes, seconds] = match;
|
||||
return Number(hours) * 3600 + Number(minutes) * 60 + Number(seconds);
|
||||
}
|
||||
|
||||
// Some "essentials"-style ffmpeg distributions (common on Windows) ship
|
||||
// ffmpeg.exe without ffprobe.exe. ffprobeDuration's caller (audio.mjs)
|
||||
// otherwise reads a spurious NaN as "the WAV file is corrupt" and drops an
|
||||
// already-successfully-synthesized TTS line, rather than "the tool for
|
||||
// measuring it is missing".
|
||||
function ffmpegDurationFallback(absPath) {
|
||||
const r = spawnSync("ffmpeg", ["-i", absPath], { encoding: "utf8" });
|
||||
return parseFfmpegDurationBanner(r.stderr);
|
||||
}
|
||||
|
||||
export function ffprobeDuration(absPath) {
|
||||
const r = spawnSync(
|
||||
"ffprobe",
|
||||
["-v", "error", "-show_entries", "format=duration", "-of", "default=nw=1:nk=1", "--", absPath],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
if (r.error?.code === "ENOENT") return ffmpegDurationFallback(absPath);
|
||||
if (r.status !== 0) return NaN;
|
||||
return parseFloat(String(r.stdout).trim());
|
||||
}
|
||||
|
||||
export function resolveNpxCliFromNpmExecPath(
|
||||
npmExecPath = process.env.npm_execpath,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
if (!npmExecPath) return null;
|
||||
const fileName = npmExecPath.replace(/\\/g, "/").split("/").pop()?.toLowerCase();
|
||||
const npxCliPath =
|
||||
fileName === "npx-cli.js" ? npmExecPath : join(dirname(npmExecPath), "npx-cli.js");
|
||||
return pathExists(npxCliPath) ? npxCliPath : null;
|
||||
}
|
||||
|
||||
export function resolveNpxCliPath(
|
||||
npmExecPath = process.env.npm_execpath,
|
||||
nodeExecPath = process.env.npm_node_execpath || process.execPath,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
const fromNpm = resolveNpxCliFromNpmExecPath(npmExecPath, pathExists);
|
||||
if (fromNpm) return fromNpm;
|
||||
const besideNode = join(dirname(nodeExecPath), "node_modules", "npm", "bin", "npx-cli.js");
|
||||
return pathExists(besideNode) ? besideNode : null;
|
||||
}
|
||||
|
||||
export function resolveSpawnCommand(
|
||||
cmd,
|
||||
args,
|
||||
opts = {},
|
||||
platform = process.platform,
|
||||
env = process.env,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
if (cmd !== "npx" || platform !== "win32") {
|
||||
return { cmd, args, opts: { stdio: "ignore", ...opts } };
|
||||
}
|
||||
|
||||
// On Windows, npx resolves to npx.cmd, which Node cannot execute directly.
|
||||
// Avoid `shell:true` and the .cmd shim entirely by invoking npm's JS CLI with
|
||||
// node, preserving request-provided values as argv data instead of shell text.
|
||||
const nodeExecPath = env.npm_node_execpath || process.execPath;
|
||||
const npxCliPath = resolveNpxCliPath(env.npm_execpath, nodeExecPath, pathExists);
|
||||
if (!npxCliPath) return null;
|
||||
return {
|
||||
cmd: nodeExecPath,
|
||||
args: [npxCliPath, ...args.map((arg) => String(arg))],
|
||||
opts: { stdio: "ignore", windowsHide: true, ...opts },
|
||||
};
|
||||
}
|
||||
|
||||
// `platform`/`spawnFn` params (default process.platform / the real spawn)
|
||||
// exist so tests can exercise the win32 branch without mocking node:child_process
|
||||
// (its ESM exports are non-configurable, so mock.method can't patch it).
|
||||
// One-shot so a whole batch of TTS lines doesn't repeat the same diagnostic.
|
||||
let _warnedNpxResolution = false;
|
||||
/** Test-only: reset the one-shot npx-resolution warning latch. */
|
||||
export function _resetNpxResolutionWarnForTests() {
|
||||
_warnedNpxResolution = false;
|
||||
}
|
||||
|
||||
export function spawnP(
|
||||
cmd,
|
||||
args,
|
||||
opts = {},
|
||||
platform = process.platform,
|
||||
spawnFn = spawn,
|
||||
env = process.env,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
const resolved = resolveSpawnCommand(cmd, args, opts, platform, env, pathExists);
|
||||
if (!resolved) {
|
||||
// resolveSpawnCommand only returns null for the npx-on-win32 case where
|
||||
// neither npm's configured CLI nor the beside-node fallback exists. Without
|
||||
// this, every call silently returns status:-1 and stdio:"ignore" hides why.
|
||||
if (!_warnedNpxResolution) {
|
||||
_warnedNpxResolution = true;
|
||||
const reason = env.npm_execpath
|
||||
? `npm_execpath (${env.npm_execpath}) and the beside-node npm fallback could not be found`
|
||||
: "npm_execpath is unset and the beside-node npm fallback could not be found";
|
||||
console.error(
|
||||
`[media-use] Cannot run "${cmd}" on Windows: ${reason}. ` +
|
||||
`Every "${cmd}" call is being skipped. Install npm with Node, or run via ` +
|
||||
`\`npx\`/\`npm run\` with a valid npm_execpath.`,
|
||||
);
|
||||
}
|
||||
return Promise.resolve({ status: -1 });
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const p = spawnFn(resolved.cmd, resolved.args, resolved.opts);
|
||||
p.on("exit", (code) => resolve({ status: code ?? -1 }));
|
||||
p.on("error", () => resolve({ status: -1 }));
|
||||
});
|
||||
}
|
||||
|
||||
// mp3/whatever bytes → wav 44.1k mono at destWav (ffmpeg detects true format).
|
||||
function transcodeToWav(bytes, destWav) {
|
||||
const td = mkdtempSync(join(tmpdir(), "hf-tts-"));
|
||||
const tmp = join(td, "a.mp3");
|
||||
writeFileSync(tmp, bytes);
|
||||
mkdirSync(dirname(destWav), { recursive: true });
|
||||
const ff = spawnSync(
|
||||
"ffmpeg",
|
||||
["-y", "-loglevel", "error", "-i", tmp, "-ar", "44100", "-ac", "1", destWav],
|
||||
{ stdio: "ignore" },
|
||||
);
|
||||
rmSync(td, { recursive: true, force: true });
|
||||
return ff.status === 0 && existsSync(destWav);
|
||||
}
|
||||
|
||||
const ELEVENLABS_PY = `
|
||||
import os, sys
|
||||
from elevenlabs.client import ElevenLabs
|
||||
from elevenlabs import save
|
||||
client = ElevenLabs(api_key=os.environ["ELEVENLABS_API_KEY"])
|
||||
text = open(sys.argv[1], encoding="utf-8").read()
|
||||
audio = client.text_to_speech.convert(
|
||||
text=text, voice_id=sys.argv[2],
|
||||
model_id="eleven_multilingual_v2", output_format="mp3_44100_128",
|
||||
)
|
||||
save(audio, sys.argv[3])
|
||||
`;
|
||||
|
||||
// ── synthesize one line ───────────────────────────────────────────────────────
|
||||
// Writes wav at wavAbs. Returns { ok, words, error } — words is the raw
|
||||
// [{text,start,end}] array for HeyGen (native), or null for ElevenLabs/Kokoro
|
||||
// (caller must transcribeWav). Never throws; failures return { ok:false, error }
|
||||
// where `error` states WHY (so the caller can surface it, not a bare "TTS failed").
|
||||
export async function synthesizeOne({
|
||||
provider,
|
||||
text,
|
||||
voiceId,
|
||||
lang = "en",
|
||||
speed = 1.0,
|
||||
model,
|
||||
style,
|
||||
wavAbs,
|
||||
hyperframesDir,
|
||||
}) {
|
||||
if (provider === "heygen") return synthesizeHeygen({ text, voiceId, lang, speed, wavAbs });
|
||||
if (provider === "gemini")
|
||||
return synthesizeGemini({ text, voiceId, model, style, speed, wavAbs });
|
||||
if (provider === "elevenlabs") {
|
||||
// The Python helper writes straight to wavAbs; unlike heygen (transcodeToWav)
|
||||
// and kokoro (the `hyperframes tts` CLI), it does NOT create the parent dir,
|
||||
// so on a fresh project (no assets/voice/ yet) the save fails and the line is
|
||||
// silently dropped as "TTS failed - omitted". Create it first, like the other
|
||||
// providers do. Guarded so a mkdir failure (EACCES/EROFS) returns
|
||||
// { ok:false } like the rest of this branch rather than throwing (the
|
||||
// function's contract is "never throws; failures return { ok:false }").
|
||||
try {
|
||||
mkdirSync(dirname(wavAbs), { recursive: true });
|
||||
} catch {
|
||||
return { ok: false, words: null };
|
||||
}
|
||||
return withTmpText(text, async (textPath) => {
|
||||
const { cmd, args } = pythonInvocation(["-c", ELEVENLABS_PY, textPath, voiceId, wavAbs]);
|
||||
const r = await spawnP(cmd, args, {});
|
||||
return synthResult(r, wavAbs, "elevenlabs (python)");
|
||||
});
|
||||
}
|
||||
// kokoro — via the published CLI; --output is relative to the project dir.
|
||||
const wavRel = relTo(hyperframesDir, wavAbs);
|
||||
return withTmpText(text, async (textPath) => {
|
||||
const args = ["hyperframes", "tts", textPath, "--voice", voiceId, "--output", wavRel];
|
||||
if (lang !== "en") args.push("--lang", lang);
|
||||
if (speed !== 1) args.push("--speed", String(speed));
|
||||
const r = await spawnP("npx", args, { cwd: hyperframesDir });
|
||||
return synthResult(r, wavAbs, "kokoro (npx hyperframes tts)");
|
||||
});
|
||||
}
|
||||
|
||||
// Shape a spawn result into { ok, words, error }, naming why on failure so the
|
||||
// caller surfaces it instead of a bare "TTS failed".
|
||||
export function synthResult(r, wavAbs, label) {
|
||||
if (r.status === 0 && existsSync(wavAbs)) return { ok: true, words: null };
|
||||
const why =
|
||||
r.status !== 0 ? `${label} exited with status ${r.status}` : `${label} produced no wav file`;
|
||||
return { ok: false, words: null, error: why };
|
||||
}
|
||||
|
||||
// `deps` is injectable for tests; production uses the real network/ffmpeg impls.
|
||||
// Every failure path returns an `error` string so the caller can surface WHY a
|
||||
// line was dropped instead of the bare "TTS failed" that hid the real cause
|
||||
// (e.g. an HTTP 402 plan_upgrade_required thrown by heygenJSON was swallowed).
|
||||
export async function synthesizeHeygen({ text, voiceId, lang, speed, wavAbs }, deps = {}) {
|
||||
const requestJSON = deps.heygenJSON ?? heygenJSON;
|
||||
const authHeaders = deps.heygenAuthHeaders ?? heygenAuthHeaders;
|
||||
const fetchImpl = deps.fetch ?? fetch;
|
||||
const transcode = deps.transcodeToWav ?? transcodeToWav;
|
||||
try {
|
||||
const body = { text, voice_id: voiceId, speed };
|
||||
if (lang !== "en") body.language = lang;
|
||||
const payload = await requestJSON(`/voices/speech`, {
|
||||
method: "POST",
|
||||
headers: authHeaders(),
|
||||
body,
|
||||
});
|
||||
const inner = payload.data ?? payload;
|
||||
if (!inner.audio_url) {
|
||||
return { ok: false, words: null, error: "HeyGen /voices/speech returned no audio_url" };
|
||||
}
|
||||
const res = await fetchMedia(inner.audio_url, { fetchImpl });
|
||||
if (!res.ok) {
|
||||
return { ok: false, words: null, error: `audio_url fetch failed: HTTP ${res.status}` };
|
||||
}
|
||||
const bytes = Buffer.from(await res.arrayBuffer());
|
||||
// .wav output → transcode to 44.1k mono; .mp3 → raw bytes (no ffmpeg). The
|
||||
// engine always asks for .wav; the standalone heygen-tts CLI may ask for .mp3.
|
||||
if (wavAbs.endsWith(".wav")) {
|
||||
if (!transcode(bytes, wavAbs)) {
|
||||
return {
|
||||
ok: false,
|
||||
words: null,
|
||||
error: "wav transcode failed (ffmpeg)",
|
||||
};
|
||||
}
|
||||
} else {
|
||||
mkdirSync(dirname(wavAbs), { recursive: true });
|
||||
writeFileSync(wavAbs, bytes);
|
||||
}
|
||||
const words = Array.isArray(inner.word_timestamps)
|
||||
? inner.word_timestamps
|
||||
.filter((w) => w && typeof w.word === "string" && isFinite(w.start) && isFinite(w.end))
|
||||
.filter((w) => !/^<.*>$/.test(w.word.trim())) // drop <start>/<end> sentinels
|
||||
.map((w) => ({ text: w.word, start: w.start, end: w.end }))
|
||||
: [];
|
||||
return { ok: true, words };
|
||||
} catch (e) {
|
||||
return { ok: false, words: null, error: heygenMessage(e) };
|
||||
}
|
||||
}
|
||||
|
||||
// ElevenLabs/Kokoro have no word timings — run Whisper over the wav. Returns the
|
||||
// flat [{id,text,start,end}] word array, or null. Each call uses a throwaway
|
||||
// --dir so parallel scenes don't collide on transcript.json.
|
||||
export async function transcribeWav({ wavRel, lang = "en", hyperframesDir }) {
|
||||
const model = lang === "en" ? "small.en" : "small";
|
||||
const td = mkdtempSync(join(tmpdir(), "hf-trans-"));
|
||||
const args = ["hyperframes", "transcribe", wavRel, "--model", model, "--dir", td];
|
||||
if (lang !== "en") args.push("--language", lang);
|
||||
const r = await spawnP("npx", args, { cwd: hyperframesDir });
|
||||
let words = null;
|
||||
if (r.status === 0) {
|
||||
const src = join(td, "transcript.json");
|
||||
if (existsSync(src)) {
|
||||
try {
|
||||
const arr = JSON.parse(readFileSync(src, "utf8"));
|
||||
if (Array.isArray(arr) && arr.length) words = arr;
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
rmSync(td, { recursive: true, force: true });
|
||||
return words;
|
||||
}
|
||||
|
||||
// ── tiny local utils ──────────────────────────────────────────────────────────
|
||||
async function withTmpText(text, run) {
|
||||
const td = mkdtempSync(join(tmpdir(), "hf-txt-"));
|
||||
try {
|
||||
const p = join(td, "line.txt");
|
||||
writeFileSync(p, text);
|
||||
return await run(p);
|
||||
} finally {
|
||||
// Cleanup must preserve the synthesis result or the original write/spawn error.
|
||||
try {
|
||||
rmSync(td, { recursive: true, force: true });
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
function relTo(base, abs) {
|
||||
return abs.startsWith(base + "/") ? abs.slice(base.length + 1) : abs;
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { EventEmitter } from "node:events";
|
||||
import {
|
||||
resolveNpxCliFromNpmExecPath,
|
||||
resolveNpxCliPath,
|
||||
resolveSpawnCommand,
|
||||
spawnP,
|
||||
_resetNpxResolutionWarnForTests,
|
||||
} from "./tts.mjs";
|
||||
|
||||
// Regression: on Windows, npx resolves to npx.cmd, which spawn() cannot exec
|
||||
// without shell:true — it fails ENOENT, silently swallowed as ok:false by the
|
||||
// caller. spawnP takes injectable platform/spawnFn params so this doesn't
|
||||
// need to touch the real process.platform or mock node:child_process (whose
|
||||
// ESM exports are non-configurable).
|
||||
function fakeSpawn(captured) {
|
||||
return (cmd, args, opts) => {
|
||||
captured.push({ cmd, args, opts });
|
||||
const p = new EventEmitter();
|
||||
setImmediate(() => p.emit("exit", 0));
|
||||
return p;
|
||||
};
|
||||
}
|
||||
|
||||
const envWithNpxCli = {
|
||||
npm_execpath: "/opt/node/lib/node_modules/npm/bin/npm-cli.js",
|
||||
npm_node_execpath: "/opt/node/bin/node",
|
||||
};
|
||||
const npxCliPath = "/opt/node/lib/node_modules/npm/bin/npx-cli.js";
|
||||
const pathExists = (path) => path === npxCliPath;
|
||||
|
||||
test("resolveNpxCliFromNpmExecPath finds npx-cli next to npm-cli", () => {
|
||||
assert.equal(resolveNpxCliFromNpmExecPath(envWithNpxCli.npm_execpath, pathExists), npxCliPath);
|
||||
});
|
||||
|
||||
test("resolveNpxCliPath finds npx-cli beside node when npm_execpath is unset", () => {
|
||||
const node = "C:/Program Files/nodejs/node.exe";
|
||||
const expected = "C:/Program Files/nodejs/node_modules/npm/bin/npx-cli.js";
|
||||
assert.equal(
|
||||
resolveNpxCliPath(undefined, node, (path) => path === expected),
|
||||
expected,
|
||||
);
|
||||
});
|
||||
|
||||
test("resolveSpawnCommand routes npx through node+npx-cli on win32 without shell:true", () => {
|
||||
const resolved = resolveSpawnCommand(
|
||||
"npx",
|
||||
["hyperframes", "tts", "C:\\Users\\Test User\\line.txt", "--voice", "am_michael"],
|
||||
{},
|
||||
"win32",
|
||||
envWithNpxCli,
|
||||
pathExists,
|
||||
);
|
||||
assert.ok(resolved);
|
||||
assert.equal(resolved.cmd, envWithNpxCli.npm_node_execpath);
|
||||
assert.deepEqual(resolved.args, [
|
||||
npxCliPath,
|
||||
"hyperframes",
|
||||
"tts",
|
||||
"C:\\Users\\Test User\\line.txt",
|
||||
"--voice",
|
||||
"am_michael",
|
||||
]);
|
||||
assert.equal(resolved.opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("resolveSpawnCommand preserves Windows npx shell metacharacters as argv data", () => {
|
||||
const resolved = resolveSpawnCommand(
|
||||
"npx",
|
||||
["hyperframes", "tts", "hello & calc"],
|
||||
{},
|
||||
"win32",
|
||||
envWithNpxCli,
|
||||
pathExists,
|
||||
);
|
||||
assert.ok(resolved);
|
||||
assert.deepEqual(resolved.args, [npxCliPath, "hyperframes", "tts", "hello & calc"]);
|
||||
});
|
||||
|
||||
test("spawnP uses the resolved node+npx-cli command for npx on win32", async () => {
|
||||
const captured = [];
|
||||
await spawnP(
|
||||
"npx",
|
||||
["hyperframes", "tts"],
|
||||
{},
|
||||
"win32",
|
||||
fakeSpawn(captured),
|
||||
envWithNpxCli,
|
||||
pathExists,
|
||||
);
|
||||
assert.equal(captured.length, 1);
|
||||
assert.equal(captured[0].cmd, envWithNpxCli.npm_node_execpath);
|
||||
assert.deepEqual(captured[0].args, [npxCliPath, "hyperframes", "tts"]);
|
||||
assert.equal(captured[0].opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("spawnP does not enable shell for npx on darwin/linux", async () => {
|
||||
const captured = [];
|
||||
await spawnP("npx", ["hyperframes", "tts"], {}, "darwin", fakeSpawn(captured));
|
||||
assert.equal(captured[0].cmd, "npx");
|
||||
assert.deepEqual(captured[0].args, ["hyperframes", "tts"]);
|
||||
assert.equal(captured[0].opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("spawnP does not enable shell for non-npx commands even on win32", async () => {
|
||||
const captured = [];
|
||||
await spawnP("python3", ["-c", "pass"], {}, "win32", fakeSpawn(captured));
|
||||
assert.equal(captured[0].cmd, "python3");
|
||||
assert.deepEqual(captured[0].args, ["-c", "pass"]);
|
||||
assert.equal(captured[0].opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("spawnP resolves npx beside node when npm_execpath is unset on win32", async () => {
|
||||
_resetNpxResolutionWarnForTests();
|
||||
const captured = [];
|
||||
const node = "C:/Program Files/nodejs/node.exe";
|
||||
const npxCli = "C:/Program Files/nodejs/node_modules/npm/bin/npx-cli.js";
|
||||
const result = await spawnP(
|
||||
"npx",
|
||||
["hyperframes", "tts"],
|
||||
{},
|
||||
"win32",
|
||||
fakeSpawn(captured),
|
||||
{ npm_node_execpath: node },
|
||||
(path) => path === npxCli,
|
||||
);
|
||||
assert.equal(result.status, 0);
|
||||
assert.equal(captured.length, 1);
|
||||
assert.equal(captured[0].cmd, node);
|
||||
assert.deepEqual(captured[0].args, [npxCli, "hyperframes", "tts"]);
|
||||
});
|
||||
|
||||
test("spawnP warns once with an accurate diagnostic when neither npx path exists", async () => {
|
||||
_resetNpxResolutionWarnForTests();
|
||||
const errors = [];
|
||||
const originalError = console.error;
|
||||
console.error = (message) => errors.push(String(message));
|
||||
try {
|
||||
const env = { npm_execpath: "C:/missing/npm-cli.js", npm_node_execpath: "C:/node/node.exe" };
|
||||
const missing = () => false;
|
||||
assert.equal(
|
||||
(await spawnP("npx", ["hyperframes", "tts"], {}, "win32", fakeSpawn([]), env, missing))
|
||||
.status,
|
||||
-1,
|
||||
);
|
||||
assert.equal(
|
||||
(await spawnP("npx", ["hyperframes", "tts"], {}, "win32", fakeSpawn([]), env, missing))
|
||||
.status,
|
||||
-1,
|
||||
);
|
||||
} finally {
|
||||
console.error = originalError;
|
||||
}
|
||||
assert.equal(errors.length, 1);
|
||||
assert.match(errors[0], /npm_execpath \(C:\/missing\/npm-cli\.js\)/);
|
||||
assert.doesNotMatch(errors[0], /npm_execpath is not set/);
|
||||
});
|
||||
@@ -0,0 +1,230 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
writeFileSync,
|
||||
chmodSync,
|
||||
rmSync,
|
||||
existsSync,
|
||||
} from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import {
|
||||
parseFfmpegDurationBanner,
|
||||
ffprobeDuration,
|
||||
synthesizeOne,
|
||||
synthesizeHeygen,
|
||||
synthResult,
|
||||
pickProvider,
|
||||
} from "./tts.mjs";
|
||||
|
||||
test("parseFfmpegDurationBanner reads ffmpeg's stderr Duration line", () => {
|
||||
const stderr = [
|
||||
"ffmpeg version 6.0",
|
||||
"Input #0, wav, from 'a.wav':",
|
||||
" Duration: 00:00:03.42, bitrate: 705 kb/s",
|
||||
"At least one output file must be specified",
|
||||
].join("\n");
|
||||
assert.equal(parseFfmpegDurationBanner(stderr), 3.42);
|
||||
});
|
||||
|
||||
test("parseFfmpegDurationBanner handles an hours component", () => {
|
||||
const stderr = " Duration: 01:02:03.50, start: 0.000000, bitrate: 128 kb/s";
|
||||
assert.equal(parseFfmpegDurationBanner(stderr), 3723.5);
|
||||
});
|
||||
|
||||
test("parseFfmpegDurationBanner returns NaN when there is no Duration line", () => {
|
||||
assert.ok(Number.isNaN(parseFfmpegDurationBanner("ffmpeg: command not found")));
|
||||
assert.ok(Number.isNaN(parseFfmpegDurationBanner("")));
|
||||
assert.ok(Number.isNaN(parseFfmpegDurationBanner(undefined)));
|
||||
});
|
||||
|
||||
// Regression for the actual bug: ffprobeDuration used to collapse "ffprobe
|
||||
// binary is missing" (ENOENT — the "essentials"-style Windows ffmpeg build
|
||||
// with no ffprobe.exe) and "file is genuinely unreadable" into the same NaN,
|
||||
// giving audio.mjs no way to tell "measure differently" from "give up".
|
||||
//
|
||||
// Builds an isolated PATH containing only a fake `ffmpeg` stub (no `ffprobe`
|
||||
// at all) so ffprobeDuration's spawnSync("ffprobe", ...) call ENOENTs for
|
||||
// real, then verifies it recovers the duration via the ffmpeg fallback
|
||||
// instead of returning NaN.
|
||||
test("ffprobeDuration falls back to ffmpeg when the ffprobe binary itself is missing", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-ffprobe-fallback-"));
|
||||
const fakeFfmpeg = join(dir, "ffmpeg");
|
||||
writeFileSync(
|
||||
fakeFfmpeg,
|
||||
"#!/bin/sh\necho 'Duration: 00:00:02.50, start: 0.000000, bitrate: 128 kb/s' 1>&2\nexit 1\n",
|
||||
);
|
||||
chmodSync(fakeFfmpeg, 0o755);
|
||||
const originalPath = process.env.PATH;
|
||||
try {
|
||||
process.env.PATH = dir; // only the fake ffmpeg resolves; no real ffprobe on this PATH
|
||||
assert.equal(ffprobeDuration("/does/not/matter.wav"), 2.5);
|
||||
} finally {
|
||||
process.env.PATH = originalPath;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("ffprobeDuration returns NaN when neither ffprobe nor ffmpeg resolve", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-no-binaries-"));
|
||||
const originalPath = process.env.PATH;
|
||||
try {
|
||||
process.env.PATH = dir; // empty directory — nothing resolves
|
||||
assert.ok(Number.isNaN(ffprobeDuration("/does/not/matter.wav")));
|
||||
} finally {
|
||||
process.env.PATH = originalPath;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("synthesizeOne(elevenlabs) creates the output dir before writing", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-el-mkdir-"));
|
||||
const wavAbs = join(dir, "assets", "voice", "line-0.wav"); // nested, not yet created
|
||||
const savedKey = process.env.ELEVENLABS_API_KEY;
|
||||
try {
|
||||
// Unset the key so the Python side fails fast — the mkdir must run before
|
||||
// the spawn regardless, which is what this guards.
|
||||
delete process.env.ELEVENLABS_API_KEY;
|
||||
await synthesizeOne({
|
||||
provider: "elevenlabs",
|
||||
text: "hi",
|
||||
voiceId: "v",
|
||||
wavAbs,
|
||||
hyperframesDir: dir,
|
||||
});
|
||||
assert.ok(existsSync(dirname(wavAbs)), "output directory should be created");
|
||||
} finally {
|
||||
if (savedKey === undefined) delete process.env.ELEVENLABS_API_KEY;
|
||||
else process.env.ELEVENLABS_API_KEY = savedKey;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("synthesizeHeygen surfaces a thrown HTTP error (e.g. 402) instead of swallowing it", async () => {
|
||||
const res = await synthesizeHeygen(
|
||||
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: "/tmp/x.wav" },
|
||||
{
|
||||
heygenAuthHeaders: () => ({}),
|
||||
heygenJSON: async () => {
|
||||
throw new Error("HeyGen POST /voices/speech → HTTP 402\nplan_upgrade_required");
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error, /402/);
|
||||
assert.match(res.error, /plan_upgrade_required/);
|
||||
});
|
||||
|
||||
test("synthesizeHeygen surfaces a failed audio_url fetch with its status", async () => {
|
||||
const res = await synthesizeHeygen(
|
||||
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: "/tmp/x.wav" },
|
||||
{
|
||||
heygenAuthHeaders: () => ({}),
|
||||
heygenJSON: async () => ({ data: { audio_url: "http://audio.example/x" } }),
|
||||
fetch: async () => ({ ok: false, status: 403 }),
|
||||
},
|
||||
);
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error, /HTTP 403/);
|
||||
});
|
||||
|
||||
test("synthesizeHeygen reports a missing audio_url", async () => {
|
||||
const res = await synthesizeHeygen(
|
||||
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: "/tmp/x.wav" },
|
||||
{ heygenAuthHeaders: () => ({}), heygenJSON: async () => ({}) },
|
||||
);
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error, /no audio_url/);
|
||||
});
|
||||
|
||||
test("synthesizeHeygen reports wav transcode failures", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-tts-test-"));
|
||||
try {
|
||||
const res = await synthesizeHeygen(
|
||||
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: join(dir, "voice.wav") },
|
||||
{
|
||||
heygenAuthHeaders: () => ({}),
|
||||
heygenJSON: async () => ({ data: { audio_url: "http://audio.example/x" } }),
|
||||
fetch: async () => ({ ok: true, status: 200, arrayBuffer: async () => new ArrayBuffer(0) }),
|
||||
transcodeToWav: () => false,
|
||||
},
|
||||
);
|
||||
assert.equal(res.ok, false);
|
||||
assert.equal(res.error, "wav transcode failed (ffmpeg)");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("synthResult names a non-zero subprocess exit", () => {
|
||||
const res = synthResult({ status: 2 }, "/tmp/none.wav", "kokoro (npx hyperframes tts)");
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error, /kokoro .* exited with status 2/);
|
||||
});
|
||||
|
||||
// Kokoro shells out to `npx hyperframes tts`; a fake npx on PATH records the argv
|
||||
// synthesizeOne really builds, so a dropped flag fails here. Windows runs npx via
|
||||
// node directly, so the stub never resolves there.
|
||||
async function kokoroArgv(options) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-kokoro-argv-"));
|
||||
const argvLog = join(dir, "argv.txt");
|
||||
writeFileSync(join(dir, "npx"), `#!/bin/sh\nprintf '%s\\n' "$@" > "${argvLog}"\n`);
|
||||
chmodSync(join(dir, "npx"), 0o755);
|
||||
const originalPath = process.env.PATH;
|
||||
try {
|
||||
process.env.PATH = dir;
|
||||
await synthesizeOne({
|
||||
provider: "kokoro",
|
||||
text: "hi",
|
||||
voiceId: "am_michael",
|
||||
wavAbs: join(dir, "line-0.wav"),
|
||||
hyperframesDir: dir,
|
||||
...options,
|
||||
});
|
||||
return readFileSync(argvLog, "utf8").trim().split("\n");
|
||||
} finally {
|
||||
process.env.PATH = originalPath;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test(
|
||||
"synthesizeOne(kokoro) forwards a non-default speed to the CLI",
|
||||
{ skip: process.platform === "win32" },
|
||||
async () => {
|
||||
const argv = await kokoroArgv({ speed: 0.8, lang: "es" });
|
||||
assert.deepEqual(argv.slice(-4), ["--lang", "es", "--speed", "0.8"]);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
"synthesizeOne(kokoro) leaves --speed off at the default",
|
||||
{ skip: process.platform === "win32" },
|
||||
async () => {
|
||||
assert.ok(!(await kokoroArgv({})).includes("--speed"));
|
||||
},
|
||||
);
|
||||
|
||||
test("pickProvider heygen with an unreadable credentials path says to fix that path", () => {
|
||||
const saved = ["HEYGEN_API_KEY", "HYPERFRAMES_API_KEY", "HEYGEN_CONFIG_DIR"].map((k) => [
|
||||
k,
|
||||
process.env[k],
|
||||
]);
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-cred-"));
|
||||
try {
|
||||
delete process.env.HEYGEN_API_KEY;
|
||||
delete process.env.HYPERFRAMES_API_KEY;
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
mkdirSync(join(dir, "credentials"));
|
||||
assert.throws(() => pickProvider("heygen"), /fix or remove that path/);
|
||||
} finally {
|
||||
for (const [k, v] of saved) {
|
||||
if (v === undefined) delete process.env[k];
|
||||
else process.env[k] = v;
|
||||
}
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate BGM using Google Lyria RealTime API.
|
||||
|
||||
Usage:
|
||||
python lyria-recipe.py --output <path> --duration <seconds> [tuning flags]
|
||||
|
||||
Requires:
|
||||
$GOOGLE_API_KEY or $GEMINI_API_KEY environment variable (treated as aliases).
|
||||
pip install google-genai python-dotenv. audio.mjs Step 4b installs these on
|
||||
demand when a key is set but google.genai is not importable; if that install
|
||||
fails it falls back to local MusicGen rather than leaving the video with no BGM.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
# Windows sizes stdio to the ANSI code page (cp1252). These scripts emit UTF-8 on
|
||||
# every platform; say so rather than depending on the console's code page. Carry
|
||||
# `errors` across: reconfigure() resets it to "strict", and CPython deliberately gives
|
||||
# stderr "backslashreplace" so the diagnostic path can never itself raise.
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(_stream, "reconfigure"):
|
||||
_stream.reconfigure(encoding="utf-8", errors=_stream.errors)
|
||||
|
||||
DEFAULT_PROMPT = "Uplifting corporate tech, bright and modern, gentle piano with synth pads"
|
||||
SAMPLE_RATE = 48000
|
||||
CHANNELS = 2
|
||||
SAMPLE_WIDTH = 2 # 16-bit
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description="Generate BGM via Google Lyria RealTime.")
|
||||
p.add_argument("--output", required=True, help="Output WAV path.")
|
||||
p.add_argument("--duration", type=float, required=True, help="Target duration in seconds.")
|
||||
p.add_argument("--prompt", default=DEFAULT_PROMPT, help="Mood / instrumentation prompt.")
|
||||
p.add_argument("--negative-prompt", default=None, help="Styles to exclude (optional).")
|
||||
p.add_argument("--bpm", type=int, default=110)
|
||||
p.add_argument("--brightness", type=float, default=0.8, help="0-1, higher = brighter mood.")
|
||||
p.add_argument("--density", type=float, default=0.5, help="0-1, higher = fuller mix.")
|
||||
p.add_argument(
|
||||
"--scale",
|
||||
default="MAJOR",
|
||||
help="MAJOR / MINOR / PENTATONIC / etc. — see google.genai.types.Scale. Pass empty string for none.",
|
||||
)
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
async def generate_bgm(args: argparse.Namespace) -> dict:
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
api_key = os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY") or ""
|
||||
if not api_key:
|
||||
raise RuntimeError("Neither GOOGLE_API_KEY nor GEMINI_API_KEY is set.")
|
||||
|
||||
client = genai.Client(
|
||||
api_key=api_key,
|
||||
http_options={"api_version": "v1alpha"},
|
||||
)
|
||||
|
||||
out_path = Path(args.output)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
target_bytes = int(args.duration * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)
|
||||
|
||||
cfg: dict = {"bpm": args.bpm, "temperature": 1.0}
|
||||
if args.density is not None:
|
||||
cfg["density"] = args.density
|
||||
if args.brightness is not None:
|
||||
cfg["brightness"] = args.brightness
|
||||
if args.scale:
|
||||
scale_enum = getattr(types.Scale, args.scale, None)
|
||||
if scale_enum:
|
||||
cfg["scale"] = scale_enum
|
||||
|
||||
prompts = [types.WeightedPrompt(text=args.prompt, weight=1.0)]
|
||||
if args.negative_prompt:
|
||||
prompts.append(types.WeightedPrompt(text=args.negative_prompt, weight=-1.0))
|
||||
|
||||
buf = bytearray()
|
||||
timeout = args.duration + 8
|
||||
|
||||
async with client.aio.live.music.connect(
|
||||
model="models/lyria-realtime-exp",
|
||||
) as session:
|
||||
await session.set_weighted_prompts(prompts=prompts)
|
||||
await session.set_music_generation_config(
|
||||
config=types.LiveMusicGenerationConfig(**cfg),
|
||||
)
|
||||
await session.play()
|
||||
|
||||
async def collect():
|
||||
while len(buf) < target_bytes:
|
||||
async for msg in session.receive():
|
||||
sc = msg.server_content
|
||||
if sc and sc.audio_chunks:
|
||||
for chunk in sc.audio_chunks:
|
||||
buf.extend(chunk.data)
|
||||
if len(buf) >= target_bytes:
|
||||
return
|
||||
await asyncio.sleep(1e-6)
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(collect(), timeout=timeout)
|
||||
except TimeoutError:
|
||||
print(f"Timeout after {timeout:.0f}s, collected {len(buf)} bytes", file=sys.stderr)
|
||||
|
||||
audio = bytes(buf[:target_bytes])
|
||||
with wave.open(str(out_path), "wb") as wf:
|
||||
wf.setnchannels(CHANNELS)
|
||||
wf.setsampwidth(SAMPLE_WIDTH)
|
||||
wf.setframerate(SAMPLE_RATE)
|
||||
wf.writeframes(audio)
|
||||
|
||||
actual_duration = len(audio) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)
|
||||
print(f"BGM: {out_path} ({actual_duration:.2f}s)")
|
||||
return {"file": str(out_path), "duration_sec": round(actual_duration, 2)}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
try:
|
||||
asyncio.run(generate_bgm(args))
|
||||
except RuntimeError as exc:
|
||||
print(f"BGM generation failed: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env node
|
||||
// Phase 4c pre-assemble helper — wait for detached BGM, then write status.
|
||||
//
|
||||
// audio.mjs may launch Lyria / MusicGen in a detached process so voice work can
|
||||
// keep moving. Before assemble-index.mjs decides whether to emit the BGM audio
|
||||
// track, this script gives the background renderer a bounded chance to finish
|
||||
// and converts log/process state into a small bgm_status.json file.
|
||||
//
|
||||
// Always exits 0 for normal pipeline use: missing/failed BGM should not block a
|
||||
// voice/captions/SFX render. Structural invocation errors still exit 1.
|
||||
//
|
||||
// Usage:
|
||||
// node wait-bgm.mjs --audio-meta ./audio_meta.json --hyperframes . \
|
||||
// [--timeout-ms 120000] [--interval-ms 2000] [--out ./bgm_status.json]
|
||||
|
||||
import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { recordInManifest } from "./lib/media-record.mjs";
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name, def) => {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : def;
|
||||
};
|
||||
|
||||
function die(msg) {
|
||||
console.error(`✗ wait-bgm.mjs: ${msg}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const audioMetaPath = resolve(flag("audio-meta", "./audio_meta.json"));
|
||||
const hyperframesDir = resolve(flag("hyperframes", "."));
|
||||
const outPath = resolve(flag("out", join(hyperframesDir, "bgm_status.json")));
|
||||
const timeoutMs = Math.max(0, Number(flag("timeout-ms", "120000")) || 0);
|
||||
const intervalMs = Math.max(250, Number(flag("interval-ms", "2000")) || 2000);
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
|
||||
}
|
||||
|
||||
function isProcessAlive(pid) {
|
||||
if (!pid || !Number.isFinite(Number(pid))) return false;
|
||||
try {
|
||||
process.kill(Number(pid), 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readTail(path, maxChars = 6000) {
|
||||
if (!path) return "";
|
||||
try {
|
||||
const txt = readFileSync(path, "utf8");
|
||||
return txt.slice(Math.max(0, txt.length - maxChars));
|
||||
} catch (error) {
|
||||
if (error.code === "ENOENT" || error.code === "ENOTDIR") return "";
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function detectFailure(logTail) {
|
||||
if (!logTail) return "";
|
||||
const lines = logTail.split("\n");
|
||||
// Bare "out of range" over-matched benign BGM-renderer logs (e.g. a "sample rate
|
||||
// out of range, resampling" notice), mislabelling a healthy track as failed and
|
||||
// silently dropping the music. Anchor to the actual crash strings instead:
|
||||
// Python "(list) index out of range" and torch "index … out of bounds".
|
||||
const idx = lines.findIndex((line) =>
|
||||
/(Traceback|IndexError|RuntimeError|Exception|Killed|No space left|Cannot allocate|index out of range|out of bounds)/i.test(
|
||||
line,
|
||||
),
|
||||
);
|
||||
if (idx < 0) return "";
|
||||
return lines.slice(idx).join("\n").trim();
|
||||
}
|
||||
|
||||
function writeStatus(status) {
|
||||
const payload = {
|
||||
generated_at: new Date().toISOString(),
|
||||
...status,
|
||||
};
|
||||
writeFileSync(outPath, JSON.stringify(payload, null, 2) + "\n");
|
||||
return payload;
|
||||
}
|
||||
|
||||
if (!existsSync(audioMetaPath)) die(`audio_meta.json missing at ${audioMetaPath}`);
|
||||
|
||||
const audioMeta = JSON.parse(readFileSync(audioMetaPath, "utf8"));
|
||||
const bgmPath = audioMeta.bgm?.path || "";
|
||||
const bgmAbsPath = bgmPath ? join(hyperframesDir, bgmPath) : "";
|
||||
const logPath = audioMeta.bgm_log || "";
|
||||
const pid = audioMeta.bgm_pid || null;
|
||||
|
||||
const base = {
|
||||
enabled: Boolean(audioMeta.bgm_pending && bgmPath),
|
||||
provider: audioMeta.bgm_provider || null,
|
||||
mode: audioMeta.bgm_mode || null,
|
||||
path: bgmPath || null,
|
||||
log: logPath || null,
|
||||
pid,
|
||||
target_duration_s: audioMeta.bgm_target_duration_s || null,
|
||||
seed_duration_s: audioMeta.bgm_seed_duration_s || null,
|
||||
loop_count: audioMeta.bgm_loop_count || null,
|
||||
timeout_ms: timeoutMs,
|
||||
};
|
||||
|
||||
if (!base.enabled) {
|
||||
const status = writeStatus({
|
||||
...base,
|
||||
status: "disabled",
|
||||
ready: false,
|
||||
waited_ms: 0,
|
||||
message: "BGM not requested or disabled in audio_meta.json.",
|
||||
});
|
||||
console.log(`✓ bgm: ${status.status} (${status.message})`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const started = Date.now();
|
||||
let lastFailure = "";
|
||||
let lastTail = "";
|
||||
|
||||
while (Date.now() - started <= timeoutMs) {
|
||||
if (existsSync(bgmAbsPath)) {
|
||||
const size = statSync(bgmAbsPath).size;
|
||||
writeStatus({
|
||||
...base,
|
||||
status: "ready",
|
||||
ready: true,
|
||||
waited_ms: Date.now() - started,
|
||||
size_bytes: size,
|
||||
message: `BGM ready at ${bgmPath}.`,
|
||||
});
|
||||
console.log(`✓ bgm: ready (${bgmPath}, ${size}B)`);
|
||||
const generated = { path: bgmPath, type: "bgm", source: "generated", provider: base.provider };
|
||||
for (const anomaly of recordInManifest(hyperframesDir, [generated]))
|
||||
console.error(` ${anomaly}`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
lastTail = readTail(logPath);
|
||||
lastFailure = detectFailure(lastTail);
|
||||
const alive = isProcessAlive(pid);
|
||||
if (lastFailure || (!alive && logPath && existsSync(logPath))) {
|
||||
const message = lastFailure
|
||||
? `BGM renderer failed; see ${logPath}.`
|
||||
: `BGM renderer exited without writing ${bgmPath}; see ${logPath}.`;
|
||||
const status = writeStatus({
|
||||
...base,
|
||||
status: "failed",
|
||||
ready: false,
|
||||
waited_ms: Date.now() - started,
|
||||
process_alive: alive,
|
||||
message,
|
||||
error_tail: lastFailure || lastTail.slice(-2000),
|
||||
});
|
||||
console.log(`! bgm: failed (${status.message})`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (timeoutMs === 0) break;
|
||||
await sleep(Math.min(intervalMs, Math.max(0, timeoutMs - (Date.now() - started))));
|
||||
}
|
||||
|
||||
const status = writeStatus({
|
||||
...base,
|
||||
status: "timeout",
|
||||
ready: false,
|
||||
waited_ms: Date.now() - started,
|
||||
process_alive: isProcessAlive(pid),
|
||||
message: `Timed out waiting for ${bgmPath}; assemble-index will skip BGM if still absent.`,
|
||||
log_tail: lastTail.slice(-2000),
|
||||
});
|
||||
console.log(`! bgm: timeout after ${status.waited_ms}ms (${bgmPath})`);
|
||||
@@ -0,0 +1,108 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { test } from "node:test";
|
||||
import { runInNewContext } from "node:vm";
|
||||
|
||||
const source = readFileSync(new URL("./wait-bgm.mjs", import.meta.url), "utf8");
|
||||
const start = source.indexOf("function readTail(");
|
||||
const end = source.indexOf("\nfunction detectFailure(", start);
|
||||
function load(read) {
|
||||
return runInNewContext(`${source.slice(start, end)}; readTail`, {
|
||||
readFileSync: read,
|
||||
existsSync: () => true,
|
||||
statSync: () => {
|
||||
throw new Error("pathname stat must not precede log read");
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
test("reads a log once without a separate pathname check", () => {
|
||||
let calls = 0;
|
||||
const read = load((path, encoding) => {
|
||||
calls++;
|
||||
assert.equal(path, "renderer.log");
|
||||
assert.equal(encoding, "utf8");
|
||||
return "prefix\nRuntimeError: failed";
|
||||
});
|
||||
assert.equal(read("renderer.log", 20), "RuntimeError: failed");
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
|
||||
test("preserves UTF-16 character tail boundaries and zero-length tails", () => {
|
||||
const txt = "🎵日本語".repeat(2000);
|
||||
const read = load(() => txt);
|
||||
assert.equal(read("log"), txt.slice(-6000));
|
||||
assert.equal(read("log", 0), "");
|
||||
});
|
||||
|
||||
for (const code of ["ENOENT", "ENOTDIR"]) {
|
||||
test(`a log removed during polling is absent (${code})`, () => {
|
||||
const read = load(() => {
|
||||
throw Object.assign(new Error(code), { code });
|
||||
});
|
||||
assert.equal(read("log"), "");
|
||||
});
|
||||
}
|
||||
|
||||
test("preserves unexpected read errors", () => {
|
||||
const error = Object.assign(new Error("denied"), { code: "EACCES" });
|
||||
assert.throws(
|
||||
() =>
|
||||
load(() => {
|
||||
throw error;
|
||||
})("log"),
|
||||
(caught) => caught === error,
|
||||
);
|
||||
});
|
||||
|
||||
test("empty log path does not access the filesystem", () => {
|
||||
assert.equal(
|
||||
load(() => {
|
||||
throw new Error("unexpected read");
|
||||
})(""),
|
||||
"",
|
||||
);
|
||||
});
|
||||
|
||||
test("records locally made music as generated once it is ready", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-wait-bgm-"));
|
||||
try {
|
||||
mkdirSync(join(dir, "assets/bgm"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets/bgm/track.wav"), "generated music");
|
||||
writeFileSync(
|
||||
join(dir, "audio_meta.json"),
|
||||
JSON.stringify({
|
||||
bgm: { path: "assets/bgm/track.wav" },
|
||||
bgm_pending: true,
|
||||
bgm_provider: "musicgen",
|
||||
bgm_mode: "detached-single",
|
||||
}),
|
||||
);
|
||||
const script = new URL("./wait-bgm.mjs", import.meta.url).pathname;
|
||||
const args = ["--audio-meta", join(dir, "audio_meta.json"), "--hyperframes", dir];
|
||||
|
||||
const run = spawnSync(process.execPath, [script, ...args, "--timeout-ms", "1000"], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
|
||||
assert.equal(run.status, 0, run.stderr);
|
||||
const records = readFileSync(join(dir, ".media/manifest.jsonl"), "utf8")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
assert.deepEqual(
|
||||
records.map(({ path, type, source, provenance }) => [
|
||||
path,
|
||||
type,
|
||||
source,
|
||||
provenance.provider,
|
||||
]),
|
||||
[["assets/bgm/track.wav", "bgm", "generated", "musicgen"]],
|
||||
);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user