Add HyperFrames skills (from heygen-com/hyperframes, Apache-2.0)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FXVsD1DgdzZLiPuwtKzMSY
This commit is contained in:
300 files changed
+49175
No files matched your search
@@ -0,0 +1,35 @@
|
||||
# SFX Credits
|
||||
|
||||
All sound effects in this directory are sourced from [Pixabay](https://pixabay.com/sound-effects/) and used under the [Pixabay Content License](https://pixabay.com/service/license-summary/).
|
||||
|
||||
The Pixabay license allows free use for commercial and non-commercial purposes without attribution, but attribution is appreciated and given here for transparency.
|
||||
|
||||
## Files
|
||||
|
||||
The following `.mp3` files are bundled with this skill:
|
||||
|
||||
- `chime.mp3`
|
||||
- `click.mp3` / `click-soft.mp3`
|
||||
- `error.mp3`
|
||||
- `glitch-1.mp3` / `glitch-2.mp3` / `glitch-3.mp3`
|
||||
- `impact-bass-1.mp3` / `impact-bass-2.mp3`
|
||||
- `key-press.mp3`
|
||||
- `notification.mp3`
|
||||
- `ping.mp3`
|
||||
- `pop.mp3`
|
||||
- `riser.mp3`
|
||||
- `sparkle.mp3`
|
||||
- `typing.mp3`
|
||||
- `whoosh.mp3` / `whoosh-short.mp3` / `whoosh-cinematic.mp3`
|
||||
|
||||
See `manifest.json` for per-file metadata (duration, energy character, recommended use).
|
||||
|
||||
## License
|
||||
|
||||
All files are distributed under the [Pixabay Content License](https://pixabay.com/service/license-summary/), which permits:
|
||||
|
||||
- Commercial and non-commercial use
|
||||
- Modification and remixing
|
||||
- Redistribution as part of derivative works (such as videos rendered with HyperFrames)
|
||||
|
||||
without any attribution requirement.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,97 @@
|
||||
{
|
||||
"chime": {
|
||||
"file": "chime.mp3",
|
||||
"duration": 2.5,
|
||||
"description": "Soft melodic chime — gentle positive beat: success/confirmation or a lighthearted transition. Sync to the visual moment."
|
||||
},
|
||||
"click-soft": {
|
||||
"file": "click-soft.mp3",
|
||||
"duration": 0.37,
|
||||
"description": "Quiet short click — low-key UI tap / soft selection. Short accent, sync exactly to the on-screen action."
|
||||
},
|
||||
"click": {
|
||||
"file": "click.mp3",
|
||||
"duration": 0.37,
|
||||
"description": "Crisp UI click — button press, toggle, selection. Short accent, sync exactly to the on-screen action."
|
||||
},
|
||||
"error": {
|
||||
"file": "error.mp3",
|
||||
"duration": 1.62,
|
||||
"description": "Negative / error tone — failure state, a 'wrong' beat, or a glitchy interruption."
|
||||
},
|
||||
"glitch-1": {
|
||||
"file": "glitch-1.mp3",
|
||||
"duration": 2.64,
|
||||
"description": "Punchy digital glitch — hard-cut accent or sudden reveal. Trigger on the hit; let the decay bleed into the next shot (J-cut)."
|
||||
},
|
||||
"glitch-2": {
|
||||
"file": "glitch-2.mp3",
|
||||
"duration": 3.5,
|
||||
"description": "Harsh, longer glitch — chaotic / jarring transition or a distorted reveal."
|
||||
},
|
||||
"glitch-3": {
|
||||
"file": "glitch-3.mp3",
|
||||
"duration": 3.1,
|
||||
"description": "Low-key glitch texture — subtle digital shift, minimal transition that sits under other audio."
|
||||
},
|
||||
"impact-bass-1": {
|
||||
"file": "impact-bass-1.mp3",
|
||||
"duration": 2.12,
|
||||
"description": "Bass impact hit — logo/hero snap, headline slam. Trigger on the visual landing; decay carries into the next shot (J-cut)."
|
||||
},
|
||||
"impact-bass-2": {
|
||||
"file": "impact-bass-2.mp3",
|
||||
"duration": 2.59,
|
||||
"description": "Bass impact with a short swell — brief anticipation then a deep hit. Place so the peak lands on the reveal."
|
||||
},
|
||||
"key-press": {
|
||||
"file": "key-press.mp3",
|
||||
"duration": 0.4,
|
||||
"description": "Single key press — one keystroke / terminal-input beat. Short accent, sync to the typed character."
|
||||
},
|
||||
"notification": {
|
||||
"file": "notification.mp3",
|
||||
"duration": 2.46,
|
||||
"description": "Notification chime — alert, message-in, toast/badge appears. Sync to the element entering."
|
||||
},
|
||||
"ping": {
|
||||
"file": "ping.mp3",
|
||||
"duration": 1.32,
|
||||
"description": "Sharp electronic ping — punchy accent on a key reveal or data point. Sync to the beat."
|
||||
},
|
||||
"pop": {
|
||||
"file": "pop.mp3",
|
||||
"duration": 0.72,
|
||||
"description": "Quick pop — element appear/spawn, chip/tag/badge in. Small precise accent, sync to the pop-in."
|
||||
},
|
||||
"riser": {
|
||||
"file": "riser.mp3",
|
||||
"duration": 10.03,
|
||||
"description": "Long cinematic riser (~10s build, peak at the end). Trigger at (climax_time − 10.03s) so it crests exactly on the reveal."
|
||||
},
|
||||
"sparkle": {
|
||||
"file": "sparkle.mp3",
|
||||
"duration": 1.8,
|
||||
"description": "Bright sparkle / shimmer — magical reveal or 'shine' highlight on a hero element. Sync to the highlight."
|
||||
},
|
||||
"typing": {
|
||||
"file": "typing.mp3",
|
||||
"duration": 1.5,
|
||||
"description": "Typing burst (~1.5s of keys) — keyboard / code typing reveal, text-being-typed beat. Start as the text begins typing."
|
||||
},
|
||||
"whoosh-cinematic": {
|
||||
"file": "whoosh-cinematic.mp3",
|
||||
"duration": 5.54,
|
||||
"description": "Cinematic whoosh build (~5.5s) — sweeping scene transition. Align so the swell peaks on the cut."
|
||||
},
|
||||
"whoosh-short": {
|
||||
"file": "whoosh-short.mp3",
|
||||
"duration": 0.57,
|
||||
"description": "Short whoosh — quick swipe/slide accent, fast element move, snappy transition. Sync to the motion."
|
||||
},
|
||||
"whoosh": {
|
||||
"file": "whoosh.mp3",
|
||||
"duration": 0.57,
|
||||
"description": "Punchy whoosh/impact — fast reveal or hard transition accent. Sync to the motion."
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,76 @@
|
||||
# Background music (BGM)
|
||||
|
||||
One music bed per composition, produced by the shared audio engine (`scripts/audio.mjs` → `scripts/lib/bgm.mjs`). Two routes, chosen by the engine's one switch — whether a HeyGen credential is present:
|
||||
|
||||
- **HeyGen retrieval — the default when credentialed.** Search HeyGen's music catalog by mood, download the top track. No generation; same `~/.heygen` / `$HEYGEN_API_KEY` credential as TTS.
|
||||
- **Local generation (Lyria → MusicGen) — the fallback when there is no credential** (or when asked for explicitly). Generate a WAV from a mood prompt. There is **no `npx hyperframes bgm` command**; the engine spawns `scripts/lyria-recipe.py` or an inline MusicGen script directly.
|
||||
|
||||
> **Run the Preflight first — no credential is not a green light to silently generate locally.** Before generating, complete the sign-in **Preflight** (see `../../SKILL.md` → Preflight): run `npx hyperframes auth status`, recommend signing in, and **STOP for the user's choice** (sign in for HeyGen's music library, or continue offline with local generation). This applies to a one-off "generate a BGM" request just as much as inside a full workflow.
|
||||
|
||||
## Driving it from the request
|
||||
|
||||
`audio_request.json` → `bgm: { mode?, query?, prompt? }`:
|
||||
|
||||
- **`mode`** — `retrieve | generate | none`. Omit for **auto** (retrieve when credentialed, else generate). An **explicit** `retrieve` is strict: no credential ⇒ skip, never a detached generate (so a caller with no `wait-bgm` step can't get a pending job it won't await).
|
||||
- **`query`** — the mood, used for retrieval and as a fallback prompt seed (e.g. a storyboard's `music:` field, falling back to `message` → `arc` → `"calm cinematic underscore"`).
|
||||
- **`prompt`** — an explicit full prompt for generation; omit and the engine infers one (see Mood inference). Optional `blob` / `archetype` / `arc` feed that inference.
|
||||
|
||||
Both routes keep a file of yours already at the output name: the engine writes the next free name (`track-2.mp3`), reports it as an anomaly, and `bgm.path` carries the real path.
|
||||
|
||||
## HeyGen retrieval (default)
|
||||
|
||||
`searchSounds(query, "music", { limit: 5 })` → `GET /audio/sounds?query=<mood>&type=music&limit=5`. Take the top result (ranked by `score`), download its presigned `audio_url` → `assets/bgm/track.mp3`. Synchronous. No match → skip (BGM is optional; never fail the render over it). Cue written to `audio_meta.json`:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"path": "assets/bgm/track.mp3",
|
||||
"volume": 0.12,
|
||||
"mode": "retrieve",
|
||||
"query": "calm cinematic underscore",
|
||||
"duration_s": 42.0,
|
||||
}
|
||||
```
|
||||
|
||||
`volume` comes from the engine's `bgmDefaultVolume()`: `BGM_BED_VOLUME` (currently `0.12` ≈ -18 dB — a bed under the voice) under narration, `BGM_SILENT_VOLUME` (currently `0.9`) for a silent film (no voice). Tune those constants in `scripts/lib/bgm.mjs`, not call sites. An explicit `volume` in `audio_meta.json` always overrides this default. `bgm_pending` is `false` — the file is on disk when the engine returns.
|
||||
|
||||
For short launch videos, do not assume the beginning of the retrieved file is the best edit point. Check the opening against later five-second sections. If the track starts with a quiet build but a later section has a stronger, clean musical entrance, trim from that section and apply a short fade-in and longer fade-out. Repeat this check whenever the composition duration changes; the final music file must cover the full cut without a silent tail.
|
||||
|
||||
## Local generation (fallback) — Lyria → MusicGen
|
||||
|
||||
Spawned **detached** so voice work isn't blocked; `audio_meta.bgm_pending: true` and `bgm_pid` / `bgm_log` are set until it finishes. **Run `scripts/wait-bgm.mjs` before assembling** — it polls the output file / process / log, detects crashes, and writes `bgm_status.json` (`status: ready | failed | timeout | disabled`). A failed/absent track is simply omitted; it never blocks voice/SFX.
|
||||
|
||||
| Order | Provider | Env / deps | Speed | Quality |
|
||||
| ----- | ------------------------------------ | ------------------------------------------------------------------------------------- | --------------------------------------- | --------------------------- |
|
||||
| 1 | Google Lyria RealTime | `$GEMINI_API_KEY` or `$GOOGLE_API_KEY` + `google-genai` (auto-installed on demand) | Real-time stream (≈ requested duration) | Production-grade |
|
||||
| 2 | MusicGen (`facebook/musicgen-small`) | Python `transformers + torch + soundfile + numpy` (~300 MB first run; auto-installed) | Slow on CPU; fast on Apple MPS / CUDA | Decent; prompt-only control |
|
||||
|
||||
Output → `assets/bgm/track.wav`, target = total voice duration. MusicGen generates **one** seed clip (≤28–30s, under the decoder's positional limit) then crossfade-loops it up to the target (or trims down if shorter), avoiding per-segment seams. Backend selection is by what can actually **run**: Lyria only when `import google.genai` succeeds, else MusicGen; if neither can be made to run, BGM is skipped (voice + SFX still render).
|
||||
|
||||
## Mood inference (the generate prompt)
|
||||
|
||||
`inferBgmPrompt()` in `scripts/lib/bgm.mjs`: an explicit `prompt` wins; otherwise industry-keyword **base** → narrative-**archetype** shape → emotional-**arc** tiebreaker.
|
||||
|
||||
| Match in `blob` / `query` | Base prompt | BPM |
|
||||
| ------------------------------------------------------ | --------------------------------------------------------------------------- | --- |
|
||||
| `crypto / nft / web3 / defi / token / blockchain` | atmospheric electronic, deep bass, futuristic synths, restrained percussion | 100 |
|
||||
| `finance / fintech / bank / payment / invest / wealth` | calm cinematic, soft strings, subtle piano, restrained percussion | 92 |
|
||||
| `creative / agency / design / studio / art / brand` | playful electronic, warm pads, light percussion | 115 |
|
||||
| _(default: SaaS / tech / platform)_ | uplifting corporate tech, bright modern piano with synth pads | 108 |
|
||||
|
||||
Archetype then reshapes the arc — PAS → "MINOR to MAJOR" build; BAB / future-pacing → aspirational rising; feature-cascade → +10 BPM driving; demo-loop → −8 BPM minimal. The emotional arc breaks remaining ties (tension→relief, excitement, trust/reassurance).
|
||||
|
||||
## Lyria knobs (direct recipe use)
|
||||
|
||||
The engine bakes BPM / scale into the **prompt text** (via the inference above) and passes only `--output` / `--duration` / `--prompt` to the recipe. If you invoke `scripts/lyria-recipe.py` directly you can also set: `--bpm` (90–110 calm, 110–130 energetic), `--brightness` (0–1, ≥0.7 promotional), `--density` (0–1, higher = fuller), `--scale` (`MAJOR` / `MINOR` / `PENTATONIC` / …), `--negative-prompt` (styles to exclude). MusicGen ignores all of these — put the mood in the prompt.
|
||||
|
||||
## Failure modes
|
||||
|
||||
| Failure | Behavior |
|
||||
| --------------------------------------------- | ---------------------------------------------------------------------------------------- |
|
||||
| No music match (retrieve) | `bgm: null`, anomaly logged. Render proceeds without BGM. |
|
||||
| Explicit `retrieve`, no credential | Skipped (no silent generate fallback). Use `mode: generate` or omit `mode` for auto. |
|
||||
| Neither Lyria nor MusicGen can run (generate) | `bgm` disabled with a `pip install …` hint. Voice + SFX still render. |
|
||||
| Generate still rendering at assemble time | `bgm_pending: true`; `wait-bgm.mjs` waits/checks and writes `bgm_status.json` first. |
|
||||
| Generate crashed | `wait-bgm.mjs` → `bgm_status.json { status: "failed" }`; the `<audio>` track is omitted. |
|
||||
|
||||
BGM failure never blocks a render.
|
||||
@@ -0,0 +1,162 @@
|
||||
# Captions
|
||||
|
||||
<!-- registry-items: allow=max-width,data-composition-src,hyperframes-registry,blend-mode,caption-style,font-family -->
|
||||
|
||||
**The live search is the source of truth for what the registry has.** The table(s) below are a hand-maintained sample and under-cover by design: run `npx hyperframes catalog --query "<what you want>" --json` — it needs nothing installed — before concluding the registry lacks something. Item names here are checked against `registry/registry.json` by `bun run lint:skills`.
|
||||
|
||||
Before authoring: confirm the transcript came from the right Whisper model. CLI default `small.en` silently translates non-English audio — see [`../transcribe.md`](../transcribe.md) → "Language Rule" and [`transcript-handling.md`](transcript-handling.md) for the mandatory quality check.
|
||||
|
||||
Analyze spoken content to determine caption style. If user specifies a style, use that. Otherwise, detect tone from the transcript.
|
||||
|
||||
## Transcript Source
|
||||
|
||||
```json
|
||||
[
|
||||
{ "id": "w0", "text": "Hello", "start": 0.0, "end": 0.5 },
|
||||
{ "id": "w1", "text": "world.", "start": 0.6, "end": 1.2 }
|
||||
]
|
||||
```
|
||||
|
||||
`id` (`w0`, `w1`, …) is the stable reference for per-word overrides and is added by `hyperframes transcribe`. It's optional for backwards compatibility with hand-authored transcripts. See [`../transcribe.md`](../transcribe.md) → "Output Shape" for how this is produced, and [`transcript-handling.md`](transcript-handling.md) for cleanup before consumption.
|
||||
|
||||
## Style Detection (When No Style Specified)
|
||||
|
||||
Read the full transcript before choosing. Four dimensions:
|
||||
|
||||
**1. Visual feel** — corporate→clean; energetic→bold; storytelling→elegant; technical→precise; social→playful.
|
||||
|
||||
**2. Color palette** — dark+bright for energy; muted for professional; high contrast for clarity; one accent color.
|
||||
|
||||
**3. Font mood** — heavy/condensed for impact; clean sans for modern; rounded for friendly; serif for elegance.
|
||||
|
||||
**4. Animation character** — scale-pop for punchy; gentle fade for calm; word-by-word for emphasis; typewriter for technical.
|
||||
|
||||
## Per-Word Styling
|
||||
|
||||
Scan for words deserving distinct treatment:
|
||||
|
||||
- **Brand/product names** — larger size, unique color
|
||||
- **ALL CAPS** — scale boost, flash, accent color
|
||||
- **Numbers/statistics** — bold weight, accent color
|
||||
- **Emotional keywords** — exaggerated animation (overshoot, bounce)
|
||||
- **Call-to-action** — highlight, underline, color pop
|
||||
- **Marker highlight** — for beyond-color emphasis (highlight sweep, circle, burst, scribble, sketchout), search the registry (`/hyperframes-registry`).
|
||||
|
||||
## Script-to-Style Mapping
|
||||
|
||||
| Tone | Font mood | Animation | Color | Size |
|
||||
| ------------ | ------------------------ | ---------------------------------- | --------------------------- | ------- |
|
||||
| Hype/launch | Heavy condensed, 800-900 | Scale-pop, back.out(1.7), 0.1-0.2s | Bright on dark | 72-96px |
|
||||
| Corporate | Clean sans, 600-700 | Fade+slide, power3.out, 0.3s | White/neutral, muted accent | 56-72px |
|
||||
| Tutorial | Mono/clean sans, 500-600 | Typewriter/fade, 0.4-0.5s | High contrast, minimal | 48-64px |
|
||||
| Storytelling | Serif/elegant, 400-500 | Slow fade, power2.out, 0.5-0.6s | Warm muted tones | 44-56px |
|
||||
| Social | Rounded sans, 700-800 | Bounce, elastic.out, word-by-word | Playful, colored pills | 56-80px |
|
||||
|
||||
## Word Grouping
|
||||
|
||||
- **High energy:** 2-3 words. Quick turnover.
|
||||
- **Conversational:** 3-5 words. Natural phrases.
|
||||
- **Measured/calm:** 4-6 words. Longer groups.
|
||||
|
||||
Break on sentence boundaries, 150ms+ pauses, or max word count.
|
||||
|
||||
## Positioning
|
||||
|
||||
- **Landscape (1920x1080):** Bottom 80-120px, centered
|
||||
- **Portrait (1080x1920):** Lower middle ~600-700px from bottom, centered
|
||||
- Never cover the subject's face
|
||||
- `position: absolute` — never relative
|
||||
- One caption group visible at a time
|
||||
|
||||
## Text Overflow Prevention
|
||||
|
||||
Use `window.__hyperframes.fitTextFontSize()`:
|
||||
|
||||
```js
|
||||
var result = window.__hyperframes.fitTextFontSize(group.text.toUpperCase(), {
|
||||
fontFamily: "Outfit",
|
||||
fontWeight: 900,
|
||||
maxWidth: 1600,
|
||||
});
|
||||
el.style.fontSize = result.fontSize + "px";
|
||||
```
|
||||
|
||||
Options: `maxWidth` (1600 landscape, 900 portrait), `baseFontSize` (78), `minFontSize` (42), `fontWeight`, `fontFamily`, `step` (2).
|
||||
|
||||
CSS safety nets: `max-width` on container, `overflow: visible` (**not** `hidden` — hidden clips scaled emphasis words and glow effects), `position: absolute`, explicit `height`. When per-word styling uses `scale > 1.0`, compute `maxWidth = safeWidth / maxScale` to leave headroom.
|
||||
|
||||
**Container pattern:** Full-width absolute container, centered. Do **not** use `left: 50%; transform: translateX(-50%)` — causes clipping at composition edges.
|
||||
|
||||
## Caption Exit Guarantee
|
||||
|
||||
Every group **must** have a hard kill after exit animation:
|
||||
|
||||
```js
|
||||
tl.to(groupEl, { opacity: 0, scale: 0.95, duration: 0.12, ease: "power2.in" }, group.end - 0.12);
|
||||
// `tl.set` is an instant flip, not a tween — safe to set `visibility` here (core's "no animating
|
||||
// visibility" rule applies to tweens, which can't smoothly interpolate non-numeric values anyway).
|
||||
tl.set(groupEl, { opacity: 0, visibility: "hidden" }, group.end);
|
||||
```
|
||||
|
||||
Self-lint after building timeline — place **before** `window.__timelines[id] = tl` so it runs at composition init:
|
||||
|
||||
```js
|
||||
GROUPS.forEach(function (group, gi) {
|
||||
var el = document.getElementById("cg-" + gi);
|
||||
if (!el) return;
|
||||
tl.seek(group.end + 0.01);
|
||||
var computed = window.getComputedStyle(el);
|
||||
if (computed.opacity !== "0" && computed.visibility !== "hidden") {
|
||||
console.warn(
|
||||
"[caption-lint] group " + gi + " still visible at t=" + (group.end + 0.01).toFixed(2) + "s",
|
||||
);
|
||||
}
|
||||
});
|
||||
tl.seek(0);
|
||||
```
|
||||
|
||||
## Pre-Built Caption Components
|
||||
|
||||
Before building caption styles from scratch, check the registry — 15 ready-to-use caption components cover the most common styles. Install with `npx hyperframes add <name>` and wire as a sub-composition via `data-composition-src` (see `hyperframes-registry`).
|
||||
|
||||
```bash
|
||||
npx hyperframes catalog --tag caption-style # list all caption components
|
||||
npx hyperframes add caption-highlight # install a specific one
|
||||
```
|
||||
|
||||
| Style | Component | Best for |
|
||||
| ------------------------- | ---------------------------- | ---------------------------- |
|
||||
| TikTok-style highlight | `caption-highlight` | Social, high-energy |
|
||||
| Karaoke pill | `caption-pill-karaoke` | Music, lyric videos |
|
||||
| Cinematic editorial | `caption-editorial-emphasis` | Documentary, storytelling |
|
||||
| Glitch / cyber | `caption-glitch-rgb` | Tech, gaming |
|
||||
| Full-screen slam | `caption-kinetic-slam` | Hype, announcements |
|
||||
| Neon glow | `caption-neon-glow` | Night, club, neon aesthetics |
|
||||
| Neon accent (multi-color) | `caption-neon-accent` | Colorful, playful |
|
||||
| Wipe reveal | `caption-clip-wipe` | Clean, modern |
|
||||
| Gradient fill | `caption-gradient-fill` | Vibrant, eye-catching |
|
||||
| Matrix decode | `caption-matrix-decode` | Sci-fi, tech reveals |
|
||||
| Emoji pop | `caption-emoji-pop` | Social, casual |
|
||||
| Parallax layers | `caption-parallax-layers` | Depth, cinematic |
|
||||
| Particle burst | `caption-particle-burst` | Celebration, impact keywords |
|
||||
| Lava texture | `caption-texture` | Bold, dramatic |
|
||||
| Weight shift | `caption-weight-shift` | Elegant, typographic |
|
||||
|
||||
Related: `caption-blend-difference` (tagged `text` / `blend-mode`, not `caption-style`, so it won't appear under the filter above) auto-inverts text against any background via `mix-blend-mode: difference` — useful when the background is busy or unpredictable.
|
||||
|
||||
Browse all with previews: [hyperframes.heygen.com/catalog](https://hyperframes.heygen.com/catalog)
|
||||
|
||||
Caption components ship with transparent backgrounds — they're pure overlays. If the underlying video is bright or busy, add a contrast layer (e.g. a semi-transparent dark div) in the host composition beneath the caption sub-composition, not inside the component itself.
|
||||
|
||||
## Further References
|
||||
|
||||
- [`motion.md`](motion.md) — karaoke, marker effects, audio-reactive modulation, scatter exits.
|
||||
- [`transcript-handling.md`](transcript-handling.md) — input formats, quality checks, cleaning, external API fallback.
|
||||
|
||||
## Constraints
|
||||
|
||||
- Deterministic. No `Math.random()`, no `Date.now()`.
|
||||
- Sync to transcript timestamps.
|
||||
- One group visible at a time.
|
||||
- Every group must have a hard `tl.set` kill at `group.end`.
|
||||
- Fonts: the compiler auto-embeds only its **built-in mapped set** (Inter, Roboto, Montserrat, …) — for those, just declare `font-family` in CSS. Any **other** font (a brand/custom font like `TT Norms Pro`, or a non-Latin CJK/Devanagari family) is **not** auto-supplied: it needs an `@font-face` pointing at a real `.woff2` shipped with the project, or the text silently falls back to a generic font in the render. Don't assume a `font-family` you can see locally will render — the render machine is a clean headless Chrome with no installed fonts.
|
||||
@@ -0,0 +1,87 @@
|
||||
# Dynamic Caption Techniques
|
||||
|
||||
You are here because SKILL.md told you to read this file before writing animation code. Pick your technique combination from the table below based on the energy level you detected from the transcript, then implement using standard GSAP patterns.
|
||||
|
||||
## Technique Selection by Energy
|
||||
|
||||
| Energy level | Highlight | Exit | Cycle pattern |
|
||||
| ------------ | ------------------------------------- | ------------------- | ----------------------------------------- |
|
||||
| High | Karaoke with accent glow + scale pop | Scatter or drop | Alternate highlight styles every 2 groups |
|
||||
| Medium-high | Karaoke with color pop | Scatter or collapse | Alternate every 3 groups |
|
||||
| Medium | Karaoke (subtle, white only) | Fade + slide | Alternate every 3 groups |
|
||||
| Medium-low | Karaoke (minimal scale change) | Fade | Single style, vary ease per group |
|
||||
| Low | Karaoke (warm tones, slow transition) | Collapse | Alternate every 4 groups |
|
||||
|
||||
**All energy levels use karaoke highlight as the baseline.** The difference is intensity — high energy gets accent color + glow + 15% scale pop on active words, low energy gets a gentle white shift with 3% scale.
|
||||
|
||||
**Emphasis words always break the pattern.** When a word is flagged as emphasis (emotional keyword, ALL CAPS, brand name), give it a stronger animation than surrounding words (larger scale, accent color, overshoot ease). This creates contrast.
|
||||
|
||||
**Marker highlight modes add a visual layer on top of karaoke.** For emphasis words that need more than color/scale, add a marker-style effect: highlight sweep, circle, burst, scribble, or sketchout (search the registry, `/hyperframes-registry`). Match mode to energy: burst for hype, circle for key terms, highlight for standard, scribble for subtle.
|
||||
|
||||
## Audio-Reactive Captions (Mandatory for Music)
|
||||
|
||||
**If the source audio is music (vocals over instrumentation, beats, any musical content), you MUST extract audio data and add audio-reactive animations.** This is not optional — music without audio reactivity looks disconnected. Even low-energy ballads get subtle bass pulse and treble glow.
|
||||
|
||||
No special wiring is needed. The group loop already iterates over every caption group to build entrance, karaoke, and exit tweens. At that point, read the audio data for each group's time range and use it to modulate the group's animation intensity with regular GSAP tweens.
|
||||
|
||||
```js
|
||||
// Load audio data inline (same pattern as TRANSCRIPT)
|
||||
var AUDIO = JSON.parse(audioDataJson); // { fps, totalFrames, frames: [{ bands: [...] }] }
|
||||
|
||||
GROUPS.forEach(function (group, gi) {
|
||||
var groupEl = document.getElementById("cg-" + gi);
|
||||
if (!groupEl) return;
|
||||
|
||||
// Read peak energy for this group's time range
|
||||
var startFrame = Math.floor(group.start * AUDIO.fps);
|
||||
var endFrame = Math.min(Math.floor(group.end * AUDIO.fps), AUDIO.totalFrames - 1);
|
||||
var peakBass = 0;
|
||||
var peakTreble = 0;
|
||||
for (var f = startFrame; f <= endFrame; f++) {
|
||||
var frame = AUDIO.frames[f];
|
||||
if (!frame) continue;
|
||||
peakBass = Math.max(peakBass, frame.bands[0] || 0, frame.bands[1] || 0);
|
||||
peakTreble = Math.max(peakTreble, frame.bands[6] || 0, frame.bands[7] || 0);
|
||||
}
|
||||
|
||||
// Modulate entrance — louder groups enter bigger and glowier
|
||||
tl.to(
|
||||
groupEl,
|
||||
{
|
||||
scale: 1 + peakBass * 0.06,
|
||||
textShadow:
|
||||
"0 0 " + Math.round(peakTreble * 12) + "px rgba(255,255,255," + peakTreble * 0.4 + ")",
|
||||
duration: 0.3,
|
||||
ease: "power2.out",
|
||||
},
|
||||
group.start,
|
||||
);
|
||||
|
||||
// Reset at exit so audio-driven values don't persist
|
||||
tl.set(groupEl, { scale: 1, textShadow: "none" }, group.end - 0.15);
|
||||
});
|
||||
```
|
||||
|
||||
This shapes the animation at build time, not playback time — no per-frame callbacks, no `tl.call()` loops, no async fetch timing issues. Loud groups come in with more weight and glow; quiet groups come in soft. The audio data modulates _how much_, the content determines _what_.
|
||||
|
||||
Keep audio reactivity subtle — 3-6% scale variation and soft glow. Heavy pulsing makes text unreadable.
|
||||
|
||||
To generate the audio data file:
|
||||
|
||||
```bash
|
||||
python3 <SKILL_DIR>/audio/scripts/extract-audio-data.py audio.mp3 --fps 30 --bands 8 -o audio-data.json
|
||||
```
|
||||
|
||||
## Combining Techniques
|
||||
|
||||
Don't use the same highlight animation on every group — cycle through styles using the group index. Don't combine multiple competing animations on the same word at the same timestamp. Vary techniques across groups to match the content's pace changes.
|
||||
|
||||
**Marker highlight effects** layer well with karaoke — use karaoke for the word-by-word reveal, then add a marker effect on emphasis words only. For example: karaoke highlights each word in white, but brand names get a yellow highlight sweep and stats get a red circle. Cycle marker modes across groups for visual variety.
|
||||
|
||||
## Runtime Tools
|
||||
|
||||
Caption motion uses standard HyperFrames runtime APIs. Use the canonical sources:
|
||||
|
||||
- **GSAP timeline + tween syntax** — `/hyperframes` → `references/runtimes.md` and `references/motion.md`
|
||||
- **Fitting caption text** — `/hyperframes` → `references/captions.md`
|
||||
- **Audio data extraction** — generate via `python3 <SKILL_DIR>/audio/scripts/extract-audio-data.py audio.mp3 --fps 30 --bands 8 -o audio-data.json`, then load inline as shown in "Audio-Reactive Captions" above
|
||||
@@ -0,0 +1,97 @@
|
||||
# Transcript Guide
|
||||
|
||||
For the `transcribe` CLI invocation, the `.en`-translates-non-English rule, and whisper model selection, see [`../transcribe.md`](../transcribe.md). This file covers what to do with the resulting transcript when authoring captions: input formats, mandatory quality checks, cleaning code, external-API fallbacks.
|
||||
|
||||
## Supported Input Formats
|
||||
|
||||
The CLI auto-detects and normalizes these formats:
|
||||
|
||||
| Format | Extension | Source | Word-level? |
|
||||
| --------------------- | --------- | --------------------------------------------------------------------------- | ----------------- |
|
||||
| whisper.cpp JSON | `.json` | `hyperframes init --video`, `hyperframes transcribe` | Yes |
|
||||
| OpenAI Whisper API | `.json` | `openai.audio.transcriptions.create({ timestamp_granularities: ["word"] })` | Yes |
|
||||
| SRT subtitles | `.srt` | Video editors, subtitle tools, YouTube | No (phrase-level) |
|
||||
| VTT subtitles | `.vtt` | Web players, YouTube, transcription services | No (phrase-level) |
|
||||
| Normalized word array | `.json` | Pre-processed by any tool | Yes |
|
||||
|
||||
**Word-level timestamps produce better captions.** SRT/VTT give phrase-level timing, which works but can't do per-word animation effects.
|
||||
|
||||
## Transcript Quality Check (Mandatory)
|
||||
|
||||
After every transcription, **read the transcript and check for quality issues before proceeding.** Bad transcripts produce nonsensical captions. Never skip this step.
|
||||
|
||||
### What to look for
|
||||
|
||||
| Signal | Example | Cause |
|
||||
| ---------------------------- | -------------------------------------- | ---------------------------------------------------------------------------- |
|
||||
| Music note tokens (`♪`, `�`) | `{ "text": "♪" }` or `{ "text": "�" }` | Whisper detected music, not speech |
|
||||
| Garbled / nonsense words | "Do a chin", "Get so gay", "huh" | Model misheard lyrics or background noise |
|
||||
| Long gaps with no words | 20+ seconds of only `♪` tokens | Instrumental section — expected, but high ratio means speech is being missed |
|
||||
| Repeated filler | Many "huh", "uh", "oh" entries | Model is hallucinating on music |
|
||||
| Very short word spans | Words with `end - start < 0.05` | Unreliable timestamp alignment |
|
||||
|
||||
### Automatic retry rules
|
||||
|
||||
**If more than 20% of entries are `♪`/`�` tokens, or the transcript contains obvious nonsense words, the transcription failed.** Do not proceed with the bad transcript. Instead:
|
||||
|
||||
1. **Retry with `medium.en`** if the original used `small.en` or smaller:
|
||||
```bash
|
||||
npx hyperframes transcribe audio.mp3 --model medium.en
|
||||
```
|
||||
2. **If `medium.en` also fails** (still >20% music tokens or garbled), tell the user the audio is too noisy for local transcription and suggest:
|
||||
- Providing lyrics manually as an SRT/VTT file
|
||||
- Using an external API (OpenAI or Groq Whisper — see below)
|
||||
3. **Always clean the transcript** before building captions — filter out `♪`/`�` tokens and entries where `text` is a single non-word character. Only real words should reach the caption composition.
|
||||
|
||||
### Cleaning a transcript
|
||||
|
||||
After transcription (even with a good model), strip non-word entries:
|
||||
|
||||
```js
|
||||
var raw = JSON.parse(transcriptJson);
|
||||
var words = raw.filter(function (w) {
|
||||
if (!w.text || w.text.trim().length === 0) return false;
|
||||
if (/^[♪�\u266a\u266b\u266c\u266d\u266e\u266f]+$/.test(w.text)) return false;
|
||||
if (/^(huh|uh|um|ah|oh)$/i.test(w.text) && w.end - w.start < 0.1) return false;
|
||||
return true;
|
||||
});
|
||||
```
|
||||
|
||||
For model-selection guidance by content type, see [`../transcribe.md`](../transcribe.md) → "Picking a model by content type".
|
||||
|
||||
## Using External Transcription APIs
|
||||
|
||||
For the best accuracy, use an external API and import the result:
|
||||
|
||||
**OpenAI Whisper API** (recommended for quality):
|
||||
|
||||
```bash
|
||||
# Generate with word timestamps, then import
|
||||
curl https://api.openai.com/v1/audio/transcriptions \
|
||||
-H "Authorization: Bearer $OPENAI_API_KEY" \
|
||||
-F file=@audio.mp3 -F model=whisper-1 \
|
||||
-F response_format=verbose_json \
|
||||
-F "timestamp_granularities[]=word" \
|
||||
-o transcript-openai.json
|
||||
|
||||
npx hyperframes transcribe transcript-openai.json
|
||||
```
|
||||
|
||||
**Groq Whisper API** (fast, free tier available):
|
||||
|
||||
```bash
|
||||
curl https://api.groq.com/openai/v1/audio/transcriptions \
|
||||
-H "Authorization: Bearer $GROQ_API_KEY" \
|
||||
-F file=@audio.mp3 -F model=whisper-large-v3 \
|
||||
-F response_format=verbose_json \
|
||||
-F "timestamp_granularities[]=word" \
|
||||
-o transcript-groq.json
|
||||
|
||||
npx hyperframes transcribe transcript-groq.json
|
||||
```
|
||||
|
||||
## If No Transcript Exists
|
||||
|
||||
1. Check the project root for `transcript.json`, `.srt`, or `.vtt` files.
|
||||
2. If none found, run [`../transcribe.md`](../transcribe.md) — pick the starting model from "Picking a model by content type" there.
|
||||
3. Run the quality check above. If it fails, retry with a larger model or fall back to manual lyrics / external API.
|
||||
@@ -0,0 +1,143 @@
|
||||
# Background Removal
|
||||
|
||||
Make a transparent overlay (typical: a talking head over an arbitrary scene). Uses `u2net_human_seg` (Apache-2.0).
|
||||
|
||||
```bash
|
||||
npx hyperframes remove-background subject.mp4 -o transparent.webm # default: VP9 + alpha
|
||||
npx hyperframes remove-background subject.mp4 -o transparent.mov # ProRes 4444 (editing)
|
||||
npx hyperframes remove-background portrait.jpg -o cutout.png # single-image cutout
|
||||
npx hyperframes remove-background subject.mp4 -o subject.webm \
|
||||
--background-output plate.webm # both layers, one pass
|
||||
npx hyperframes remove-background subject.mp4 -o transparent.webm --device cpu
|
||||
npx hyperframes remove-background --info # detected providers
|
||||
```
|
||||
|
||||
## Output Format
|
||||
|
||||
- **`.webm` (VP9 alpha)** — default. Plug straight into `<video>` for Chrome-native transparent playback (~1 MB / 4s @ 1080p).
|
||||
- **`.mov` (ProRes 4444)** — round-trip in editors (Premiere / Resolve / DaVinci). ~50 MB / 4s.
|
||||
- **`.png`** — single-image cutout.
|
||||
|
||||
## Quality (`--quality`)
|
||||
|
||||
Controls VP9 encoder CRF only — segmentation quality is fixed. Higher quality keeps the cutout's RGB closer to the source MP4 (important when overlaying the cutout on its own source).
|
||||
|
||||
| Preset | CRF | When |
|
||||
| ---------- | --- | --------------------------------------------- |
|
||||
| `fast` | 30 | Iterating, smaller files, looser color match |
|
||||
| `balanced` | 18 | **Default**; visually identical for most uses |
|
||||
| `best` | 12 | Master / final delivery, tightest color match |
|
||||
|
||||
## Device (`--device`)
|
||||
|
||||
`auto` (default) picks CoreML on Apple Silicon, CUDA when available, otherwise CPU. Force with `--device cpu | coreml | cuda`. CUDA requires `HYPERFRAMES_CUDA=1` plus a GPU-enabled `onnxruntime-node` build. Use `--info` to inspect detected providers without rendering.
|
||||
|
||||
## Compositing patterns — pick the right one
|
||||
|
||||
The cutout WebM is a **re-encoded copy** of the source MP4's RGB. What sits behind it matters.
|
||||
|
||||
| Pattern | Behind the cutout | Result |
|
||||
| -------------------------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------------------------------------------- |
|
||||
| **Cutout over a different scene** (most common) | Static image, gradient, unrelated video | Looks great. Single RGB source for the subject. |
|
||||
| **Cutout over its own source mp4** (text-behind-subject) | Same mp4 the cutout came from | At `balanced` doubling is barely visible; at `fast` you'll see color shift / edge halo. Use `best` for masters. |
|
||||
| **Cutout over a different take of the same person** | Footage of the same subject | **Two overlapping people. Don't do this.** |
|
||||
|
||||
## Text-behind-subject pattern (two non-obvious rules)
|
||||
|
||||
Putting a headline behind a presenter cutout:
|
||||
|
||||
```html
|
||||
<video
|
||||
src="presenter.mp4"
|
||||
id="bg"
|
||||
data-start="0"
|
||||
data-duration="6"
|
||||
data-track-index="0"
|
||||
muted
|
||||
playsinline
|
||||
></video>
|
||||
|
||||
<h1 id="headline" style="z-index:2; ...">MAKE IT IN HYPERFRAMES</h1>
|
||||
|
||||
<div class="cutout-wrap" style="position:absolute; inset:0; z-index:3; opacity:0">
|
||||
<video
|
||||
src="presenter.webm"
|
||||
data-start="0"
|
||||
data-duration="6"
|
||||
data-track-index="1"
|
||||
muted
|
||||
playsinline
|
||||
></video>
|
||||
</div>
|
||||
```
|
||||
|
||||
```js
|
||||
// Flip the wrapper's opacity at the cut, NOT the video's
|
||||
tl.set(".cutout-wrap", { opacity: 1 }, 3.3);
|
||||
```
|
||||
|
||||
Two rules that are easy to miss:
|
||||
|
||||
1. **Wrap the cutout `<video>` in a non-timed `<div>` and animate the wrapper's opacity, not the video element's.** The framework forces `opacity: 1` on active clips (any element with `data-start` / `data-duration`), so animating the video's opacity directly is silently overridden. The wrapper has no `data-*` attributes, so it's owned by your CSS / GSAP.
|
||||
2. **Both videos use `data-start="0"` and `data-media-start="0"`** so the framework decodes them in sync from t=0. Late-mounting the cutout (`data-start=3.3`) introduces a seek + warm-up that lands a frame off the base mp4 — visible as one frame of misalignment at the cut.
|
||||
|
||||
## Layer separation (`--background-output`)
|
||||
|
||||
Emits a **second** transparent video alongside the cutout: same source RGB, alpha is `255 - mask` instead of `mask`. The cutout has the subject opaque; the plate has the surroundings opaque (with a transparent hole where the subject was). Use it when text / graphics need to live **between** the two layers.
|
||||
|
||||
| File | Alpha is… | Use it for |
|
||||
| -------------------------------- | ------------------------------------------------------- | ---------------------------------------------------------------- |
|
||||
| `-o subject.webm` | mask — subject opaque, background transparent | Foreground layer (top) |
|
||||
| `--background-output plate.webm` | inverse mask — surroundings opaque, subject transparent | Bottom layer; place text / graphics between this and the subject |
|
||||
|
||||
Both share the same `--quality` and run from a single inference pass — only encode cost roughly doubles. Only valid for video inputs with `.webm` / `.mov` outputs.
|
||||
|
||||
**Hole-cut, not inpainted.** The subject region in `plate.webm` is fully transparent — composite something opaque under it to fill the hole.
|
||||
|
||||
**Single test for whether `--background-output` is the right tool:** _will anything ever be visible through the subject's silhouette where the subject used to be?_ If no, you don't need the plate — `subject.webm` alone over a different background is enough.
|
||||
|
||||
### Use case → right tool
|
||||
|
||||
| Use case | Right tool |
|
||||
| ----------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- |
|
||||
| Text/graphics between the cutout and the plate (this command's reason for existing) | **Hole-cut** (`--background-output`) |
|
||||
| Subject onto an unrelated scene | Just `subject.webm`; ignore the plate |
|
||||
| Show the room _without_ the person, alone over no other content | **Clean plate** — needs an inpainter (LaMa, ProPainter, E2FGVI). Not this command. |
|
||||
| Replace the subject with a different subject | **Clean plate** — same as above |
|
||||
|
||||
### Canonical 3-layer template (plate + content + cutout)
|
||||
|
||||
Ship just the two transparent layers and let arbitrary content live between them — no original mp4 needed:
|
||||
|
||||
```html
|
||||
<!-- z=1 plate: surroundings opaque, subject silhouette transparent -->
|
||||
<video
|
||||
src="plate.webm"
|
||||
data-start="0"
|
||||
data-duration="6"
|
||||
data-track-index="0"
|
||||
muted
|
||||
playsinline
|
||||
></video>
|
||||
|
||||
<!-- z=2 your content lives between the layers -->
|
||||
<h1 id="headline" style="z-index:2; ...">MAKE IT IN HYPERFRAMES</h1>
|
||||
|
||||
<!-- z=3 cutout floats the subject back on top -->
|
||||
<div class="cutout-wrap" style="position:absolute; inset:0; z-index:3">
|
||||
<video
|
||||
src="subject.webm"
|
||||
data-start="0"
|
||||
data-duration="6"
|
||||
data-track-index="1"
|
||||
muted
|
||||
playsinline
|
||||
></video>
|
||||
</div>
|
||||
```
|
||||
|
||||
Functionally equivalent to the text-behind-subject pattern above, but doesn't require shipping the original mp4 — the plate replaces it. Use this when delivering just the two transparent layers as a reusable asset.
|
||||
|
||||
## When `remove-background` is NOT the right tool
|
||||
|
||||
If a user asks for "the room **without** the person, displayed standalone" (no subject anywhere, no compositing on top), `--background-output` is wrong — its plate has a transparent hole, not a filled-in clean plate. They need an **inpainter**: LaMa, ProPainter, or E2FGVI. Tell them this command can't do it.
|
||||
@@ -0,0 +1,29 @@
|
||||
# Requirements & Caches
|
||||
|
||||
## Credential & key priority
|
||||
|
||||
Run `npx hyperframes auth status` to see what's configured and which engines a workflow will use (see the skill's **Preflight** section). Keys resolve in this order — **first match wins**:
|
||||
|
||||
| Provider | Resolution order (first non-empty wins) | Local deps when used |
|
||||
| -------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ |
|
||||
| **HeyGen** (TTS + BGM retrieval) | `$HEYGEN_API_KEY` → `$HYPERFRAMES_API_KEY` → `~/.heygen/credentials` (shared with heygen-cli; `$HEYGEN_CONFIG_DIR` overrides the dir; written by `hyperframes auth login`) | none (REST) |
|
||||
| **ElevenLabs** (TTS fallback) | `$ELEVENLABS_API_KEY` | `pip install elevenlabs` |
|
||||
| **Lyria** (BGM fallback) | `$GEMINI_API_KEY` → `$GOOGLE_API_KEY` | `pip install google-genai` |
|
||||
| **Kokoro** (TTS, no key) | always — final voice fallback | `pip install kokoro-onnx soundfile` |
|
||||
| **MusicGen** (BGM, no key) | always — final music fallback | `pip install transformers torch soundfile numpy` |
|
||||
|
||||
`hyperframes auth login` (browser OAuth) is the recommended setup: one sign-in, every project, no per-repo `.env`. An OAuth login is sent as `Authorization: Bearer`; an API key as `X-Api-Key`; both are tagged with `X-HeyGen-Source: cli`. OAuth CLI users can consume the web-plan free allowance for HeyGen TTS (10 min/month); API keys follow the normal API billing path. With no HeyGen credential, voice/BGM run fully locally (Kokoro / MusicGen) — `hyperframes auth status` and `hyperframes doctor` both report whether those local deps are installed.
|
||||
|
||||
## Model caches & system dependencies
|
||||
|
||||
Each command downloads its own model on first run and caches it under `~/.cache/hyperframes/`:
|
||||
|
||||
- **TTS (HeyGen)** — no local deps; needs a HeyGen credential + `ffmpeg` on PATH (to transcode the mp3 response to `.wav`). Credential resolves like the CLI: `$HEYGEN_API_KEY` → `$HYPERFRAMES_API_KEY` → `~/.heygen/credentials` (shared with heygen-cli; run `npx hyperframes auth login`). An OAuth login is sent as `Authorization: Bearer`; an API key as `X-Api-Key`; both include `X-HeyGen-Source: cli` so the backend can apply CLI OAuth free usage.
|
||||
- **TTS (ElevenLabs)** — same as HeyGen: API key + `ffmpeg`.
|
||||
- **TTS (Kokoro)** — Kokoro-82M (~311 MB) + voices (~27 MB) in `tts/`. Requires Python 3.8+ with `kokoro-onnx` and `soundfile` (`pip install kokoro-onnx soundfile`). Non-English text also needs `espeak-ng` system-wide.
|
||||
- **BGM (Lyria)** — needs `$GEMINI_API_KEY` or `$GOOGLE_API_KEY` + `pip install google-genai`. No local model cache.
|
||||
- **BGM (MusicGen)** — `pip install transformers torch soundfile`. `facebook/musicgen-small` (~300 MB) cached under `~/.cache/huggingface/` on first run.
|
||||
- **Transcribe** — Whisper model size depending on choice (75 MB – 3.1 GB) in `whisper/`, downloaded from HuggingFace on first use. `whisper.cpp` itself is NOT bundled: the CLI resolves it from PATH, installs via Homebrew (macOS), or builds it from source with git+cmake on first use (`$HYPERFRAMES_WHISPER_PATH` overrides).
|
||||
- **Remove-background** — `u2net_human_seg` (~168 MB ONNX) in `background-removal/models/`. Peak inference RAM ~1.5 GB.
|
||||
|
||||
Run `npx hyperframes doctor` if a command fails because of a missing dependency.
|
||||
@@ -0,0 +1,48 @@
|
||||
# Sound effects (SFX)
|
||||
|
||||
Named sound effects, produced by the shared audio engine (`scripts/audio.mjs` → `scripts/lib/sfx.mjs`). No account or key is needed:
|
||||
|
||||
- **Openverse first, per cue.** Each cue is searched on [Openverse](https://openverse.org)'s free audio API (most sound effects come from Freesound). Search-and-download, **not** generation.
|
||||
- **The bundled 21-file library is the fallback** (`assets/sfx/` + `manifest.json`): a cue Openverse has no usable match for, and every cue once Openverse is offline or rate limited, is matched against it and the file copied into the project.
|
||||
|
||||
There is no `npx hyperframes sfx` command. SFX is never generated — it is retrieved from Openverse (online) or taken from the bundled library (offline). `npx hyperframes media-use resolve --type sfx` follows the same order (`--local-only` skips Openverse).
|
||||
|
||||
## Licences
|
||||
|
||||
Only **CC0** and **CC BY** sounds are used: never NonCommercial, ShareAlike or NoDerivatives. Each CC BY sound is credited in the project's `assets/sfx/CREDITS.md` (title, creator, licence, source link; for `resolve`, in a `CREDITS.md` beside the sound files it saves) and its `audio_meta.sfx[]` entry carries `license` and `credit`. Tell the person to include those credits wherever the video is published.
|
||||
|
||||
## Cues — request → meta
|
||||
|
||||
Each line names the effects it wants: `lines[].sfx: ["whoosh", "ui click"]`. The engine flattens these into cues, resolves each, dedupes identical `(id, name)` pairs (the same effect named twice downloads/copies once), and writes `audio_meta.sfx[]`:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"id": "3", // joins the cue to the caller's model (frame / scene / segment)
|
||||
"name": "whoosh",
|
||||
"file": "assets/sfx/whoosh.mp3", // downloaded or copied, relative to project root
|
||||
"source": "openverse" | "local", // which route resolved it
|
||||
"offset_s": 0, // delay from the line's start
|
||||
"duration_s": 0.57,
|
||||
"volume": 0.35, // SFX sit UNDER voice + BGM
|
||||
"license": "by", // Openverse only: "cc0" or "by"
|
||||
"credit": "\"Swoosh\" by Ana is licensed under CC BY 4.0. …" // Openverse only
|
||||
}
|
||||
```
|
||||
|
||||
A cue that matches nothing is **skipped** (recorded as an anomaly); SFX never blocks a render. Neither route replaces a file of yours already at the output name: the cue gets the next free name (`whoosh-2.mp3`), reported as an anomaly, and `file` carries the real path.
|
||||
|
||||
## Openverse
|
||||
|
||||
One search per distinct effect name (20 results, `license=cc0,by`; each result's licence is checked again). The pick keeps Openverse's relevance order, prefers clips that fit a cue (15 s at most; past ~4 s ranks lower) and CC0 over CC BY when they are close. The file lands at `assets/sfx/<slug>.<ext>` and `duration_s` comes from the result. Name effects concretely (`glass shatter`, not `dramatic sound`); a vague query returns a poor match. Anonymous use allows 20 searches a minute and 200 a day: past that, the rest of the run uses the bundled library.
|
||||
|
||||
## Bundled library (fallback)
|
||||
|
||||
21 curated files in `assets/sfx/`, indexed by `manifest.json` — `{ file, duration, description }` per key (e.g. `whoosh`, `pop`, `click`, `chime`, `riser`, `impact-bass-1`, `glitch-1`, `typing`, …). A cue name resolves by **manifest key, file basename, or slug**, so `whoosh`, `whoosh.mp3`, or `"ui click"` (→ slug) all match. Matched files are copied into the project's `assets/sfx/`; `duration_s` comes from the manifest, so timing is known **offline** — e.g. `riser` is 10.03s, so trigger it at `climax − 10.03s`. The manifest's `description` field carries placement hints per effect; read `assets/sfx/manifest.json` for the full set and usage.
|
||||
|
||||
## Rules
|
||||
|
||||
- **Volume ~0.35.** SFX must sit under narration and BGM, not fight them.
|
||||
- **No match → skip, don't fail.** A missing effect logs an anomaly and moves on; never a render blocker.
|
||||
- **Retrieval (Openverse) or bundled library (offline) — never generation.**
|
||||
- **One asset per distinct name.** Reuse across lines is deduped to a single search and download, many cues.
|
||||
- **Credit CC BY sounds.** `CREDITS.md` lists them; keep it with the project.
|
||||
@@ -0,0 +1,52 @@
|
||||
# Transcription
|
||||
|
||||
Create normalized word-level timestamps. **Always specify `--model` explicitly** — the CLI default is `small.en`, which silently translates non-English audio into English.
|
||||
|
||||
```bash
|
||||
npx hyperframes transcribe audio.mp3 --model small.en # known English
|
||||
npx hyperframes transcribe video.mp4 --model small --language es # known Spanish
|
||||
npx hyperframes transcribe audio.mp3 --model small # unknown language (auto-detect)
|
||||
npx hyperframes transcribe subtitles.srt # import existing
|
||||
npx hyperframes transcribe subtitles.vtt
|
||||
npx hyperframes transcribe openai-response.json
|
||||
```
|
||||
|
||||
## Language Rule (Non-Negotiable)
|
||||
|
||||
`.en` models (`tiny.en` / `base.en` / `small.en` / `medium.en`) **translate** non-English audio into English. This silently destroys the original language.
|
||||
|
||||
1. **Known English** → `--model small.en` (or `medium.en` for music / noisy audio)
|
||||
2. **Known non-English** → `--model small --language <iso-code>` (no `.en` suffix)
|
||||
3. **Unknown language** → `--model small` (whisper auto-detects)
|
||||
|
||||
**CLI default is `small.en`** — do not rely on it; always pass `--model` to make the choice explicit. `--language` also filters out non-target-language segments from mixed-language audio.
|
||||
|
||||
## Model Sizes
|
||||
|
||||
| Model | Size | Speed | When |
|
||||
| ---------- | ------ | -------- | ------------------------------------- |
|
||||
| `tiny` | 75 MB | Fastest | Quick previews, smoke tests |
|
||||
| `base` | 142 MB | Fast | Short clips, clear audio |
|
||||
| `small` | 466 MB | Moderate | Default for most multilingual content |
|
||||
| `medium` | 1.5 GB | Slow | Music with vocals, noisy audio |
|
||||
| `large-v3` | 3.1 GB | Slowest | Production quality |
|
||||
|
||||
### Picking a model by content type
|
||||
|
||||
1. Speech over silence / light background → `small.en`
|
||||
2. Speech over music, or music with vocals → start with `medium.en`
|
||||
3. Produced music track (vocals + full instrumentation) → start with `medium.en`; expect to need manual lyrics or an external API ([`captions/transcript-handling.md`](captions/transcript-handling.md) → "Using External Transcription APIs")
|
||||
4. Multilingual → `medium` or `large-v3` (no `.en` suffix), pair with `--language`
|
||||
|
||||
## Output Shape
|
||||
|
||||
Compositions consume a flat array of word objects. The `id` (`w0`, `w1`, …) is added during normalization for stable references in caption overrides; optional for backwards compatibility.
|
||||
|
||||
```json
|
||||
[
|
||||
{ "id": "w0", "text": "Hello", "start": 0.0, "end": 0.5 },
|
||||
{ "id": "w1", "text": "world.", "start": 0.6, "end": 1.2 }
|
||||
]
|
||||
```
|
||||
|
||||
For mandatory caption-quality checks, retry rules, and the OpenAI/Groq Whisper API import path, see `captions/transcript-handling.md`.
|
||||
@@ -0,0 +1,35 @@
|
||||
# TTS → Captions
|
||||
|
||||
When no recorded voiceover exists, generate one and obtain word-level caption timing. Two paths depending on which TTS provider is in use:
|
||||
|
||||
## Path A — HeyGen (single call, no Whisper)
|
||||
|
||||
HeyGen returns word timestamps in the same response as the audio. Use the
|
||||
bundled REST helper (the `hyperframes tts` command is Kokoro-only):
|
||||
|
||||
```bash
|
||||
node skills/media-use/audio/scripts/heygen-tts.mjs \
|
||||
script.txt --output narration.wav --words narration.words.json
|
||||
```
|
||||
|
||||
`narration.words.json` is already in the `[{ id, text, start, end }]` shape the captions pipeline consumes — no separate transcribe pass.
|
||||
|
||||
## Path B — Gemini / ElevenLabs / Kokoro (TTS → transcription)
|
||||
|
||||
These adapters supply audio without word data. The shared audio engine runs
|
||||
transcription automatically when timings are absent. For Gemini, use the
|
||||
request in [Text to speech](tts.md#gemini-narration), then consume
|
||||
`audio_meta.json` → `voices[].words`.
|
||||
|
||||
For a standalone local Kokoro generation, generate the audio, then transcribe:
|
||||
|
||||
```bash
|
||||
npx hyperframes tts script.txt --voice af_heart --output narration.wav
|
||||
npx hyperframes transcribe narration.wav --model small.en # voice af_heart is American English
|
||||
```
|
||||
|
||||
Whisper extracts precise word boundaries from the generated audio, so caption timing matches delivery without hand-tuning. Match `--model` to the voice's language (use `small.en` for `a`/`b` prefixes, `small --language <code>` otherwise). Then consume `transcript.json` via the caption references in `captions/`.
|
||||
|
||||
For Gemini, verify that transcription preserved the script, especially names,
|
||||
numbers, and delivery pauses. If `words` is empty, resolve the transcription
|
||||
failure before captioning. Generate and align again after changing the read.
|
||||
@@ -0,0 +1,257 @@
|
||||
# Text To Speech
|
||||
|
||||
`npx hyperframes tts` synthesizes locally with Kokoro. It does not accept a
|
||||
`--provider` or `--words` flag. For HeyGen audio plus word timestamps, use the
|
||||
bundled `heygen-tts.mjs` script below.
|
||||
|
||||
> **Run the Preflight first — no credential is not a green light to silently use the local voice.** Before generating a voiceover, complete the sign-in **Preflight** (see `../../SKILL.md` → Preflight): run `npx hyperframes auth status`, recommend signing in, and **STOP for the user's choice** (sign in for HeyGen voices, or continue offline with local Kokoro). This applies to a one-off "generate a voiceover" request just as much as inside a full workflow.
|
||||
|
||||
## Narrating a HyperFrames docs video
|
||||
|
||||
Anything that will sit on hyperframes.heygen.com uses one narrator, so the site
|
||||
does not sound like two products.
|
||||
|
||||
| | |
|
||||
| -------------- | ---------------------------------------------- |
|
||||
| Voice | **River** — "Relaxed, Neutral, Informative" |
|
||||
| Provider | ElevenLabs |
|
||||
| `voice_id` | `SAz9YHcvj6GT2YYXdXww` |
|
||||
| Model | `eleven_multilingual_v2` |
|
||||
| Pace | 145–155 words per minute, with room to breathe |
|
||||
| Music under it | about −31 LUFS, never masking the voice |
|
||||
|
||||
```bash
|
||||
curl -s -X POST "https://api.elevenlabs.io/v1/text-to-speech/SAz9YHcvj6GT2YYXdXww" \
|
||||
-H "xi-api-key: $ELEVENLABS_API_KEY" -H "Content-Type: application/json" \
|
||||
-d '{"text":"...","model_id":"eleven_multilingual_v2"}' -o take.mp3
|
||||
```
|
||||
|
||||
This is the voice every user-journey film on the docs site already uses. Falling
|
||||
back to local Kokoro because a key was not to hand produces a film that sounds
|
||||
wrong beside the others — three docs videos were built that way and had to be
|
||||
re-voiced. If you cannot reach ElevenLabs, say so and stop rather than
|
||||
substituting a different voice.
|
||||
|
||||
Use another voice only for a documented reason, and write the reason down.
|
||||
|
||||
## Available routes
|
||||
|
||||
Gemini is an explicit alternative to the automatic provider order below. A
|
||||
request to use Gemini already chooses the provider; do not redirect that user
|
||||
to HeyGen sign-in. Read the Gemini section for its credential requirement.
|
||||
|
||||
| Order | Provider | Env trigger | Voice IDs | Word timestamps | Audio format |
|
||||
| ----- | ----------------- | -------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | ----------------------------------------- | -------------------- |
|
||||
| 1 | HeyGen (Starfish) | `$HEYGEN_ACCESS_TOKEN` / `$HEYGEN_API_KEY` / `~/.heygen/credentials` | UUIDs from `GET /v3/voices?engine=starfish`, or your cloned voice ids (`heygen-voice.mjs`) | **Yes** (`word_timestamps[]` in response) | mp3 → wav via ffmpeg |
|
||||
| 2 | ElevenLabs | `$ELEVENLABS_API_KEY` | UUIDs from elevenlabs.io dashboard | No | mp3 → wav via ffmpeg |
|
||||
| 3 | Kokoro-82M | always (local fallback) | `am_michael`, `af_heart`, … (54 voices) | No | wav direct |
|
||||
|
||||
```bash
|
||||
# Local Kokoro CLI
|
||||
npx hyperframes tts "Welcome to HyperFrames" -o narration.wav
|
||||
```
|
||||
|
||||
## Self-contained HeyGen (no CLI) — `scripts/heygen-tts.mjs`
|
||||
|
||||
The published `hyperframes tts` CLI synthesizes locally with Kokoro only. When you
|
||||
want HeyGen specifically — best quality **plus** word timestamps in one call — use
|
||||
the skill's bundled script, which calls the HeyGen v3 REST API directly and needs
|
||||
no CLI provider plumbing:
|
||||
|
||||
The script resolves a HeyGen credential the same way the CLI does — first source
|
||||
wins: a host-injected OAuth `$HEYGEN_ACCESS_TOKEN` → `$HEYGEN_API_KEY` →
|
||||
`$HYPERFRAMES_API_KEY` → a project `.env` (auto-loaded,
|
||||
walks up ≤5 dirs) → `~/.heygen/credentials` (shared with heygen-cli;
|
||||
`$HEYGEN_CONFIG_DIR` overrides the dir). An OAuth login is sent as
|
||||
`Authorization: Bearer`; an API key as `X-Api-Key`; both include
|
||||
`X-HeyGen-Source: cli`. OAuth CLI users can consume the web-plan free allowance
|
||||
(10 min/month) before paid usage; API keys follow normal API billing. If the
|
||||
only credential is an expired OAuth token it stops with a hint to run
|
||||
`npx hyperframes auth refresh`.
|
||||
|
||||
```bash
|
||||
# Only needed if you haven't run `npx hyperframes auth login`:
|
||||
export HEYGEN_API_KEY=... # or put it in a project .env
|
||||
|
||||
# Synthesize + capture word timestamps in one call (skips a Whisper pass)
|
||||
node skills/media-use/audio/scripts/heygen-tts.mjs \
|
||||
"Welcome to HyperFrames." -o narration.wav --words narration.words.json
|
||||
|
||||
node skills/media-use/audio/scripts/heygen-tts.mjs ./script.txt -o narration.wav
|
||||
node skills/media-use/audio/scripts/heygen-tts.mjs --list # public starfish voices
|
||||
```
|
||||
|
||||
- **Voice:** `--voice <id>` must be a **starfish** voice_id (`--list`, or `GET /v3/voices?engine=starfish`) or one of your cloned voice ids (below). v2-catalog ids are rejected with HTTP 400. Omit `--voice` (English) and it defaults to **Marcia** (`05f19352e8f74b0392a8f411eba40de1`, a fixed default so the choice is deterministic). Non-English with no `--voice` falls back to the first matching catalog voice.
|
||||
- **Output:** `.wav` → transcoded to 44.1k mono via ffmpeg; `.mp3` → raw bytes (no ffmpeg needed).
|
||||
- **Words:** `--words <path>` writes the flat `[{id,text,start,end}]` shape below, drop-in for the captions pipeline. HeyGen's `<start>`/`<end>` boundary sentinels are filtered out and ids are re-contiguous.
|
||||
- **Non-English:** `--lang <code>` (anything but `en`) is sent as the request `language`.
|
||||
|
||||
## Cloned voices — `scripts/heygen-voice.mjs`
|
||||
|
||||
Clone a voice from an mp3 or wav recording, then pass the printed id to
|
||||
`heygen-tts.mjs --voice`. Same credential resolution as above. Any API refusal
|
||||
(clone limit reached, plan upgrade required, voice not found) prints HeyGen's own
|
||||
message to stderr and exits 1.
|
||||
|
||||
```bash
|
||||
node skills/media-use/audio/scripts/heygen-voice.mjs clone take.mp3 --name "My voice" # → {"voice_id":"..."}
|
||||
node skills/media-use/audio/scripts/heygen-voice.mjs list --prefix "My" # → [{"voice_id","name","created_at"}]
|
||||
node skills/media-use/audio/scripts/heygen-voice.mjs delete <voice_id>
|
||||
```
|
||||
|
||||
`clone` waits up to 120 s for the clone to finish. `list` reads
|
||||
`GET /v3/voices?type=private`.
|
||||
|
||||
## When to use which provider
|
||||
|
||||
| Goal | Use |
|
||||
| --------------------------------------------------------- | --------------------------------------------------------- |
|
||||
| Best voice quality + word timestamps in one call | **HeyGen** |
|
||||
| Drop-in cloud TTS, big voice catalog | **ElevenLabs** |
|
||||
| Offline, no API key, fast iteration | **Kokoro** |
|
||||
| Directed delivery with Gemini prebuilt or custom voices | **Gemini** (explicit selection; transcription for timing) |
|
||||
| Non-English multilingual with deterministic phonemization | **Kokoro** (`ef_dora`, `jf_alpha`, `zf_xiaobei`, …) |
|
||||
|
||||
## Gemini narration
|
||||
|
||||
Use the shared audio engine, not `hyperframes tts`. Authenticate with either:
|
||||
|
||||
- `GEMINI_API_KEY` or `GOOGLE_API_KEY` (first set key wins).
|
||||
- A service-account JSON file at `GOOGLE_APPLICATION_CREDENTIALS`, or injected
|
||||
JSON in `GCS_CREDS`. Install `google-auth requests` in the Python 3 environment
|
||||
used by the helper. A configured file takes precedence over injected JSON.
|
||||
|
||||
API keys take precedence over service accounts: unset both key variables to use
|
||||
OAuth. Never put credentials in a request file or composition. The helper
|
||||
obtains a fresh OAuth token for each generation with the
|
||||
`generative-language.retriever` scope. The quota project resolves from
|
||||
`GOOGLE_CLOUD_PROJECT`, then `GCLOUD_PROJECT_ID`, then the service-account JSON.
|
||||
User ADC files and metadata-server authentication are not supported.
|
||||
|
||||
Both routes call the Gemini Developer Interactions API, not Cloud TTS or Vertex
|
||||
AI. Those APIs have separate model catalogs and access requirements.
|
||||
|
||||
Save this as `audio_request.json` in the project:
|
||||
|
||||
```json
|
||||
{
|
||||
"provider": "gemini",
|
||||
"tts_model": "gemini-3.8-flash-tts",
|
||||
"voice": "Kore",
|
||||
"lang": "en",
|
||||
"style": "Warm, clear, conversational. Leave a short pause between sentences.",
|
||||
"lines": [
|
||||
{ "id": "intro", "text": "Every word has a moment. Let the picture follow the voice." }
|
||||
],
|
||||
"bgm": { "mode": "none" }
|
||||
}
|
||||
```
|
||||
|
||||
```bash
|
||||
node <SKILL_DIR>/audio/scripts/audio.mjs \
|
||||
--request ./audio_request.json --hyperframes . --out ./audio_meta.json --only tts
|
||||
```
|
||||
|
||||
The engine saves `assets/voice/intro.wav` (or `intro-2.wav` when a file of yours already has that name; `voices[].path` says which), measures its duration, and transcribes
|
||||
it into `voices[].words` in `audio_meta.json`. Check that every requested line
|
||||
has audio and nonempty word timings before building a captioned video. Review
|
||||
the timings against the actual audio; transcription is estimated alignment,
|
||||
not native TTS timestamps. Do not distribute words evenly across a clip.
|
||||
|
||||
- **Models:** `gemini-3.8-flash-tts` (default), `gemini-3.8-flash-lite-tts`,
|
||||
`gemini-3.1-flash-tts-preview`, `gemini-2.5-pro-preview-tts`, and
|
||||
`gemini-2.5-flash-preview-tts`. Use these exact Developer API IDs; Cloud TTS
|
||||
aliases such as `gemini-2.5-flash-tts` are not accepted.
|
||||
- **Voice:** `Kore` by default; pass another prebuilt voice or an existing custom
|
||||
voice ID for 3.8. Older models require prebuilt voices. Creating or replicating
|
||||
voices is outside this helper.
|
||||
- **Delivery:** Put directions in `style`, not in spoken `text`. Each line can
|
||||
override `style`. 3.8 uses structured annotations; older models receive a
|
||||
delivery prompt before the transcript. Check that directions were not spoken.
|
||||
Use style for pacing; numeric `speed` must be omitted or 1.
|
||||
- **Audio:** 3.8 returns a complete WAV. Older models return mono 16-bit PCM,
|
||||
which the helper wraps as WAV at the returned sample rate without resampling.
|
||||
No generation calls run during playback or rendering.
|
||||
- **Timing:** This adapter requests no native word timestamps. It uses the
|
||||
existing transcription pass. `lang` selects transcription language; Gemini
|
||||
infers speech language from the text.
|
||||
|
||||
An API error is reported as a failed line; an explicitly chosen Gemini voice
|
||||
never silently falls back to another provider. Check the engine's anomalies
|
||||
and output metadata, not only its exit code.
|
||||
|
||||
API contract: [Google's speech generation guide](https://ai.google.dev/gemini-api/docs/speech-generation).
|
||||
|
||||
## ffmpeg requirement
|
||||
|
||||
HeyGen + ElevenLabs return mp3. The bundled HeyGen helper transcodes to wav
|
||||
when `--output` ends in `.wav` (the default and what downstream `ffprobe` +
|
||||
Whisper expect). If you'd rather skip the transcode, pass `-o file.mp3`.
|
||||
Without `ffmpeg` on PATH, wav output from cloud providers fails; the local
|
||||
Kokoro CLI writes wav directly.
|
||||
|
||||
## Voice selection (Kokoro)
|
||||
|
||||
Default `af_heart`. Curated picks:
|
||||
|
||||
| Content type | Voice |
|
||||
| ----------------- | ---------------------- |
|
||||
| Product demo | `af_heart`, `af_nova` |
|
||||
| Tutorial / how-to | `am_adam`, `bf_emma` |
|
||||
| Marketing / promo | `af_sky`, `am_michael` |
|
||||
| Documentation | `bf_emma`, `bm_george` |
|
||||
| Casual / social | `af_heart`, `af_sky` |
|
||||
|
||||
Run `npx hyperframes tts --list` for the bundled set.
|
||||
|
||||
## Multilingual (Kokoro voice prefix → language)
|
||||
|
||||
The first letter of a Kokoro voice ID picks the phonemizer language; `--lang` overrides auto-detection.
|
||||
|
||||
| Prefix | Language |
|
||||
| ------ | -------------------- |
|
||||
| `a` | American English |
|
||||
| `b` | British English |
|
||||
| `e` | Spanish |
|
||||
| `f` | French |
|
||||
| `h` | Hindi |
|
||||
| `i` | Italian |
|
||||
| `j` | Japanese |
|
||||
| `p` | Brazilian Portuguese |
|
||||
| `z` | Mandarin |
|
||||
|
||||
```bash
|
||||
npx hyperframes tts "La reunión empieza a las nueve" --voice ef_dora
|
||||
npx hyperframes tts "Today is a nice day" --voice af_heart
|
||||
```
|
||||
|
||||
Valid `--lang` codes (only needed to override the voice's auto-detected language): `en-us`, `en-gb`, `es`, `fr-fr`, `hi`, `it`, `pt-br`, `ja`, `zh`.
|
||||
|
||||
Non-English phonemization requires `espeak-ng` system-wide (`brew install espeak-ng` / `apt-get install espeak-ng`).
|
||||
|
||||
## Speed
|
||||
|
||||
- `0.7-0.8` — tutorial, complex content, accessibility
|
||||
- `1.0` — natural pace (default)
|
||||
- `1.1-1.2` — intros, transitions, upbeat content
|
||||
- `1.5+` — rarely appropriate, test carefully
|
||||
|
||||
The `hyperframes tts` command honors `--speed` for Kokoro. Provider-specific
|
||||
helpers document their own pacing controls.
|
||||
|
||||
## Long scripts
|
||||
|
||||
Past a few paragraphs, write the text to a `.txt` file and pass the path. Inputs over ~5 minutes of speech may benefit from splitting into segments.
|
||||
|
||||
## HeyGen word-timestamp shape
|
||||
|
||||
When `--words <path>` is passed to a HeyGen call, the file is written in the same flat shape `transcribe` produces — drop-in compatible with the captions pipeline:
|
||||
|
||||
```json
|
||||
[
|
||||
{ "id": "w0", "text": "Hi", "start": 0.0, "end": 0.21 },
|
||||
{ "id": "w1", "text": "there", "start": 0.22, "end": 0.55 }
|
||||
]
|
||||
```
|
||||
|
||||
For ElevenLabs / Kokoro, run `npx hyperframes transcribe narration.wav --model small.en` to get the same shape.
|
||||
@@ -0,0 +1,310 @@
|
||||
#!/usr/bin/env node
|
||||
// audio.mjs — the shared HyperFrames audio engine. ONE implementation of TTS +
|
||||
// BGM + SFX. Callers do NOT vendor a copy: they write a neutral
|
||||
// `audio_request.json` (mapping their storyboard/scenes into it) and call:
|
||||
//
|
||||
// node <MEDIA_DIR>/scripts/audio.mjs --request ./audio_request.json --hyperframes . --out ./audio_meta.json
|
||||
//
|
||||
// TTS and BGM degrade on ONE switch — whether HeyGen is configured (credential
|
||||
// present, NOT the CLI). SFX needs no credential. This mirrors ../SKILL.md:
|
||||
//
|
||||
// TTS : HeyGen REST → ElevenLabs → Kokoro (CLI)
|
||||
// BGM : HeyGen retrieve → (no credential) Lyria/MusicGen generate
|
||||
// SFX : Openverse (CC0 / CC BY) → (offline, rate limited, no match) bundled library
|
||||
//
|
||||
// ── audio_request.json (input) ────────────────────────────────────────────────
|
||||
// {
|
||||
// "provider": "auto", // auto|heygen|elevenlabs|kokoro|gemini (override: --provider)
|
||||
// "lang": "en", "speed": 1.0,
|
||||
// "lines": [ // one TTS unit each; id joins back to the caller's model
|
||||
// { "id": "01", "text": "...", "sfx": ["whoosh", "ui click"] }
|
||||
// ],
|
||||
// "bgm": { "mode": "retrieve", // retrieve|generate|none (override: --bgm-mode / --no-bgm)
|
||||
// "query": "calm cinematic underscore", // mood for retrieval
|
||||
// "prompt": null, // full prompt for generation (else inferred)
|
||||
// "blob": "...", "archetype": "...", "arc": "..." } // optional mood-inference hints
|
||||
// }
|
||||
//
|
||||
// ── audio_meta.json (output, id-keyed) ───────────────────────────────────────
|
||||
// { tts_provider, voice_id,
|
||||
// bgm: { path, volume, mode, query?, duration_s? } | null,
|
||||
// bgm_pending, bgm_provider, bgm_pid, bgm_log, bgm_mode, bgm_target_duration_s, …,
|
||||
// voices: [ { id, path, duration_s, words: [{id,text,start,end}] } ],
|
||||
// sfx: [ { id, name, file, source, offset_s, duration_s, volume, license?, credit? } ],
|
||||
// total_duration_s }
|
||||
//
|
||||
// --only tts,bgm,sfx runs a subset and MERGES into an existing --out (so a
|
||||
// caller can do TTS+BGM early, then SFX later once cues exist). When BGM uses
|
||||
// the generate path it is spawned detached (bgm_pending:true) — run wait-bgm.mjs
|
||||
// before assembling.
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { heygenAuthHeaders, heygenCredential, loadEnvFromDir } from "./lib/heygen.mjs";
|
||||
import {
|
||||
ffprobeDuration,
|
||||
pickProvider,
|
||||
resolveVoiceId,
|
||||
synthesizeOne,
|
||||
transcribeWav,
|
||||
withWordIds,
|
||||
} from "./lib/tts.mjs";
|
||||
import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs";
|
||||
import { BUNDLED_SFX_DIR, resolveSfx } from "./lib/sfx.mjs";
|
||||
import { mapWithConcurrency } from "./lib/concurrency.mjs";
|
||||
import { openAudioMeta } from "./lib/audio-meta.mjs";
|
||||
import { recordInManifest, voicePaths, writtenAssets } from "./lib/media-record.mjs";
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name, def) => {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : def;
|
||||
};
|
||||
const has = (name) => argv.includes(`--${name}`);
|
||||
const die = (m) => {
|
||||
console.error(`✗ audio engine: ${m}`);
|
||||
process.exit(1);
|
||||
};
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
|
||||
// Two independent reports of an unbounded Promise.all over TTS lines
|
||||
// overwhelming a machine: one OOM'd 12/13 concurrent Kokoro TTS +
|
||||
// whisper-transcribe lines on a resource-constrained laptop, the other saw
|
||||
// 7/8 lines fail on first run (concurrent cold-start model loads) and pass on
|
||||
// retry once the model was cached. Kokoro/Whisper each load their own local
|
||||
// model per subprocess, so firing every line at once multiplies that cost by
|
||||
// the line count. mapWithConcurrency caps how many run at once — still
|
||||
// parallel, just bounded.
|
||||
const ttsConcurrency = Math.max(1, Number(process.env.HYPERFRAMES_TTS_CONCURRENCY) || 4);
|
||||
|
||||
const hyperframesDir = resolve(flag("hyperframes", "."));
|
||||
const requestPath = resolve(flag("request", join(hyperframesDir, "audio_request.json")));
|
||||
const outPath = resolve(flag("out", join(hyperframesDir, "audio_meta.json")));
|
||||
const sfxLibDir = resolve(flag("sfx-lib", BUNDLED_SFX_DIR));
|
||||
const lyriaRecipe = resolve(flag("lyria-recipe", join(HERE, "lyria-recipe.py")));
|
||||
const onlyArg = flag("only", "tts,bgm,sfx");
|
||||
const only = new Set(
|
||||
onlyArg
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean),
|
||||
);
|
||||
const providerOverride = flag("provider", null);
|
||||
const bgmModeOverride = flag("bgm-mode", null);
|
||||
const noBgm = has("no-bgm");
|
||||
const voiceOverride = flag("voice", null);
|
||||
const speedOverride = flag("speed", null);
|
||||
const langOverride = flag("lang", null);
|
||||
const seedSeconds = Number(flag("seed-seconds", "28")) || 28;
|
||||
|
||||
if (!existsSync(requestPath)) die(`audio_request.json not found at ${requestPath}`);
|
||||
let request;
|
||||
try {
|
||||
request = JSON.parse(readFileSync(requestPath, "utf8"));
|
||||
} catch (e) {
|
||||
die(`audio_request.json parse: ${e.message}`);
|
||||
}
|
||||
const lines = Array.isArray(request.lines) ? request.lines : [];
|
||||
const lang = langOverride || request.lang || "en";
|
||||
const speedInput = speedOverride ?? request.speed ?? 1;
|
||||
const speed = Number(speedInput);
|
||||
if (!(speed > 0 && speed <= 3))
|
||||
die(`speed must be above 0 and at most 3, got ${JSON.stringify(speedInput)}`);
|
||||
|
||||
// ── env + HeyGen availability (the single switch) ─────────────────────────────
|
||||
loadEnvFromDir(hyperframesDir);
|
||||
const heygenOK = heygenCredential() !== null;
|
||||
|
||||
// ── merge base: preserve sections not selected by --only ──────────────────────
|
||||
const audioMeta = openAudioMeta(outPath);
|
||||
const prev = audioMeta.value;
|
||||
const anomalies = [];
|
||||
|
||||
// ── TTS ───────────────────────────────────────────────────────────────────────
|
||||
let voices = prev.voices ?? [];
|
||||
let ttsProvider = prev.tts_provider ?? null;
|
||||
let voiceId = prev.voice_id ?? null;
|
||||
if (only.has("tts") && lines.length) {
|
||||
try {
|
||||
ttsProvider = pickProvider(
|
||||
providerOverride || (request.provider === "auto" ? null : request.provider),
|
||||
);
|
||||
} catch (e) {
|
||||
die(e.message);
|
||||
}
|
||||
voiceId = await resolveVoiceId({
|
||||
provider: ttsProvider,
|
||||
userVoice: voiceOverride || request.voice,
|
||||
lang,
|
||||
});
|
||||
console.error(`· tts: ${ttsProvider} · voice ${voiceId} · ${lines.length} line(s)`);
|
||||
const paths = voicePaths(hyperframesDir, lines, anomalies);
|
||||
const synthLine = async (line) => {
|
||||
const id = String(line.id);
|
||||
const text = String(line.text ?? "").trim();
|
||||
if (!text) {
|
||||
anomalies.push(`line ${id}: empty text — skipped`);
|
||||
return null;
|
||||
}
|
||||
const rel = paths.get(id);
|
||||
const abs = join(hyperframesDir, rel);
|
||||
const { ok, words, error } = await synthesizeOne({
|
||||
provider: ttsProvider,
|
||||
text,
|
||||
voiceId,
|
||||
lang,
|
||||
speed,
|
||||
model: flag("tts-model", request.tts_model),
|
||||
style: flag("style", line.style ?? request.style),
|
||||
wavAbs: abs,
|
||||
hyperframesDir,
|
||||
});
|
||||
if (!ok) {
|
||||
anomalies.push(`line ${id}: TTS failed — omitted${error ? ` (${error})` : ""}`);
|
||||
return null;
|
||||
}
|
||||
let wordArr = words; // heygen: native; else transcribe
|
||||
if (!wordArr) wordArr = await transcribeWav({ wavRel: rel, lang, hyperframesDir });
|
||||
const dur = ffprobeDuration(abs);
|
||||
if (!isFinite(dur) || dur <= 0) {
|
||||
anomalies.push(`line ${id}: bad voice duration — omitted`);
|
||||
return null;
|
||||
}
|
||||
return { id, path: rel, duration_s: r3(dur), words: withWordIds(wordArr) };
|
||||
};
|
||||
const results = await mapWithConcurrency(lines, ttsConcurrency, synthLine);
|
||||
voices = results.filter(Boolean);
|
||||
for (const v of voices)
|
||||
console.error(` voice ${v.id}: ${v.path} (${v.duration_s}s, ${v.words.length} words)`);
|
||||
}
|
||||
const hasVoice = voices.length > 0;
|
||||
const totalDuration = r3(voices.reduce((a, v) => a + (v.duration_s || 0), 0));
|
||||
|
||||
// ── BGM ─────────────────────────────────────────────────────────────────────
|
||||
let bgm = prev.bgm ?? null;
|
||||
const bgmFields = {
|
||||
bgm_pending: prev.bgm_pending ?? false,
|
||||
bgm_provider: prev.bgm_provider ?? null,
|
||||
bgm_pid: prev.bgm_pid ?? null,
|
||||
bgm_log: prev.bgm_log ?? null,
|
||||
bgm_mode: prev.bgm_mode ?? null,
|
||||
bgm_target_duration_s: prev.bgm_target_duration_s ?? null,
|
||||
bgm_seed_duration_s: prev.bgm_seed_duration_s ?? null,
|
||||
bgm_loop_count: prev.bgm_loop_count ?? null,
|
||||
};
|
||||
if (only.has("bgm")) {
|
||||
bgm = null;
|
||||
Object.keys(bgmFields).forEach((k) => (bgmFields[k] = k === "bgm_pending" ? false : null));
|
||||
// Mode resolution. An EXPLICIT mode (flag or request.bgm.mode) is strict:
|
||||
// "retrieve" means retrieve-or-nothing — it never silently starts a detached
|
||||
// generate (a caller with no wait-bgm step must not get a pending job it
|
||||
// can't await). Only the UNSET/auto default picks generate
|
||||
// when HeyGen is absent.
|
||||
const explicitMode = bgmModeOverride || request.bgm?.mode || null;
|
||||
let mode = noBgm ? "none" : explicitMode || (heygenOK ? "retrieve" : "generate");
|
||||
if (mode === "retrieve" && !heygenOK) {
|
||||
anomalies.push(
|
||||
"bgm: retrieve requires a HeyGen credential — skipped (no generate fallback for an explicit retrieve)",
|
||||
);
|
||||
mode = "none";
|
||||
}
|
||||
|
||||
if (mode === "none") {
|
||||
console.error(`· bgm: disabled`);
|
||||
} else if (mode === "retrieve") {
|
||||
try {
|
||||
bgm = await retrieveBgm({
|
||||
query: request.bgm?.query,
|
||||
headers: heygenAuthHeaders(),
|
||||
hyperframesDir,
|
||||
hasVoice,
|
||||
anomalies,
|
||||
});
|
||||
if (bgm) {
|
||||
bgmFields.bgm_provider = "heygen";
|
||||
bgmFields.bgm_mode = "retrieve";
|
||||
console.error(` bgm: ${bgm.path} (retrieve "${bgm.query}")`);
|
||||
} else {
|
||||
anomalies.push(`bgm: no music match for "${request.bgm?.query ?? ""}" — skipped`);
|
||||
}
|
||||
} catch (e) {
|
||||
anomalies.push(`bgm retrieve failed: ${e.message} — skipped`);
|
||||
}
|
||||
} else {
|
||||
// generate
|
||||
const prompt = inferBgmPrompt({
|
||||
userPrompt: request.bgm?.prompt,
|
||||
blob: request.bgm?.blob || request.bgm?.query,
|
||||
archetype: request.bgm?.archetype,
|
||||
arc: request.bgm?.arc,
|
||||
});
|
||||
const gen = generateBgmDetached({
|
||||
prompt,
|
||||
durationS: totalDuration || 30,
|
||||
hyperframesDir,
|
||||
lyriaRecipe: existsSync(lyriaRecipe) ? lyriaRecipe : null,
|
||||
seedSeconds,
|
||||
hasVoice,
|
||||
anomalies,
|
||||
});
|
||||
if (gen.disabled) {
|
||||
anomalies.push(`bgm: ${gen.reason}`);
|
||||
} else {
|
||||
bgm = { path: gen.path, volume: gen.volume, mode: gen.mode, duration_s: null };
|
||||
bgmFields.bgm_pending = true;
|
||||
bgmFields.bgm_provider = gen.provider;
|
||||
bgmFields.bgm_pid = gen.pid;
|
||||
bgmFields.bgm_log = gen.log;
|
||||
bgmFields.bgm_mode = gen.mode;
|
||||
bgmFields.bgm_target_duration_s = gen.target_duration_s ?? null;
|
||||
bgmFields.bgm_seed_duration_s = gen.seed_duration_s ?? null;
|
||||
bgmFields.bgm_loop_count = gen.loop_count ?? null;
|
||||
console.error(` bgm: launched ${gen.provider} (detached, pid ${gen.pid}) → ${gen.path}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── SFX ─────────────────────────────────────────────────────────────────────
|
||||
let sfx = prev.sfx ?? [];
|
||||
if (only.has("sfx")) {
|
||||
const cues = lines.flatMap((l) =>
|
||||
(Array.isArray(l.sfx) ? l.sfx : [])
|
||||
.map((name) => ({ id: String(l.id), name: String(name).trim() }))
|
||||
.filter((c) => c.name),
|
||||
);
|
||||
const res = await resolveSfx({ cues, hyperframesDir, sfxLibDir });
|
||||
sfx = res.sfx;
|
||||
anomalies.push(...res.anomalies);
|
||||
const openverse = sfx.filter((s) => s.source === "openverse").length;
|
||||
console.error(
|
||||
`· sfx: ${sfx.length} cue(s) resolved (${openverse} Openverse, ${sfx.length - openverse} bundled library)`,
|
||||
);
|
||||
}
|
||||
|
||||
// ── write audio_meta.json ─────────────────────────────────────────────────────
|
||||
const meta = {
|
||||
tts_provider: ttsProvider,
|
||||
voice_id: voiceId,
|
||||
bgm,
|
||||
...bgmFields,
|
||||
voices,
|
||||
sfx,
|
||||
total_duration_s: totalDuration,
|
||||
};
|
||||
mkdirSync(dirname(outPath), { recursive: true });
|
||||
audioMeta.write(meta);
|
||||
const written = writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields, sfx });
|
||||
anomalies.push(...recordInManifest(hyperframesDir, written));
|
||||
|
||||
console.log(`✓ audio engine → ${outPath}`);
|
||||
console.log(` heygen: ${heygenOK ? "yes" : "no"} · ran: ${[...only].join(",")}`);
|
||||
console.log(
|
||||
` voices: ${voices.length} · bgm: ${bgm ? `${bgmFields.bgm_provider}${bgmFields.bgm_pending ? " (pending)" : ""}` : "none"} · sfx: ${sfx.length}`,
|
||||
);
|
||||
console.log(` total voice duration: ${totalDuration}s`);
|
||||
if (anomalies.length) {
|
||||
console.log(`\nanomalies (non-fatal):`);
|
||||
for (const a of anomalies) console.log(` - ${a}`);
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import { test } from "node:test";
|
||||
import { mkdtempSync, mkdirSync, readFileSync, rmSync, existsSync, writeFileSync } from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { join, dirname } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { recordInManifest } from "./lib/media-record.mjs";
|
||||
import { resolveSfx } from "./lib/sfx.mjs";
|
||||
|
||||
// Proves the relocated engine (skills/media-use/audio/) still resolves its
|
||||
// bundled SFX library from the moved location — the path most likely to break
|
||||
// on a subtree move. Openverse is unreachable unless a test serves it.
|
||||
const offline = async () => {
|
||||
throw new TypeError("fetch failed");
|
||||
};
|
||||
globalThis.fetch = offline;
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const sfxLibDir = join(HERE, "..", "assets", "sfx"); // same offset the engine uses
|
||||
|
||||
test("bundled SFX library resolves from the relocated path", async () => {
|
||||
assert.ok(existsSync(join(sfxLibDir, "manifest.json")), "moved manifest is present");
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "1", name: "whoosh" }],
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
assert.equal(sfx.length, 1, `expected 1 resolved cue, got anomalies: ${anomalies.join("; ")}`);
|
||||
assert.equal(sfx[0].source, "local");
|
||||
assert.match(sfx[0].file, /assets\/sfx\//);
|
||||
assert.ok(existsSync(join(dir, sfx[0].file)), "matched SFX copied into the project");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("an unknown cue is reported, not fatal", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "1", name: "definitely-not-a-real-sfx" }],
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
assert.equal(sfx.length, 0);
|
||||
assert.ok(anomalies.some((a) => /not in bundled library/.test(a)));
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("a person's own file under a bundled name survives, and the cue gets the next free name", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
const own = join(dir, "assets", "sfx", "whoosh.mp3");
|
||||
mkdirSync(dirname(own), { recursive: true });
|
||||
writeFileSync(own, "the person's own whoosh");
|
||||
const resolve = () =>
|
||||
resolveSfx({
|
||||
cues: [{ id: "1", name: "whoosh" }],
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
|
||||
const first = await resolve();
|
||||
const second = await resolve();
|
||||
|
||||
assert.equal(readFileSync(own, "utf8"), "the person's own whoosh");
|
||||
assert.deepEqual(
|
||||
[first, second].map(({ sfx }) => [sfx[0].file, sfx[0].source]),
|
||||
[
|
||||
["assets/sfx/whoosh-2.mp3", "local"],
|
||||
["assets/sfx/whoosh-2.mp3", "local"],
|
||||
],
|
||||
);
|
||||
assert.deepEqual(
|
||||
readFileSync(join(dir, "assets/sfx/whoosh-2.mp3")),
|
||||
readFileSync(join(sfxLibDir, "whoosh.mp3")),
|
||||
);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("two effects never share a file when one's name is taken by the person, run after run", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
mkdirSync(join(dir, "assets", "sfx"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets", "sfx", "glitch.mp3"), "the person's own glitch");
|
||||
globalThis.fetch = async (url) => {
|
||||
const query = new URL(url).searchParams.get("q");
|
||||
if (query)
|
||||
return Response.json({
|
||||
results: [
|
||||
{
|
||||
license: "cc0",
|
||||
filetype: "mp3",
|
||||
duration: 500,
|
||||
url: `https://sound.test/${encodeURIComponent(query)}`,
|
||||
},
|
||||
],
|
||||
});
|
||||
return new Response(`bytes of ${new URL(url).pathname}`);
|
||||
};
|
||||
const run = async (names) => {
|
||||
const { sfx } = await resolveSfx({
|
||||
cues: names.map((name, index) => ({ id: String(index), name })),
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
const files = sfx.map(({ file }) => file);
|
||||
recordInManifest(
|
||||
dir,
|
||||
[...new Set(files)].map((path) => ({ path, type: "sfx", source: "search" })),
|
||||
);
|
||||
return files;
|
||||
};
|
||||
try {
|
||||
assert.deepEqual(await run(["glitch"]), ["assets/sfx/glitch-2.mp3"]);
|
||||
assert.deepEqual(await run(["glitch", "glitch 2", "glitch"]), [
|
||||
"assets/sfx/glitch-2.mp3",
|
||||
"assets/sfx/glitch-2-2.mp3",
|
||||
"assets/sfx/glitch-2.mp3",
|
||||
]);
|
||||
} finally {
|
||||
globalThis.fetch = offline;
|
||||
}
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// The engine owns speed parsing; a fallback before the check would turn 0 or junk into 1.
|
||||
function runEngineWith(request, ...args) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "audio-speed-range-"));
|
||||
try {
|
||||
writeFileSync(join(dir, "audio_request.json"), JSON.stringify({ lines: [], ...request }));
|
||||
return spawnSync(process.execPath, [join(HERE, "audio.mjs"), "--hyperframes", dir, ...args], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
for (const [label, request, args, shown] of [
|
||||
["request speed 5", { speed: 5 }, [], "5"],
|
||||
["request speed 0", { speed: 0 }, [], "0"],
|
||||
["request speed -1", { speed: -1 }, [], "-1"],
|
||||
["request speed text", { speed: "fast" }, [], '"fast"'],
|
||||
["--speed 0", {}, ["--speed", "0"], '"0"'],
|
||||
["--speed text", { speed: 1 }, ["--speed", "abc"], '"abc"'],
|
||||
]) {
|
||||
test(`${label} stops the engine before any TTS runs`, () => {
|
||||
const r = runEngineWith(request, ...args);
|
||||
assert.equal(r.status, 1);
|
||||
assert.ok(r.stderr.includes(`speed must be above 0 and at most 3, got ${shown}`), r.stderr);
|
||||
});
|
||||
}
|
||||
|
||||
test("a numeric string speed, as the adapters forward it, passes the check", () => {
|
||||
assert.doesNotMatch(runEngineWith({ speed: "0.8" }, "--only", "tts").stderr, /speed must/);
|
||||
});
|
||||
@@ -0,0 +1,560 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Apply a voiceover carve to a composition, from the command line.
|
||||
*
|
||||
* The carve is an analysis: it listens to a voice track, finds the bands it
|
||||
* occupies, and writes a chain of dips into the music bed plus a level match. In
|
||||
* Studio a panel runs it. This is the same analysis for an agent that has no
|
||||
* panel to click — identical functions from `@hyperframes/core`, identical
|
||||
* output, so a composition carved here and one carved in Studio are the same
|
||||
* three attributes.
|
||||
*
|
||||
* node carve.mjs --comp index.html
|
||||
* node carve.mjs --comp index.html --bed music-bed --voice narration \
|
||||
* --voice interview-guest --strength 0.45
|
||||
*
|
||||
* With no --bed/--voice it works out the tracks itself: the bed, and every voice
|
||||
* playing over it. `--voice` may be repeated to name them instead. Every named
|
||||
* voice is analysed together, so a bed running under a narrator and an answer makes
|
||||
* room for both.
|
||||
*
|
||||
* Needs `ffmpeg` on PATH (to decode the audio) and `@hyperframes/core` resolvable
|
||||
* from the composition's project (`npm i -D @hyperframes/core`) — the CLI bundles
|
||||
* core inline rather than shipping it as a package, so it cannot be borrowed from
|
||||
* there.
|
||||
*/
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createRequire } from "node:module";
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { isMainModule } from "./lib/main-module.mjs";
|
||||
|
||||
/** Sample rate the analysis runs at. Matches Studio's own decode rate, so the
|
||||
* bands and envelopes come out the same either way. */
|
||||
const SAMPLE_RATE = 48000;
|
||||
|
||||
/** Default carve strength. 0.25 kept the bed present but still fighting the voice. */
|
||||
export const DEFAULT_STRENGTH = 0.8;
|
||||
|
||||
const usage = `carve.mjs --comp <file.html> [--bed <elementId>] [--voice <elementId> ...]
|
||||
[--strength 0..1] [--dry-run] [--core <dir>]
|
||||
|
||||
--bed id of the music track that gets carved (detected if omitted)
|
||||
--voice id of a voice to make room for; repeatable (detected if omitted)
|
||||
--strength how hard to carve, 0..1 (default ${DEFAULT_STRENGTH})
|
||||
--dry-run report what it would write, touch nothing
|
||||
--core directory to resolve @hyperframes/core from (default: the comp's)`;
|
||||
|
||||
export function parseArgs(argv) {
|
||||
const args = { strength: DEFAULT_STRENGTH, dryRun: false, voices: [] };
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const flag = argv[i];
|
||||
const next = () => {
|
||||
const value = argv[i + 1];
|
||||
if (value === undefined) fail(`${flag} needs a value`);
|
||||
i += 1;
|
||||
return value;
|
||||
};
|
||||
if (flag === "--comp") args.comp = next();
|
||||
else if (flag === "--bed") args.bed = next();
|
||||
else if (flag === "--voice") args.voices.push(next());
|
||||
else if (flag === "--strength") args.strength = Number(next());
|
||||
else if (flag === "--core") args.core = next();
|
||||
else if (flag === "--dry-run") args.dryRun = true;
|
||||
else if (flag === "-h" || flag === "--help") fail(usage, 0);
|
||||
else fail(`unknown flag: ${flag}\n\n${usage}`);
|
||||
}
|
||||
if (!args.comp) fail(`--comp is required\n\n${usage}`);
|
||||
if (!Number.isFinite(args.strength) || args.strength < 0 || args.strength > 1) {
|
||||
fail("--strength must be a number from 0 to 1");
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function fail(message, code = 1) {
|
||||
process.stderr.write(`${message}\n`);
|
||||
process.exit(code);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the carve analysis out of `@hyperframes/core`.
|
||||
*
|
||||
* Resolved from the project rather than from this script, which lives wherever
|
||||
* the skill was installed — a sibling of the composition is what has the
|
||||
* dependency.
|
||||
*/
|
||||
export async function loadCore(fromDir) {
|
||||
const require = createRequire(pathToFileURL(resolve(fromDir, "package.json")));
|
||||
|
||||
/*
|
||||
* Two constraints at once, and satisfying either alone is broken:
|
||||
*
|
||||
* 1. Anchored at the PROJECT, not at this script. This file lives wherever
|
||||
* the skill was installed, which has no @hyperframes/core; the
|
||||
* composition's project is what holds the dependency. So a bare
|
||||
* `import("@hyperframes/core/audio-carve")` from here cannot work — bare
|
||||
* specifiers resolve relative to the importing module.
|
||||
* 2. Honouring the package's export CONDITIONS. `require.resolve` asks for
|
||||
* "require"/"node". The workspace manifest declares `node`, so this
|
||||
* resolved fine inside the monorepo — but the PUBLISHED manifest carries
|
||||
* only `import` + `types`, so every consumer of the released package got
|
||||
* ERR_PACKAGE_PATH_NOT_EXPORTED for a package that ships the file. That
|
||||
* is the audience this skill is shipped to, so the script was broken
|
||||
* everywhere except where it was developed.
|
||||
*
|
||||
* Keep the project anchor; fall back to the package's declared `import`
|
||||
* target when no require-resolvable condition exists.
|
||||
*/
|
||||
const load = async (subpath) => {
|
||||
const spec = `@hyperframes/core/${subpath}`;
|
||||
try {
|
||||
return await import(pathToFileURL(require.resolve(spec)).href);
|
||||
} catch (error) {
|
||||
if (error?.code !== "ERR_PACKAGE_PATH_NOT_EXPORTED") throw error;
|
||||
// `./package.json` is exported by every manifest, so this always resolves
|
||||
// and gives us the package root without guessing at node_modules layout.
|
||||
const pkgPath = require.resolve("@hyperframes/core/package.json");
|
||||
const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
|
||||
const entry = pkg.exports?.[`./${subpath}`];
|
||||
const target = typeof entry === "string" ? entry : (entry?.import ?? entry?.default ?? null);
|
||||
if (!target) {
|
||||
fail(
|
||||
`@hyperframes/core does not export ./${subpath}\n` +
|
||||
` found at: ${pkgPath} (version ${pkg.version})\n` +
|
||||
` update it: npm i -D @hyperframes/core`,
|
||||
);
|
||||
}
|
||||
return import(pathToFileURL(resolve(dirname(pkgPath), target)).href);
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
return {
|
||||
carve: await load("audio-carve"),
|
||||
fx: await load("audio-fx"),
|
||||
};
|
||||
} catch (error) {
|
||||
fail(
|
||||
`cannot load @hyperframes/core from ${fromDir}\n` +
|
||||
` is it installed there? npm i -D @hyperframes/core\n` +
|
||||
` or point at one: --core <dir containing node_modules/@hyperframes/core>\n` +
|
||||
` (${error.code ?? "error"}: ${error.message.split("\n")[0]})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The `sources` a carve should record for these voices, on this bed.
|
||||
*
|
||||
* SKILL.md states the invariant: "A carve against more than one clip id is
|
||||
* wrong. Group the clips and carve against the group." Naming the group lets
|
||||
* `resolveCarveSourceIds` resolve membership at analysis time, so a voice added
|
||||
* later is covered without editing `sources` — whereas a list of clip ids rots
|
||||
* silently the moment a fourth narration clip appears. The lint rule
|
||||
* `audio_carve_ungrouped_sources` enforces exactly this.
|
||||
*
|
||||
* This script was writing clip ids unconditionally, so it violated its own
|
||||
* skill's invariant and tripped its own lint rule on every run. When every
|
||||
* voice shares one group, record the group. Mixed or ungrouped voices keep
|
||||
* their ids, and the lint rule then correctly tells the author to group them.
|
||||
*
|
||||
* The bed has to be part of the decision, because the group form resolves
|
||||
* LATER and wider than it looks. If the bed is itself a member of the voices'
|
||||
* group, `resolveCarveSourceIds` expands that id to every current member on the
|
||||
* next analysis — including the bed — and the bed ends up carved against
|
||||
* itself, which SKILL.md calls a bug rather than a mix choice. This run cannot
|
||||
* see it: `main()` sums the voice list it detected and never round-trips
|
||||
* through group resolution, so the first pass is correct and only the next
|
||||
* re-analysis in Studio is wrong. So decline the group form there and fall back
|
||||
* to clip ids, which is exactly the case `audio_carve_ungrouped_sources` exists
|
||||
* to put in front of the author.
|
||||
*
|
||||
* Only an `<audio>` bed can trip it: group membership is audio-only
|
||||
* (`audioGroupOf`), so `data-audio-group` on a `<video>` bed is ignored by core
|
||||
* and expanding a group can never pull it in.
|
||||
*/
|
||||
export function carveSources(voices, bed, members) {
|
||||
const group = sharedVoiceGroup(voices);
|
||||
return group && !groupSourceRefusal(voices, bed, members) ? [group] : voices.map((v) => v.id);
|
||||
}
|
||||
|
||||
/** The one group every voice belongs to, or null if they do not share exactly one. */
|
||||
function sharedVoiceGroup(voices) {
|
||||
const groups = voices.map((v) => attrOf(v.tag, "data-audio-group"));
|
||||
const first = groups[0];
|
||||
return Boolean(first) && groups.every((g) => g === first) ? first : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Why naming the voices' shared group would persist something this run did not
|
||||
* analyse — or null when the group is safe to name.
|
||||
*
|
||||
* `members` is every `<audio>` in the composition as `{id, group, nameKind}`,
|
||||
* with `nameKind` from core's `classifyAudioName`, so this and Studio's picker
|
||||
* classify the same way.
|
||||
*
|
||||
* Required, deliberately not defaulting to `[]`. With an empty list the `mixed`
|
||||
* refusal below cannot fire, so a call that forgot the argument would return the
|
||||
* group form and restore the exact behaviour this function exists to prevent —
|
||||
* silently, because the first CLI pass is correct either way and only a later
|
||||
* Studio re-analysis is wrong. A missing argument throws on `members.filter`
|
||||
* instead.
|
||||
*
|
||||
* Two refusals, and both exist because the group form resolves LATER and WIDER
|
||||
* than the analysis: `resolveCarveSourceIds` expands a group id to every current
|
||||
* member on every analysis, and `resolveCarveVoices` keeps any audio member with
|
||||
* a src. `main()` meanwhile sums the voice list `detectTracks` returned, so the
|
||||
* first pass looks correct however wrong the persisted attribute is.
|
||||
*
|
||||
* `bed` — the bed is a member, so it would be handed to itself as a voice
|
||||
* and carved against its own content.
|
||||
* `mixed` — a member classified music or sfx is not a voice this run measured,
|
||||
* so it would enter the sidechain on the next analysis and duck the
|
||||
* bed under a whoosh.
|
||||
*
|
||||
* Deliberately NOT a refusal: a member classified `voice` or `unknown` that this
|
||||
* run left out. That is the group form working as designed — `detectTracks` only
|
||||
* takes voices that overlap the bed, and picking up a clip that starts playing
|
||||
* later without an edit to `sources` is the whole reason SKILL.md says to name
|
||||
* the group. Refusing there would collapse the group form into clip ids for
|
||||
* every ordinary narration sequence.
|
||||
*/
|
||||
export function groupSourceRefusal(voices, bed, members) {
|
||||
const group = sharedVoiceGroup(voices);
|
||||
if (!group) return null;
|
||||
if (bed?.kind === "audio" && attrOf(bed.tag, "data-audio-group") === group) {
|
||||
return { group, reason: "bed", ids: [bed.id] };
|
||||
}
|
||||
const analysed = new Set(voices.map((v) => v.id));
|
||||
const strays = members
|
||||
.filter(
|
||||
(m) =>
|
||||
m.group === group &&
|
||||
!analysed.has(m.id) &&
|
||||
(m.nameKind === "music" || m.nameKind === "sfx"),
|
||||
)
|
||||
.map((m) => m.id);
|
||||
return strays.length > 0 ? { group, reason: "mixed", ids: strays } : null;
|
||||
}
|
||||
|
||||
/** Mono float PCM for one media file, via ffmpeg. */
|
||||
function decode(path) {
|
||||
let raw;
|
||||
try {
|
||||
raw = execFileSync(
|
||||
"ffmpeg",
|
||||
[
|
||||
"-v",
|
||||
"error",
|
||||
"-i",
|
||||
path,
|
||||
"-vn",
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
String(SAMPLE_RATE),
|
||||
"-f",
|
||||
"f32le",
|
||||
"-",
|
||||
],
|
||||
{ maxBuffer: 1 << 30 },
|
||||
);
|
||||
} catch (error) {
|
||||
fail(`could not decode ${path}\n ${error.message.split("\n")[0]}`);
|
||||
}
|
||||
if (raw.length === 0) fail(`no audio in ${path}`);
|
||||
return new Float32Array(raw.buffer, raw.byteOffset, raw.length / 4);
|
||||
}
|
||||
|
||||
const attrOf = (tag, name) => tag.match(new RegExp(`\\s${name}="([^"]*)"`, "i"))?.[1] ?? null;
|
||||
|
||||
const unescapeAttr = (value) =>
|
||||
value
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, "&");
|
||||
|
||||
const escapeAttr = (value) => value.replace(/&/g, "&").replace(/"/g, """);
|
||||
|
||||
/** Every media element with a src, as {id, tag, kind}. */
|
||||
function mediaElements(html) {
|
||||
const found = [];
|
||||
for (const match of html.matchAll(/<(audio|video)\b[^>]*>/gi)) {
|
||||
const tag = match[0];
|
||||
// `\sid=` and not `id=`: `data-hf-id` would match first.
|
||||
const id = tag.match(/\sid="([^"]+)"/)?.[1];
|
||||
if (id && attrOf(tag, "src")) found.push({ id, tag, kind: match[1].toLowerCase() });
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/**
|
||||
* Work out which track is the bed and which tracks are its voices.
|
||||
*
|
||||
* Names first, because they are what the author already told us and the answer is
|
||||
* explainable: a track whose id or filename looks like music is the bed, ones that
|
||||
* look like speech are voices, SFX-shaped names are neither. `classifyAudioName`
|
||||
* comes from core so Studio's own picker and this cannot disagree.
|
||||
*
|
||||
* EVERY voice over the bed, not one of them. A bed usually runs under a whole
|
||||
* sequence, and they are analysed together — so there is nothing to disambiguate,
|
||||
* which is why this no longer refuses when several tracks look like speech.
|
||||
*
|
||||
* Only tracks that actually play while the bed does: one somewhere else on the
|
||||
* timeline cannot mask it. It still refuses when it cannot find a bed at all, or
|
||||
* finds no voice to make room for.
|
||||
*/
|
||||
function detectTracks(html, given, classify, overlaps) {
|
||||
const all = mediaElements(html);
|
||||
const kindOf = (el) => classify(el.id, unescapeAttr(attrOf(el.tag, "src") ?? ""));
|
||||
const spanOf = (el) => {
|
||||
const raw = attrOf(el.tag, "data-duration");
|
||||
const n = raw === null ? Number.NaN : Number(raw);
|
||||
return {
|
||||
start: startOf(el.tag),
|
||||
duration: Number.isFinite(n) ? n : null,
|
||||
};
|
||||
};
|
||||
const pick = (id, what) => {
|
||||
const found = all.find((el) => el.id === id);
|
||||
if (!found) fail(`no <audio>/<video> with id="${id}" in the composition`);
|
||||
return { ...found, why: `--${what}` };
|
||||
};
|
||||
|
||||
let bed = given.bed ? pick(given.bed, "bed") : null;
|
||||
const named = given.voices.map((id) => pick(id, "voice"));
|
||||
|
||||
if (!bed) {
|
||||
const others = all.filter((el) => !named.some((v) => v.id === el.id));
|
||||
const music = others.filter((el) => kindOf(el) === "music");
|
||||
if (music.length === 1) bed = { ...music[0], why: "name looks like music" };
|
||||
else if (music.length > 1) {
|
||||
fail(
|
||||
`several tracks look like music (${music.map((el) => el.id).join(", ")}) — name one with --bed`,
|
||||
);
|
||||
} else if (others.length === 1) {
|
||||
bed = { ...others[0], why: "only track left" };
|
||||
} else {
|
||||
fail(
|
||||
`cannot tell which track is the music bed\n` +
|
||||
` media in the composition: ${all.map((el) => el.id).join(", ") || "none"}\n` +
|
||||
` name it with --bed`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const bedSpan = spanOf(bed);
|
||||
const overlapping = (el) => overlaps(bedSpan, spanOf(el));
|
||||
const plausible = all
|
||||
.filter((el) => el.id !== bed.id && kindOf(el) !== "music" && kindOf(el) !== "sfx")
|
||||
.filter(overlapping);
|
||||
// A voiceover is normally its own <audio>. Video counts only when no audio track
|
||||
// is left to be the voice — a talking-head recut — because otherwise every B-roll
|
||||
// clip in the composition reads as somebody talking.
|
||||
const spoken = plausible.filter((el) => el.kind === "audio");
|
||||
const pool = spoken.length > 0 ? spoken : plausible;
|
||||
const voices = named.length
|
||||
? named
|
||||
: pool.map((el) => ({
|
||||
...el,
|
||||
why: kindOf(el) === "voice" ? "name looks like a voice" : "plays over the bed",
|
||||
}));
|
||||
|
||||
const usable = voices.filter((el) => attrOf(el.tag, "src"));
|
||||
if (usable.length === 0) {
|
||||
fail(
|
||||
`no voice to make room for on ${bed.id}\n` +
|
||||
` media in the composition: ${all.map((el) => el.id).join(", ") || "none"}\n` +
|
||||
` name one with --voice`,
|
||||
);
|
||||
}
|
||||
return { bed, voices: usable, all };
|
||||
}
|
||||
|
||||
const startOf = (tag) => {
|
||||
const raw = Number(attrOf(tag, "data-start"));
|
||||
return Number.isFinite(raw) ? raw : 0;
|
||||
};
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const compPath = resolve(args.comp);
|
||||
const compDir = dirname(compPath);
|
||||
const { carve: carveApi, fx: fxApi } = await loadCore(args.core ? resolve(args.core) : compDir);
|
||||
|
||||
const html = readFileSync(compPath, "utf-8");
|
||||
const {
|
||||
bed: bedEl,
|
||||
voices,
|
||||
all: media,
|
||||
} = detectTracks(html, args, carveApi.classifyAudioName, carveApi.clipsOverlap);
|
||||
// Group membership + name classification for every audio track, so the source
|
||||
// decision can see what the group will resolve to later and not just what this
|
||||
// run analysed.
|
||||
const members = media
|
||||
.filter((el) => el.kind === "audio")
|
||||
.map((el) => ({
|
||||
id: el.id,
|
||||
group: attrOf(el.tag, "data-audio-group"),
|
||||
nameKind: carveApi.classifyAudioName(el.id, unescapeAttr(attrOf(el.tag, "src") ?? "")),
|
||||
}));
|
||||
const bedTag = bedEl.tag;
|
||||
const bedSrc = attrOf(bedTag, "src");
|
||||
process.stdout.write(
|
||||
`bed ${bedEl.id} (${bedEl.why})\n` +
|
||||
voices.map((v) => `voice ${v.id} (${v.why})`).join("\n") +
|
||||
"\n",
|
||||
);
|
||||
|
||||
const profile = carveApi.carveProfile(args.strength);
|
||||
// Every voice summed onto the BED's clock before anything is measured. One
|
||||
// question — where and when is speech masking this bed — with one answer, even
|
||||
// when the answer comes from several people at different times.
|
||||
const voice = carveApi.mixCarveSources(
|
||||
voices.map((v) => ({
|
||||
samples: decode(resolve(compDir, unescapeAttr(attrOf(v.tag, "src")))),
|
||||
offsetSeconds: startOf(v.tag) - startOf(bedTag),
|
||||
})),
|
||||
SAMPLE_RATE,
|
||||
);
|
||||
if (voice.length === 0) fail("the voices do not overlap the bed, so there is nothing to carve");
|
||||
const bands = carveApi.analyseCarveBands(voice, SAMPLE_RATE, profile);
|
||||
|
||||
// The level half of the carve needs both sides: "how far over the speech is this
|
||||
// bed" cannot be answered by listening to one of them. No offset — the mix is
|
||||
// already on the bed's clock.
|
||||
const bed = profile.duckDb > 0 ? decode(resolve(compDir, unescapeAttr(bedSrc))) : null;
|
||||
const duck = bed ? carveApi.analyseCarveDuck(voice, bed, SAMPLE_RATE, profile, 0) : [];
|
||||
|
||||
// Anything the author built by hand survives a carve; only the previous
|
||||
// carve's own nodes are replaced. That is what `fromCarve` is for.
|
||||
const existingChain = attrOf(bedTag, "data-fx-chain");
|
||||
const existingNodes = existingChain
|
||||
? fxApi.parseAudioFxChain(unescapeAttr(existingChain)).nodes
|
||||
: [];
|
||||
const kept = existingNodes.filter((n) => !n.fromCarve);
|
||||
// Lanes belonging to the carve being replaced, addressed by the ids the OLD
|
||||
// nodes had. Taken before anything is minted: those ids are freed by the
|
||||
// replacement and a new node can be handed one of them, so reading them off the
|
||||
// new chain would keep exactly the stale lanes it is supposed to drop.
|
||||
const stalePrefixes = existingNodes.filter((n) => n.fromCarve && n.id).map((n) => `fx.${n.id}.`);
|
||||
|
||||
let claimed = { version: 1, nodes: kept };
|
||||
const mint = (node) => {
|
||||
const withId = { ...node, id: fxApi.mintAudioFxNodeId(claimed), fromCarve: true };
|
||||
claimed = { version: 1, nodes: [...claimed.nodes, withId] };
|
||||
return withId;
|
||||
};
|
||||
const bandNodes = bands.map((band) => mint(carveApi.carveBandsToChain([band]).nodes[0]));
|
||||
|
||||
const duckNode =
|
||||
duck.length > 0
|
||||
? mint({
|
||||
type: "gain",
|
||||
enabled: true,
|
||||
params: {
|
||||
...fxApi.defaultAudioFxParams("gain"),
|
||||
gain: 0,
|
||||
},
|
||||
})
|
||||
: null;
|
||||
const chain = {
|
||||
version: 1,
|
||||
nodes: [...bandNodes, ...(duckNode ? [duckNode] : []), ...kept],
|
||||
};
|
||||
|
||||
/**
|
||||
* One carve envelope as a lane on the BED's clock.
|
||||
*
|
||||
* Nothing to shift: the voices were summed onto that clock before the analysis
|
||||
* ran. A lane does hold its first value backwards to the start of its clip, so an
|
||||
* envelope that begins later needs an explicit "no cut" at zero or the bed starts
|
||||
* out ducked.
|
||||
*/
|
||||
const laneFor = (id, points) => {
|
||||
const timed = points
|
||||
.map((p) => ({ t: Number(p.t.toFixed(3)), v: p.v }))
|
||||
.filter((p) => p.t >= 0);
|
||||
if ((timed[0]?.t ?? 0) > 0) timed.unshift({ t: 0, v: 0 });
|
||||
return timed.length > 1 ? [{ target: `fx.${id}.gain`, points: timed }] : [];
|
||||
};
|
||||
|
||||
// Every carve follows the speech: a fixed depth thins the bed through every pause.
|
||||
const carvedLanes = [
|
||||
...carveApi
|
||||
.analyseCarveDynamics(voice, SAMPLE_RATE, bands)
|
||||
.flatMap((dyn, i) => (bandNodes[i]?.id ? laneFor(bandNodes[i].id, dyn.points) : [])),
|
||||
...(duckNode?.id && duck.length > 0 ? laneFor(duckNode.id, duck) : []),
|
||||
];
|
||||
|
||||
// Hand-drawn lanes are kept the same way hand-built nodes are: by dropping only
|
||||
// the ones that addressed the previous carve's nodes.
|
||||
const existingAutomation = attrOf(bedTag, "data-automation");
|
||||
const carriedLanes = existingAutomation
|
||||
? (JSON.parse(unescapeAttr(existingAutomation)).lanes ?? []).filter(
|
||||
(lane) => !stalePrefixes.some((prefix) => String(lane.target).startsWith(prefix)),
|
||||
)
|
||||
: [];
|
||||
const lanes = [...carriedLanes, ...carvedLanes];
|
||||
|
||||
const settings = {
|
||||
enabled: true,
|
||||
sources: carveSources(voices, bedEl, members),
|
||||
strength: args.strength,
|
||||
};
|
||||
// Say why the group form was declined, or the lint rule tells the author to
|
||||
// group clips they have already grouped.
|
||||
const refusal = groupSourceRefusal(voices, bedEl, members);
|
||||
if (refusal) {
|
||||
process.stderr.write(
|
||||
refusal.reason === "bed"
|
||||
? `note bed ${bedEl.id} is in group "${refusal.group}" with the voices, so\n` +
|
||||
` sources are clip ids: naming that group would carve the bed\n` +
|
||||
` against itself on the next analysis. Move the bed to its own group.\n`
|
||||
: `note group "${refusal.group}" also holds ${refusal.ids.join(", ")}, which this run\n` +
|
||||
` did not analyse (music/sfx by name), so sources are clip ids: naming\n` +
|
||||
` the group would pull them into the sidechain on the next analysis.\n` +
|
||||
` Move them out of the voice group.\n`,
|
||||
);
|
||||
}
|
||||
const written =
|
||||
` data-fx-carve="${escapeAttr(JSON.stringify(settings))}"` +
|
||||
` data-fx-chain="${escapeAttr(fxApi.serializeAudioFxChain(chain))}"` +
|
||||
(lanes.length > 0
|
||||
? ` data-automation="${escapeAttr(JSON.stringify({ version: 1, lanes }))}"`
|
||||
: "");
|
||||
|
||||
process.stdout.write(
|
||||
`carve strength ${args.strength}, ${voices.length} voice${voices.length === 1 ? "" : "s"}\n` +
|
||||
`bands ${bands.map((b) => `${b.freq}Hz ${b.gainDb}dB q${b.q}`).join(", ")}\n` +
|
||||
`level ${
|
||||
duckNode
|
||||
? `${duck.length}-point envelope, floor ${Math.min(...duck.map((p) => p.v))} dB`
|
||||
: "no level match at this strength"
|
||||
}\n` +
|
||||
`lanes ${carvedLanes.length} carve${carriedLanes.length ? ` + ${carriedLanes.length} kept` : ""}\n`,
|
||||
);
|
||||
if (args.dryRun) {
|
||||
process.stdout.write("dry run: nothing written\n");
|
||||
return;
|
||||
}
|
||||
|
||||
let stripped = bedTag;
|
||||
for (const attr of ["data-fx-carve", "data-fx-chain", "data-automation"]) {
|
||||
stripped = stripped.replace(new RegExp(`\\s${attr}="[^"]*"`, "i"), "");
|
||||
}
|
||||
// Inserted before the tag's own closing ">", which is the only place they can
|
||||
// go: `stripped` is the opening tag alone, so appending would land outside it.
|
||||
const nextTag = stripped.replace(/\/?>$/, (close) => `${written}${close}`);
|
||||
if (nextTag === stripped) fail("attribute write produced no change — refusing to save");
|
||||
writeFileSync(compPath, html.replace(bedTag, nextTag));
|
||||
process.stdout.write(`wrote ${args.comp} (id="${bedEl.id}")\n`);
|
||||
}
|
||||
|
||||
// Only run as a CLI. Guarded so the pure helpers above can be unit-tested by
|
||||
// importing this module (`skills/**/*.test.mjs`, run by `bun run test:skills`).
|
||||
if (isMainModule(import.meta.url)) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
DEFAULT_STRENGTH,
|
||||
carveSources,
|
||||
groupSourceRefusal,
|
||||
loadCore,
|
||||
parseArgs,
|
||||
} from "./carve.mjs";
|
||||
|
||||
const SCRIPTS_DIR = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
/** A voice as `detectTracks` yields it: an id plus its raw opening tag. */
|
||||
function voice(id, group) {
|
||||
const attr = group === undefined ? "" : ` data-audio-group="${group}"`;
|
||||
return { id, tag: `<audio id="${id}"${attr} src="${id}.wav"></audio>` };
|
||||
}
|
||||
|
||||
/** An audio member as `main` describes it for the source decision. */
|
||||
function member(id, group, nameKind) {
|
||||
return { id, group, nameKind };
|
||||
}
|
||||
|
||||
/** A bed as `mediaElements` yields it: `kind` is what decides group membership. */
|
||||
function bed(id, group, kind = "audio") {
|
||||
const attr = group === undefined ? "" : ` data-audio-group="${group}"`;
|
||||
return { id, kind, tag: `<${kind} id="${id}"${attr} src="${id}.mp3"></${kind}>` };
|
||||
}
|
||||
|
||||
async function inTempDir(run) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "carve-test-"));
|
||||
try {
|
||||
return await run(dir);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test("voices sharing one group carve against the group, not their ids", () => {
|
||||
// SKILL.md's invariant: "A carve against more than one clip id is wrong.
|
||||
// Group the clips and carve against the group." Naming the group lets
|
||||
// membership resolve at analysis time, so a voice added later is covered.
|
||||
const voices = [voice("vo1", "voiceover"), voice("vo2", "voiceover"), voice("vo3", "voiceover")];
|
||||
assert.deepEqual(carveSources(voices, undefined, []), ["voiceover"]);
|
||||
});
|
||||
|
||||
test("a single grouped voice still records the group", () => {
|
||||
assert.deepEqual(carveSources([voice("vo1", "voiceover")], undefined, []), ["voiceover"]);
|
||||
});
|
||||
|
||||
test("ungrouped voices keep their ids, so the lint rule can still say so", () => {
|
||||
// Not silently inventing a group: `audio_carve_ungrouped_sources` is the
|
||||
// right signal here, and it needs the ids to fire on.
|
||||
assert.deepEqual(carveSources([voice("vo1"), voice("vo2")], undefined, []), ["vo1", "vo2"]);
|
||||
});
|
||||
|
||||
test("voices in DIFFERENT groups keep their ids", () => {
|
||||
// One carve cannot name two groups, and picking either would silently drop
|
||||
// the other's members from the analysis.
|
||||
const voices = [voice("vo1", "narration"), voice("vo2", "interview")];
|
||||
assert.deepEqual(carveSources(voices, undefined, []), ["vo1", "vo2"]);
|
||||
});
|
||||
|
||||
test("a partially grouped set keeps its ids", () => {
|
||||
const voices = [voice("vo1", "voiceover"), voice("vo2")];
|
||||
assert.deepEqual(carveSources(voices, undefined, []), ["vo1", "vo2"]);
|
||||
});
|
||||
|
||||
test("an empty group attribute is not a group", () => {
|
||||
assert.deepEqual(carveSources([voice("vo1", ""), voice("vo2", "")], undefined, []), [
|
||||
"vo1",
|
||||
"vo2",
|
||||
]);
|
||||
});
|
||||
|
||||
// The nine cases above pass `[]` explicitly because they predate the membership
|
||||
// check and are about the bed and the group attributes alone. `members` is a
|
||||
// required argument: with `[]` the `mixed` refusal cannot fire, so a default
|
||||
// would let a call that forgot it return the group form and silently undo the
|
||||
// widening fix — see the wiring test at the bottom of this file.
|
||||
|
||||
test("a bed inside the voices' group keeps clip ids, so it is never its own source", () => {
|
||||
// `resolveCarveSourceIds` expands a group to every CURRENT member, and it gets
|
||||
// no host element to exclude — so naming a group the bed belongs to puts the
|
||||
// bed in its own voice list on the next analysis, and it is carved against
|
||||
// itself. This run cannot see it (main sums the detected voices directly), so
|
||||
// the check has to happen here.
|
||||
const voices = [voice("vo1", "mix"), voice("vo2", "mix")];
|
||||
assert.deepEqual(carveSources(voices, bed("bgm", "mix"), []), ["vo1", "vo2"]);
|
||||
});
|
||||
|
||||
test("a bed in a DIFFERENT group leaves the group form alone", () => {
|
||||
const voices = [voice("vo1", "voiceover"), voice("vo2", "voiceover")];
|
||||
assert.deepEqual(carveSources(voices, bed("bgm", "music"), []), ["voiceover"]);
|
||||
});
|
||||
|
||||
test("an ungrouped bed leaves the group form alone", () => {
|
||||
assert.deepEqual(carveSources([voice("vo1", "voiceover")], bed("bgm"), []), ["voiceover"]);
|
||||
});
|
||||
|
||||
test("a VIDEO bed is immune — group membership is audio-only", () => {
|
||||
// `data-audio-group` on a <video> is ignored by core, so expanding the group
|
||||
// can never pull this bed in and declining would be a false positive.
|
||||
const voices = [voice("vo1", "mix"), voice("vo2", "mix")];
|
||||
assert.deepEqual(carveSources(voices, bed("clip", "mix", "video"), []), ["mix"]);
|
||||
});
|
||||
|
||||
test("an sfx member of the voices' group blocks the group form", () => {
|
||||
// The group resolves wider than the analysis: `resolveCarveSourceIds` expands
|
||||
// it to every current member and `resolveCarveVoices` keeps any audio with a
|
||||
// src, so an sfx clip sharing the voice group enters the sidechain on the next
|
||||
// analysis and ducks the bed under a whoosh. This run cannot see it — it sums
|
||||
// the voices `detectTracks` returned, which correctly excluded the sfx.
|
||||
const voices = [voice("vo1", "voiceover"), voice("vo2", "voiceover")];
|
||||
const members = [
|
||||
member("vo1", "voiceover", "voice"),
|
||||
member("vo2", "voiceover", "voice"),
|
||||
member("whoosh", "voiceover", "sfx"),
|
||||
member("bgm", "music", "music"),
|
||||
];
|
||||
assert.deepEqual(carveSources(voices, bed("bgm", "music"), members), ["vo1", "vo2"]);
|
||||
});
|
||||
|
||||
test("a music member of the voices' group blocks it too", () => {
|
||||
const voices = [voice("vo1", "voiceover")];
|
||||
const members = [member("vo1", "voiceover", "voice"), member("pad", "voiceover", "music")];
|
||||
assert.deepEqual(carveSources(voices, bed("bgm", "music"), members), ["vo1"]);
|
||||
});
|
||||
|
||||
test("a voice member this run did NOT analyse keeps the group form", () => {
|
||||
// The designed case, and the one a membership check must not break: a voice
|
||||
// that does not overlap the bed is left out of the analysis on purpose, and
|
||||
// covering it on a later analysis without editing `sources` is the entire
|
||||
// reason SKILL.md says to name the group. Refusing here would collapse the
|
||||
// group form into clip ids for every ordinary narration sequence.
|
||||
const voices = [voice("vo1", "voiceover"), voice("vo2", "voiceover")];
|
||||
const members = [
|
||||
member("vo1", "voiceover", "voice"),
|
||||
member("vo2", "voiceover", "voice"),
|
||||
member("vo-outro", "voiceover", "voice"),
|
||||
];
|
||||
assert.deepEqual(carveSources(voices, bed("bgm", "music"), members), ["voiceover"]);
|
||||
});
|
||||
|
||||
test("an unclassifiable member keeps the group form", () => {
|
||||
// `unknown` is what `detectTracks` itself treats as a possible voice, so it is
|
||||
// not evidence of a non-voice member — loose in the safe direction, same as
|
||||
// detection.
|
||||
const voices = [voice("vo1", "voiceover")];
|
||||
const members = [member("vo1", "voiceover", "voice"), member("track7", "voiceover", "unknown")];
|
||||
assert.deepEqual(carveSources(voices, bed("bgm", "music"), members), ["voiceover"]);
|
||||
});
|
||||
|
||||
test("an sfx member of a DIFFERENT group is irrelevant", () => {
|
||||
const voices = [voice("vo1", "voiceover")];
|
||||
const members = [member("vo1", "voiceover", "voice"), member("whoosh", "sfx", "sfx")];
|
||||
assert.deepEqual(carveSources(voices, bed("bgm", "music"), members), ["voiceover"]);
|
||||
});
|
||||
|
||||
test("groupSourceRefusal names which member blocked the group, and why", () => {
|
||||
// The stderr note is built from this, so it has to carry the ids: "sources are
|
||||
// clip ids" plus `audio_carve_ungrouped_sources` reads as nonsense to an author
|
||||
// who did group their clips.
|
||||
const voices = [voice("vo1", "voiceover"), voice("vo2", "voiceover")];
|
||||
assert.deepEqual(
|
||||
groupSourceRefusal(voices, bed("bgm", "music"), [member("whoosh", "voiceover", "sfx")]),
|
||||
{ group: "voiceover", reason: "mixed", ids: ["whoosh"] },
|
||||
);
|
||||
assert.deepEqual(groupSourceRefusal(voices, bed("bgm", "voiceover"), []), {
|
||||
group: "voiceover",
|
||||
reason: "bed",
|
||||
ids: ["bgm"],
|
||||
});
|
||||
assert.equal(groupSourceRefusal(voices, bed("bgm", "music"), []), null);
|
||||
});
|
||||
|
||||
test("the CLI still runs when invoked through a symlinked path", () => {
|
||||
// argv[1] keeps the invoked spelling while import.meta.url is the realpath, so
|
||||
// a raw compare in the entry guard skips main() and the CLI exits 0 having
|
||||
// written nothing. macOS /tmp -> /private/tmp reaches this with no symlink of
|
||||
// one's own; so does any skill install placed behind a link.
|
||||
return inTempDir((dir) => {
|
||||
const link = join(dir, "scripts-link");
|
||||
symlinkSync(SCRIPTS_DIR, link, "junction");
|
||||
let status = 0;
|
||||
let stderr = "";
|
||||
try {
|
||||
execFileSync(process.execPath, [join(link, "carve.mjs")], { encoding: "utf-8" });
|
||||
} catch (error) {
|
||||
status = error.status;
|
||||
stderr = String(error.stderr);
|
||||
}
|
||||
// Reaching parseArgs is the proof main() ran at all: no args is an error, and
|
||||
// the broken guard's symptom is a silent exit 0 with no output.
|
||||
assert.equal(status, 1, `expected the usage error, got status ${status}`);
|
||||
assert.match(stderr, /--comp is required/);
|
||||
});
|
||||
});
|
||||
|
||||
test("loadCore honours an import-only export map, as the published core has", () => {
|
||||
// The published manifest carries only `import` + `types` for these subpaths, so
|
||||
// `require.resolve` cannot resolve them at all and the fallback has to read the
|
||||
// export map itself. A fixture pins that without depending on npm.
|
||||
return inTempDir(async (dir) => {
|
||||
const pkgDir = join(dir, "node_modules", "@hyperframes", "core");
|
||||
mkdirSync(join(pkgDir, "dist"), { recursive: true });
|
||||
writeFileSync(join(dir, "package.json"), JSON.stringify({ name: "fixture-project" }));
|
||||
writeFileSync(
|
||||
join(pkgDir, "package.json"),
|
||||
JSON.stringify({
|
||||
name: "@hyperframes/core",
|
||||
version: "0.0.0-fixture",
|
||||
type: "module",
|
||||
exports: {
|
||||
"./package.json": "./package.json",
|
||||
"./audio-carve": { import: "./dist/audioCarve.js" },
|
||||
"./audio-fx": { import: "./dist/audioFx.js" },
|
||||
},
|
||||
}),
|
||||
);
|
||||
writeFileSync(join(pkgDir, "dist", "audioCarve.js"), 'export const marker = "carve";\n');
|
||||
writeFileSync(join(pkgDir, "dist", "audioFx.js"), 'export const marker = "fx";\n');
|
||||
|
||||
const core = await loadCore(dir);
|
||||
assert.equal(core.carve.marker, "carve");
|
||||
assert.equal(core.fx.marker, "fx");
|
||||
});
|
||||
});
|
||||
|
||||
test("members is required, so dropping it cannot silently restore the group form", () => {
|
||||
// The gap this closes: `main()` is the only code that BUILDS `members`, and no
|
||||
// test runs `main()` (the symlink test stops at the usage error, a real run
|
||||
// needs ffmpeg). With `members = []` defaulted, a refactor that dropped the
|
||||
// third argument would return the group form again with the whole suite green
|
||||
// — the same signature as the bug itself: first pass correct, persisted
|
||||
// attribute wrong, nothing red. Omitting it now throws instead.
|
||||
const voices = [voice("vo1", "voiceover"), voice("vo2", "voiceover")];
|
||||
assert.throws(() => carveSources(voices, bed("bgm", "music")), TypeError);
|
||||
assert.throws(() => groupSourceRefusal(voices, bed("bgm", "music")), TypeError);
|
||||
});
|
||||
|
||||
test("the default strength is 0.8, and --strength still overrides it", () => {
|
||||
// 0.25 left the bed audibly fighting the voice on narrated builds; a carve at
|
||||
// the default has to already be a finished mix. Pinned so it cannot drift back
|
||||
// unnoticed — core pins its own DEFAULT_CARVE the same way.
|
||||
assert.equal(DEFAULT_STRENGTH, 0.8);
|
||||
assert.equal(parseArgs(["--comp", "x.html"]).strength, 0.8);
|
||||
assert.equal(parseArgs(["--comp", "x.html", "--strength", "0.25"]).strength, 0.25);
|
||||
});
|
||||
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Extract per-frame audio visualization data from an audio or video file.
|
||||
|
||||
Outputs JSON with RMS amplitude and frequency band data at the target FPS,
|
||||
ready to embed in a HyperFrames composition.
|
||||
|
||||
Usage:
|
||||
python extract-audio-data.py input.mp3 -o audio-data.json
|
||||
python extract-audio-data.py input.mp4 --fps 30 --bands 16 -o audio-data.json
|
||||
|
||||
Requirements:
|
||||
- Python 3.9+
|
||||
- ffmpeg (for decoding audio)
|
||||
- numpy (pip install numpy)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
# Windows sizes stdio to the ANSI code page (cp1252). These scripts emit UTF-8 on
|
||||
# every platform; say so rather than depending on the console's code page. Carry
|
||||
# `errors` across: reconfigure() resets it to "strict", and CPython deliberately gives
|
||||
# stderr "backslashreplace" so the diagnostic path can never itself raise.
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(_stream, "reconfigure"):
|
||||
_stream.reconfigure(encoding="utf-8", errors=_stream.errors)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FFT parameters
|
||||
#
|
||||
# A 4096-sample window gives ~10.8 Hz per bin at 44100Hz — enough to resolve
|
||||
# low-frequency bands cleanly. The per-frame audio slice (44100/30 = 1470
|
||||
# samples at 30fps) is too small and causes low bands to map to the same bins.
|
||||
#
|
||||
# Frequency range 30Hz–16kHz covers the useful range for music. Below 30Hz is
|
||||
# sub-bass most speakers can't reproduce; above 16kHz is noise/harmonics that
|
||||
# don't contribute to perceived rhythm or melody.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
FFT_SIZE = 4096
|
||||
MIN_FREQ = 30.0
|
||||
MAX_FREQ = 16000.0
|
||||
|
||||
|
||||
def decode_audio(path: str) -> np.ndarray:
|
||||
"""Decode audio to mono float32 samples via ffmpeg."""
|
||||
cmd = [
|
||||
"ffmpeg", "-i", path,
|
||||
"-vn", "-ac", "1", "-ar", str(SAMPLE_RATE),
|
||||
"-f", "s16le", "-acodec", "pcm_s16le",
|
||||
"-loglevel", "error",
|
||||
"pipe:1",
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True)
|
||||
if result.returncode != 0:
|
||||
# Decoding ffmpeg's diagnostics strictly makes the reporter the thing that
|
||||
# crashes: a Windows ffmpeg emits cp1252 bytes, and UnicodeDecodeError here
|
||||
# would bury the actual failure it was trying to report.
|
||||
print(f"ffmpeg error: {result.stderr.decode('utf-8', errors='replace')}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return np.frombuffer(result.stdout, dtype=np.int16).astype(np.float32) / 32768.0
|
||||
|
||||
|
||||
def compute_band_edges(n_bands: int) -> np.ndarray:
|
||||
"""Logarithmically-spaced frequency band edges from MIN_FREQ to MAX_FREQ."""
|
||||
return np.array([
|
||||
MIN_FREQ * (MAX_FREQ / MIN_FREQ) ** (i / n_bands)
|
||||
for i in range(n_bands + 1)
|
||||
])
|
||||
|
||||
|
||||
def compute_fft_bands(
|
||||
windowed: np.ndarray, freq_per_bin: float, n_bins: int,
|
||||
band_edges: np.ndarray, n_bands: int,
|
||||
) -> np.ndarray:
|
||||
"""Compute peak magnitude in logarithmically-spaced frequency bands."""
|
||||
magnitudes = np.abs(np.fft.rfft(windowed))
|
||||
|
||||
bands = np.zeros(n_bands)
|
||||
for b in range(n_bands):
|
||||
low_bin = max(0, int(band_edges[b] / freq_per_bin))
|
||||
high_bin = min(n_bins, int(band_edges[b + 1] / freq_per_bin))
|
||||
if high_bin <= low_bin:
|
||||
high_bin = low_bin + 1
|
||||
# Clamp to valid range to avoid empty slices
|
||||
low_bin = min(low_bin, n_bins - 1)
|
||||
high_bin = min(high_bin, n_bins)
|
||||
bands[b] = np.max(magnitudes[low_bin:high_bin])
|
||||
|
||||
return bands
|
||||
|
||||
|
||||
def extract(path: str, fps: int, n_bands: int) -> dict:
|
||||
"""Extract per-frame audio data."""
|
||||
print(f"Decoding audio from {path}...", file=sys.stderr)
|
||||
samples = decode_audio(path)
|
||||
duration = len(samples) / SAMPLE_RATE
|
||||
frame_step = SAMPLE_RATE // fps
|
||||
total_frames = int(duration * fps)
|
||||
|
||||
print(f"Duration: {duration:.1f}s, {total_frames} frames at {fps}fps", file=sys.stderr)
|
||||
print(f"FFT window: {FFT_SIZE} samples ({SAMPLE_RATE / FFT_SIZE:.1f} Hz/bin)", file=sys.stderr)
|
||||
print(f"Frequency range: {MIN_FREQ:.0f}-{MAX_FREQ:.0f} Hz, {n_bands} bands", file=sys.stderr)
|
||||
|
||||
# Precompute constants
|
||||
hann = np.hanning(FFT_SIZE)
|
||||
band_edges = compute_band_edges(n_bands)
|
||||
freq_per_bin = SAMPLE_RATE / FFT_SIZE
|
||||
n_bins = FFT_SIZE // 2 + 1
|
||||
half_fft = FFT_SIZE // 2
|
||||
|
||||
# Pass 1: extract raw values
|
||||
rms_values = np.zeros(total_frames)
|
||||
band_values = np.zeros((total_frames, n_bands))
|
||||
|
||||
for f in range(total_frames):
|
||||
# RMS from the frame's audio slice
|
||||
rms_start = f * frame_step
|
||||
rms_end = rms_start + frame_step
|
||||
frame_slice = samples[rms_start:min(rms_end, len(samples))]
|
||||
if len(frame_slice) > 0:
|
||||
rms_values[f] = np.sqrt(np.mean(frame_slice ** 2))
|
||||
|
||||
# FFT from a centered 4096-sample window
|
||||
center = rms_start + frame_step // 2
|
||||
win_start = center - half_fft
|
||||
win_end = center + half_fft
|
||||
|
||||
if win_start >= 0 and win_end <= len(samples):
|
||||
window = samples[win_start:win_end] * hann
|
||||
else:
|
||||
# Zero-pad at edges
|
||||
padded = np.zeros(FFT_SIZE)
|
||||
src_start = max(0, win_start)
|
||||
src_end = min(len(samples), win_end)
|
||||
dst_start = src_start - win_start
|
||||
dst_end = dst_start + (src_end - src_start)
|
||||
padded[dst_start:dst_end] = samples[src_start:src_end]
|
||||
window = padded * hann
|
||||
|
||||
band_values[f] = compute_fft_bands(window, freq_per_bin, n_bins, band_edges, n_bands)
|
||||
|
||||
# Pass 2: normalize
|
||||
peak_rms = rms_values.max() if total_frames > 0 else 1.0
|
||||
if peak_rms > 0:
|
||||
rms_values /= peak_rms
|
||||
|
||||
# Per-band normalization so treble is visible alongside louder bass
|
||||
band_peaks = band_values.max(axis=0)
|
||||
band_peaks[band_peaks == 0] = 1.0
|
||||
band_values /= band_peaks
|
||||
|
||||
# Build output
|
||||
frames = []
|
||||
for f in range(total_frames):
|
||||
frames.append({
|
||||
"time": round(f / fps, 4),
|
||||
"rms": round(float(rms_values[f]), 4),
|
||||
"bands": [round(float(b), 4) for b in band_values[f]],
|
||||
})
|
||||
|
||||
return {
|
||||
"duration": round(duration, 4),
|
||||
"fps": fps,
|
||||
"bands": n_bands,
|
||||
"totalFrames": total_frames,
|
||||
"frames": frames,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Extract per-frame audio visualization data")
|
||||
parser.add_argument("input", help="Audio or video file")
|
||||
parser.add_argument("-o", "--output", default="audio-data.json", help="Output JSON path")
|
||||
parser.add_argument("--fps", type=int, default=30, help="Frames per second (default: 30)")
|
||||
parser.add_argument("--bands", type=int, default=16, help="Number of frequency bands (default: 16)")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.fps < 1:
|
||||
parser.error("--fps must be at least 1")
|
||||
if args.bands < 1:
|
||||
parser.error("--bands must be at least 1")
|
||||
|
||||
data = extract(args.input, args.fps, args.bands)
|
||||
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f)
|
||||
|
||||
print(f"Wrote {args.output} ({data['totalFrames']} frames, {data['bands']} bands)", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,131 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { mkdtempSync, writeFileSync, readFileSync, mkdirSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
// Exercise the actual engine and provider together; only the paid API and
|
||||
// external audio tools are fixtures. This is not a live synthesis/render test.
|
||||
for (const expired of [false, true]) {
|
||||
for (const only of ["tts", "tts,bgm,sfx"]) {
|
||||
// With expired credentials the person already has an assets/voice/intro.wav, which the engine must keep.
|
||||
const voicePath = expired ? "assets/voice/intro-2.wav" : "assets/voice/intro.wav";
|
||||
test(`Gemini engine returns caption metadata with ${expired ? "expired" : "absent"} HeyGen credentials (${only})`, (t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-gemini-pipeline-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
if (expired) {
|
||||
mkdirSync(join(dir, "assets/voice"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets/voice/intro.wav"), "the person's own intro");
|
||||
}
|
||||
const config = join(dir, "heygen");
|
||||
mkdirSync(config);
|
||||
if (expired)
|
||||
writeFileSync(
|
||||
join(config, "credentials"),
|
||||
JSON.stringify({
|
||||
oauth: { access_token: "expired-fixture", expires_at: "2000-01-01T00:00:00Z" },
|
||||
}),
|
||||
);
|
||||
const bin = join(dir, "bin");
|
||||
mkdirSync(bin);
|
||||
const preload = join(dir, "fetch.mjs");
|
||||
writeFileSync(
|
||||
preload,
|
||||
`
|
||||
import assert from 'node:assert/strict';
|
||||
globalThis.fetch = async (url, options) => {
|
||||
assert.equal(url, 'https://generativelanguage.googleapis.com/v1beta/interactions');
|
||||
const body = JSON.parse(options.body);
|
||||
assert.equal(body.model, 'gemini-3.8-flash-lite-tts');
|
||||
assert.equal(body.input[0].content[0].annotations[0].style, 'Line direction');
|
||||
const bytes = Buffer.alloc(48);
|
||||
bytes.write('RIFF'); bytes.write('WAVE', 8);
|
||||
return Response.json({status:'completed', steps:[{type:'model_output', content:[
|
||||
{type:'audio', mime_type:'audio/wav', data:bytes.toString('base64')}
|
||||
]}]});
|
||||
};
|
||||
`,
|
||||
);
|
||||
writeFileSync(
|
||||
join(bin, "npx"),
|
||||
`#!${process.execPath}
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const args = process.argv.slice(2);
|
||||
assert.deepEqual(args.slice(0, 3), ['hyperframes', 'transcribe', '${voicePath}']);
|
||||
assert.ok(fs.existsSync(args[2]));
|
||||
assert.equal(args[args.indexOf('--model')+1], 'small.en');
|
||||
fs.writeFileSync(path.join(args[args.indexOf('--dir')+1], 'transcript.json'), JSON.stringify([
|
||||
{text:'Hello',start:0.1,end:0.4}, {text:'there.',start:0.5,end:0.9}
|
||||
]));
|
||||
`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
writeFileSync(join(bin, "ffprobe"), "#!/bin/sh\necho 1.25\n", { mode: 0o755 });
|
||||
writeFileSync(join(dir, ".env"), "");
|
||||
writeFileSync(
|
||||
join(dir, "audio_request.json"),
|
||||
JSON.stringify({
|
||||
provider: "gemini",
|
||||
tts_model: "gemini-3.8-flash-lite-tts",
|
||||
style: "Global direction",
|
||||
lines: [{ id: "intro", text: "Hello there.", style: "Line direction" }],
|
||||
bgm: { mode: "none" },
|
||||
}),
|
||||
);
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
preload,
|
||||
new URL("./audio.mjs", import.meta.url).pathname,
|
||||
"--hyperframes",
|
||||
dir,
|
||||
"--only",
|
||||
only,
|
||||
],
|
||||
{
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
HEYGEN_CONFIG_DIR: config,
|
||||
HEYGEN_API_KEY: "",
|
||||
HYPERFRAMES_API_KEY: "",
|
||||
GEMINI_API_KEY: "fixture-key",
|
||||
PATH: `${bin}:${process.env.PATH}`,
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8"));
|
||||
assert.equal(meta.tts_provider, "gemini");
|
||||
assert.equal(meta.voice_id, "Kore");
|
||||
assert.deepEqual(meta.voices, [
|
||||
{
|
||||
id: "intro",
|
||||
path: voicePath,
|
||||
duration_s: 1.25,
|
||||
words: [
|
||||
{ id: "w0", text: "Hello", start: 0.1, end: 0.4 },
|
||||
{ id: "w1", text: "there.", start: 0.5, end: 0.9 },
|
||||
],
|
||||
},
|
||||
]);
|
||||
assert.equal(meta.total_duration_s, 1.25);
|
||||
const manifest = readFileSync(join(dir, ".media/manifest.jsonl"), "utf8").trim().split("\n");
|
||||
assert.deepEqual(
|
||||
manifest
|
||||
.map((line) => JSON.parse(line))
|
||||
.map(({ path, type, source }) => [path, type, source]),
|
||||
[[voicePath, "voice", "generated"]],
|
||||
);
|
||||
if (expired)
|
||||
assert.equal(
|
||||
readFileSync(join(dir, "assets/voice/intro.wav"), "utf8"),
|
||||
"the person's own intro",
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env node
|
||||
// Self-contained HeyGen TTS — single text in → one wav (+ optional words JSON)
|
||||
// out. A thin CLI over lib/tts.mjs (the same code the audio engine uses), so the
|
||||
// HeyGen REST call, starfish voice pick, mp3→wav transcode, and word-timestamp
|
||||
// filtering live in exactly one place. Bypasses the `hyperframes` CLI, which in
|
||||
// the published build is Kokoro-only.
|
||||
//
|
||||
// Usage:
|
||||
// node heygen-tts.mjs "Text to speak" -o narration.wav [--words narration.words.json]
|
||||
// node heygen-tts.mjs ./script.txt -o narration.wav --words narration.words.json
|
||||
// node heygen-tts.mjs "Bonjour" -o fr.wav --lang fr --voice <id>
|
||||
// node heygen-tts.mjs --list # list starfish voices and exit
|
||||
//
|
||||
// Flags: -o/--output (.wav → ffmpeg transcode; .mp3 → raw bytes), --words,
|
||||
// --voice (starfish id), --speed, --lang, --list.
|
||||
// Requires: $HEYGEN_API_KEY / OAuth ~/.heygen credentials and ffmpeg for .wav output.
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { heygenAuthHeaders, heygenJSON, loadEnvFromDir } from "./lib/heygen.mjs";
|
||||
import { ffprobeDuration, resolveVoiceId, synthesizeOne, withWordIds } from "./lib/tts.mjs";
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
function flag(name, def) {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
if (i < 0) return def;
|
||||
if (i + 1 >= argv.length) return true;
|
||||
const v = argv[i + 1];
|
||||
return v.startsWith("--") ? true : v;
|
||||
}
|
||||
const die = (m) => {
|
||||
console.error(`✗ heygen-tts: ${m}`);
|
||||
process.exit(1);
|
||||
};
|
||||
|
||||
// First arg that isn't a flag or the -o value is the text / .txt path.
|
||||
const positional = (() => {
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a.startsWith("--")) {
|
||||
const next = argv[i + 1];
|
||||
if (next && !next.startsWith("--")) i++;
|
||||
continue;
|
||||
}
|
||||
if (a === "-o") {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
return a;
|
||||
}
|
||||
return null;
|
||||
})();
|
||||
|
||||
const output = resolve(
|
||||
(typeof flag("output") === "string" && flag("output")) ||
|
||||
(argv.includes("-o") && argv[argv.indexOf("-o") + 1]) ||
|
||||
"narration.wav",
|
||||
);
|
||||
const wordsPath = typeof flag("words") === "string" ? resolve(flag("words")) : null;
|
||||
const userVoice = typeof flag("voice") === "string" ? flag("voice") : null;
|
||||
const speedRaw = typeof flag("speed") === "string" ? Number(flag("speed")) : 1.0;
|
||||
const speed = isFinite(speedRaw) && speedRaw > 0 ? speedRaw : 1.0;
|
||||
const lang = typeof flag("lang") === "string" ? flag("lang") : "en";
|
||||
const listOnly = flag("list") === true;
|
||||
|
||||
loadEnvFromDir(process.cwd());
|
||||
let authHeaders;
|
||||
try {
|
||||
authHeaders = heygenAuthHeaders();
|
||||
} catch (e) {
|
||||
die(e.message);
|
||||
}
|
||||
|
||||
// ---------- --list ----------
|
||||
if (listOnly) {
|
||||
const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, {
|
||||
headers: authHeaders,
|
||||
});
|
||||
for (const v of payload.data ?? payload.voices ?? []) {
|
||||
console.log(`${v.voice_id}\t${v.name}\t${v.language ?? ""}`);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// ---------- resolve text + voice ----------
|
||||
if (!positional) die("no text given. Pass a string or a .txt path, or use --list.");
|
||||
const text =
|
||||
positional.endsWith(".txt") && existsSync(resolve(positional))
|
||||
? readFileSync(resolve(positional), "utf8").trim()
|
||||
: positional;
|
||||
if (!text) die("input text is empty");
|
||||
|
||||
const voiceId = await resolveVoiceId({ provider: "heygen", userVoice, lang });
|
||||
if (!userVoice) console.error(`· using voice ${voiceId}`);
|
||||
|
||||
// ---------- synthesize (shared engine code) ----------
|
||||
const { ok, words, error } = await synthesizeOne({
|
||||
provider: "heygen",
|
||||
text,
|
||||
voiceId,
|
||||
lang,
|
||||
speed,
|
||||
wavAbs: output,
|
||||
hyperframesDir: process.cwd(),
|
||||
});
|
||||
// HeyGen's own refusal, verbatim: a caller shows it or acts on it (a deleted voice, a limit).
|
||||
if (!ok) die(error ?? "synthesis failed (HeyGen request/transcode error)");
|
||||
|
||||
let wordCount = 0;
|
||||
if (wordsPath) {
|
||||
if (words && words.length) {
|
||||
mkdirSync(dirname(wordsPath), { recursive: true });
|
||||
writeFileSync(wordsPath, JSON.stringify(withWordIds(words), null, 2));
|
||||
wordCount = words.length;
|
||||
} else {
|
||||
console.error("⚠ no word_timestamps in response — run `hyperframes transcribe` instead");
|
||||
}
|
||||
}
|
||||
|
||||
const dur = ffprobeDuration(output);
|
||||
const durStr = isFinite(dur) ? ` (${dur.toFixed(2)}s)` : "";
|
||||
console.log(`✓ ${output}${durStr}${wordCount ? ` · ${wordsPath} (${wordCount} words)` : ""}`);
|
||||
@@ -0,0 +1,38 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const script = fileURLToPath(new URL("./heygen-tts.mjs", import.meta.url));
|
||||
// Every fetch answers as HeyGen does for a voice that no longer exists.
|
||||
const refusing = `data:text/javascript,globalThis.fetch = async () => new Response(${JSON.stringify(
|
||||
JSON.stringify({ error: { code: "voice_not_found", message: "Voice not found: vc_gone" } }),
|
||||
)}, { status: 404, headers: { "content-type": "application/json" } });`;
|
||||
|
||||
test("a refused speech prints HeyGen's own message, so a caller can act on it", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-tts-"));
|
||||
const run = spawnSync(
|
||||
process.execPath,
|
||||
["--import", refusing, script, "Hello", "--voice", "vc_gone", "-o", join(dir, "out.mp3")],
|
||||
{
|
||||
cwd: dir,
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
HEYGEN_API_KEY: "",
|
||||
HYPERFRAMES_API_KEY: "",
|
||||
HEYGEN_ACCESS_TOKEN: "t",
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(run.status, 1);
|
||||
assert.match(run.stderr, /✗ heygen-tts: Voice not found: vc_gone\n/);
|
||||
assert.doesNotMatch(
|
||||
run.stderr,
|
||||
/HTTP 404|"error"/,
|
||||
"HeyGen's message alone, not the raw response",
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env node
|
||||
// HeyGen voice clones: `clone <audio> --name <n>`, `delete <voice_id>`, `list --prefix <p>` (references/tts.md).
|
||||
// The clone id works as `heygen-tts.mjs --voice <id>`. A refusal prints HeyGen's own message and exits 1.
|
||||
// Auth: same resolver as heygen-tts.mjs (lib/heygen.mjs).
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { extname } from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import { heygenAuthHeaders, heygenJSON, heygenMessage, loadEnvFromDir } from "./lib/heygen.mjs";
|
||||
import { isMainModule } from "./lib/main-module.mjs";
|
||||
|
||||
const MEDIA_TYPES = { ".mp3": "audio/mpeg", ".wav": "audio/wav" };
|
||||
const POLL_MS = 2_000;
|
||||
const DEADLINE_MS = 120_000;
|
||||
|
||||
async function clone(file, name, headers, { sleep, now }) {
|
||||
const mediaType = MEDIA_TYPES[extname(file).toLowerCase()];
|
||||
if (!mediaType) throw new Error(`unsupported audio file ${file}: use .mp3 or .wav`);
|
||||
const data = readFileSync(file).toString("base64");
|
||||
const created = await heygenJSON("/voices/clone", {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: { voice_name: name, audio: { type: "base64", media_type: mediaType, data } },
|
||||
});
|
||||
const id = created.data.voice_clone_id;
|
||||
const deadline = now() + DEADLINE_MS;
|
||||
for (;;) {
|
||||
const { data: voice } = await heygenJSON(`/voices/${id}`, { headers });
|
||||
if (voice.status === "complete") return { voice_id: voice.voice_id ?? id };
|
||||
if (voice.status === "failed") throw new Error(voice.failure_message || "voice clone failed");
|
||||
if (now() >= deadline) throw new Error(`voice clone ${id} not ready after 120 s`);
|
||||
await sleep(POLL_MS);
|
||||
}
|
||||
}
|
||||
|
||||
async function list(prefix, headers) {
|
||||
const mine = [];
|
||||
const seen = new Set();
|
||||
let token;
|
||||
do {
|
||||
if (token && seen.has(token)) throw new Error(`HeyGen returned page token ${token} twice`);
|
||||
if (token) seen.add(token);
|
||||
const query = new URLSearchParams({ type: "private", limit: "100" });
|
||||
if (token) query.set("token", token);
|
||||
const page = await heygenJSON(`/voices?${query}`, { headers });
|
||||
mine.push(...page.data.filter((v) => (v.name ?? "").startsWith(prefix)));
|
||||
token = page.has_more ? page.next_token : null;
|
||||
} while (token);
|
||||
// ponytail: the list response has no created_at, so one GET per match; bounded by the clone limit.
|
||||
return Promise.all(
|
||||
mine.map(async ({ voice_id, name }) => {
|
||||
const { data } = await heygenJSON(`/voices/${voice_id}`, { headers });
|
||||
return { voice_id, name, created_at: data.created_at ?? null };
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function main(
|
||||
argv,
|
||||
{
|
||||
sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
|
||||
now = Date.now,
|
||||
out = (s) => process.stdout.write(`${s}\n`),
|
||||
err = (s) => process.stderr.write(`${s}\n`),
|
||||
} = {},
|
||||
) {
|
||||
try {
|
||||
const { positionals, values } = parseArgs({
|
||||
args: argv,
|
||||
allowPositionals: true,
|
||||
options: { name: { type: "string" }, prefix: { type: "string", default: "" } },
|
||||
});
|
||||
const [command, arg] = positionals;
|
||||
if (command === "clone") {
|
||||
if (!arg || !values.name) throw new Error("usage: clone <audio-file> --name <name>");
|
||||
out(JSON.stringify(await clone(arg, values.name, heygenAuthHeaders(), { sleep, now })));
|
||||
} else if (command === "delete") {
|
||||
if (!arg) throw new Error("usage: delete <voice_id>");
|
||||
await heygenJSON(`/voices/${encodeURIComponent(arg)}`, {
|
||||
method: "DELETE",
|
||||
headers: heygenAuthHeaders(),
|
||||
});
|
||||
} else if (command === "list") {
|
||||
out(JSON.stringify(await list(values.prefix, heygenAuthHeaders())));
|
||||
} else {
|
||||
throw new Error("usage: heygen-voice.mjs clone|delete|list (see the header of this file)");
|
||||
}
|
||||
return 0;
|
||||
} catch (e) {
|
||||
err(heygenMessage(e));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (isMainModule(import.meta.url)) {
|
||||
loadEnvFromDir(process.cwd());
|
||||
process.exitCode = await main(process.argv.slice(2));
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
import { afterEach, beforeEach, test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { main } from "./heygen-voice.mjs";
|
||||
|
||||
const realFetch = globalThis.fetch;
|
||||
const savedEnv = {
|
||||
HEYGEN_ACCESS_TOKEN: process.env.HEYGEN_ACCESS_TOKEN,
|
||||
HEYGEN_API_KEY: process.env.HEYGEN_API_KEY,
|
||||
};
|
||||
let dir;
|
||||
let calls;
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), "heygen-voice-"));
|
||||
delete process.env.HEYGEN_ACCESS_TOKEN;
|
||||
process.env.HEYGEN_API_KEY = "hg_test";
|
||||
calls = [];
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = realFetch;
|
||||
for (const [k, v] of Object.entries(savedEnv)) {
|
||||
if (v === undefined) delete process.env[k];
|
||||
else process.env[k] = v;
|
||||
}
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// routes: "METHOD /path" → array of [status, body] answered in order (last one repeats).
|
||||
function mockFetch(routes) {
|
||||
globalThis.fetch = async (url, opts) => {
|
||||
const key = `${opts.method} ${String(url).replace("https://api.heygen.com/v3", "")}`;
|
||||
calls.push({ key, opts });
|
||||
const queue = routes[key];
|
||||
if (!queue) throw new Error(`unexpected request ${key}`);
|
||||
const [status, body] = queue.length > 1 ? queue.shift() : queue[0];
|
||||
return new Response(body === undefined ? null : JSON.stringify(body), { status });
|
||||
};
|
||||
}
|
||||
|
||||
async function run(argv, { sleep = async () => {}, now = () => 0 } = {}) {
|
||||
const out = [];
|
||||
const err = [];
|
||||
const code = await main(argv, {
|
||||
sleep,
|
||||
now,
|
||||
out: (s) => out.push(s),
|
||||
err: (s) => err.push(s),
|
||||
});
|
||||
return { code, out: out.join("\n"), err: err.join("\n") };
|
||||
}
|
||||
|
||||
function audioFile(name = "sample.mp3") {
|
||||
const path = join(dir, name);
|
||||
writeFileSync(path, Buffer.from("fake-audio"));
|
||||
return path;
|
||||
}
|
||||
|
||||
test("clone uploads base64 audio, polls until complete, prints the voice id", async () => {
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[200, { data: { voice_clone_id: "vc1" } }]],
|
||||
"GET /voices/vc1": [
|
||||
[200, { data: { status: "processing", voice_id: "vc1" } }],
|
||||
[200, { data: { status: "complete", voice_id: "vc1", name: "Me" } }],
|
||||
],
|
||||
});
|
||||
const sleeps = [];
|
||||
const r = await run(["clone", audioFile(), "--name", "Me"], {
|
||||
sleep: async (ms) => sleeps.push(ms),
|
||||
});
|
||||
assert.deepEqual(r, { code: 0, out: '{"voice_id":"vc1"}', err: "" });
|
||||
assert.deepEqual(sleeps, [2000]);
|
||||
const post = calls[0];
|
||||
assert.equal(post.opts.headers["X-Api-Key"], "hg_test");
|
||||
assert.deepEqual(JSON.parse(post.opts.body), {
|
||||
voice_name: "Me",
|
||||
audio: {
|
||||
type: "base64",
|
||||
media_type: "audio/mpeg",
|
||||
data: Buffer.from("fake-audio").toString("base64"),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("clone sends audio/wav for a .wav file", async () => {
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[200, { data: { voice_clone_id: "vc1" } }]],
|
||||
"GET /voices/vc1": [[200, { data: { status: "complete", voice_id: "vc1" } }]],
|
||||
});
|
||||
const r = await run(["clone", audioFile("take.WAV"), "--name", "Me"]);
|
||||
assert.equal(r.code, 0);
|
||||
assert.equal(JSON.parse(calls[0].opts.body).audio.media_type, "audio/wav");
|
||||
});
|
||||
|
||||
test("clone prints HeyGen's refusal verbatim and exits 1", async () => {
|
||||
const message =
|
||||
"Voice clone limit reached (10). Delete unused clones or contact support to increase your limit.";
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[400, { error: { code: "resource_limit_reached", message } }]],
|
||||
});
|
||||
const r = await run(["clone", audioFile(), "--name", "Me"]);
|
||||
assert.deepEqual(r, { code: 1, out: "", err: message });
|
||||
});
|
||||
|
||||
test("clone surfaces a failed clone's failure_message", async () => {
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[200, { data: { voice_clone_id: "vc1" } }]],
|
||||
"GET /voices/vc1": [[200, { data: { status: "failed", failure_message: "Audio too noisy" } }]],
|
||||
});
|
||||
const r = await run(["clone", audioFile(), "--name", "Me"]);
|
||||
assert.deepEqual(r, { code: 1, out: "", err: "Audio too noisy" });
|
||||
});
|
||||
|
||||
test("clone gives up after the 120 s deadline", async () => {
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[200, { data: { voice_clone_id: "vc1" } }]],
|
||||
"GET /voices/vc1": [[200, { data: { status: "processing" } }]],
|
||||
});
|
||||
let clock = 0;
|
||||
const r = await run(["clone", audioFile(), "--name", "Me"], {
|
||||
now: () => clock,
|
||||
sleep: async (ms) => {
|
||||
clock += ms;
|
||||
},
|
||||
});
|
||||
assert.deepEqual(r, { code: 1, out: "", err: "voice clone vc1 not ready after 120 s" });
|
||||
assert.equal(calls.filter((c) => c.key === "GET /voices/vc1").length, 61);
|
||||
});
|
||||
|
||||
test("clone rejects an audio type other than mp3 or wav without calling HeyGen", async () => {
|
||||
mockFetch({});
|
||||
const r = await run(["clone", audioFile("take.ogg"), "--name", "Me"]);
|
||||
assert.equal(r.code, 1);
|
||||
assert.match(r.err, /use \.mp3 or \.wav/);
|
||||
assert.equal(calls.length, 0);
|
||||
});
|
||||
|
||||
test("delete calls DELETE /voices/<id> and exits 0", async () => {
|
||||
mockFetch({ "DELETE /voices/vc1": [[200, { data: {} }]] });
|
||||
const r = await run(["delete", "vc1"]);
|
||||
assert.deepEqual(r, { code: 0, out: "", err: "" });
|
||||
});
|
||||
|
||||
test("delete exits 0 on a 204 with no body, and escapes the id in the path", async () => {
|
||||
mockFetch({ "DELETE /voices/a%2Fb": [[204]] });
|
||||
const r = await run(["delete", "a/b"]);
|
||||
assert.deepEqual(r, { code: 0, out: "", err: "" });
|
||||
});
|
||||
|
||||
test("clone falls back to the clone id when the finished voice has no voice_id", async () => {
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[200, { data: { voice_clone_id: "vc9" } }]],
|
||||
"GET /voices/vc9": [[200, { data: { status: "complete" } }]],
|
||||
});
|
||||
const r = await run(["clone", audioFile(), "--name", "Me"]);
|
||||
assert.deepEqual(r, { code: 0, out: '{"voice_id":"vc9"}', err: "" });
|
||||
});
|
||||
|
||||
test("list skips a nameless voice and stops on a page token HeyGen repeats", async () => {
|
||||
mockFetch({
|
||||
"GET /voices?type=private&limit=100": [
|
||||
[200, { data: [{ voice_id: "n", name: null }], has_more: true, next_token: "t2" }],
|
||||
],
|
||||
"GET /voices?type=private&limit=100&token=t2": [
|
||||
[200, { data: [], has_more: true, next_token: "t2" }],
|
||||
],
|
||||
});
|
||||
// A fake answers at once, so a list that never stops would starve any timeout: cap the calls instead.
|
||||
const answer = globalThis.fetch;
|
||||
globalThis.fetch = (...args) =>
|
||||
calls.length >= 10 ? Promise.reject(new Error("paged forever")) : answer(...args);
|
||||
const r = await run(["list", "--prefix", "desk-"]);
|
||||
assert.deepEqual(r, { code: 1, out: "", err: "HeyGen returned page token t2 twice" });
|
||||
});
|
||||
|
||||
test("delete prints HeyGen's 404 message verbatim and exits 1", async () => {
|
||||
mockFetch({
|
||||
"DELETE /voices/gone": [
|
||||
[404, { error: { code: "voice_not_found", message: "Voice not found: gone" } }],
|
||||
],
|
||||
});
|
||||
const r = await run(["delete", "gone"]);
|
||||
assert.deepEqual(r, { code: 1, out: "", err: "Voice not found: gone" });
|
||||
});
|
||||
|
||||
test("list pages through private voices and returns prefix matches with created_at", async () => {
|
||||
mockFetch({
|
||||
"GET /voices?type=private&limit=100": [
|
||||
[
|
||||
200,
|
||||
{
|
||||
data: [
|
||||
{ voice_id: "a", name: "desk-1" },
|
||||
{ voice_id: "b", name: "other" },
|
||||
],
|
||||
has_more: true,
|
||||
next_token: "t2",
|
||||
},
|
||||
],
|
||||
],
|
||||
"GET /voices?type=private&limit=100&token=t2": [
|
||||
[200, { data: [{ voice_id: "c", name: "desk-2" }], has_more: false }],
|
||||
],
|
||||
"GET /voices/a": [[200, { data: { voice_id: "a", created_at: 1700000000 } }]],
|
||||
"GET /voices/c": [[200, { data: { voice_id: "c", created_at: 1700000100 } }]],
|
||||
});
|
||||
const r = await run(["list", "--prefix", "desk-"]);
|
||||
assert.equal(r.code, 0);
|
||||
assert.deepEqual(JSON.parse(r.out), [
|
||||
{ voice_id: "a", name: "desk-1", created_at: 1700000000 },
|
||||
{ voice_id: "c", name: "desk-2", created_at: 1700000100 },
|
||||
]);
|
||||
});
|
||||
|
||||
test("list prints HeyGen's refusal verbatim and exits 1", async () => {
|
||||
const message = "Invalid or expired API key. Verify your x-api-key header.";
|
||||
mockFetch({
|
||||
"GET /voices?type=private&limit=100": [[401, { error: { code: "unauthorized", message } }]],
|
||||
});
|
||||
const r = await run(["list", "--prefix", "desk-"]);
|
||||
assert.deepEqual(r, { code: 1, out: "", err: message });
|
||||
});
|
||||
|
||||
test("a host-injected HEYGEN_ACCESS_TOKEN is sent as Bearer", async () => {
|
||||
process.env.HEYGEN_ACCESS_TOKEN = "at_host";
|
||||
mockFetch({ "DELETE /voices/vc1": [[200, { data: {} }]] });
|
||||
await run(["delete", "vc1"]);
|
||||
assert.equal(calls[0].opts.headers.Authorization, "Bearer at_host");
|
||||
assert.equal(calls[0].opts.headers["X-Api-Key"], undefined);
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
import { closeSync, ftruncateSync, openSync, readFileSync, writeSync } from "node:fs";
|
||||
|
||||
// Keep the merge base and output on the same file even if its pathname changes
|
||||
// while audio generation runs. The CLI owns this handle until write or exit.
|
||||
export function openAudioMeta(path) {
|
||||
let fd;
|
||||
try {
|
||||
fd = openSync(path, "r+");
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT") throw error;
|
||||
}
|
||||
let value = {};
|
||||
if (fd !== undefined) {
|
||||
try {
|
||||
value = JSON.parse(readFileSync(fd, "utf8"));
|
||||
} catch (error) {
|
||||
closeSync(fd);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return {
|
||||
value,
|
||||
write(meta) {
|
||||
const bytes = Buffer.from(JSON.stringify(meta, null, 2));
|
||||
// Defer new-file creation until generation succeeds. Never overwrite a
|
||||
// file (or follow a link) that appeared since the missing merge base.
|
||||
if (fd === undefined) fd = openSync(path, "wx");
|
||||
try {
|
||||
let offset = 0;
|
||||
while (offset < bytes.length) {
|
||||
offset += writeSync(fd, bytes, offset, bytes.length - offset, offset);
|
||||
}
|
||||
ftruncateSync(fd, bytes.length);
|
||||
} finally {
|
||||
closeSync(fd);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import {
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { openAudioMeta } from "./audio-meta.mjs";
|
||||
|
||||
function fixture(t) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "audio-meta-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
}
|
||||
|
||||
test("updates the read file from byte zero, truncates, and retains its mode", (t) => {
|
||||
const path = join(fixture(t), "meta.json");
|
||||
writeFileSync(path, JSON.stringify({ voices: ["long existing voice metadata"] }), {
|
||||
mode: 0o600,
|
||||
});
|
||||
const originalMode = statSync(path).mode;
|
||||
const handle = openAudioMeta(path);
|
||||
assert.deepEqual(handle.value, { voices: ["long existing voice metadata"] });
|
||||
handle.write({ voices: ["声"] });
|
||||
assert.equal(readFileSync(path, "utf8"), JSON.stringify({ voices: ["声"] }, null, 2));
|
||||
assert.equal(statSync(path).mode, originalMode);
|
||||
});
|
||||
|
||||
test("pathname replacement cannot redirect the existing-file write", (t) => {
|
||||
const dir = fixture(t);
|
||||
const path = join(dir, "meta.json");
|
||||
const moved = join(dir, "original.json");
|
||||
writeFileSync(path, "{}");
|
||||
const handle = openAudioMeta(path);
|
||||
renameSync(path, moved);
|
||||
writeFileSync(path, "replacement must survive");
|
||||
handle.write({ voices: [] });
|
||||
assert.equal(readFileSync(path, "utf8"), "replacement must survive");
|
||||
assert.deepEqual(JSON.parse(readFileSync(moved, "utf8")), { voices: [] });
|
||||
});
|
||||
|
||||
test("existing symlink outputs retain their original target even after retargeting", (t) => {
|
||||
const dir = fixture(t);
|
||||
const path = join(dir, "meta.json");
|
||||
const target = join(dir, "target.json");
|
||||
const victim = join(dir, "victim.json");
|
||||
writeFileSync(target, "{}");
|
||||
writeFileSync(victim, "untouched");
|
||||
symlinkSync(target, path);
|
||||
const handle = openAudioMeta(path);
|
||||
rmSync(path);
|
||||
symlinkSync(victim, path);
|
||||
handle.write({ bgm: null });
|
||||
assert.equal(readFileSync(victim, "utf8"), "untouched");
|
||||
assert.deepEqual(JSON.parse(readFileSync(target, "utf8")), { bgm: null });
|
||||
});
|
||||
|
||||
test("missing output is created only when saved", (t) => {
|
||||
const path = join(fixture(t), "meta.json");
|
||||
const handle = openAudioMeta(path);
|
||||
assert.deepEqual(handle.value, {});
|
||||
assert.throws(() => statSync(path), { code: "ENOENT" });
|
||||
handle.write({ sfx: [] });
|
||||
assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), { sfx: [] });
|
||||
});
|
||||
|
||||
for (const replacement of ["file", "symlink"]) {
|
||||
test(`new output does not overwrite a racing ${replacement}`, (t) => {
|
||||
const dir = fixture(t);
|
||||
const path = join(dir, "meta.json");
|
||||
const victim = join(dir, "victim.json");
|
||||
const handle = openAudioMeta(path);
|
||||
writeFileSync(victim, "untouched");
|
||||
if (replacement === "symlink") symlinkSync(victim, path);
|
||||
else writeFileSync(path, "untouched");
|
||||
assert.throws(() => handle.write({}), { code: "EEXIST" });
|
||||
assert.equal(readFileSync(path, "utf8"), "untouched");
|
||||
assert.equal(readFileSync(victim, "utf8"), "untouched");
|
||||
});
|
||||
}
|
||||
|
||||
test("malformed merge base remains an error without changing bytes", (t) => {
|
||||
const path = join(fixture(t), "meta.json");
|
||||
writeFileSync(path, "bad json");
|
||||
assert.throws(() => openAudioMeta(path), SyntaxError);
|
||||
assert.equal(readFileSync(path, "utf8"), "bad json");
|
||||
});
|
||||
|
||||
test("CLI partial run retains unselected voices, BGM and SFX", (t) => {
|
||||
const dir = fixture(t);
|
||||
const path = join(dir, "meta.json");
|
||||
const request = join(dir, "request.json");
|
||||
const previous = {
|
||||
voices: [{ id: "a", duration_s: 2 }],
|
||||
bgm: { path: "bgm.wav" },
|
||||
sfx: [{ name: "click" }],
|
||||
tts_provider: "kokoro",
|
||||
voice_id: "voice-a",
|
||||
};
|
||||
writeFileSync(path, JSON.stringify(previous));
|
||||
writeFileSync(request, "{}");
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
fileURLToPath(new URL("../audio.mjs", import.meta.url)),
|
||||
"--request",
|
||||
request,
|
||||
"--hyperframes",
|
||||
dir,
|
||||
"--out",
|
||||
path,
|
||||
"--only",
|
||||
"none",
|
||||
],
|
||||
{
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
HEYGEN_CONFIG_DIR: dir,
|
||||
HEYGEN_API_KEY: "",
|
||||
HYPERFRAMES_API_KEY: "",
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const actual = JSON.parse(readFileSync(path, "utf8"));
|
||||
for (const key of Object.keys(previous)) assert.deepEqual(actual[key], previous[key]);
|
||||
});
|
||||
@@ -0,0 +1,4 @@
|
||||
// Music sits under narration; a silent film brings music forward.
|
||||
export const BGM_BED_VOLUME = 0.12;
|
||||
export const BGM_SILENT_VOLUME = 0.9;
|
||||
export const bgmDefaultVolume = (hasVoice) => (hasVoice ? BGM_BED_VOLUME : BGM_SILENT_VOLUME);
|
||||
@@ -0,0 +1,257 @@
|
||||
// bgm.mjs — background music for the media audio engine. Two routes, gated the
|
||||
// same way as TTS/SFX:
|
||||
//
|
||||
// retrieve (default when HeyGen is configured) — search HeyGen's music library
|
||||
// by mood, download the top track. Synchronous. assets/bgm/track.mp3.
|
||||
// generate (the alternative; the automatic choice when HeyGen is absent) —
|
||||
// Lyria (cloud, $GEMINI_API_KEY/$GOOGLE_API_KEY + google-genai) preferred,
|
||||
// else local MusicGen (facebook/musicgen-small via transformers). Spawned
|
||||
// DETACHED so the engine can return while audio renders; the caller marks
|
||||
// bgm_pending and runs wait-bgm.mjs before assembling. assets/bgm/track.wav.
|
||||
//
|
||||
// Missing/failed BGM never blocks a render.
|
||||
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, openSync, closeSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { downloadTo, searchSounds } from "./heygen.mjs";
|
||||
import { agentWritePath } from "./media-record.mjs";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
const lyriaKey = () => process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || "";
|
||||
|
||||
import { bgmDefaultVolume } from "./bgm-volume.mjs";
|
||||
export { BGM_BED_VOLUME, BGM_SILENT_VOLUME, bgmDefaultVolume } from "./bgm-volume.mjs";
|
||||
|
||||
const BGM_PY_DEPS = ["transformers", "torch", "soundfile", "numpy"];
|
||||
const BGM_PY_PROBE =
|
||||
"import transformers, soundfile, torch, numpy; from transformers import MusicgenForConditionalGeneration";
|
||||
const LYRIA_PY_DEPS = ["google-genai", "python-dotenv"];
|
||||
const LYRIA_PY_PROBE = "import google.genai";
|
||||
|
||||
function pyOk(probe) {
|
||||
const { cmd, args } = pythonInvocation(["-c", probe]);
|
||||
return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;
|
||||
}
|
||||
// `python -m pip`, not a bare `pip` binary: a Homebrew/system Python often
|
||||
// exposes only `python3`/`pip3` on PATH, so a plain `pip` spawn silently
|
||||
// no-ops (ENOENT) and the documented "auto-installed on demand" path never
|
||||
// actually installs. `-m pip` also guarantees the packages land in the SAME
|
||||
// interpreter pyOk() probes — a bare `pip`/`pip3` could resolve to a
|
||||
// different Python installation than `python3` if more than one is on PATH.
|
||||
function pipInstall(deps) {
|
||||
const { cmd, args } = pythonInvocation(["-m", "pip", "install", "-q", ...deps]);
|
||||
return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;
|
||||
}
|
||||
|
||||
// ── retrieval (HeyGen music library) ──────────────────────────────────────────
|
||||
export async function retrieveBgm({ query, headers, hyperframesDir, hasVoice, anomalies }) {
|
||||
const q = query || "calm cinematic underscore";
|
||||
const results = await searchSounds(q, "music", headers, { limit: 5 });
|
||||
if (!results.length) return null;
|
||||
const top = results[0];
|
||||
const rel = agentWritePath(hyperframesDir, "assets/bgm/track.mp3", { anomalies });
|
||||
await downloadTo(top.audio_url, join(hyperframesDir, rel));
|
||||
return {
|
||||
path: rel,
|
||||
volume: bgmDefaultVolume(hasVoice),
|
||||
query: q,
|
||||
mode: "retrieve",
|
||||
duration_s: typeof top.duration === "number" ? r3(top.duration) : null,
|
||||
};
|
||||
}
|
||||
|
||||
// ── mood inference (for the generate path's prompt) ──────────────────────────
|
||||
// Industry base → archetype shape → emotional-arc tiebreaker. Exported so a
|
||||
// workflow adapter can build a rich prompt from its own narrative metadata; the
|
||||
// engine also calls it when generate has only a plain mood query.
|
||||
export function inferBgmPrompt({ blob = "", archetype = "", arc = "", userPrompt = "" } = {}) {
|
||||
if (userPrompt) return userPrompt;
|
||||
const b = String(blob).toLowerCase();
|
||||
let base;
|
||||
let bpm;
|
||||
if (/\b(crypto|nft|web3|defi|token|blockchain|exchange|wallet|dao)\b/.test(b)) {
|
||||
base = "atmospheric electronic, deep bass, futuristic synths, restrained percussion";
|
||||
bpm = 100;
|
||||
} else if (/\b(finance|fintech|bank|payment|invest|wealth|insurance|treasury)\b/.test(b)) {
|
||||
base = "calm cinematic, soft strings, subtle piano, restrained percussion";
|
||||
bpm = 92;
|
||||
} else if (/\b(creative|agency|design|studio|art|brand|marketing|content)\b/.test(b)) {
|
||||
base = "playful electronic, warm pads, light percussion";
|
||||
bpm = 115;
|
||||
} else {
|
||||
base = "uplifting corporate tech, bright modern piano with synth pads";
|
||||
bpm = 108;
|
||||
}
|
||||
const at = String(archetype).toLowerCase();
|
||||
const ar = String(arc).toLowerCase();
|
||||
if (/\bpas\b|pain.agitate|pain.+solve/.test(at))
|
||||
return `${base}, starts with subtle tension then builds to resolution, BPM ${bpm}, transitions from MINOR to MAJOR`;
|
||||
if (/\bbab\b|before.after|future.pac|vision/.test(at))
|
||||
return `${base}, cinematic and aspirational, steady build with rising energy, BPM ${bpm}, MAJOR`;
|
||||
if (/cascade|feature.benefit/.test(at))
|
||||
return `${base}, energetic and driving, consistent momentum, BPM ${Math.min(bpm + 10, 128)}, MAJOR`;
|
||||
if (/demo.loop|question.+answer/.test(at))
|
||||
return `${base}, clean and focused, minimal arrangement, BPM ${Math.max(bpm - 8, 88)}`;
|
||||
if (/frustrat|anxiety|overwhelm|tension/.test(ar) && /relief|excite|triumph/.test(ar))
|
||||
return `${base}, builds from understated tension to uplifting resolution, BPM ${bpm}, MINOR to MAJOR`;
|
||||
if (/excit|awe|power|triumph/.test(ar))
|
||||
return `${base}, energetic and confident, BPM ${bpm}, MAJOR`;
|
||||
if (/trust|ease|clarity|reassur/.test(ar))
|
||||
return `${base}, warm and reassuring, BPM ${Math.max(bpm - 5, 85)}`;
|
||||
return `${base}, BPM ${bpm}, MAJOR`;
|
||||
}
|
||||
|
||||
// ── generation (Lyria → MusicGen, detached) ──────────────────────────────────
|
||||
// Returns a bgmMeta the caller folds into audio_meta:
|
||||
// { path, mode, volume, provider, pid, log, target_duration_s, seed_duration_s,
|
||||
// loop_count, pending:true } on success, or { disabled:true, reason }.
|
||||
export function generateBgmDetached({
|
||||
prompt,
|
||||
durationS,
|
||||
hyperframesDir,
|
||||
lyriaRecipe,
|
||||
seedSeconds = 28,
|
||||
hasVoice,
|
||||
anomalies,
|
||||
}) {
|
||||
const rel = agentWritePath(hyperframesDir, "assets/bgm/track.wav", { anomalies });
|
||||
const abs = join(hyperframesDir, rel);
|
||||
mkdirSync(join(hyperframesDir, "assets", "bgm"), { recursive: true });
|
||||
const log = join(hyperframesDir, "assets", "bgm", `bgm-${Date.now()}.log`);
|
||||
const targetS = Math.max(1, durationS);
|
||||
const baseMeta = { path: rel, mode: null, volume: bgmDefaultVolume(hasVoice), pending: true };
|
||||
|
||||
const lyriaConfigured = !!lyriaKey() && !!lyriaRecipe && existsSync(lyriaRecipe);
|
||||
|
||||
// Make a backend runnable: prefer Lyria when configured (install google-genai
|
||||
// on demand), else ensure local MusicGen deps. Installs are synchronous here —
|
||||
// generation itself is detached, so the engine still returns promptly.
|
||||
if (lyriaConfigured && !pyOk(LYRIA_PY_PROBE)) pipInstall(LYRIA_PY_DEPS);
|
||||
const useLyria = lyriaConfigured && pyOk(LYRIA_PY_PROBE);
|
||||
if (!useLyria && !pyOk(BGM_PY_PROBE)) pipInstall(BGM_PY_DEPS);
|
||||
|
||||
const fd = openSync(log, "w");
|
||||
if (useLyria) {
|
||||
const { cmd, args } = pythonInvocation([
|
||||
lyriaRecipe,
|
||||
"--output",
|
||||
abs,
|
||||
"--duration",
|
||||
String(targetS),
|
||||
"--prompt",
|
||||
prompt,
|
||||
]);
|
||||
rmSync(abs, { force: true }); // wait-bgm takes any file here as the finished track
|
||||
const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] });
|
||||
proc.unref();
|
||||
closeSync(fd);
|
||||
return {
|
||||
...baseMeta,
|
||||
mode: "detached-single",
|
||||
provider: "lyria",
|
||||
pid: proc.pid,
|
||||
log,
|
||||
target_duration_s: r3(targetS),
|
||||
};
|
||||
}
|
||||
|
||||
if (pyOk(BGM_PY_PROBE)) {
|
||||
const seedS = Math.min(Math.max(seedSeconds, 10), 30);
|
||||
const loops = targetS > seedS ? Math.ceil(targetS / seedS) : 1;
|
||||
const script = musicgenScript({ prompt, abs, targetS, seedS });
|
||||
const { cmd, args } = pythonInvocation(["-c", script]);
|
||||
rmSync(abs, { force: true });
|
||||
const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] });
|
||||
proc.unref();
|
||||
closeSync(fd);
|
||||
return {
|
||||
...baseMeta,
|
||||
mode: targetS > seedS ? "detached-seed-loop" : "detached-seed-trim",
|
||||
provider: "musicgen",
|
||||
pid: proc.pid,
|
||||
log,
|
||||
target_duration_s: r3(targetS),
|
||||
seed_duration_s: seedS,
|
||||
loop_count: loops,
|
||||
};
|
||||
}
|
||||
|
||||
closeSync(fd);
|
||||
return {
|
||||
disabled: true,
|
||||
reason: lyriaConfigured
|
||||
? `Lyria configured but google-genai uninstallable, and local MusicGen unavailable (pip install ${BGM_PY_DEPS.join(" ")})`
|
||||
: `no Lyria key/recipe and local MusicGen deps unavailable (pip install ${BGM_PY_DEPS.join(" ")})`,
|
||||
};
|
||||
}
|
||||
|
||||
// Inline MusicGen: generate ONE seed clip (≤30s to stay under the decoder's
|
||||
// positional limit), then trim it down or crossfade-loop it up to the target.
|
||||
function musicgenScript({ prompt, abs, targetS, seedS }) {
|
||||
return `
|
||||
import math, os, sys, traceback
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from transformers import MusicgenForConditionalGeneration, AutoProcessor
|
||||
|
||||
prompt = ${JSON.stringify(prompt)}
|
||||
out_path = ${JSON.stringify(abs)}
|
||||
target_s = float(${targetS.toFixed(3)})
|
||||
seed_s = float(${seedS.toFixed(3)})
|
||||
token_rate = 50
|
||||
crossfade_s = 0.3
|
||||
|
||||
def apply_fade(arr, sr, fade_in_s=0.08, fade_out_s=0.5):
|
||||
n_in = min(int(round(fade_in_s * sr)), arr.shape[0] // 2)
|
||||
n_out = min(int(round(fade_out_s * sr)), arr.shape[0] // 2)
|
||||
if n_in > 1: arr[:n_in] *= np.linspace(0.0, 1.0, n_in, dtype="float32")
|
||||
if n_out > 1: arr[-n_out:] *= np.linspace(1.0, 0.0, n_out, dtype="float32")
|
||||
return arr
|
||||
|
||||
def loop_crossfade(seed, target_len, xf):
|
||||
if seed.shape[0] >= target_len: return seed[:target_len]
|
||||
xf = min(xf, seed.shape[0] // 2)
|
||||
if xf < 1:
|
||||
reps = int(math.ceil(target_len / seed.shape[0]))
|
||||
return np.tile(seed, reps)[:target_len]
|
||||
t = np.linspace(0.0, 1.0, xf, dtype="float32")
|
||||
fade_out = np.cos(t * (math.pi / 2)); fade_in = np.sin(t * (math.pi / 2))
|
||||
out = seed.copy()
|
||||
while out.shape[0] < target_len:
|
||||
tail = out[-xf:] * fade_out; head = seed[:xf] * fade_in
|
||||
out = np.concatenate([out[:-xf], tail + head, seed[xf:]])
|
||||
return out[:target_len]
|
||||
|
||||
try:
|
||||
Path(os.path.dirname(out_path)).mkdir(parents=True, exist_ok=True)
|
||||
processor = AutoProcessor.from_pretrained("facebook/musicgen-small")
|
||||
model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small")
|
||||
model.eval()
|
||||
sr = int(model.config.audio_encoder.sampling_rate)
|
||||
gen_s = min(seed_s, target_s)
|
||||
tokens = max(1, int(math.ceil(gen_s * token_rate)))
|
||||
print(f"[musicgen] seed dur={gen_s:.2f}s tokens={tokens}", flush=True)
|
||||
inputs = processor(text=[prompt], padding=True, return_tensors="pt")
|
||||
audio = model.generate(**inputs, max_new_tokens=tokens)
|
||||
seed = audio[0, 0].detach().cpu().numpy().astype("float32")
|
||||
peak = float(np.max(np.abs(seed)))
|
||||
if peak > 1e-6: seed = seed * (0.89 / peak)
|
||||
want = max(1, int(round(target_s * sr)))
|
||||
if seed.shape[0] >= want:
|
||||
final = seed[:want].copy()
|
||||
else:
|
||||
final = loop_crossfade(seed, want, int(round(crossfade_s * sr)))
|
||||
if final.shape[0] < want: final = np.pad(final, (0, want - final.shape[0]))
|
||||
else: final = final[:want]
|
||||
final = apply_fade(final, sr)
|
||||
peak = float(np.max(np.abs(final)))
|
||||
if peak > 1.0: final = final / peak
|
||||
sf.write(out_path, final, sr)
|
||||
print(f"[musicgen] wrote {out_path} samples={final.shape[0]} sr={sr}", flush=True)
|
||||
except Exception:
|
||||
traceback.print_exc(); sys.exit(1)
|
||||
`;
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { appendRecord } from "../../../scripts/lib/manifest.mjs";
|
||||
import {
|
||||
BGM_BED_VOLUME,
|
||||
BGM_SILENT_VOLUME,
|
||||
bgmDefaultVolume,
|
||||
generateBgmDetached,
|
||||
} from "./bgm.mjs";
|
||||
|
||||
// Regression: narrated pipelines used to ship BGM at 0.8 (≈ -2 dB), ~16 dB
|
||||
// hotter than a music bed under a voice should be. The default under narration
|
||||
// must be a proper bed (≈ -18 dB); a silent film keeps the louder default.
|
||||
|
||||
const dbfs = (linear) => 20 * Math.log10(linear);
|
||||
|
||||
test("BGM under narration is a bed near -18 dB", () => {
|
||||
assert.equal(bgmDefaultVolume(true), BGM_BED_VOLUME);
|
||||
assert.equal(BGM_BED_VOLUME, 0.12);
|
||||
const db = dbfs(BGM_BED_VOLUME);
|
||||
assert.ok(db < -17 && db > -19, `bed should be ≈ -18 dB, got ${db.toFixed(1)} dB`);
|
||||
});
|
||||
|
||||
test("a silent film (no voice) keeps BGM forward", () => {
|
||||
assert.equal(bgmDefaultVolume(false), BGM_SILENT_VOLUME);
|
||||
assert.equal(BGM_SILENT_VOLUME, 0.9);
|
||||
});
|
||||
|
||||
test("the narrated default is well below the voice (≈ 0 dBFS)", () => {
|
||||
// Voice sits at data-volume="1" (0 dBFS); the bed must be ~16+ dB under it.
|
||||
const separation = dbfs(1) - dbfs(bgmDefaultVolume(true));
|
||||
assert.ok(
|
||||
separation >= 16,
|
||||
`bed should sit ≥16 dB under the voice, got ${separation.toFixed(1)} dB`,
|
||||
);
|
||||
});
|
||||
|
||||
test(
|
||||
"generating music again clears the engine's old track, so waiting cannot mistake it for the new one",
|
||||
{ skip: process.platform === "win32" && "the fake python is a shell script" },
|
||||
(t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-bgm-"));
|
||||
const path = process.env.PATH;
|
||||
t.after(() => {
|
||||
process.env.PATH = path;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
mkdirSync(join(dir, "bin"));
|
||||
for (const name of ["python3", "python"])
|
||||
writeFileSync(join(dir, "bin", name), "#!/bin/sh\nexit 0\n", { mode: 0o755 });
|
||||
process.env.PATH = `${join(dir, "bin")}:${path}`;
|
||||
mkdirSync(join(dir, "assets/bgm"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets/bgm/track.wav"), "last run's track");
|
||||
appendRecord(dir, {
|
||||
id: "bgm_001",
|
||||
type: "bgm",
|
||||
path: "assets/bgm/track.wav",
|
||||
source: "generated",
|
||||
});
|
||||
|
||||
const gen = generateBgmDetached({
|
||||
prompt: "calm",
|
||||
durationS: 5,
|
||||
hyperframesDir: dir,
|
||||
anomalies: [],
|
||||
});
|
||||
|
||||
assert.equal(gen.path, "assets/bgm/track.wav");
|
||||
assert.equal(existsSync(join(dir, "assets/bgm/track.wav")), false);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,14 @@
|
||||
// mapWithConcurrency — run `fn` over `items` with at most `limit` in flight at
|
||||
// once. Preserves input order in the result array regardless of completion order.
|
||||
export async function mapWithConcurrency(items, limit, fn) {
|
||||
const results = new Array(items.length);
|
||||
let next = 0;
|
||||
async function worker() {
|
||||
while (next < items.length) {
|
||||
const i = next++;
|
||||
results[i] = await fn(items[i], i);
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
||||
return results;
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mapWithConcurrency } from "./concurrency.mjs";
|
||||
|
||||
// Regression: audio.mjs used a bare Promise.all(lines.map(synthLine)) to
|
||||
// synthesize every TTS line at once, spawning one Kokoro/whisper model load
|
||||
// per line concurrently. Two independent reports of this overwhelming a
|
||||
// machine (OOM, and cold-start contention causing spurious failures).
|
||||
// mapWithConcurrency is the extracted cap; test it in isolation since
|
||||
// audio.mjs itself is a script (runs CLI/exit side effects on import).
|
||||
test("processes every item and preserves input order regardless of completion order", async () => {
|
||||
const order = [5, 1, 3, 2, 4];
|
||||
const results = await mapWithConcurrency(order, 2, async (n) => {
|
||||
await new Promise((r) => setTimeout(r, n));
|
||||
return n * 10;
|
||||
});
|
||||
assert.deepEqual(results, [50, 10, 30, 20, 40]);
|
||||
});
|
||||
|
||||
test("never runs more than `limit` at once", async () => {
|
||||
let inFlight = 0;
|
||||
let maxInFlight = 0;
|
||||
const items = Array.from({ length: 10 }, (_, i) => i);
|
||||
await mapWithConcurrency(items, 3, async () => {
|
||||
inFlight++;
|
||||
maxInFlight = Math.max(maxInFlight, inFlight);
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
inFlight--;
|
||||
});
|
||||
assert.equal(maxInFlight, 3);
|
||||
});
|
||||
|
||||
test("limit larger than the item count runs everything without hanging", async () => {
|
||||
const results = await mapWithConcurrency([1, 2], 10, async (n) => n * 2);
|
||||
assert.deepEqual(results, [2, 4]);
|
||||
});
|
||||
|
||||
test("empty input resolves to an empty array", async () => {
|
||||
const results = await mapWithConcurrency([], 4, async (n) => n);
|
||||
assert.deepEqual(results, []);
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
|
||||
export function geminiConfigured() {
|
||||
return Boolean(
|
||||
process.env.GEMINI_API_KEY ||
|
||||
process.env.GOOGLE_API_KEY ||
|
||||
process.env.GOOGLE_APPLICATION_CREDENTIALS ||
|
||||
process.env.GCS_CREDS,
|
||||
);
|
||||
}
|
||||
|
||||
export function geminiAuth({ run = spawnSync } = {}) {
|
||||
const key = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
|
||||
if (key) return { headers: { "x-goog-api-key": key }, secret: key };
|
||||
if (!geminiConfigured()) {
|
||||
throw new Error(
|
||||
"Gemini TTS needs GEMINI_API_KEY or GOOGLE_API_KEY, or service-account credentials (GOOGLE_APPLICATION_CREDENTIALS or GCS_CREDS)",
|
||||
);
|
||||
}
|
||||
const { cmd, args } = pythonInvocation([
|
||||
fileURLToPath(new URL("./gemini-auth.py", import.meta.url)),
|
||||
]);
|
||||
const result = run(cmd, args, { encoding: "utf8", timeout: 60_000, maxBuffer: 1024 * 1024 });
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse(result.stdout);
|
||||
} catch {
|
||||
throw new Error(
|
||||
"Gemini service-account authentication failed; check Python and install google-auth requests",
|
||||
);
|
||||
}
|
||||
if (result.status !== 0 || !data.token) {
|
||||
// Do not echo subprocess output: it could contain credentials.
|
||||
throw new Error(
|
||||
"Gemini service-account authentication failed; check service-account JSON, permissions, and Python packages google-auth requests",
|
||||
);
|
||||
}
|
||||
const headers = { Authorization: `Bearer ${data.token}` };
|
||||
if (data.project) headers["x-goog-user-project"] = data.project;
|
||||
return { headers, secret: data.token };
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Obtain Gemini OAuth headers without persisting credentials or tokens."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Keep the credential subprocess protocol UTF-8 on every platform.
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(_stream, "reconfigure"):
|
||||
_stream.reconfigure(encoding="utf-8", errors=_stream.errors)
|
||||
|
||||
|
||||
def authenticate():
|
||||
try:
|
||||
from google.auth.transport.requests import Request
|
||||
from google.oauth2 import service_account
|
||||
except ImportError:
|
||||
return {"error": "Service-account auth needs Python packages: pip install google-auth requests"}
|
||||
|
||||
try:
|
||||
path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")
|
||||
if path:
|
||||
with open(path, encoding="utf-8") as source:
|
||||
info = json.load(source)
|
||||
else:
|
||||
info = json.loads(os.environ["GCS_CREDS"])
|
||||
if info.get("type") != "service_account":
|
||||
return {"error": "Gemini requires service-account JSON, not user ADC credentials"}
|
||||
# Only exchange credentials with Google's OAuth server.
|
||||
info["token_uri"] = "https://oauth2.googleapis.com/token"
|
||||
credentials = service_account.Credentials.from_service_account_info(
|
||||
info,
|
||||
scopes=[
|
||||
"https://www.googleapis.com/auth/generative-language.retriever",
|
||||
],
|
||||
)
|
||||
credentials.refresh(Request())
|
||||
project = (
|
||||
os.environ.get("GOOGLE_CLOUD_PROJECT")
|
||||
or os.environ.get("GCLOUD_PROJECT_ID")
|
||||
or info.get("project_id")
|
||||
)
|
||||
return {"token": credentials.token, "project": project}
|
||||
except Exception:
|
||||
# Library exceptions can contain credential data. Never forward them.
|
||||
return {"error": "Gemini service-account authentication failed; check credentials, permissions, and network access"}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
result = authenticate()
|
||||
print(json.dumps(result))
|
||||
sys.exit(1 if "error" in result else 0)
|
||||
@@ -0,0 +1,86 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { geminiAuth, geminiConfigured } from "./gemini-auth.mjs";
|
||||
import { pickProvider } from "./tts.mjs";
|
||||
|
||||
function env(t) {
|
||||
const saved = { ...process.env };
|
||||
for (const key of [
|
||||
"GEMINI_API_KEY",
|
||||
"GOOGLE_API_KEY",
|
||||
"GOOGLE_APPLICATION_CREDENTIALS",
|
||||
"GCS_CREDS",
|
||||
])
|
||||
delete process.env[key];
|
||||
t.after(() => {
|
||||
process.env = saved;
|
||||
});
|
||||
}
|
||||
|
||||
test("API keys take precedence over service accounts without invoking Python", (t) => {
|
||||
env(t);
|
||||
process.env.GEMINI_API_KEY = "gemini-key";
|
||||
process.env.GOOGLE_API_KEY = "google-key";
|
||||
process.env.GCS_CREDS = "private-json";
|
||||
const run = () => assert.fail("must not spawn Python");
|
||||
assert.deepEqual(geminiAuth({ run }).headers, { "x-goog-api-key": "gemini-key" });
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
assert.deepEqual(geminiAuth({ run }).headers, { "x-goog-api-key": "google-key" });
|
||||
});
|
||||
|
||||
test("service-account configuration selects Gemini and returns scoped helper headers", (t) => {
|
||||
env(t);
|
||||
assert.equal(geminiConfigured(), false);
|
||||
assert.throws(() => geminiAuth(), /needs GEMINI_API_KEY/);
|
||||
for (const key of ["GOOGLE_APPLICATION_CREDENTIALS", "GCS_CREDS"]) {
|
||||
process.env[key] = "credential-source";
|
||||
assert.equal(pickProvider("gemini"), "gemini");
|
||||
const auth = geminiAuth({
|
||||
run: (_, args, options) => {
|
||||
assert.ok(args.at(-1).endsWith("gemini-auth.py"));
|
||||
assert.ok(!args.includes("credential-source"));
|
||||
assert.equal(options.timeout, 60000);
|
||||
return {
|
||||
status: 0,
|
||||
stdout: JSON.stringify({ token: "access-token", project: "quota-project" }),
|
||||
};
|
||||
},
|
||||
});
|
||||
assert.deepEqual(auth.headers, {
|
||||
Authorization: "Bearer access-token",
|
||||
"x-goog-user-project": "quota-project",
|
||||
});
|
||||
delete process.env[key];
|
||||
}
|
||||
});
|
||||
|
||||
test("failed, malformed, or missing Python output cannot leak secrets or fall back", (t) => {
|
||||
env(t);
|
||||
process.env.GCS_CREDS = "private-json";
|
||||
for (const result of [
|
||||
{ status: 1, stdout: '{"error":"private-json"}', stderr: "private-json" },
|
||||
{ status: null, stdout: "private-json", error: new Error("private-json") },
|
||||
{ status: 0, stdout: "{}" },
|
||||
]) {
|
||||
assert.throws(
|
||||
() => geminiAuth({ run: () => result }),
|
||||
(error) => {
|
||||
assert.match(error.message, /authentication failed/);
|
||||
assert.ok(!error.message.includes("private-json"));
|
||||
return true;
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// Keep the Python credential-boundary suite reachable from the normal Node CI runner.
|
||||
test("Python service-account boundary validates credentials, scope, transport and errors", () => {
|
||||
const { cmd, args } = pythonInvocation([
|
||||
fileURLToPath(new URL("./gemini-auth_test.py", import.meta.url)),
|
||||
]);
|
||||
const result = spawnSync(cmd, args, { encoding: "utf8", timeout: 30_000 });
|
||||
assert.equal(result.status, 0, result.stderr || result.error?.message);
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Credential-boundary tests; stub Google's transport, never use real secrets."""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
|
||||
# Keep the credential subprocess protocol UTF-8 on every platform.
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(_stream, "reconfigure"):
|
||||
_stream.reconfigure(encoding="utf-8", errors=_stream.errors)
|
||||
|
||||
spec = importlib.util.spec_from_file_location("gemini_auth", Path(__file__).with_name("gemini-auth.py"))
|
||||
auth = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(auth)
|
||||
|
||||
|
||||
class AuthenticationTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.info = {"type": "service_account", "project_id": "account-project", "token_uri": "https://untrusted.invalid/token"}
|
||||
self.credentials = Mock(token="fixture-token")
|
||||
self.factory = Mock(return_value=self.credentials)
|
||||
self.request = Mock(return_value=object())
|
||||
transport = types.ModuleType("google.auth.transport.requests")
|
||||
transport.Request = self.request
|
||||
oauth = types.ModuleType("google.oauth2")
|
||||
oauth.service_account = types.SimpleNamespace(Credentials=types.SimpleNamespace(from_service_account_info=self.factory))
|
||||
modules = {name: types.ModuleType(name) for name in ("google", "google.auth", "google.auth.transport")}
|
||||
modules.update({"google.auth.transport.requests": transport, "google.oauth2": oauth})
|
||||
self.addCleanup(patch.stopall)
|
||||
patch.dict("sys.modules", modules).start()
|
||||
patch.dict(os.environ, {"GCS_CREDS": json.dumps(self.info)}, clear=True).start()
|
||||
|
||||
def test_scope_endpoint_and_refresh(self):
|
||||
self.assertEqual(auth.authenticate(), {"token": "fixture-token", "project": "account-project"})
|
||||
info = self.factory.call_args.args[0]
|
||||
self.assertEqual(info["token_uri"], "https://oauth2.googleapis.com/token")
|
||||
self.assertEqual(self.factory.call_args.kwargs["scopes"], ["https://www.googleapis.com/auth/generative-language.retriever"])
|
||||
self.credentials.refresh.assert_called_once_with(self.request.return_value)
|
||||
|
||||
def test_file_takes_precedence_over_json_environment(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "service.json"
|
||||
path.write_text(json.dumps({**self.info, "project_id": "file-project"}), encoding="utf-8")
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = str(path)
|
||||
os.environ["GCS_CREDS"] = "invalid-json"
|
||||
self.assertEqual(auth.authenticate()["project"], "file-project")
|
||||
|
||||
def test_quota_project_precedence(self):
|
||||
os.environ["GCLOUD_PROJECT_ID"] = "alias-project"
|
||||
self.assertEqual(auth.authenticate()["project"], "alias-project")
|
||||
os.environ["GOOGLE_CLOUD_PROJECT"] = "explicit-project"
|
||||
self.assertEqual(auth.authenticate()["project"], "explicit-project")
|
||||
|
||||
def test_user_adc_is_rejected_before_exchange(self):
|
||||
os.environ["GCS_CREDS"] = json.dumps({"type": "authorized_user"})
|
||||
self.assertIn("service-account JSON", auth.authenticate()["error"])
|
||||
self.factory.assert_not_called()
|
||||
|
||||
def test_invalid_json_or_missing_file_does_not_fall_back(self):
|
||||
os.environ["GCS_CREDS"] = "private-invalid-json"
|
||||
self.assertIn("error", auth.authenticate())
|
||||
os.environ["GCS_CREDS"] = json.dumps(self.info)
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/missing/service-account-fixture.json"
|
||||
self.assertIn("error", auth.authenticate())
|
||||
self.factory.assert_not_called()
|
||||
|
||||
def test_library_errors_are_redacted(self):
|
||||
for boundary in (self.factory, self.credentials.refresh):
|
||||
with self.subTest(boundary=boundary):
|
||||
boundary.side_effect = RuntimeError("private-key-and-token")
|
||||
result = auth.authenticate()
|
||||
self.assertIn("error", result)
|
||||
self.assertNotIn("private-key-and-token", json.dumps(result))
|
||||
boundary.side_effect = None
|
||||
|
||||
def test_missing_dependency_has_actionable_error(self):
|
||||
with patch.dict("sys.modules", {"google.auth.transport.requests": None}):
|
||||
self.assertIn("pip install google-auth requests", auth.authenticate()["error"])
|
||||
self.factory.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,122 @@
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { geminiAuth } from "./gemini-auth.mjs";
|
||||
|
||||
export const GEMINI_TTS_MODEL = "gemini-3.8-flash-tts";
|
||||
|
||||
export const GEMINI_TTS_MODELS = [
|
||||
GEMINI_TTS_MODEL,
|
||||
"gemini-3.8-flash-lite-tts",
|
||||
"gemini-3.1-flash-tts-preview",
|
||||
"gemini-2.5-pro-preview-tts",
|
||||
"gemini-2.5-flash-preview-tts",
|
||||
];
|
||||
|
||||
// 3.8 returns WAV; older models return PCM that we wrap without resampling.
|
||||
// The shared engine transcribes the saved audio for word timings.
|
||||
export async function synthesizeGemini(
|
||||
{ text, voiceId = "Kore", model = GEMINI_TTS_MODEL, style, speed = 1, wavAbs },
|
||||
{ fetchImpl = fetch, authenticate = geminiAuth } = {},
|
||||
) {
|
||||
let secret;
|
||||
try {
|
||||
if (!GEMINI_TTS_MODELS.includes(model)) {
|
||||
throw new Error(`Unsupported Gemini TTS model: ${model}`);
|
||||
}
|
||||
if (speed !== 1) throw new Error("Gemini TTS uses style for pacing; omit speed or use 1");
|
||||
const modern = model.startsWith("gemini-3.8-");
|
||||
if (!modern && /^(voice_|voicekey_)/.test(voiceId)) {
|
||||
throw new Error(
|
||||
"Custom Gemini voices require a 3.8 TTS model; use a prebuilt voice with older models",
|
||||
);
|
||||
}
|
||||
const auth = authenticate();
|
||||
secret = auth.secret;
|
||||
const content = {
|
||||
type: "text",
|
||||
text:
|
||||
!modern && style ? `Read the following text with this delivery: ${style}\n\n${text}` : text,
|
||||
};
|
||||
if (modern && style) content.annotations = [{ type: "speech_metadata", style }];
|
||||
const response = await fetchImpl(
|
||||
"https://generativelanguage.googleapis.com/v1beta/interactions",
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", ...auth.headers },
|
||||
signal: AbortSignal.timeout(120_000),
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
input: [{ type: "user_input", content: [content] }],
|
||||
response_format: modern ? { type: "audio", mime_type: "audio/wav" } : { type: "audio" },
|
||||
generation_config: { speech_config: [{ voice: voiceId }] },
|
||||
store: false,
|
||||
}),
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
const detail = await response.text();
|
||||
throw new Error(`Gemini TTS HTTP ${response.status}: ${detail}`);
|
||||
}
|
||||
const payload = await response.json();
|
||||
if (payload.status !== "completed") {
|
||||
throw new Error(`Gemini TTS did not complete (${payload.status ?? "missing status"})`);
|
||||
}
|
||||
const audio = (payload.steps ?? [])
|
||||
.filter((step) => step.type === "model_output")
|
||||
.flatMap((step) => step.content ?? [])
|
||||
.filter((part) => part.type === "audio");
|
||||
if (audio.length !== 1 || !audio[0].data) {
|
||||
throw new Error("Gemini TTS returned no single audio block");
|
||||
}
|
||||
let bytes = Buffer.from(audio[0].data, "base64");
|
||||
if (!modern && /^audio\/l16(?:;|$)/i.test(audio[0].mime_type ?? "")) {
|
||||
bytes = pcmToWav(bytes, audio[0].mime_type);
|
||||
} else if (audio[0].mime_type !== "audio/wav") {
|
||||
throw new Error("Gemini TTS returned an unsupported audio format");
|
||||
}
|
||||
if (
|
||||
bytes.length <= 44 ||
|
||||
bytes.toString("ascii", 0, 4) !== "RIFF" ||
|
||||
bytes.toString("ascii", 8, 12) !== "WAVE"
|
||||
) {
|
||||
throw new Error("Gemini TTS returned invalid WAV audio");
|
||||
}
|
||||
mkdirSync(dirname(wavAbs), { recursive: true });
|
||||
writeFileSync(wavAbs, bytes);
|
||||
return { ok: true, words: null };
|
||||
} catch (error) {
|
||||
// Error responses must never echo the credential into logs or metadata.
|
||||
const message = String(error?.message ?? error);
|
||||
return {
|
||||
ok: false,
|
||||
words: null,
|
||||
error: secret ? message.split(secret).join("[redacted]") : message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function pcmToWav(pcm, mime) {
|
||||
const rate = Number(/(?:^|;)\s*rate=(\d+)(?:;|$)/i.exec(mime)?.[1]);
|
||||
const channels = /(?:^|;)\s*channels=([^;]+)/i.exec(mime)?.[1];
|
||||
const codec = /(?:^|;)\s*codec=([^;]+)/i.exec(mime)?.[1];
|
||||
if ((channels && channels.trim() !== "1") || (codec && codec.trim() !== "pcm")) {
|
||||
throw new Error("Gemini TTS returned unsupported PCM channels or codec");
|
||||
}
|
||||
if (!Number.isInteger(rate) || rate < 8000 || rate > 96000 || !pcm.length || pcm.length % 2) {
|
||||
throw new Error("Gemini TTS returned invalid PCM audio or sample rate");
|
||||
}
|
||||
const header = Buffer.alloc(44);
|
||||
header.write("RIFF");
|
||||
header.writeUInt32LE(36 + pcm.length, 4);
|
||||
header.write("WAVEfmt ", 8);
|
||||
header.writeUInt32LE(16, 16);
|
||||
header.writeUInt16LE(1, 20);
|
||||
header.writeUInt16LE(1, 22);
|
||||
header.writeUInt32LE(rate, 24);
|
||||
header.writeUInt32LE(rate * 2, 28);
|
||||
header.writeUInt16LE(2, 32);
|
||||
header.writeUInt16LE(16, 34);
|
||||
header.write("data", 36);
|
||||
header.writeUInt32LE(pcm.length, 40);
|
||||
return Buffer.concat([header, pcm]);
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, rmSync, existsSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { synthesizeGemini, GEMINI_TTS_MODEL } from "./gemini-tts.mjs";
|
||||
import { pickProvider, resolveVoiceId, synthesizeOne } from "./tts.mjs";
|
||||
|
||||
function fixture(t) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "gemini-tts-"));
|
||||
const saved = { ...process.env };
|
||||
process.env.GEMINI_API_KEY = "test-gemini-key";
|
||||
delete process.env.GOOGLE_API_KEY;
|
||||
delete process.env.GOOGLE_APPLICATION_CREDENTIALS;
|
||||
delete process.env.GCS_CREDS;
|
||||
t.after(() => {
|
||||
process.env = saved;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
const wav = Buffer.alloc(48);
|
||||
wav.write("RIFF");
|
||||
wav.writeUInt32LE(40, 4);
|
||||
wav.write("WAVEfmt ", 8);
|
||||
wav.writeUInt32LE(16, 16);
|
||||
wav.writeUInt16LE(1, 20);
|
||||
wav.writeUInt16LE(1, 22);
|
||||
wav.writeUInt32LE(24000, 24);
|
||||
wav.writeUInt32LE(48000, 28);
|
||||
wav.writeUInt16LE(2, 32);
|
||||
wav.writeUInt16LE(16, 34);
|
||||
wav.write("data", 36);
|
||||
wav.writeUInt32LE(4, 40);
|
||||
const audio = { type: "audio", mime_type: "audio/wav", data: wav.toString("base64") };
|
||||
const payload = { status: "completed", steps: [{ type: "model_output", content: [audio] }] };
|
||||
return {
|
||||
wav,
|
||||
audio,
|
||||
payload,
|
||||
args: { text: "Hello there.", voiceId: "Kore", wavAbs: join(dir, "voice", "one.wav") },
|
||||
};
|
||||
}
|
||||
|
||||
test("Gemini preserves verbatim text, directs style separately, saves WAV and requests transcription", async (t) => {
|
||||
const { wav, payload, args } = fixture(t);
|
||||
t.mock.method(globalThis, "fetch", async (url, options) => {
|
||||
assert.equal(url, "https://generativelanguage.googleapis.com/v1beta/interactions");
|
||||
assert.equal(options.headers["x-goog-api-key"], "test-gemini-key");
|
||||
const body = JSON.parse(options.body);
|
||||
assert.equal(body.model, GEMINI_TTS_MODEL);
|
||||
assert.equal(body.store, false);
|
||||
assert.deepEqual(body.input[0].content, [
|
||||
{
|
||||
type: "text",
|
||||
text: args.text,
|
||||
annotations: [{ type: "speech_metadata", style: "Warm and calm" }],
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(body.generation_config.speech_config, [{ voice: "Kore" }]);
|
||||
assert.equal(body.response_format.mime_type, "audio/wav");
|
||||
return Response.json(payload);
|
||||
});
|
||||
const result = await synthesizeOne({ ...args, provider: "gemini", style: "Warm and calm" });
|
||||
assert.deepEqual(result, { ok: true, words: null });
|
||||
assert.deepEqual(readFileSync(args.wavAbs), wav);
|
||||
});
|
||||
|
||||
test("Flash-Lite and GOOGLE_API_KEY work without a Gemini key", async (t) => {
|
||||
const { payload, args } = fixture(t);
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
process.env.GOOGLE_API_KEY = "test-google-key";
|
||||
assert.equal(pickProvider("gemini"), "gemini");
|
||||
assert.equal(await resolveVoiceId({ provider: "gemini" }), "Kore");
|
||||
const result = await synthesizeGemini(
|
||||
{ ...args, model: "gemini-3.8-flash-lite-tts" },
|
||||
{
|
||||
fetchImpl: async (_, options) => {
|
||||
assert.equal(options.headers["x-goog-api-key"], "test-google-key");
|
||||
const body = JSON.parse(options.body);
|
||||
assert.equal(body.model, "gemini-3.8-flash-lite-tts");
|
||||
assert.equal(body.input[0].content[0].annotations, undefined);
|
||||
return Response.json(payload);
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.ok, true);
|
||||
});
|
||||
|
||||
test("Gemini remains opt-in even with a key and preserves explicitly chosen voices", async (t) => {
|
||||
fixture(t);
|
||||
assert.notEqual(pickProvider(), "gemini");
|
||||
assert.equal(
|
||||
await resolveVoiceId({ provider: "gemini", userVoice: "voice_custom" }),
|
||||
"voice_custom",
|
||||
);
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
assert.throws(() => pickProvider("gemini"), /GEMINI_API_KEY or GOOGLE_API_KEY/);
|
||||
});
|
||||
|
||||
test("invalid config fails before spending a generation request", async (t) => {
|
||||
const { args } = fixture(t);
|
||||
const deps = { fetchImpl: () => assert.fail("must not call the API") };
|
||||
for (const change of [{ model: "gemini-2.5-flash" }, { speed: 1.2 }]) {
|
||||
const result = await synthesizeGemini({ ...args, ...change }, deps);
|
||||
assert.equal(result.ok, false);
|
||||
assert.ok(result.error);
|
||||
}
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
assert.match((await synthesizeGemini(args, deps)).error, /needs GEMINI_API_KEY/);
|
||||
assert.equal(existsSync(args.wavAbs), false);
|
||||
});
|
||||
|
||||
test("HTTP and network failures stay actionable without leaking the key", async (t) => {
|
||||
const { args } = fixture(t);
|
||||
const result = await synthesizeGemini(args, {
|
||||
fetchImpl: async () => new Response("quota exceeded test-gemini-key", { status: 429 }),
|
||||
});
|
||||
assert.equal(result.ok, false);
|
||||
assert.match(result.error, /HTTP 429: quota exceeded/);
|
||||
assert.ok(!result.error.includes("test-gemini-key"));
|
||||
const timeout = await synthesizeGemini(args, {
|
||||
fetchImpl: async () => {
|
||||
throw new Error("timed out");
|
||||
},
|
||||
});
|
||||
assert.match(timeout.error, /timed out/);
|
||||
assert.equal(existsSync(args.wavAbs), false);
|
||||
});
|
||||
|
||||
test("incomplete, missing, raw PCM and malformed audio cannot become a successful WAV", async (t) => {
|
||||
const { args, payload, audio } = fixture(t);
|
||||
const cases = [
|
||||
{ ...payload, status: "incomplete" },
|
||||
{ status: "completed", steps: [] },
|
||||
...[{ ...audio, mime_type: "audio/l16" }, { ...audio, data: "not audio" }, audio].map(
|
||||
(part, i) => ({
|
||||
status: "completed",
|
||||
steps: [{ type: "model_output", content: i === 2 ? [part, part] : [part] }],
|
||||
}),
|
||||
),
|
||||
];
|
||||
for (const body of cases) {
|
||||
const result = await synthesizeGemini(args, { fetchImpl: async () => Response.json(body) });
|
||||
assert.equal(result.ok, false);
|
||||
assert.equal(existsSync(args.wavAbs), false);
|
||||
}
|
||||
});
|
||||
|
||||
for (const model of [
|
||||
"gemini-3.1-flash-tts-preview",
|
||||
"gemini-2.5-pro-preview-tts",
|
||||
"gemini-2.5-flash-preview-tts",
|
||||
]) {
|
||||
test(`${model} uses legacy delivery prompts and wraps PCM as mono WAV`, async (t) => {
|
||||
const { args, wav } = fixture(t);
|
||||
const pcm = wav.subarray(44);
|
||||
const result = await synthesizeGemini(
|
||||
{ ...args, model, style: "Warm and clear" },
|
||||
{
|
||||
fetchImpl: async (_, options) => {
|
||||
const body = JSON.parse(options.body);
|
||||
assert.deepEqual(body.response_format, { type: "audio" });
|
||||
assert.equal(body.input[0].content[0].annotations, undefined);
|
||||
assert.ok(body.input[0].content[0].text.endsWith(args.text));
|
||||
assert.ok(body.input[0].content[0].text.includes("Warm and clear"));
|
||||
return Response.json({
|
||||
status: "completed",
|
||||
steps: [
|
||||
{
|
||||
type: "model_output",
|
||||
content: [
|
||||
{
|
||||
type: "audio",
|
||||
mime_type: "audio/L16;codec=pcm;rate=24000",
|
||||
data: pcm.toString("base64"),
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.ok, true, result.error);
|
||||
assert.deepEqual(readFileSync(args.wavAbs), wav);
|
||||
});
|
||||
}
|
||||
|
||||
test("service-account bearer and quota project reach synthesis, and token errors are redacted", async (t) => {
|
||||
const { args, payload } = fixture(t);
|
||||
const authenticate = () => ({
|
||||
headers: { Authorization: "Bearer secret-token", "x-goog-user-project": "test-project" },
|
||||
secret: "secret-token",
|
||||
});
|
||||
const ok = await synthesizeGemini(args, {
|
||||
authenticate,
|
||||
fetchImpl: async (_, options) => {
|
||||
assert.equal(options.headers.Authorization, "Bearer secret-token");
|
||||
assert.equal(options.headers["x-goog-user-project"], "test-project");
|
||||
assert.equal(options.headers["x-goog-api-key"], undefined);
|
||||
return Response.json(payload);
|
||||
},
|
||||
});
|
||||
assert.equal(ok.ok, true);
|
||||
const failed = await synthesizeGemini(args, {
|
||||
authenticate,
|
||||
fetchImpl: async () => new Response("denied secret-token", { status: 403 }),
|
||||
});
|
||||
assert.match(failed.error, /HTTP 403/);
|
||||
assert.ok(!failed.error.includes("secret-token"));
|
||||
});
|
||||
|
||||
test("older models reject unsupported PCM and custom voices", async (t) => {
|
||||
const { args } = fixture(t);
|
||||
for (const mime of [
|
||||
"audio/l16",
|
||||
"audio/l16;rate=0",
|
||||
"audio/l16;rate=24000;channels=2",
|
||||
"audio/l16;rate=24000;codec=other",
|
||||
]) {
|
||||
const result = await synthesizeGemini(
|
||||
{ ...args, model: "gemini-3.1-flash-tts-preview" },
|
||||
{
|
||||
fetchImpl: async () =>
|
||||
Response.json({
|
||||
status: "completed",
|
||||
steps: [
|
||||
{
|
||||
type: "model_output",
|
||||
content: [{ type: "audio", mime_type: mime, data: "AAAAAA==" }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
);
|
||||
assert.equal(result.ok, false);
|
||||
assert.equal(existsSync(args.wavAbs), false);
|
||||
}
|
||||
const result = await synthesizeGemini(
|
||||
{ ...args, model: "gemini-2.5-pro-preview-tts", voiceId: "voice_custom" },
|
||||
{ fetchImpl: () => assert.fail("must not generate") },
|
||||
);
|
||||
assert.match(result.error, /Custom Gemini voices require/);
|
||||
});
|
||||
@@ -0,0 +1,240 @@
|
||||
import { fetchMedia } from "../../../scripts/lib/media-fetch.mjs";
|
||||
// heygen.mjs — vendored HeyGen REST helpers (auth + transport) for the audio
|
||||
// pipeline. The credential resolver matches the hyperframes CLI auth: first
|
||||
// usable source wins — a host gateway ($HEYGEN_API_BASE with its own
|
||||
// $HEYGEN_API_KEY) → a host-injected OAuth $HEYGEN_ACCESS_TOKEN (Bearer) →
|
||||
// $HEYGEN_API_KEY / $HYPERFRAMES_API_KEY → a nearby .env → ~/.heygen/
|
||||
// credentials (oauth → Bearer, else api_key → X-Api-Key; $HEYGEN_CONFIG_DIR
|
||||
// overrides the dir). $HEYGEN_API_BASE moves every request to that host, as it
|
||||
// does for the heygen CLI; plain HTTP needs $HEYGEN_ALLOW_HTTP=1, as there.
|
||||
// Vendored so the skill ships standalone. Pure node.
|
||||
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
|
||||
export const HEYGEN_BASE = "https://api.heygen.com/v3";
|
||||
|
||||
// The v3 base every request goes to: $HEYGEN_API_BASE when a host app names its own gateway (HyperFrames Desktop
|
||||
// forwards it to HeyGen with the API key saved in its Settings), else HeyGen's public API. Plain HTTP carries the key
|
||||
// in the clear, so it needs $HEYGEN_ALLOW_HTTP=1, the heygen CLI's own rule.
|
||||
export function heygenBase() {
|
||||
const host = process.env.HEYGEN_API_BASE?.trim().replace(/\/+$/, "");
|
||||
if (!host) return HEYGEN_BASE;
|
||||
if (host.startsWith("http://") && process.env.HEYGEN_ALLOW_HTTP !== "1")
|
||||
throw new Error(
|
||||
`HEYGEN_API_BASE (${host}) uses HTTP, which sends the key in plaintext. Set HEYGEN_ALLOW_HTTP=1 to allow it.`,
|
||||
);
|
||||
return `${host}/v3`;
|
||||
}
|
||||
|
||||
// No base override, or one on HeyGen's own hosts (a canary or dev API).
|
||||
function heygenOwnBase() {
|
||||
const host = process.env.HEYGEN_API_BASE?.trim();
|
||||
if (!host) return true;
|
||||
try {
|
||||
const name = new URL(host).hostname;
|
||||
return name === "heygen.com" || name.endsWith(".heygen.com");
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// A host gateway: the host named its own API base and the key that base accepts. It pays for every call, so it wins
|
||||
// over any other credential the environment carries.
|
||||
const hostGatewayKey = () =>
|
||||
process.env.HEYGEN_API_BASE?.trim() && process.env.HEYGEN_API_KEY
|
||||
? process.env.HEYGEN_API_KEY
|
||||
: null;
|
||||
export const HEYGEN_CLI_SOURCE_HEADERS = { "X-HeyGen-Source": "cli" };
|
||||
// Tool-attribution sent on EVERY media-use HeyGen call regardless of auth type, so
|
||||
// the backend can isolate media-use consumption from other free TTS / avatar video.
|
||||
// Unconditional — a paying user's media-use call is still media-use — unlike the
|
||||
// OAuth-only cli-source header above, which also gates the free allowance.
|
||||
export const HEYGEN_CLIENT_SOURCE_HEADERS = { "X-HeyGen-Client-Source": "media-use" };
|
||||
|
||||
// A missing `.env`, or a `.env` folder (some home dirs have one), is no env file: null. Read without checking first,
|
||||
// so the file cannot change between a check and the read.
|
||||
function envFileText(path) {
|
||||
try {
|
||||
return readFileSync(path, "utf8");
|
||||
} catch (error) {
|
||||
if (["ENOENT", "ENOTDIR", "EISDIR"].includes(error.code)) return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// A host app sets these in the environment it spawns, never in a project file: a project's .env naming its own base
|
||||
// would send the person's shell HEYGEN_API_KEY to that host.
|
||||
const HOST_ONLY = new Set(["HEYGEN_API_BASE", "HEYGEN_ALLOW_HTTP"]);
|
||||
|
||||
// Walk up ≤5 dirs from startDir; load the first .env (shell env always wins).
|
||||
export function loadEnvFromDir(startDir) {
|
||||
let dir = resolve(startDir);
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const text = envFileText(join(dir, ".env"));
|
||||
if (text != null) {
|
||||
for (const raw of text.split("\n")) {
|
||||
let line = raw.trim();
|
||||
if (!line || line.startsWith("#")) continue;
|
||||
if (line.startsWith("export ")) line = line.slice(7).trim();
|
||||
const eq = line.indexOf("=");
|
||||
if (eq < 1) continue;
|
||||
const key = line.slice(0, eq).trim();
|
||||
let val = line.slice(eq + 1).trim();
|
||||
if (val.startsWith('"') || val.startsWith("'")) {
|
||||
const q = val[0];
|
||||
const end = val.indexOf(q, 1);
|
||||
val = end > 0 ? val.slice(1, end) : val.slice(1);
|
||||
}
|
||||
if (!HOST_ONLY.has(key) && !(key in process.env)) process.env[key] = val;
|
||||
}
|
||||
return;
|
||||
}
|
||||
const parent = dirname(dir);
|
||||
if (parent === dir) break;
|
||||
dir = parent;
|
||||
}
|
||||
}
|
||||
|
||||
// → { headers } | { expired: true } | null. Never throws.
|
||||
export function heygenCredential() {
|
||||
const cred = resolveCredential();
|
||||
return cred?.unreadable ? null : cred;
|
||||
}
|
||||
|
||||
// heygenCredential's answer, or { unreadable: { file, code } } when the credentials path exists but cannot be read
|
||||
// (a folder, a locked ~/.heygen), so heygenAuthHeaders can say to fix that path: logging in again would fail there too.
|
||||
// Read without checking first, so the file cannot change between a check and the read.
|
||||
function resolveCredential() {
|
||||
const gatewayKey = hostGatewayKey();
|
||||
if (gatewayKey) return { headers: { "X-Api-Key": gatewayKey } };
|
||||
// Every other credential belongs to HeyGen: a base on any other host gets none of them.
|
||||
if (!heygenOwnBase()) return null;
|
||||
const accessToken = process.env.HEYGEN_ACCESS_TOKEN;
|
||||
if (accessToken) return { headers: { Authorization: `Bearer ${accessToken}` } };
|
||||
const envKey = process.env.HEYGEN_API_KEY || process.env.HYPERFRAMES_API_KEY;
|
||||
if (envKey) return { headers: { "X-Api-Key": envKey } };
|
||||
|
||||
const file = join(process.env.HEYGEN_CONFIG_DIR || join(homedir(), ".heygen"), "credentials");
|
||||
let raw;
|
||||
try {
|
||||
raw = readFileSync(file, "utf8").trim();
|
||||
} catch (error) {
|
||||
return error.code === "ENOENT" ? null : { unreadable: { file, code: error.code } };
|
||||
}
|
||||
if (!raw) return null;
|
||||
if (!raw.startsWith("{")) return { headers: { "X-Api-Key": raw } };
|
||||
|
||||
// A malformed credentials file (partial write / wrong shape) must degrade to
|
||||
// "no credential", not crash the engine at startup.
|
||||
let cred;
|
||||
try {
|
||||
cred = JSON.parse(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const oauth = cred.oauth;
|
||||
if (oauth?.access_token) {
|
||||
const expired = oauth.expires_at && new Date(oauth.expires_at).getTime() - 60_000 < Date.now();
|
||||
if (!expired) return { headers: { Authorization: `Bearer ${oauth.access_token}` } };
|
||||
if (!cred.api_key) return { expired: true };
|
||||
}
|
||||
if (cred.api_key) return { headers: { "X-Api-Key": cred.api_key } };
|
||||
return null;
|
||||
}
|
||||
|
||||
// → "oauth" | "api_key" | null. Same oauth-vs-api-key check heygenAuthHeaders()
|
||||
// makes internally, exposed on its own so callers that only need to *tag* the
|
||||
// auth path (telemetry) don't have to parse headers back apart. Never throws:
|
||||
// no credential (or an expired one) is just `null`, same as a fresh resolve
|
||||
// with nothing to tag.
|
||||
export function heygenAuthMethod() {
|
||||
const cred = heygenCredential();
|
||||
if (!cred?.headers) return null;
|
||||
return "Authorization" in cred.headers ? "oauth" : "api_key";
|
||||
}
|
||||
|
||||
// → auth headers object, or throw with a fix hint.
|
||||
export function heygenAuthHeaders() {
|
||||
const cred = resolveCredential();
|
||||
if (cred?.headers) {
|
||||
// Only tag OAuth (Bearer) traffic as cli-source — the backend uses it to
|
||||
// grant the free allowance for OAuth requests and ignores it for API-key
|
||||
// (X-Api-Key) traffic, where it's dead metadata.
|
||||
const isOauth = "Authorization" in cred.headers;
|
||||
return isOauth
|
||||
? { ...cred.headers, ...HEYGEN_CLI_SOURCE_HEADERS, ...HEYGEN_CLIENT_SOURCE_HEADERS }
|
||||
: { ...cred.headers, ...HEYGEN_CLIENT_SOURCE_HEADERS };
|
||||
}
|
||||
if (cred?.unreadable)
|
||||
throw new Error(
|
||||
`HeyGen credentials at ${cred.unreadable.file} can't be read (${cred.unreadable.code}) — fix or remove that path, then run \`npx hyperframes auth login\``,
|
||||
);
|
||||
if (cred?.expired)
|
||||
throw new Error(
|
||||
"HeyGen OAuth token expired — run `npx hyperframes auth refresh` (or `npx hyperframes auth login`)",
|
||||
);
|
||||
throw new Error(
|
||||
"no HeyGen credentials — set $HEYGEN_API_KEY, or run `npx hyperframes auth login` (writes ~/.heygen/credentials)",
|
||||
);
|
||||
}
|
||||
|
||||
// Authed JSON request against the v3 API; throws on a non-OK status.
|
||||
export async function heygenJSON(path, { method = "GET", headers = {}, body } = {}) {
|
||||
const opts = { method, headers: { ...HEYGEN_CLIENT_SOURCE_HEADERS, ...headers } };
|
||||
if (body !== undefined) {
|
||||
opts.headers["Content-Type"] = "application/json";
|
||||
opts.body = JSON.stringify(body);
|
||||
}
|
||||
const res = await fetch(`${heygenBase()}${path}`, opts);
|
||||
if (!res.ok) {
|
||||
const detail = await res.text().catch(() => "");
|
||||
const message = `HeyGen ${method} ${path} → HTTP ${res.status}${detail ? `\n${detail.slice(0, 300)}` : ""}`;
|
||||
throw Object.assign(new Error(message), { status: res.status, body: detail });
|
||||
}
|
||||
// A DELETE may answer 204 with no body.
|
||||
const text = await res.text();
|
||||
return text ? JSON.parse(text) : {};
|
||||
}
|
||||
|
||||
// HeyGen's own words for a failed call: its {"error":{"message"}}, else the raw body, else the error's message.
|
||||
export function heygenMessage(e) {
|
||||
if (!e?.body) return e?.message ? String(e.message) : String(e);
|
||||
try {
|
||||
return JSON.parse(e.body).error?.message ?? e.body;
|
||||
} catch {
|
||||
return e.body;
|
||||
}
|
||||
}
|
||||
|
||||
// Download a (presigned) URL to destPath; returns byte length.
|
||||
export async function downloadTo(url, destPath) {
|
||||
const res = await fetchMedia(url);
|
||||
if (!res.ok) throw new Error(`download HTTP ${res.status}: ${String(url).slice(0, 80)}`);
|
||||
const bytes = Buffer.from(await res.arrayBuffer());
|
||||
mkdirSync(dirname(destPath), { recursive: true });
|
||||
writeFileSync(destPath, bytes);
|
||||
return bytes.length;
|
||||
}
|
||||
|
||||
// Retrieval search over HeyGen's audio catalog (NOT generation). type =
|
||||
// "music" | "sound_effects". Returns the ranked results array (best first); each
|
||||
// item has a presigned `audio_url` (+ `duration`, `description`, `name`, `score`).
|
||||
// `query` is required (≥1 char, empty → HTTP 400) and `limit` is capped at 50.
|
||||
// `minScore`: omit to use the server default (0.7). That default is TOO HIGH for
|
||||
// sound_effects — good SFX hits score ~0.5–0.67, so callers wanting SFX should
|
||||
// pass a lower floor (~0.4); music scores high and is fine at the default.
|
||||
export async function searchSounds(query, type, headers, { limit = 5, minScore } = {}) {
|
||||
const params = new URLSearchParams({ query, type, limit: String(limit) });
|
||||
if (minScore != null) params.set("min_score", String(minScore));
|
||||
const payload = await heygenJSON(`/audio/sounds?${params.toString()}`, { headers });
|
||||
// `data` comes back as a ranked array (best first). Older responses keyed it by
|
||||
// numeric index ("0","1",…); normalize both shapes to an array (empty → []).
|
||||
const data = payload?.data ?? payload;
|
||||
if (Array.isArray(data)) return data;
|
||||
if (data && typeof data === "object") return Object.values(data);
|
||||
throw new Error(
|
||||
`unexpected /audio/sounds shape — top keys: ${Object.keys(payload ?? {}).join(", ")}`,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { createServer } from "node:http";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
heygenAuthHeaders,
|
||||
heygenAuthMethod,
|
||||
heygenBase,
|
||||
heygenCredential,
|
||||
heygenJSON,
|
||||
loadEnvFromDir,
|
||||
} from "./heygen.mjs";
|
||||
|
||||
const HEYGEN_ENV = [
|
||||
"HEYGEN_ACCESS_TOKEN",
|
||||
"HEYGEN_API_KEY",
|
||||
"HYPERFRAMES_API_KEY",
|
||||
"HEYGEN_CONFIG_DIR",
|
||||
"HEYGEN_API_BASE",
|
||||
"HEYGEN_ALLOW_HTTP",
|
||||
];
|
||||
|
||||
// Runs fn with every HeyGen variable unset, then puts them back; an async fn restores once it settles.
|
||||
function withCleanHeygenEnv(fn) {
|
||||
const previous = Object.fromEntries(HEYGEN_ENV.map((name) => [name, process.env[name]]));
|
||||
const restore = () => {
|
||||
for (const [name, value] of Object.entries(previous)) {
|
||||
if (value === undefined) delete process.env[name];
|
||||
else process.env[name] = value;
|
||||
}
|
||||
};
|
||||
for (const name of HEYGEN_ENV) delete process.env[name];
|
||||
let result;
|
||||
try {
|
||||
result = fn();
|
||||
} catch (error) {
|
||||
restore();
|
||||
throw error;
|
||||
}
|
||||
if (result && typeof result.then === "function") return result.finally(restore);
|
||||
restore();
|
||||
return result;
|
||||
}
|
||||
|
||||
test("heygenAuthHeaders does not tag API-key requests as CLI traffic, but still carries the media-use tool tag", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_API_KEY = "hg_test";
|
||||
// API-key requests use normal billing; the backend ignores the cli-source
|
||||
// header for them, so it's not sent. The tool-attribution header IS sent on
|
||||
// every media-use call (any auth type) so the backend can isolate media-use.
|
||||
assert.deepEqual(heygenAuthHeaders(), {
|
||||
"X-Api-Key": "hg_test",
|
||||
"X-HeyGen-Client-Source": "media-use",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthHeaders tags OAuth requests as CLI traffic and with the media-use tool tag", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
writeFileSync(
|
||||
join(dir, "credentials"),
|
||||
JSON.stringify({
|
||||
oauth: {
|
||||
access_token: "at_test",
|
||||
expires_at: "2099-01-01T00:00:00Z",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.deepEqual(heygenAuthHeaders(), {
|
||||
Authorization: "Bearer at_test",
|
||||
"X-HeyGen-Source": "cli",
|
||||
"X-HeyGen-Client-Source": "media-use",
|
||||
});
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("a host-injected HEYGEN_ACCESS_TOKEN wins over an API key and is treated as OAuth", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_ACCESS_TOKEN = "at_host";
|
||||
process.env.HEYGEN_API_KEY = "hg_test";
|
||||
assert.deepEqual(heygenAuthHeaders(), {
|
||||
Authorization: "Bearer at_host",
|
||||
"X-HeyGen-Source": "cli",
|
||||
"X-HeyGen-Client-Source": "media-use",
|
||||
});
|
||||
assert.equal(heygenAuthMethod(), "oauth");
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns api_key for an env API key, without tagging headers", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_API_KEY = "hg_test";
|
||||
assert.equal(heygenAuthMethod(), "api_key");
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns oauth for a live OAuth credential", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
writeFileSync(
|
||||
join(dir, "credentials"),
|
||||
JSON.stringify({
|
||||
oauth: {
|
||||
access_token: "at_test",
|
||||
expires_at: "2099-01-01T00:00:00Z",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.equal(heygenAuthMethod(), "oauth");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns null with no credential at all", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir; // no credentials file written
|
||||
assert.equal(heygenAuthMethod(), null);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("loadEnvFromDir skips a .env folder and loads the .env file above it", () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "heygen-env-"));
|
||||
const project = join(root, "project");
|
||||
mkdirSync(join(project, ".env"), { recursive: true });
|
||||
writeFileSync(join(root, ".env"), "MEDIA_USE_ENV_DIR_TEST=from-parent\n");
|
||||
try {
|
||||
loadEnvFromDir(project);
|
||||
assert.equal(process.env.MEDIA_USE_ENV_DIR_TEST, "from-parent");
|
||||
} finally {
|
||||
delete process.env.MEDIA_USE_ENV_DIR_TEST;
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("a project's .env cannot name the HeyGen base, so a shell key never leaves for its host", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const project = mkdtempSync(join(tmpdir(), "heygen-env-"));
|
||||
writeFileSync(
|
||||
join(project, ".env"),
|
||||
"HEYGEN_API_BASE=https://proxy.example.com\nHEYGEN_ALLOW_HTTP=1\nMEDIA_USE_ENV_BASE_TEST=loaded\n",
|
||||
);
|
||||
try {
|
||||
process.env.HEYGEN_API_KEY = "hg_shell_real";
|
||||
loadEnvFromDir(project);
|
||||
assert.equal(process.env.MEDIA_USE_ENV_BASE_TEST, "loaded");
|
||||
assert.equal(process.env.HEYGEN_API_BASE, undefined);
|
||||
assert.equal(process.env.HEYGEN_ALLOW_HTTP, undefined);
|
||||
assert.equal(heygenBase(), "https://api.heygen.com/v3");
|
||||
} finally {
|
||||
delete process.env.MEDIA_USE_ENV_BASE_TEST;
|
||||
rmSync(project, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns null when the credentials path is a folder", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
mkdirSync(join(dir, "credentials"));
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
assert.equal(heygenAuthMethod(), null);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns null when the credentials path is a symlink loop", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
symlinkSync(join(dir, "credentials"), join(dir, "credentials"));
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
assert.equal(heygenAuthMethod(), null);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthHeaders says to fix an unreadable credentials path, and to log in when there is none", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
assert.throws(() => heygenAuthHeaders(), /no HeyGen credentials/);
|
||||
mkdirSync(join(dir, "credentials"));
|
||||
assert.equal(heygenCredential(), null);
|
||||
assert.throws(
|
||||
() => heygenAuthHeaders(),
|
||||
(error) =>
|
||||
error.message.includes(join(dir, "credentials")) &&
|
||||
/fix or remove that path/.test(error.message),
|
||||
);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenBase is HeyGen's public API unless a host names another", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
assert.equal(heygenBase(), "https://api.heygen.com/v3");
|
||||
process.env.HEYGEN_API_BASE = "https://api-canary.heygen.com/";
|
||||
assert.equal(heygenBase(), "https://api-canary.heygen.com/v3");
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenBase refuses a plain-HTTP host unless HEYGEN_ALLOW_HTTP is set, as the heygen CLI does", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_API_BASE = "http://127.0.0.1:4100";
|
||||
assert.throws(() => heygenBase(), /HEYGEN_ALLOW_HTTP=1/);
|
||||
process.env.HEYGEN_ALLOW_HTTP = "1";
|
||||
assert.equal(heygenBase(), "http://127.0.0.1:4100/v3");
|
||||
});
|
||||
});
|
||||
|
||||
test("a host gateway (HEYGEN_API_BASE with its own HEYGEN_API_KEY) wins over a host OAuth token", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_API_BASE = "http://127.0.0.1:4100";
|
||||
process.env.HEYGEN_ALLOW_HTTP = "1";
|
||||
process.env.HEYGEN_API_KEY = "gateway-token";
|
||||
process.env.HEYGEN_ACCESS_TOKEN = "at_host";
|
||||
assert.deepEqual(heygenAuthHeaders(), {
|
||||
"X-Api-Key": "gateway-token",
|
||||
"X-HeyGen-Client-Source": "media-use",
|
||||
});
|
||||
assert.equal(heygenAuthMethod(), "api_key");
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenJSON sends its request to the host's HEYGEN_API_BASE with the host's key", async () => {
|
||||
/** @type {{ url?: string, key?: string | string[] }} */
|
||||
const seen = {};
|
||||
const server = createServer((req, res) => {
|
||||
seen.url = req.url;
|
||||
seen.key = req.headers["x-api-key"];
|
||||
res.writeHead(200, { "content-type": "application/json" }).end('{"data":[]}');
|
||||
});
|
||||
await new Promise((done) => server.listen(0, "127.0.0.1", done));
|
||||
const { port } = /** @type {import("node:net").AddressInfo} */ (server.address());
|
||||
try {
|
||||
await withCleanHeygenEnv(async () => {
|
||||
process.env.HEYGEN_API_BASE = `http://127.0.0.1:${port}`;
|
||||
process.env.HEYGEN_ALLOW_HTTP = "1";
|
||||
process.env.HEYGEN_API_KEY = "gateway-token";
|
||||
const reply = await heygenJSON("/voices?limit=1", { headers: heygenAuthHeaders() });
|
||||
assert.deepEqual(reply, { data: [] });
|
||||
});
|
||||
assert.equal(seen.url, "/v3/voices?limit=1");
|
||||
assert.equal(seen.key, "gateway-token");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("a base that isn't HeyGen's gets no stored or host OAuth credential, only a key named for it", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
writeFileSync(join(dir, "credentials"), JSON.stringify({ api_key: "hg_stored" }));
|
||||
process.env.HEYGEN_ACCESS_TOKEN = "at_host";
|
||||
process.env.HEYGEN_API_BASE = "https://proxy.example.com";
|
||||
assert.equal(heygenCredential(), null);
|
||||
assert.throws(() => heygenAuthHeaders(), /no HeyGen credentials/);
|
||||
// HeyGen's own hosts keep every credential source.
|
||||
process.env.HEYGEN_API_BASE = "https://api-canary.heygen.com";
|
||||
assert.equal(heygenAuthMethod(), "oauth");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,13 @@
|
||||
import { realpathSync } from "node:fs";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
// Node resolves symlinks in the main module's import.meta.url but argv[1] keeps the invoked
|
||||
// spelling (a symlinked skills dir, or /tmp vs /private/tmp on macOS), so compare real paths.
|
||||
export function isMainModule(importMetaUrl) {
|
||||
if (!process.argv[1]) return false;
|
||||
try {
|
||||
return pathToFileURL(realpathSync(process.argv[1])).href === importMetaUrl;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { extname, join } from "node:path";
|
||||
import { AGENT_SOURCES, latestRecordFor, recordInPlace } from "../../../scripts/lib/manifest.mjs";
|
||||
import { regenerateIndex } from "../../../scripts/lib/index-gen.mjs";
|
||||
|
||||
/**
|
||||
* Where the engine may write `rel`: there, unless another file of this run has `taken` it or the file there is
|
||||
* the person's (not `reusable` and not recorded as agent-made). Then the first free `name-2.ext`, with an anomaly.
|
||||
*/
|
||||
export function agentWritePath(
|
||||
hyperframesDir,
|
||||
rel,
|
||||
{ anomalies, taken = new Set(), reusable = () => false },
|
||||
) {
|
||||
const free = (path) =>
|
||||
!taken.has(path) &&
|
||||
(!existsSync(join(hyperframesDir, path)) ||
|
||||
reusable(path) ||
|
||||
AGENT_SOURCES.includes(latestRecordFor(hyperframesDir, path)?.source));
|
||||
if (free(rel)) return rel;
|
||||
const ext = extname(rel);
|
||||
const stem = rel.slice(0, rel.length - ext.length);
|
||||
let n = 2;
|
||||
while (!free(`${stem}-${n}${ext}`)) n++;
|
||||
const path = `${stem}-${n}${ext}`;
|
||||
const why = taken.has(rel)
|
||||
? "another file of this run goes there"
|
||||
: "the file there is yours (the media manifest does not record it as made by the engine)";
|
||||
const note = `${rel}: kept, because ${why}; writing ${path} instead (audio_meta.json has the path used)`;
|
||||
if (!anomalies.includes(note)) anomalies.push(note);
|
||||
return path;
|
||||
}
|
||||
|
||||
/** Each spoken line's file, picked before lines synthesize concurrently so two never land on one free name. */
|
||||
export function voicePaths(hyperframesDir, lines, anomalies) {
|
||||
const taken = new Set();
|
||||
const paths = new Map();
|
||||
for (const line of lines.filter((l) => String(l.text ?? "").trim())) {
|
||||
const rel = agentWritePath(hyperframesDir, `assets/voice/${line.id}.wav`, { anomalies, taken });
|
||||
taken.add(rel);
|
||||
paths.set(String(line.id), rel);
|
||||
}
|
||||
return paths;
|
||||
}
|
||||
|
||||
// A cue's source as the manifest records it, and who provided it. A source missing here records as the person's
|
||||
// file, so every rerun would write a numbered copy beside it.
|
||||
const SFX_SOURCES = {
|
||||
openverse: { source: "search", provider: "openverse.audio" },
|
||||
local: { source: "bundled", provider: "bundled.sfx" },
|
||||
};
|
||||
|
||||
// The files one engine run wrote, each with how it was made, for the project's media manifest.
|
||||
export function writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields, sfx }) {
|
||||
const textById = new Map(lines.map((line) => [String(line.id), String(line.text ?? "").trim()]));
|
||||
const assets = [];
|
||||
if (only.has("tts")) {
|
||||
for (const voice of voices) {
|
||||
assets.push({
|
||||
path: voice.path,
|
||||
type: "voice",
|
||||
source: "generated",
|
||||
intent: textById.get(voice.id),
|
||||
duration: voice.duration_s,
|
||||
provider: ttsProvider,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (only.has("bgm") && bgm && !bgmFields.bgm_pending) {
|
||||
assets.push({
|
||||
path: bgm.path,
|
||||
type: "bgm",
|
||||
source: bgmFields.bgm_mode === "retrieve" ? "search" : "generated",
|
||||
intent: bgm.query,
|
||||
duration: bgm.duration_s,
|
||||
provider: bgmFields.bgm_provider,
|
||||
});
|
||||
}
|
||||
if (only.has("sfx")) {
|
||||
for (const cue of new Map(sfx.map((entry) => [entry.file, entry])).values()) {
|
||||
const { source, provider } = SFX_SOURCES[cue.source];
|
||||
assets.push({
|
||||
path: cue.file,
|
||||
type: "sfx",
|
||||
source,
|
||||
intent: cue.name,
|
||||
duration: cue.duration_s,
|
||||
provider,
|
||||
});
|
||||
}
|
||||
}
|
||||
return assets;
|
||||
}
|
||||
|
||||
/** Records each asset where it lies; returns one anomaly per file left unrecorded. */
|
||||
export function recordInManifest(hyperframesDir, assets) {
|
||||
const anomalies = [];
|
||||
for (const { path, type, source, intent, duration, provider } of assets) {
|
||||
try {
|
||||
recordInPlace(hyperframesDir, {
|
||||
type,
|
||||
path,
|
||||
source,
|
||||
description: intent,
|
||||
duration,
|
||||
provenance: { provider: provider || "local", ...(intent && { prompt: intent }) },
|
||||
});
|
||||
} catch (error) {
|
||||
anomalies.push(`${path}: not recorded in the media manifest (${error.message})`);
|
||||
}
|
||||
}
|
||||
try {
|
||||
if (anomalies.length < assets.length) regenerateIndex(hyperframesDir);
|
||||
} catch (error) {
|
||||
anomalies.push(`.media/index.md: not refreshed (${error.message})`);
|
||||
}
|
||||
return anomalies;
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import { test } from "node:test";
|
||||
import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { findByPrompt, readManifest } from "../../../scripts/lib/manifest.mjs";
|
||||
import { agentWritePath, recordInManifest, voicePaths, writtenAssets } from "./media-record.mjs";
|
||||
|
||||
const noBgm = { bgm: null, bgmFields: { bgm_pending: false } };
|
||||
|
||||
function project(t) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-record-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
}
|
||||
|
||||
test("a voice run leaves a manifest entry marked generated", (t) => {
|
||||
const dir = project(t);
|
||||
const assets = writtenAssets({
|
||||
only: new Set(["tts"]),
|
||||
lines: [{ id: "01", text: "Welcome to the launch" }],
|
||||
voices: [{ id: "01", path: "assets/voice/01.wav", duration_s: 2.34 }],
|
||||
ttsProvider: "kokoro",
|
||||
sfx: [],
|
||||
...noBgm,
|
||||
});
|
||||
|
||||
assert.deepEqual(recordInManifest(dir, assets), []);
|
||||
|
||||
assert.deepEqual(
|
||||
readManifest(dir).map(({ path, type, source, description, duration, provenance }) => ({
|
||||
path,
|
||||
type,
|
||||
source,
|
||||
description,
|
||||
duration,
|
||||
provenance,
|
||||
})),
|
||||
[
|
||||
{
|
||||
path: "assets/voice/01.wav",
|
||||
type: "voice",
|
||||
source: "generated",
|
||||
description: "Welcome to the launch",
|
||||
duration: 2.3,
|
||||
provenance: { provider: "kokoro", prompt: "Welcome to the launch" },
|
||||
},
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
const voiceLine = (intent, duration) => [
|
||||
{ path: "assets/voice/01.wav", type: "voice", source: "generated", intent, duration },
|
||||
];
|
||||
|
||||
test("a rerun that writes the same take keeps one record, and a new length is a new take", (t) => {
|
||||
const dir = project(t);
|
||||
|
||||
recordInManifest(dir, voiceLine("Hello world", 1.25));
|
||||
recordInManifest(dir, voiceLine("Hello world", 1.25));
|
||||
assert.equal(readManifest(dir).length, 1);
|
||||
|
||||
recordInManifest(dir, voiceLine("Hello world", 2));
|
||||
assert.deepEqual(
|
||||
readManifest(dir).map(({ duration }) => duration),
|
||||
[1.3, 2],
|
||||
);
|
||||
});
|
||||
|
||||
test("a rerun with new text records the new take, and the old text no longer finds the file", (t) => {
|
||||
const dir = project(t);
|
||||
|
||||
recordInManifest(dir, voiceLine("Hello world", 1.25));
|
||||
recordInManifest(dir, voiceLine("Welcome back to the show", 3.5));
|
||||
|
||||
assert.deepEqual(
|
||||
readManifest(dir).map(({ description, duration }) => [description, duration]),
|
||||
[
|
||||
["Hello world", 1.3],
|
||||
["Welcome back to the show", 3.5],
|
||||
],
|
||||
);
|
||||
assert.equal(findByPrompt(dir, "Hello world", "voice"), null);
|
||||
const index = readFileSync(join(dir, ".media/index.md"), "utf8");
|
||||
assert.match(index, /Welcome back to the show/);
|
||||
assert.doesNotMatch(index, /Hello world/);
|
||||
assert.equal(findByPrompt(dir, "Welcome back to the show", "voice")?.path, "assets/voice/01.wav");
|
||||
});
|
||||
|
||||
test("music and sound effects are marked by where they came from", () => {
|
||||
const assets = writtenAssets({
|
||||
only: new Set(["bgm", "sfx"]),
|
||||
lines: [],
|
||||
voices: [{ id: "01", path: "assets/voice/01.wav" }],
|
||||
bgm: { path: "assets/bgm/track.mp3", query: "calm" },
|
||||
bgmFields: { bgm_pending: false, bgm_mode: "retrieve" },
|
||||
sfx: [
|
||||
{ file: "assets/sfx/whoosh.mp3", name: "whoosh", source: "local" },
|
||||
{ file: "assets/sfx/whoosh.mp3", name: "whoosh", source: "local" },
|
||||
{ file: "assets/sfx/glass.mp3", name: "glass", source: "openverse" },
|
||||
],
|
||||
});
|
||||
|
||||
assert.deepEqual(
|
||||
assets.map(({ path, source, provider }) => [path, source, provider]),
|
||||
[
|
||||
["assets/bgm/track.mp3", "search", undefined],
|
||||
["assets/sfx/whoosh.mp3", "bundled", "bundled.sfx"],
|
||||
["assets/sfx/glass.mp3", "search", "openverse.audio"],
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test("a rerun of an Openverse sound effect writes over its own file, not a numbered copy", (t) => {
|
||||
const dir = project(t);
|
||||
mkdirSync(join(dir, "assets/sfx"), { recursive: true });
|
||||
const anomalies = [];
|
||||
for (let run = 0; run < 3; run++) {
|
||||
const file = agentWritePath(dir, "assets/sfx/door-knock.mp3", { anomalies });
|
||||
writeFileSync(join(dir, file), "knock");
|
||||
const sfx = [{ file, name: "door knock", source: "openverse", duration_s: 1 }];
|
||||
recordInManifest(
|
||||
dir,
|
||||
writtenAssets({ only: new Set(["sfx"]), lines: [], voices: [], sfx, ...noBgm }),
|
||||
);
|
||||
}
|
||||
|
||||
assert.deepEqual(readdirSync(join(dir, "assets/sfx")), ["door-knock.mp3"]);
|
||||
assert.deepEqual(anomalies, []);
|
||||
assert.equal(readManifest(dir).length, 1);
|
||||
});
|
||||
|
||||
test("music made locally is marked generated once it is ready, not while pending", () => {
|
||||
const written = (bgm_pending) =>
|
||||
writtenAssets({
|
||||
only: new Set(["bgm"]),
|
||||
lines: [],
|
||||
voices: [],
|
||||
sfx: [],
|
||||
bgm: { path: "assets/bgm/track.wav" },
|
||||
bgmFields: { bgm_pending, bgm_mode: "detached-single" },
|
||||
}).map(({ source }) => source);
|
||||
|
||||
assert.deepEqual(written(true), []);
|
||||
assert.deepEqual(written(false), ["generated"]);
|
||||
});
|
||||
|
||||
test("a file that cannot be recorded becomes an anomaly, not a failure", (t) => {
|
||||
const dir = project(t);
|
||||
writeFileSync(join(dir, ".media"), "a file where the media folder should be");
|
||||
|
||||
const anomalies = recordInManifest(dir, [
|
||||
{ path: "assets/voice/01.wav", type: "voice", source: "generated" },
|
||||
]);
|
||||
|
||||
assert.equal(anomalies.length, 1);
|
||||
assert.match(anomalies[0], /^assets\/voice\/01\.wav: not recorded in the media manifest/);
|
||||
});
|
||||
|
||||
test("the engine writes over only its own files, else the next free name", (t) => {
|
||||
const dir = project(t);
|
||||
mkdirSync(join(dir, "assets/sfx"), { recursive: true });
|
||||
const names = ["mine", "made", "adopted", "twice", "twice-2", "kept", "kept-2"];
|
||||
for (const name of names) writeFileSync(join(dir, `assets/sfx/${name}.mp3`), name);
|
||||
for (const path of ["assets/sfx/made.mp3", "assets/sfx/kept-2.mp3"])
|
||||
recordInManifest(dir, [{ path, type: "sfx", source: "search" }]);
|
||||
recordInManifest(dir, [{ path: "assets/sfx/adopted.mp3", type: "sfx", source: "existing" }]);
|
||||
const anomalies = [];
|
||||
const at = (name, reusable) =>
|
||||
agentWritePath(dir, `assets/sfx/${name}.mp3`, { anomalies, reusable });
|
||||
|
||||
assert.equal(at("new"), "assets/sfx/new.mp3");
|
||||
assert.equal(at("made"), "assets/sfx/made.mp3");
|
||||
assert.equal(
|
||||
at("mine", (path) => path === "assets/sfx/mine.mp3"),
|
||||
"assets/sfx/mine.mp3",
|
||||
);
|
||||
assert.deepEqual(anomalies, []);
|
||||
assert.equal(at("mine"), "assets/sfx/mine-2.mp3");
|
||||
assert.equal(at("adopted"), "assets/sfx/adopted-2.mp3");
|
||||
assert.equal(at("twice"), "assets/sfx/twice-3.mp3");
|
||||
assert.equal(at("kept"), "assets/sfx/kept-2.mp3");
|
||||
assert.equal(anomalies.length, 4);
|
||||
assert.match(
|
||||
anomalies[0],
|
||||
/^assets\/sfx\/mine\.mp3: kept, .* writing assets\/sfx\/mine-2\.mp3 instead/,
|
||||
);
|
||||
});
|
||||
|
||||
test("two spoken lines never share a file when one's name is taken by the person", (t) => {
|
||||
const dir = project(t);
|
||||
mkdirSync(join(dir, "assets/voice"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets/voice/hook.wav"), "the person's own hook");
|
||||
const anomalies = [];
|
||||
|
||||
const paths = voicePaths(
|
||||
dir,
|
||||
[
|
||||
{ id: "hook", text: "First" },
|
||||
{ id: "hook-2", text: "Second" },
|
||||
{ id: "blank", text: " " },
|
||||
],
|
||||
anomalies,
|
||||
);
|
||||
|
||||
assert.deepEqual(Object.fromEntries(paths), {
|
||||
hook: "assets/voice/hook-2.wav",
|
||||
"hook-2": "assets/voice/hook-2-2.wav",
|
||||
});
|
||||
assert.equal(anomalies.length, 2);
|
||||
});
|
||||
|
||||
test("a record never takes the id of a download still in flight", (t) => {
|
||||
const dir = project(t);
|
||||
mkdirSync(join(dir, ".media/audio/bgm"), { recursive: true });
|
||||
writeFileSync(join(dir, ".media/audio/bgm/bgm_001.mp3"), "");
|
||||
|
||||
recordInManifest(dir, [{ path: "assets/bgm/track.wav", type: "bgm", source: "generated" }]);
|
||||
|
||||
assert.deepEqual(
|
||||
readManifest(dir).map(({ id }) => id),
|
||||
["bgm_002"],
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,134 @@
|
||||
// Sound effects from Openverse (https://api.openverse.org): free, no key.
|
||||
// Anonymous use is rate limited (20 requests a minute, 200 a day); callers fall
|
||||
// back to the bundled library when it is unavailable.
|
||||
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
|
||||
const OPENVERSE_AUDIO_URL = "https://api.openverse.org/v1/audio/";
|
||||
// The only licences media-use accepts: no NonCommercial, ShareAlike or NoDerivatives.
|
||||
export const OPENVERSE_SFX_LICENSES = new Set(["cc0", "by"]);
|
||||
const PLAYABLE_FILETYPES = new Set(["mp3", "wav", "ogg", "flac"]);
|
||||
const MAX_SFX_MS = 15000;
|
||||
const IDEAL_SFX_MS = 4000;
|
||||
const BY_PENALTY = 2;
|
||||
const SEARCH_TIMEOUT_MS = 10000;
|
||||
|
||||
// Openverse text lands in markdown and JSON: one line, no control or markdown-structure characters, bounded.
|
||||
const oneLine = (text) =>
|
||||
String(text ?? "")
|
||||
.replace(/[\p{Cc}`[\]<>\\]+/gu, " ")
|
||||
.trim()
|
||||
.slice(0, 300);
|
||||
// Openverse's audio providers (api.openverse.org/v1/audio/stats/); a new provider's link is dropped until listed.
|
||||
const SOURCE_HOSTS = new Set(["freesound.org", "www.jamendo.com", "commons.wikimedia.org"]);
|
||||
function httpsUrl(url, hosts) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return parsed.protocol === "https:" && (!hosts || hosts.has(parsed.hostname))
|
||||
? oneLine(parsed.href)
|
||||
: "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
/** Openverse could not answer (offline, rate limited, server error): stop asking it this run. */
|
||||
export class OpenverseUnavailableError extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
this.name = "OpenverseUnavailableError";
|
||||
}
|
||||
}
|
||||
|
||||
function usable(result) {
|
||||
return (
|
||||
OPENVERSE_SFX_LICENSES.has(result?.license) &&
|
||||
PLAYABLE_FILETYPES.has(result.filetype) &&
|
||||
typeof result.url === "string" &&
|
||||
result.url.startsWith("https://") &&
|
||||
Number.isFinite(result.duration) &&
|
||||
result.duration > 0 &&
|
||||
result.duration <= MAX_SFX_MS
|
||||
);
|
||||
}
|
||||
|
||||
// Lower is better: relevance position, then seconds past a typical cue length, then CC0 over CC BY.
|
||||
function penalty(result, position) {
|
||||
const longBy = Math.max(0, result.duration - IDEAL_SFX_MS) / 1000;
|
||||
return position + longBy + (result.license === "by" ? BY_PENALTY : 0);
|
||||
}
|
||||
|
||||
/** Best usable sound in Openverse's relevance-ordered results, or null. */
|
||||
export function pickOpenverseSfx(results) {
|
||||
let best = null;
|
||||
results.forEach((result, position) => {
|
||||
if (!usable(result)) return;
|
||||
const score = penalty(result, position);
|
||||
if (!best || score < best.score) best = { result, score };
|
||||
});
|
||||
if (!best) return null;
|
||||
const r = best.result;
|
||||
return {
|
||||
id: r.id,
|
||||
url: r.url,
|
||||
ext: `.${r.filetype}`,
|
||||
title: oneLine(r.title),
|
||||
creator: oneLine(r.creator),
|
||||
license: r.license,
|
||||
licenseUrl: httpsUrl(r.license_url),
|
||||
sourceUrl: httpsUrl(r.foreign_landing_url, SOURCE_HOSTS),
|
||||
credit: oneLine(r.attribution),
|
||||
durationS: Math.round(r.duration) / 1000,
|
||||
};
|
||||
}
|
||||
|
||||
/** Best licensed sound effect for `query`, null on no match; throws OpenverseUnavailableError. */
|
||||
export async function searchOpenverseSfx(query, { fetchImpl = fetch } = {}) {
|
||||
const params = new URLSearchParams({
|
||||
q: query,
|
||||
page_size: "20",
|
||||
license: [...OPENVERSE_SFX_LICENSES].join(","),
|
||||
});
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(`${OPENVERSE_AUDIO_URL}?${params}`, {
|
||||
headers: { Accept: "application/json" },
|
||||
signal: AbortSignal.timeout(SEARCH_TIMEOUT_MS),
|
||||
});
|
||||
} catch (error) {
|
||||
throw new OpenverseUnavailableError(`Openverse unreachable (${error.message})`);
|
||||
}
|
||||
if (response.status === 429)
|
||||
throw new OpenverseUnavailableError("Openverse rate limit reached (HTTP 429)");
|
||||
if (!response.ok) throw new OpenverseUnavailableError(`Openverse HTTP ${response.status}`);
|
||||
let body;
|
||||
try {
|
||||
body = await response.json();
|
||||
} catch {
|
||||
throw new OpenverseUnavailableError("Openverse returned non-JSON");
|
||||
}
|
||||
return pickOpenverseSfx(Array.isArray(body?.results) ? body.results : []);
|
||||
}
|
||||
|
||||
/** Record a CC BY sound's credit for `file` in the project's credits file (one line per file). */
|
||||
export function writeSfxCredit(creditsPath, file, sound) {
|
||||
if (sound.license !== "by") return;
|
||||
// The sound may come from a saved record, not this run's pick: clean it again here.
|
||||
const sourceUrl = httpsUrl(sound.sourceUrl, SOURCE_HOSTS);
|
||||
const source = sourceUrl ? ` Source: ${sourceUrl}` : "";
|
||||
const line = `- \`${file}\`: ${oneLine(sound.credit)}${source}`;
|
||||
let prior = "";
|
||||
try {
|
||||
prior = readFileSync(creditsPath, "utf8");
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT") throw error;
|
||||
}
|
||||
const kept = prior
|
||||
.split("\n")
|
||||
.filter((l) => l.startsWith("- ") && !l.startsWith(`- \`${file}\`:`));
|
||||
const header =
|
||||
"# Sound effect credits\n\nThese sounds are licensed CC BY: credit them wherever the video is published.\n\n";
|
||||
mkdirSync(dirname(creditsPath), { recursive: true });
|
||||
writeFileSync(creditsPath, `${header}${[...kept, line].join("\n")}\n`);
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
// python.mjs — resolve which Python 3 executable to spawn, per platform.
|
||||
//
|
||||
// The audio engine (tts.mjs, bgm.mjs) shells out to `python3` for ElevenLabs
|
||||
// TTS and the local Lyria/MusicGen BGM paths. `python3` is the right name on
|
||||
// macOS/Linux, but on Windows the python.org installer only creates
|
||||
// `python.exe` plus the `py` launcher — there is no `python3.exe` (only the
|
||||
// Microsoft Store build adds one). So a bare `spawn("python3", …)` ENOENTs on a
|
||||
// standard Windows Python install, silently disabling every Python-backed audio
|
||||
// feature until the user hand-creates a `python3.exe` shim (reported twice).
|
||||
//
|
||||
// Resolve once, per process: probe the platform's candidates in order and take
|
||||
// the first that actually runs. `py` is the launcher, so it needs a `-3` arg to
|
||||
// select Python 3 — hence candidates are argv PREFIXES, not bare names.
|
||||
//
|
||||
// `HYPERFRAMES_PYTHON` (the documented PEP 668 venv setup) wins over the PATH probe, matching
|
||||
// the CLI's `findPython()`; ignoring it made BGM silently disable itself (#4614).
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
// Accept only an interpreter whose `--version` reports Python 3, as the CLI's
|
||||
// `validatePythonOverride()` does. Exit status alone would take a Python 2
|
||||
// `python` (or any executable that exits 0) that `doctor` rejects. Python 2
|
||||
// prints its version to stderr, so read both streams.
|
||||
export function defaultProbe(cmd, args) {
|
||||
try {
|
||||
const result = spawnSync(cmd, args, { encoding: "utf-8", timeout: 5000 });
|
||||
return result.status === 0 && /Python 3/.test(`${result.stdout}${result.stderr}`);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the argv prefix that launches Python 3 on this platform.
|
||||
* Returns e.g. `["python3"]`, `["python"]`, or `["py", "-3"]`.
|
||||
*
|
||||
* Pure except for `probe` (which runs `<cmd> … --version`); `platform`, `probe`
|
||||
* and `env` are injectable so every branch is unit-testable without spawning.
|
||||
* If nothing probes OK, falls back to the canonical name for the platform so
|
||||
* the eventual spawn fails loudly exactly as it did before — never worse.
|
||||
*/
|
||||
export function resolvePythonCommand(
|
||||
platform = process.platform,
|
||||
probe = defaultProbe,
|
||||
env = process.env,
|
||||
) {
|
||||
const override = env.HYPERFRAMES_PYTHON;
|
||||
if (override && probe(override, ["--version"])) return [override];
|
||||
const candidates =
|
||||
platform === "win32" ? [["python3"], ["python"], ["py", "-3"]] : [["python3"], ["python"]];
|
||||
for (const prefix of candidates) {
|
||||
if (probe(prefix[0], [...prefix.slice(1), "--version"])) return prefix;
|
||||
}
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
let cached = null;
|
||||
|
||||
/** Cached `resolvePythonCommand()` — probing spawns, so resolve at most once. */
|
||||
export function pythonCommand() {
|
||||
if (!cached) cached = resolvePythonCommand();
|
||||
return cached;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a `{ cmd, args }` for running Python 3 with `extraArgs`, using the
|
||||
* resolved (or supplied) prefix. Keeps the launcher's `-3` (and any future
|
||||
* prefix args) ahead of the caller's own arguments.
|
||||
*/
|
||||
export function pythonInvocation(extraArgs, prefix = pythonCommand()) {
|
||||
return { cmd: prefix[0], args: [...prefix.slice(1), ...extraArgs] };
|
||||
}
|
||||
|
||||
/** Test-only: clear the cached resolution so a test can re-probe. */
|
||||
export function _resetPythonCommandCacheForTests() {
|
||||
cached = null;
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { defaultProbe, resolvePythonCommand, pythonInvocation } from "./python.mjs";
|
||||
|
||||
// Regression: on Windows a standard python.org install has no `python3.exe`
|
||||
// (only `python.exe` + the `py` launcher), so `spawn("python3", …)` ENOENTs and
|
||||
// every Python-backed audio feature silently no-ops. resolvePythonCommand takes
|
||||
// injectable platform/probe params so all branches are testable without
|
||||
// spawning a real interpreter.
|
||||
|
||||
// probeFor(names): a probe that reports success only for the given argv-0 names.
|
||||
function probeFor(...names) {
|
||||
const ok = new Set(names);
|
||||
return (cmd) => ok.has(cmd);
|
||||
}
|
||||
|
||||
test("non-win32 uses python3 when it runs", () => {
|
||||
assert.deepEqual(resolvePythonCommand("linux", probeFor("python3")), ["python3"]);
|
||||
assert.deepEqual(resolvePythonCommand("darwin", probeFor("python3")), ["python3"]);
|
||||
});
|
||||
|
||||
test("win32 prefers python3 when the Microsoft Store build provides it", () => {
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor("python3", "python", "py")), ["python3"]);
|
||||
});
|
||||
|
||||
test("win32 falls back to python.exe when python3 is absent (python.org install)", () => {
|
||||
// The exact reported scenario: no python3, but `python` exists.
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor("python", "py")), ["python"]);
|
||||
});
|
||||
|
||||
test("win32 falls back to the py launcher with -3 when only py exists", () => {
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor("py")), ["py", "-3"]);
|
||||
});
|
||||
|
||||
test("py launcher is probed as `py -3 --version`, not bare `py`", () => {
|
||||
const seen = [];
|
||||
const probe = (cmd, args) => {
|
||||
seen.push([cmd, ...args]);
|
||||
return cmd === "py";
|
||||
};
|
||||
resolvePythonCommand("win32", probe);
|
||||
assert.deepEqual(seen.at(-1), ["py", "-3", "--version"]);
|
||||
});
|
||||
|
||||
test("falls back to the canonical name (loud failure, unchanged) when nothing runs", () => {
|
||||
// No interpreter anywhere — must not throw, and must return python3 so the
|
||||
// eventual spawn fails exactly as it did before this fix, never worse.
|
||||
assert.deepEqual(
|
||||
resolvePythonCommand("win32", () => false),
|
||||
["python3"],
|
||||
);
|
||||
assert.deepEqual(
|
||||
resolvePythonCommand("linux", () => false),
|
||||
["python3"],
|
||||
);
|
||||
});
|
||||
|
||||
// Regression (#4614): on PEP 668 systems the documented setup is a venv named
|
||||
// by HYPERFRAMES_PYTHON; the audio engine ignored it and probed the bare
|
||||
// system python3, so MusicGen BGM was skipped while `doctor` said installed.
|
||||
const VENV = "/home/u/.venvs/hf/bin/python";
|
||||
|
||||
test("HYPERFRAMES_PYTHON wins over python3 on PATH when it runs", () => {
|
||||
const env = { HYPERFRAMES_PYTHON: VENV };
|
||||
assert.deepEqual(resolvePythonCommand("darwin", probeFor(VENV, "python3"), env), [VENV]);
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor(VENV, "python", "py"), env), [VENV]);
|
||||
});
|
||||
|
||||
test("a HYPERFRAMES_PYTHON that doesn't run falls through to the PATH probe", () => {
|
||||
const env = { HYPERFRAMES_PYTHON: "/nope/python" };
|
||||
assert.deepEqual(resolvePythonCommand("linux", probeFor("python3"), env), ["python3"]);
|
||||
});
|
||||
|
||||
test("an empty HYPERFRAMES_PYTHON is ignored without being probed", () => {
|
||||
const seen = [];
|
||||
const probe = (cmd) => {
|
||||
seen.push(cmd);
|
||||
return cmd === "python3";
|
||||
};
|
||||
assert.deepEqual(resolvePythonCommand("linux", probe, { HYPERFRAMES_PYTHON: "" }), ["python3"]);
|
||||
assert.deepEqual(seen, ["python3"]);
|
||||
});
|
||||
|
||||
// defaultProbe must match the CLI's validatePythonOverride(): exiting 0 is not
|
||||
// enough, `--version` has to report Python 3. Stubs are shell scripts, so
|
||||
// these run on POSIX only.
|
||||
function withStub(script, fn) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-python-probe-"));
|
||||
try {
|
||||
const stub = join(dir, "python");
|
||||
writeFileSync(stub, `#!/bin/sh\n${script}\n`);
|
||||
chmodSync(stub, 0o755);
|
||||
fn(stub);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
const posixOnly = { skip: process.platform === "win32" && "stub scripts need a POSIX shell" };
|
||||
|
||||
test("defaultProbe accepts an interpreter reporting Python 3", posixOnly, () => {
|
||||
withStub('echo "Python 3.12.4"', (stub) => assert.equal(defaultProbe(stub, ["--version"]), true));
|
||||
});
|
||||
|
||||
test("defaultProbe rejects Python 2, which prints its version to stderr", posixOnly, () => {
|
||||
withStub('echo "Python 2.7.18" >&2', (stub) =>
|
||||
assert.equal(defaultProbe(stub, ["--version"]), false),
|
||||
);
|
||||
});
|
||||
|
||||
test(
|
||||
"defaultProbe rejects an executable that exits 0 without a Python 3 version",
|
||||
posixOnly,
|
||||
() => {
|
||||
withStub("exit 0", (stub) => assert.equal(defaultProbe(stub, ["--version"]), false));
|
||||
},
|
||||
);
|
||||
|
||||
test("defaultProbe rejects a command that doesn't exist", () => {
|
||||
assert.equal(defaultProbe("/nonexistent/hf-python", ["--version"]), false);
|
||||
});
|
||||
|
||||
test("pythonInvocation prepends the resolved prefix ahead of caller args", () => {
|
||||
assert.deepEqual(pythonInvocation(["-c", "import x"], ["python"]), {
|
||||
cmd: "python",
|
||||
args: ["-c", "import x"],
|
||||
});
|
||||
// The py launcher's -3 must stay ahead of the caller's own arguments.
|
||||
assert.deepEqual(pythonInvocation(["-c", "import x"], ["py", "-3"]), {
|
||||
cmd: "py",
|
||||
args: ["-3", "-c", "import x"],
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,196 @@
|
||||
// sfx.mjs — sound effects for the media audio engine. Each cue is searched on
|
||||
// Openverse (CC0 / CC BY only, see openverse.mjs). A cue Openverse cannot answer,
|
||||
// and every cue after Openverse is unreachable or rate limited, resolves against
|
||||
// the bundled 21-file library (assets/sfx/manifest.json) instead.
|
||||
//
|
||||
// A cue that matches nothing is skipped (recorded as an anomaly); SFX never
|
||||
// blocks a render. Every cue sits at volume ~0.35, under voice + BGM.
|
||||
|
||||
import { copyFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { downloadTo } from "./heygen.mjs";
|
||||
import { OpenverseUnavailableError, searchOpenverseSfx, writeSfxCredit } from "./openverse.mjs";
|
||||
import { latestRecordFor } from "../../../scripts/lib/manifest.mjs";
|
||||
import { agentWritePath } from "./media-record.mjs";
|
||||
|
||||
export const BUNDLED_SFX_DIR = join(
|
||||
dirname(fileURLToPath(import.meta.url)),
|
||||
"..",
|
||||
"..",
|
||||
"assets",
|
||||
"sfx",
|
||||
);
|
||||
const SFX_VOLUME = 0.35;
|
||||
const slug = (s) =>
|
||||
s
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.slice(0, 40) || "x";
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
|
||||
// Lookups by manifest key, file basename, and slug of either, so a cue can name
|
||||
// "whoosh", "whoosh.mp3", or "ui click" (→ slug match). { error } when unusable.
|
||||
function loadBundledLibrary(sfxLibDir) {
|
||||
const manifestPath = join(sfxLibDir, "manifest.json");
|
||||
if (!existsSync(manifestPath)) return { error: `no bundled SFX library at ${sfxLibDir}` };
|
||||
let manifest;
|
||||
try {
|
||||
manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
|
||||
} catch (e) {
|
||||
return { error: `bundled SFX manifest parse failed (${e.message})` };
|
||||
}
|
||||
const byKey = new Map();
|
||||
for (const [key, entry] of Object.entries(manifest)) {
|
||||
if (!entry?.file || !isFinite(entry.duration)) continue;
|
||||
const rec = { key, file: entry.file, duration: entry.duration };
|
||||
byKey.set(key, rec);
|
||||
byKey.set(entry.file, rec);
|
||||
byKey.set(slug(key), rec);
|
||||
byKey.set(slug(entry.file.replace(/\.\w+$/, "")), rec);
|
||||
}
|
||||
return { byKey };
|
||||
}
|
||||
|
||||
// cues: [{ id, name }] (id = the line/frame/scene the cue fires in). Returns
|
||||
// { sfx: [{ id, name, file, source, offset_s, duration_s, volume, license?, credit? }], anomalies }.
|
||||
export async function resolveSfx({ cues, hyperframesDir, sfxLibDir = BUNDLED_SFX_DIR }) {
|
||||
const sfx = [];
|
||||
const anomalies = [];
|
||||
const destDir = join(hyperframesDir, "assets", "sfx");
|
||||
// Each effect's file this run: one effect named twice gets one file, and two effects never share one.
|
||||
const fileFor = new Map();
|
||||
const taken = () => new Set(fileFor.values());
|
||||
// One Openverse sound per effect name this run, so a repeated cue costs no second request.
|
||||
const openverseFor = new Map();
|
||||
let openverseDown = false;
|
||||
let library;
|
||||
|
||||
// Dedupe identical (id,name) cues — the same effect named twice in one line
|
||||
// downloads/copies once.
|
||||
const seen = new Set();
|
||||
const uniq = cues.filter((c) => {
|
||||
const k = `${c.id}:${c.name}`;
|
||||
if (seen.has(k)) return false;
|
||||
seen.add(k);
|
||||
return true;
|
||||
});
|
||||
|
||||
async function fromOpenverse({ id, name }) {
|
||||
const key = `openverse:${slug(name)}`;
|
||||
let entry = openverseFor.get(key);
|
||||
if (!entry) {
|
||||
const sound = await searchOpenverseSfx(name);
|
||||
if (!sound) return null;
|
||||
const file = agentWritePath(hyperframesDir, `assets/sfx/${slug(name)}${sound.ext}`, {
|
||||
anomalies,
|
||||
taken: taken(),
|
||||
});
|
||||
try {
|
||||
await downloadTo(sound.url, join(hyperframesDir, file));
|
||||
} catch (e) {
|
||||
anomalies.push(`sfx "${name}" (id ${id}): Openverse download failed — ${e.message}`);
|
||||
return null;
|
||||
}
|
||||
fileFor.set(key, file);
|
||||
writeSfxCredit(join(destDir, "CREDITS.md"), file, sound);
|
||||
entry = {
|
||||
file,
|
||||
source: "openverse",
|
||||
duration_s: r3(sound.durationS),
|
||||
license: sound.license,
|
||||
credit: sound.credit,
|
||||
};
|
||||
openverseFor.set(key, entry);
|
||||
}
|
||||
return { id, name, ...entry, offset_s: 0, volume: SFX_VOLUME };
|
||||
}
|
||||
|
||||
function fromBundled({ id, name }) {
|
||||
library ??= loadBundledLibrary(sfxLibDir);
|
||||
if (library.error) {
|
||||
anomalies.push(`sfx "${name}" (id ${id}): ${library.error} — skipped`);
|
||||
return null;
|
||||
}
|
||||
const { byKey } = library;
|
||||
const hit = byKey.get(name) ?? byKey.get(slug(name));
|
||||
if (!hit) {
|
||||
const known = [...new Set([...byKey.values()].map((v) => v.key))].slice(0, 8).join(", ");
|
||||
anomalies.push(
|
||||
`sfx "${name}" (id ${id}): not in bundled library — skipped (have: ${known}…)`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
mkdirSync(destDir, { recursive: true });
|
||||
const src = join(sfxLibDir, hit.file);
|
||||
const bytes = existsSync(src) ? readFileSync(src) : null;
|
||||
const isLibraryCopy = (rel) =>
|
||||
bytes &&
|
||||
existsSync(join(hyperframesDir, rel)) &&
|
||||
readFileSync(join(hyperframesDir, rel)).equals(bytes);
|
||||
const destRel =
|
||||
fileFor.get(hit.file) ??
|
||||
agentWritePath(hyperframesDir, `assets/sfx/${hit.file}`, {
|
||||
anomalies,
|
||||
taken: taken(),
|
||||
// An unrecorded copy of the library file is one an engine run made before the manifest had it.
|
||||
reusable: (rel) => !latestRecordFor(hyperframesDir, rel) && isLibraryCopy(rel),
|
||||
});
|
||||
const dest = join(hyperframesDir, destRel);
|
||||
// Some installs of the skill ship manifest.json without the mp3s. An entry
|
||||
// pointing at a file we never copied would silently drop downstream, so
|
||||
// skip the cue with a loud anomaly instead.
|
||||
if (bytes) {
|
||||
if (!isLibraryCopy(destRel)) copyFileSync(src, dest);
|
||||
} else if (!existsSync(dest)) {
|
||||
anomalies.push(
|
||||
`sfx "${name}" (id ${id}): bundled file ${hit.file} missing from the offline ` +
|
||||
`library (${sfxLibDir}) — skipped. Reinstall the media-use skill to ` +
|
||||
`restore assets/sfx/*.mp3.`,
|
||||
);
|
||||
return null;
|
||||
} // else the engine's earlier copy at dest stands in for the file this install lacks
|
||||
fileFor.set(hit.file, destRel);
|
||||
return {
|
||||
id,
|
||||
name,
|
||||
file: destRel,
|
||||
source: "local",
|
||||
offset_s: 0,
|
||||
duration_s: r3(hit.duration),
|
||||
volume: SFX_VOLUME,
|
||||
};
|
||||
}
|
||||
|
||||
for (const cue of uniq) {
|
||||
let entry = null;
|
||||
if (!openverseDown) {
|
||||
try {
|
||||
entry = await fromOpenverse(cue);
|
||||
} catch (e) {
|
||||
if (!(e instanceof OpenverseUnavailableError)) throw e;
|
||||
openverseDown = true;
|
||||
anomalies.push(`${e.message}: the remaining SFX cues use the bundled library`);
|
||||
}
|
||||
}
|
||||
entry ??= fromBundled(cue);
|
||||
if (entry) sfx.push(entry);
|
||||
}
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
|
||||
/**
|
||||
* One sound effect for `query`, saved under the project's assets/sfx/ (the entry point for hosts).
|
||||
* { sound: { file, source, duration_s, license?, credit? } | null, anomalies }.
|
||||
*/
|
||||
export async function findSoundEffect(query, { hyperframesDir, sfxLibDir = BUNDLED_SFX_DIR }) {
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "1", name: query }],
|
||||
hyperframesDir,
|
||||
sfxLibDir,
|
||||
});
|
||||
if (!sfx.length) return { sound: null, anomalies };
|
||||
const { file, source, duration_s, license, credit } = sfx[0];
|
||||
return { sound: { file, source, duration_s, ...(license && { license, credit }) }, anomalies };
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, existsSync, readFileSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { appendRecord } from "../../../scripts/lib/manifest.mjs";
|
||||
import { BUNDLED_SFX_DIR, findSoundEffect, resolveSfx } from "./sfx.mjs";
|
||||
|
||||
// No real network: each test serves Openverse's answers; by default it is unreachable.
|
||||
const offline = async () => {
|
||||
throw new TypeError("fetch failed");
|
||||
};
|
||||
let serve = offline;
|
||||
const UNREACHABLE =
|
||||
"Openverse unreachable (fetch failed): the remaining SFX cues use the bundled library";
|
||||
globalThis.fetch = (url, init) => serve(String(url), init);
|
||||
|
||||
// Offline SFX resolution: the bundled library may ship manifest.json
|
||||
// without the actual mp3s. The old code copied only when the source existed but
|
||||
// pushed the sfx entry unconditionally — producing a dangling reference that
|
||||
// silently dropped downstream ("not on disk"). These tests lock in the loud
|
||||
// behavior: a present file is copied + referenced; a missing file yields an
|
||||
// anomaly and NO dangling entry.
|
||||
|
||||
async function withDirs(fn) {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-sfx-"));
|
||||
const libDir = join(root, "lib");
|
||||
const projDir = join(root, "proj");
|
||||
mkdirSync(libDir, { recursive: true });
|
||||
mkdirSync(projDir, { recursive: true });
|
||||
try {
|
||||
// `await` is load-bearing: without it the finally cleanup runs before the
|
||||
// async test body resolves, deleting the temp dir mid-assertion.
|
||||
return await fn({ libDir, projDir });
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test("offline: copies and references a present bundled file", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
writeFileSync(
|
||||
join(libDir, "manifest.json"),
|
||||
JSON.stringify({ whoosh: { file: "whoosh.mp3", duration: 0.8 } }),
|
||||
);
|
||||
writeFileSync(join(libDir, "whoosh.mp3"), "ID3-fake-bytes");
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
assert.equal(sfx.length, 1);
|
||||
assert.equal(sfx[0].file, "assets/sfx/whoosh.mp3");
|
||||
assert.equal(sfx[0].source, "local");
|
||||
assert.ok(existsSync(join(projDir, "assets/sfx/whoosh.mp3")), "mp3 copied into project");
|
||||
assert.deepEqual(anomalies, [UNREACHABLE]);
|
||||
});
|
||||
});
|
||||
|
||||
test("offline: a matched-but-missing bundled file yields an anomaly and NO dangling entry", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
// Manifest names whoosh.mp3, but the mp3 was never shipped (the reported bug).
|
||||
writeFileSync(
|
||||
join(libDir, "manifest.json"),
|
||||
JSON.stringify({ whoosh: { file: "whoosh.mp3", duration: 0.8 } }),
|
||||
);
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
assert.equal(sfx.length, 0, "no dangling entry for a file that was never copied");
|
||||
assert.equal(anomalies.length, 2);
|
||||
assert.match(anomalies[1], /missing from the offline library/);
|
||||
assert.ok(!existsSync(join(projDir, "assets/sfx/whoosh.mp3")), "nothing copied");
|
||||
});
|
||||
});
|
||||
|
||||
const whooshLibrary = (libDir, withFile) => {
|
||||
writeFileSync(
|
||||
join(libDir, "manifest.json"),
|
||||
JSON.stringify({ whoosh: { file: "whoosh.mp3", duration: 0.8 } }),
|
||||
);
|
||||
if (withFile) writeFileSync(join(libDir, "whoosh.mp3"), "ID3-library-bytes");
|
||||
};
|
||||
const record = (projDir, source) =>
|
||||
appendRecord(projDir, { id: "sfx_001", type: "sfx", path: "assets/sfx/whoosh.mp3", source });
|
||||
const bundledOnly = (libDir, projDir) =>
|
||||
resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
|
||||
test("offline: the engine's earlier copy stands in for a library file this install lacks", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
whooshLibrary(libDir, false);
|
||||
mkdirSync(join(projDir, "assets/sfx"), { recursive: true });
|
||||
writeFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "engine copy");
|
||||
record(projDir, "bundled");
|
||||
|
||||
const { sfx, anomalies } = await bundledOnly(libDir, projDir);
|
||||
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file }) => file),
|
||||
["assets/sfx/whoosh.mp3"],
|
||||
);
|
||||
assert.deepEqual(anomalies, [UNREACHABLE]);
|
||||
});
|
||||
});
|
||||
|
||||
test("offline: a person's file with the library's bytes stays theirs once recorded as their own", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
whooshLibrary(libDir, true);
|
||||
mkdirSync(join(projDir, "assets/sfx"), { recursive: true });
|
||||
writeFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "ID3-library-bytes");
|
||||
record(projDir, "existing");
|
||||
|
||||
const { sfx } = await bundledOnly(libDir, projDir);
|
||||
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file }) => file),
|
||||
["assets/sfx/whoosh-2.mp3"],
|
||||
);
|
||||
assert.equal(readFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "utf8"), "ID3-library-bytes");
|
||||
});
|
||||
});
|
||||
|
||||
const libraryWith = (libDir, names) => {
|
||||
const manifest = {};
|
||||
for (const name of names) {
|
||||
manifest[name] = { file: `${name}.mp3`, duration: 0.5 };
|
||||
writeFileSync(join(libDir, `${name}.mp3`), `library ${name}`);
|
||||
}
|
||||
writeFileSync(join(libDir, "manifest.json"), JSON.stringify(manifest));
|
||||
};
|
||||
const sound = (overrides = {}) => ({
|
||||
id: "ov-1",
|
||||
title: "Swoosh",
|
||||
creator: "Ana",
|
||||
license: "by",
|
||||
foreign_landing_url: "https://freesound.org/s/1",
|
||||
url: "https://cdn.freesound.test/swoosh.mp3",
|
||||
filetype: "mp3",
|
||||
duration: 820,
|
||||
attribution: '"Swoosh" by Ana is licensed under CC BY 4.0.',
|
||||
...overrides,
|
||||
});
|
||||
// Openverse stand-in: `answer(query)` gives a search response; any other URL is the sound file.
|
||||
function openverse(answer) {
|
||||
const searches = [];
|
||||
serve = async (url) => {
|
||||
const query = new URL(url).searchParams.get("q");
|
||||
if (!url.startsWith("https://api.openverse.org/")) return new Response(`bytes of ${url}`);
|
||||
searches.push(query);
|
||||
return answer(query);
|
||||
};
|
||||
return searches;
|
||||
}
|
||||
const run = (libDir, projDir, names) =>
|
||||
resolveSfx({
|
||||
cues: names.map((name, index) => ({ id: String(index + 1), name })),
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
|
||||
test("openverse: each cue gets the best licensed sound, with its licence and credit", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
libraryWith(libDir, ["whoosh"]);
|
||||
openverse(() => Response.json({ results: [sound()] }));
|
||||
try {
|
||||
const { sfx, anomalies } = await run(libDir, projDir, ["whoosh"]);
|
||||
assert.deepEqual(sfx, [
|
||||
{
|
||||
id: "1",
|
||||
name: "whoosh",
|
||||
file: "assets/sfx/whoosh.mp3",
|
||||
source: "openverse",
|
||||
duration_s: 0.82,
|
||||
license: "by",
|
||||
credit: '"Swoosh" by Ana is licensed under CC BY 4.0.',
|
||||
offset_s: 0,
|
||||
volume: 0.35,
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(anomalies, []);
|
||||
assert.equal(
|
||||
readFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "utf8"),
|
||||
"bytes of https://cdn.freesound.test/swoosh.mp3",
|
||||
);
|
||||
assert.match(
|
||||
readFileSync(join(projDir, "assets/sfx/CREDITS.md"), "utf8"),
|
||||
/- `assets\/sfx\/whoosh\.mp3`: "Swoosh" by Ana is licensed under CC BY 4\.0\. Source: https:\/\/freesound\.org\/s\/1/,
|
||||
);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("openverse: a rate limit sends this and every later cue to the bundled library, asking once", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
libraryWith(libDir, ["whoosh", "pop"]);
|
||||
const searches = openverse(() => new Response("slow down", { status: 429 }));
|
||||
try {
|
||||
const { sfx, anomalies } = await run(libDir, projDir, ["whoosh", "pop"]);
|
||||
assert.deepEqual(searches, ["whoosh"]);
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file, source }) => [file, source]),
|
||||
[
|
||||
["assets/sfx/whoosh.mp3", "local"],
|
||||
["assets/sfx/pop.mp3", "local"],
|
||||
],
|
||||
);
|
||||
assert.equal(readFileSync(join(projDir, "assets/sfx/pop.mp3"), "utf8"), "library pop");
|
||||
assert.deepEqual(anomalies, [
|
||||
"Openverse rate limit reached (HTTP 429): the remaining SFX cues use the bundled library",
|
||||
]);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("openverse: a cue with no licensed match falls back to the bundled library alone", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
libraryWith(libDir, ["pop"]);
|
||||
const searches = openverse((query) =>
|
||||
Response.json({ results: query === "pop" ? [sound({ license: "by-nc" })] : [sound()] }),
|
||||
);
|
||||
try {
|
||||
const { sfx } = await run(libDir, projDir, ["pop", "swoosh"]);
|
||||
assert.deepEqual(searches, ["pop", "swoosh"]);
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file, source }) => [file, source]),
|
||||
[
|
||||
["assets/sfx/pop.mp3", "local"],
|
||||
["assets/sfx/swoosh.mp3", "openverse"],
|
||||
],
|
||||
);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("openverse: a failed download falls back to the bundled file with an anomaly", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
libraryWith(libDir, ["whoosh"]);
|
||||
serve = async (url) =>
|
||||
url.startsWith("https://api.openverse.org/")
|
||||
? Response.json({ results: [sound()] })
|
||||
: new Response("gone", { status: 404 });
|
||||
try {
|
||||
const { sfx, anomalies } = await run(libDir, projDir, ["whoosh"]);
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file, source }) => [file, source]),
|
||||
[["assets/sfx/whoosh.mp3", "local"]],
|
||||
);
|
||||
assert.match(anomalies[0], /Openverse download failed — download HTTP 404/);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("openverse: one effect named in two lines is searched and downloaded once", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
libraryWith(libDir, []);
|
||||
const searches = openverse(() => Response.json({ results: [sound({ license: "cc0" })] }));
|
||||
try {
|
||||
const { sfx } = await run(libDir, projDir, ["whoosh", "whoosh"]);
|
||||
assert.deepEqual(searches, ["whoosh"]);
|
||||
assert.deepEqual(
|
||||
sfx.map(({ id, file }) => [id, file]),
|
||||
[
|
||||
["1", "assets/sfx/whoosh.mp3"],
|
||||
["2", "assets/sfx/whoosh.mp3"],
|
||||
],
|
||||
);
|
||||
assert.equal(
|
||||
existsSync(join(projDir, "assets/sfx/CREDITS.md")),
|
||||
false,
|
||||
"CC0 needs no credit",
|
||||
);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("findSoundEffect: a host gets the saved file, licence, credit and duration for one query", async () => {
|
||||
await withDirs(async ({ projDir }) => {
|
||||
const searches = openverse(() => Response.json({ results: [sound()] }));
|
||||
try {
|
||||
assert.deepEqual(await findSoundEffect("whoosh", { hyperframesDir: projDir }), {
|
||||
sound: {
|
||||
file: "assets/sfx/whoosh.mp3",
|
||||
source: "openverse",
|
||||
duration_s: 0.82,
|
||||
license: "by",
|
||||
credit: '"Swoosh" by Ana is licensed under CC BY 4.0.',
|
||||
},
|
||||
anomalies: [],
|
||||
});
|
||||
assert.deepEqual(searches, ["whoosh"]);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("findSoundEffect: offline, the skill's own bundled library answers", async () => {
|
||||
await withDirs(async ({ projDir }) => {
|
||||
const { sound, anomalies } = await findSoundEffect("whoosh", { hyperframesDir: projDir });
|
||||
assert.deepEqual(sound, {
|
||||
file: "assets/sfx/whoosh.mp3",
|
||||
source: "local",
|
||||
duration_s: sound.duration_s,
|
||||
});
|
||||
assert.deepEqual(
|
||||
readFileSync(join(projDir, sound.file)),
|
||||
readFileSync(join(BUNDLED_SFX_DIR, "whoosh.mp3")),
|
||||
);
|
||||
assert.deepEqual(anomalies, [UNREACHABLE]);
|
||||
assert.equal(
|
||||
(await findSoundEffect("no such sound at all", { hyperframesDir: projDir })).sound,
|
||||
null,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,175 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const ttsUrl = new URL("./tts.mjs", import.meta.url).href;
|
||||
const stub = `#!${process.execPath}
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const args = process.argv.slice(2);
|
||||
if (args[0] === "--version") {
|
||||
console.log("Python 3.12.0");
|
||||
process.exit(0);
|
||||
}
|
||||
const input = args[2];
|
||||
const output = args[0] === "-c" ? args[4] : args[args.indexOf("--output") + 1];
|
||||
const text = fs.readFileSync(input, "utf8");
|
||||
fs.writeFileSync(output + ".started", input);
|
||||
const finish = () => {
|
||||
if (text === "slow" && !fs.existsSync(process.env.RELEASE_FILE)) {
|
||||
setTimeout(finish, 5);
|
||||
return;
|
||||
}
|
||||
const consumed = fs.readFileSync(input, "utf8");
|
||||
fs.writeFileSync(output + ".json", JSON.stringify({ input, text, consumed, args }));
|
||||
if (process.env.CASE_MODE === "nonzero") process.exit(7);
|
||||
if (process.env.CASE_MODE !== "no-output") fs.writeFileSync(output, consumed);
|
||||
};
|
||||
setTimeout(finish, 10);
|
||||
`;
|
||||
|
||||
const exercise = `
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import childProcess from "node:child_process";
|
||||
import { syncBuiltinESMExports } from "node:module";
|
||||
import { join, dirname } from "node:path";
|
||||
const { CASE_ROOT: root, CASE_MODE: mode, CASE_PROVIDER: provider } = process.env;
|
||||
const temp = join(root, "tmp");
|
||||
const preexisting = join(temp, "hf-txt-preexisting");
|
||||
fs.mkdirSync(preexisting);
|
||||
fs.writeFileSync(join(preexisting, "marker"), "keep");
|
||||
const injected = Object.assign(new Error("synthetic " + mode), { code: "EACCES" });
|
||||
const originalMkdtemp = fs.mkdtempSync;
|
||||
const originalWrite = fs.writeFileSync;
|
||||
const originalRm = fs.rmSync;
|
||||
if (mode === "allocation") fs.mkdtempSync = (prefix, ...args) => {
|
||||
if (prefix.startsWith(join(temp, "hf-txt-"))) throw injected;
|
||||
return originalMkdtemp(prefix, ...args);
|
||||
};
|
||||
if (mode === "write") fs.writeFileSync = (file, ...args) => {
|
||||
if (String(file).endsWith("line.txt")) {
|
||||
originalWrite(file, "partial");
|
||||
throw injected;
|
||||
}
|
||||
return originalWrite(file, ...args);
|
||||
};
|
||||
if (mode === "spawn-throw") childProcess.spawn = () => { throw injected; };
|
||||
if (mode === "cleanup") fs.rmSync = (dir, ...args) => {
|
||||
if (String(dir).startsWith(join(temp, "hf-txt-"))) throw injected;
|
||||
return originalRm(dir, ...args);
|
||||
};
|
||||
syncBuiltinESMExports();
|
||||
const { synthesizeOne } = await import(${JSON.stringify(ttsUrl)});
|
||||
const project = join(root, "project");
|
||||
const wavAbs = join(project, "line.wav");
|
||||
const request = (chosen = provider, text = "synthetic café", output = wavAbs) =>
|
||||
synthesizeOne({ provider: chosen, text, voiceId: "synthetic-voice", lang: "es",
|
||||
speed: 0.8, wavAbs: output, hyperframesDir: project });
|
||||
const owned = () => fs.readdirSync(temp).filter(name => name !== "hf-txt-preexisting");
|
||||
if (["allocation", "write", "spawn-throw"].includes(mode)) {
|
||||
await assert.rejects(request(), error => error === injected);
|
||||
assert.equal(fs.existsSync(wavAbs), false);
|
||||
} else if (mode === "concurrent") {
|
||||
const slowWav = join(project, "slow.wav");
|
||||
const slow = request("elevenlabs", "slow", slowWav);
|
||||
try {
|
||||
const deadline = Date.now() + 5000;
|
||||
while (!fs.existsSync(slowWav + ".started")) {
|
||||
assert.ok(Date.now() < deadline, "slow subprocess must start");
|
||||
await new Promise(resolve => setTimeout(resolve, 5));
|
||||
}
|
||||
const slowInput = fs.readFileSync(slowWav + ".started", "utf8");
|
||||
assert.deepEqual(await request("kokoro"), { ok: true, words: null });
|
||||
assert.deepEqual(owned(), [dirname(slowInput).split(/[\\\\/]/).pop()]);
|
||||
assert.equal(fs.readFileSync(slowInput, "utf8"), "slow");
|
||||
} finally {
|
||||
fs.writeFileSync(process.env.RELEASE_FILE, "release");
|
||||
await slow;
|
||||
}
|
||||
assert.equal(fs.readFileSync(slowWav, "utf8"), "slow");
|
||||
} else {
|
||||
const result = await request();
|
||||
const label = provider === "elevenlabs" ? "elevenlabs (python)" : "kokoro (npx hyperframes tts)";
|
||||
if (["nonzero", "enoent", "no-output"].includes(mode)) {
|
||||
const error = mode === "no-output" ? label + " produced no wav file" :
|
||||
label + " exited with status " + (mode === "enoent" ? -1 : 7);
|
||||
assert.deepEqual(result, { ok: false, words: null, error });
|
||||
assert.equal(fs.existsSync(wavAbs), false);
|
||||
} else {
|
||||
assert.deepEqual(result, { ok: true, words: null });
|
||||
assert.equal(fs.readFileSync(wavAbs, "utf8"), "synthetic café");
|
||||
}
|
||||
if (mode !== "enoent") {
|
||||
const log = JSON.parse(fs.readFileSync(wavAbs + ".json", "utf8"));
|
||||
assert.equal(log.text, "synthetic café");
|
||||
assert.equal(log.consumed, log.text);
|
||||
if (provider === "kokoro") assert.deepEqual(log.args.slice(-4), ["--lang", "es", "--speed", "0.8"]);
|
||||
}
|
||||
}
|
||||
if (mode === "cleanup") assert.equal(owned().length, 1);
|
||||
else assert.deepEqual(owned(), [], "owned text directories must be removed");
|
||||
assert.equal(fs.readFileSync(join(preexisting, "marker"), "utf8"), "keep");
|
||||
`;
|
||||
|
||||
function runCase(provider, mode) {
|
||||
const root = mkdtempSync(join(tmpdir(), "tts-cleanup-test-"));
|
||||
try {
|
||||
for (const dir of ["home", "tmp", "bin", "project"]) mkdirSync(join(root, dir));
|
||||
if (mode !== "enoent") {
|
||||
for (const name of ["npx", "python3"]) {
|
||||
const executable = join(root, "bin", name);
|
||||
writeFileSync(executable, stub);
|
||||
chmodSync(executable, 0o755);
|
||||
}
|
||||
}
|
||||
const result = spawnSync(process.execPath, ["--input-type=module", "-e", exercise], {
|
||||
encoding: "utf8",
|
||||
timeout: 15000,
|
||||
env: {
|
||||
HOME: join(root, "home"),
|
||||
USERPROFILE: join(root, "home"),
|
||||
TMPDIR: join(root, "tmp"),
|
||||
TMP: join(root, "tmp"),
|
||||
TEMP: join(root, "tmp"),
|
||||
PATH: join(root, "bin"),
|
||||
CASE_ROOT: root,
|
||||
CASE_PROVIDER: provider,
|
||||
CASE_MODE: mode,
|
||||
RELEASE_FILE: join(root, "release"),
|
||||
},
|
||||
});
|
||||
assert.equal(result.error, undefined);
|
||||
assert.equal(result.status, 0, result.stdout + result.stderr);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
for (const provider of ["elevenlabs", "kokoro"]) {
|
||||
for (const mode of [
|
||||
"success",
|
||||
"nonzero",
|
||||
"no-output",
|
||||
"enoent",
|
||||
"spawn-throw",
|
||||
"allocation",
|
||||
"write",
|
||||
"cleanup",
|
||||
]) {
|
||||
test(
|
||||
`synthesizeOne(${provider}) preserves ${mode} behavior and cleans only its text directory`,
|
||||
{ skip: process.platform === "win32" },
|
||||
() => runCase(provider, mode),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
test(
|
||||
"concurrent providers retain each other's text until their subprocess finishes",
|
||||
{ skip: process.platform === "win32" },
|
||||
() => runCase("kokoro", "concurrent"),
|
||||
);
|
||||
@@ -0,0 +1,394 @@
|
||||
import { fetchMedia } from "../../../scripts/lib/media-fetch.mjs";
|
||||
// tts.mjs — multi-provider TTS for the media audio engine. The provider chain,
|
||||
// auto-detected from env, is the one documented in ../SKILL.md:
|
||||
//
|
||||
// 1. HeyGen (Starfish) — $HEYGEN_API_KEY / $HYPERFRAMES_API_KEY / ~/.heygen.
|
||||
// Direct v3 REST (NOT `hyperframes tts`, which in the published build is
|
||||
// Kokoro-only and silently ignores a HeyGen key). Returns word_timestamps
|
||||
// in the same call, so no separate transcribe pass.
|
||||
// 2. ElevenLabs — $ELEVENLABS_API_KEY + `pip install elevenlabs`. No
|
||||
// word timings → caller chains transcribeWav().
|
||||
// 3. Kokoro-82M (local) — always available, via the published `hyperframes tts`
|
||||
// CLI. No word timings → caller chains transcribeWav().
|
||||
//
|
||||
// "HeyGen available" is decided by CREDENTIAL presence (heygenCredential), never
|
||||
// by the CLI — see the note above.
|
||||
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { heygenAuthHeaders, heygenCredential, heygenJSON, heygenMessage } from "./heygen.mjs";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
import { synthesizeGemini } from "./gemini-tts.mjs";
|
||||
import { geminiConfigured } from "./gemini-auth.mjs";
|
||||
|
||||
// ── provider detection ────────────────────────────────────────────────────────
|
||||
export function heygenAvailable() {
|
||||
return heygenCredential() !== null;
|
||||
}
|
||||
export function elevenlabsAvailable() {
|
||||
if (!process.env.ELEVENLABS_API_KEY) return false;
|
||||
const { cmd, args } = pythonInvocation(["-c", "import elevenlabs"]);
|
||||
const r = spawnSync(cmd, args, {
|
||||
stdio: "ignore",
|
||||
});
|
||||
return r.status === 0;
|
||||
}
|
||||
|
||||
// First available provider wins; an explicit choice is honored (and validated).
|
||||
export function pickProvider(userProvider) {
|
||||
if (userProvider) {
|
||||
if (!["heygen", "elevenlabs", "kokoro", "gemini"].includes(userProvider))
|
||||
throw new Error(`invalid provider "${userProvider}" (heygen | elevenlabs | kokoro | gemini)`);
|
||||
if (userProvider === "gemini" && !geminiConfigured())
|
||||
throw new Error(
|
||||
"provider=gemini needs GEMINI_API_KEY or GOOGLE_API_KEY, or service-account credentials (GOOGLE_APPLICATION_CREDENTIALS or GCS_CREDS)",
|
||||
);
|
||||
// heygenAuthHeaders owns the reason and its fix: no credential, or a credentials path that cannot be read.
|
||||
if (userProvider === "heygen" && !heygenAvailable()) heygenAuthHeaders();
|
||||
if (userProvider === "elevenlabs" && !process.env.ELEVENLABS_API_KEY)
|
||||
throw new Error("provider=elevenlabs but $ELEVENLABS_API_KEY is not set");
|
||||
return userProvider;
|
||||
}
|
||||
return heygenAvailable() ? "heygen" : elevenlabsAvailable() ? "elevenlabs" : "kokoro";
|
||||
}
|
||||
|
||||
// ── voice resolution ──────────────────────────────────────────────────────────
|
||||
// HeyGen /v3/voices/speech only accepts STARFISH voice_ids; auto-pick the first
|
||||
// English public starfish voice when none is pinned. ElevenLabs/Kokoro have
|
||||
// their own defaults.
|
||||
export async function resolveVoiceId({ provider, userVoice, lang = "en" }) {
|
||||
if (userVoice) return userVoice;
|
||||
if (provider === "gemini") return "Kore";
|
||||
if (provider === "elevenlabs") return "21m00Tcm4TlvDq8ikWAM"; // Rachel
|
||||
if (provider === "kokoro") {
|
||||
if (lang === "en") return "am_michael";
|
||||
throw new Error("Kokoro non-English needs an explicit --voice (see references/tts.md)");
|
||||
}
|
||||
// heygen — pin a fixed English default so the choice is deterministic. The old
|
||||
// "first English voice the API returns" drifts whenever HeyGen re-sorts the
|
||||
// public catalog. Marcia (mature, low female). Override with --voice / request.voice.
|
||||
if (lang === "en") return "05f19352e8f74b0392a8f411eba40de1"; // Marcia · English · female
|
||||
// Non-English: no fixed default — fall back to the first matching catalog voice.
|
||||
const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, {
|
||||
headers: heygenAuthHeaders(),
|
||||
});
|
||||
const voices = payload.data ?? payload.voices ?? [];
|
||||
const pick = voices.find((v) => v.language === "English") ?? voices[0];
|
||||
if (!pick) throw new Error("no public starfish voice to default to — pass --voice");
|
||||
return pick.voice_id;
|
||||
}
|
||||
|
||||
// ── helpers ─────────────────────────────────────────────────────────────────
|
||||
export function withWordIds(words) {
|
||||
return (words ?? []).map((w, i) => ({
|
||||
id: `w${i}`,
|
||||
text: w.text,
|
||||
start: w.start,
|
||||
end: w.end,
|
||||
}));
|
||||
}
|
||||
|
||||
// `ffmpeg -i <file>` prints a `Duration: HH:MM:SS.ms` line to stderr even
|
||||
// though it exits non-zero with no output requested. Parsing pulled out as
|
||||
// a pure function so the ENOENT fallback below can be tested without
|
||||
// depending on whether ffprobe/ffmpeg are actually installed on the
|
||||
// machine running the tests.
|
||||
export function parseFfmpegDurationBanner(stderrText) {
|
||||
const match = /Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/.exec(stderrText ?? "");
|
||||
if (!match) return NaN;
|
||||
const [, hours, minutes, seconds] = match;
|
||||
return Number(hours) * 3600 + Number(minutes) * 60 + Number(seconds);
|
||||
}
|
||||
|
||||
// Some "essentials"-style ffmpeg distributions (common on Windows) ship
|
||||
// ffmpeg.exe without ffprobe.exe. ffprobeDuration's caller (audio.mjs)
|
||||
// otherwise reads a spurious NaN as "the WAV file is corrupt" and drops an
|
||||
// already-successfully-synthesized TTS line, rather than "the tool for
|
||||
// measuring it is missing".
|
||||
function ffmpegDurationFallback(absPath) {
|
||||
const r = spawnSync("ffmpeg", ["-i", absPath], { encoding: "utf8" });
|
||||
return parseFfmpegDurationBanner(r.stderr);
|
||||
}
|
||||
|
||||
export function ffprobeDuration(absPath) {
|
||||
const r = spawnSync(
|
||||
"ffprobe",
|
||||
["-v", "error", "-show_entries", "format=duration", "-of", "default=nw=1:nk=1", "--", absPath],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
if (r.error?.code === "ENOENT") return ffmpegDurationFallback(absPath);
|
||||
if (r.status !== 0) return NaN;
|
||||
return parseFloat(String(r.stdout).trim());
|
||||
}
|
||||
|
||||
export function resolveNpxCliFromNpmExecPath(
|
||||
npmExecPath = process.env.npm_execpath,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
if (!npmExecPath) return null;
|
||||
const fileName = npmExecPath.replace(/\\/g, "/").split("/").pop()?.toLowerCase();
|
||||
const npxCliPath =
|
||||
fileName === "npx-cli.js" ? npmExecPath : join(dirname(npmExecPath), "npx-cli.js");
|
||||
return pathExists(npxCliPath) ? npxCliPath : null;
|
||||
}
|
||||
|
||||
export function resolveNpxCliPath(
|
||||
npmExecPath = process.env.npm_execpath,
|
||||
nodeExecPath = process.env.npm_node_execpath || process.execPath,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
const fromNpm = resolveNpxCliFromNpmExecPath(npmExecPath, pathExists);
|
||||
if (fromNpm) return fromNpm;
|
||||
const besideNode = join(dirname(nodeExecPath), "node_modules", "npm", "bin", "npx-cli.js");
|
||||
return pathExists(besideNode) ? besideNode : null;
|
||||
}
|
||||
|
||||
export function resolveSpawnCommand(
|
||||
cmd,
|
||||
args,
|
||||
opts = {},
|
||||
platform = process.platform,
|
||||
env = process.env,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
if (cmd !== "npx" || platform !== "win32") {
|
||||
return { cmd, args, opts: { stdio: "ignore", ...opts } };
|
||||
}
|
||||
|
||||
// On Windows, npx resolves to npx.cmd, which Node cannot execute directly.
|
||||
// Avoid `shell:true` and the .cmd shim entirely by invoking npm's JS CLI with
|
||||
// node, preserving request-provided values as argv data instead of shell text.
|
||||
const nodeExecPath = env.npm_node_execpath || process.execPath;
|
||||
const npxCliPath = resolveNpxCliPath(env.npm_execpath, nodeExecPath, pathExists);
|
||||
if (!npxCliPath) return null;
|
||||
return {
|
||||
cmd: nodeExecPath,
|
||||
args: [npxCliPath, ...args.map((arg) => String(arg))],
|
||||
opts: { stdio: "ignore", windowsHide: true, ...opts },
|
||||
};
|
||||
}
|
||||
|
||||
// `platform`/`spawnFn` params (default process.platform / the real spawn)
|
||||
// exist so tests can exercise the win32 branch without mocking node:child_process
|
||||
// (its ESM exports are non-configurable, so mock.method can't patch it).
|
||||
// One-shot so a whole batch of TTS lines doesn't repeat the same diagnostic.
|
||||
let _warnedNpxResolution = false;
|
||||
/** Test-only: reset the one-shot npx-resolution warning latch. */
|
||||
export function _resetNpxResolutionWarnForTests() {
|
||||
_warnedNpxResolution = false;
|
||||
}
|
||||
|
||||
export function spawnP(
|
||||
cmd,
|
||||
args,
|
||||
opts = {},
|
||||
platform = process.platform,
|
||||
spawnFn = spawn,
|
||||
env = process.env,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
const resolved = resolveSpawnCommand(cmd, args, opts, platform, env, pathExists);
|
||||
if (!resolved) {
|
||||
// resolveSpawnCommand only returns null for the npx-on-win32 case where
|
||||
// neither npm's configured CLI nor the beside-node fallback exists. Without
|
||||
// this, every call silently returns status:-1 and stdio:"ignore" hides why.
|
||||
if (!_warnedNpxResolution) {
|
||||
_warnedNpxResolution = true;
|
||||
const reason = env.npm_execpath
|
||||
? `npm_execpath (${env.npm_execpath}) and the beside-node npm fallback could not be found`
|
||||
: "npm_execpath is unset and the beside-node npm fallback could not be found";
|
||||
console.error(
|
||||
`[media-use] Cannot run "${cmd}" on Windows: ${reason}. ` +
|
||||
`Every "${cmd}" call is being skipped. Install npm with Node, or run via ` +
|
||||
`\`npx\`/\`npm run\` with a valid npm_execpath.`,
|
||||
);
|
||||
}
|
||||
return Promise.resolve({ status: -1 });
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const p = spawnFn(resolved.cmd, resolved.args, resolved.opts);
|
||||
p.on("exit", (code) => resolve({ status: code ?? -1 }));
|
||||
p.on("error", () => resolve({ status: -1 }));
|
||||
});
|
||||
}
|
||||
|
||||
// mp3/whatever bytes → wav 44.1k mono at destWav (ffmpeg detects true format).
|
||||
function transcodeToWav(bytes, destWav) {
|
||||
const td = mkdtempSync(join(tmpdir(), "hf-tts-"));
|
||||
const tmp = join(td, "a.mp3");
|
||||
writeFileSync(tmp, bytes);
|
||||
mkdirSync(dirname(destWav), { recursive: true });
|
||||
const ff = spawnSync(
|
||||
"ffmpeg",
|
||||
["-y", "-loglevel", "error", "-i", tmp, "-ar", "44100", "-ac", "1", destWav],
|
||||
{ stdio: "ignore" },
|
||||
);
|
||||
rmSync(td, { recursive: true, force: true });
|
||||
return ff.status === 0 && existsSync(destWav);
|
||||
}
|
||||
|
||||
const ELEVENLABS_PY = `
|
||||
import os, sys
|
||||
from elevenlabs.client import ElevenLabs
|
||||
from elevenlabs import save
|
||||
client = ElevenLabs(api_key=os.environ["ELEVENLABS_API_KEY"])
|
||||
text = open(sys.argv[1], encoding="utf-8").read()
|
||||
audio = client.text_to_speech.convert(
|
||||
text=text, voice_id=sys.argv[2],
|
||||
model_id="eleven_multilingual_v2", output_format="mp3_44100_128",
|
||||
)
|
||||
save(audio, sys.argv[3])
|
||||
`;
|
||||
|
||||
// ── synthesize one line ───────────────────────────────────────────────────────
|
||||
// Writes wav at wavAbs. Returns { ok, words, error } — words is the raw
|
||||
// [{text,start,end}] array for HeyGen (native), or null for ElevenLabs/Kokoro
|
||||
// (caller must transcribeWav). Never throws; failures return { ok:false, error }
|
||||
// where `error` states WHY (so the caller can surface it, not a bare "TTS failed").
|
||||
export async function synthesizeOne({
|
||||
provider,
|
||||
text,
|
||||
voiceId,
|
||||
lang = "en",
|
||||
speed = 1.0,
|
||||
model,
|
||||
style,
|
||||
wavAbs,
|
||||
hyperframesDir,
|
||||
}) {
|
||||
if (provider === "heygen") return synthesizeHeygen({ text, voiceId, lang, speed, wavAbs });
|
||||
if (provider === "gemini")
|
||||
return synthesizeGemini({ text, voiceId, model, style, speed, wavAbs });
|
||||
if (provider === "elevenlabs") {
|
||||
// The Python helper writes straight to wavAbs; unlike heygen (transcodeToWav)
|
||||
// and kokoro (the `hyperframes tts` CLI), it does NOT create the parent dir,
|
||||
// so on a fresh project (no assets/voice/ yet) the save fails and the line is
|
||||
// silently dropped as "TTS failed - omitted". Create it first, like the other
|
||||
// providers do. Guarded so a mkdir failure (EACCES/EROFS) returns
|
||||
// { ok:false } like the rest of this branch rather than throwing (the
|
||||
// function's contract is "never throws; failures return { ok:false }").
|
||||
try {
|
||||
mkdirSync(dirname(wavAbs), { recursive: true });
|
||||
} catch {
|
||||
return { ok: false, words: null };
|
||||
}
|
||||
return withTmpText(text, async (textPath) => {
|
||||
const { cmd, args } = pythonInvocation(["-c", ELEVENLABS_PY, textPath, voiceId, wavAbs]);
|
||||
const r = await spawnP(cmd, args, {});
|
||||
return synthResult(r, wavAbs, "elevenlabs (python)");
|
||||
});
|
||||
}
|
||||
// kokoro — via the published CLI; --output is relative to the project dir.
|
||||
const wavRel = relTo(hyperframesDir, wavAbs);
|
||||
return withTmpText(text, async (textPath) => {
|
||||
const args = ["hyperframes", "tts", textPath, "--voice", voiceId, "--output", wavRel];
|
||||
if (lang !== "en") args.push("--lang", lang);
|
||||
if (speed !== 1) args.push("--speed", String(speed));
|
||||
const r = await spawnP("npx", args, { cwd: hyperframesDir });
|
||||
return synthResult(r, wavAbs, "kokoro (npx hyperframes tts)");
|
||||
});
|
||||
}
|
||||
|
||||
// Shape a spawn result into { ok, words, error }, naming why on failure so the
|
||||
// caller surfaces it instead of a bare "TTS failed".
|
||||
export function synthResult(r, wavAbs, label) {
|
||||
if (r.status === 0 && existsSync(wavAbs)) return { ok: true, words: null };
|
||||
const why =
|
||||
r.status !== 0 ? `${label} exited with status ${r.status}` : `${label} produced no wav file`;
|
||||
return { ok: false, words: null, error: why };
|
||||
}
|
||||
|
||||
// `deps` is injectable for tests; production uses the real network/ffmpeg impls.
|
||||
// Every failure path returns an `error` string so the caller can surface WHY a
|
||||
// line was dropped instead of the bare "TTS failed" that hid the real cause
|
||||
// (e.g. an HTTP 402 plan_upgrade_required thrown by heygenJSON was swallowed).
|
||||
export async function synthesizeHeygen({ text, voiceId, lang, speed, wavAbs }, deps = {}) {
|
||||
const requestJSON = deps.heygenJSON ?? heygenJSON;
|
||||
const authHeaders = deps.heygenAuthHeaders ?? heygenAuthHeaders;
|
||||
const fetchImpl = deps.fetch ?? fetch;
|
||||
const transcode = deps.transcodeToWav ?? transcodeToWav;
|
||||
try {
|
||||
const body = { text, voice_id: voiceId, speed };
|
||||
if (lang !== "en") body.language = lang;
|
||||
const payload = await requestJSON(`/voices/speech`, {
|
||||
method: "POST",
|
||||
headers: authHeaders(),
|
||||
body,
|
||||
});
|
||||
const inner = payload.data ?? payload;
|
||||
if (!inner.audio_url) {
|
||||
return { ok: false, words: null, error: "HeyGen /voices/speech returned no audio_url" };
|
||||
}
|
||||
const res = await fetchMedia(inner.audio_url, { fetchImpl });
|
||||
if (!res.ok) {
|
||||
return { ok: false, words: null, error: `audio_url fetch failed: HTTP ${res.status}` };
|
||||
}
|
||||
const bytes = Buffer.from(await res.arrayBuffer());
|
||||
// .wav output → transcode to 44.1k mono; .mp3 → raw bytes (no ffmpeg). The
|
||||
// engine always asks for .wav; the standalone heygen-tts CLI may ask for .mp3.
|
||||
if (wavAbs.endsWith(".wav")) {
|
||||
if (!transcode(bytes, wavAbs)) {
|
||||
return {
|
||||
ok: false,
|
||||
words: null,
|
||||
error: "wav transcode failed (ffmpeg)",
|
||||
};
|
||||
}
|
||||
} else {
|
||||
mkdirSync(dirname(wavAbs), { recursive: true });
|
||||
writeFileSync(wavAbs, bytes);
|
||||
}
|
||||
const words = Array.isArray(inner.word_timestamps)
|
||||
? inner.word_timestamps
|
||||
.filter((w) => w && typeof w.word === "string" && isFinite(w.start) && isFinite(w.end))
|
||||
.filter((w) => !/^<.*>$/.test(w.word.trim())) // drop <start>/<end> sentinels
|
||||
.map((w) => ({ text: w.word, start: w.start, end: w.end }))
|
||||
: [];
|
||||
return { ok: true, words };
|
||||
} catch (e) {
|
||||
return { ok: false, words: null, error: heygenMessage(e) };
|
||||
}
|
||||
}
|
||||
|
||||
// ElevenLabs/Kokoro have no word timings — run Whisper over the wav. Returns the
|
||||
// flat [{id,text,start,end}] word array, or null. Each call uses a throwaway
|
||||
// --dir so parallel scenes don't collide on transcript.json.
|
||||
export async function transcribeWav({ wavRel, lang = "en", hyperframesDir }) {
|
||||
const model = lang === "en" ? "small.en" : "small";
|
||||
const td = mkdtempSync(join(tmpdir(), "hf-trans-"));
|
||||
const args = ["hyperframes", "transcribe", wavRel, "--model", model, "--dir", td];
|
||||
if (lang !== "en") args.push("--language", lang);
|
||||
const r = await spawnP("npx", args, { cwd: hyperframesDir });
|
||||
let words = null;
|
||||
if (r.status === 0) {
|
||||
const src = join(td, "transcript.json");
|
||||
if (existsSync(src)) {
|
||||
try {
|
||||
const arr = JSON.parse(readFileSync(src, "utf8"));
|
||||
if (Array.isArray(arr) && arr.length) words = arr;
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
rmSync(td, { recursive: true, force: true });
|
||||
return words;
|
||||
}
|
||||
|
||||
// ── tiny local utils ──────────────────────────────────────────────────────────
|
||||
async function withTmpText(text, run) {
|
||||
const td = mkdtempSync(join(tmpdir(), "hf-txt-"));
|
||||
try {
|
||||
const p = join(td, "line.txt");
|
||||
writeFileSync(p, text);
|
||||
return await run(p);
|
||||
} finally {
|
||||
// Cleanup must preserve the synthesis result or the original write/spawn error.
|
||||
try {
|
||||
rmSync(td, { recursive: true, force: true });
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
function relTo(base, abs) {
|
||||
return abs.startsWith(base + "/") ? abs.slice(base.length + 1) : abs;
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { EventEmitter } from "node:events";
|
||||
import {
|
||||
resolveNpxCliFromNpmExecPath,
|
||||
resolveNpxCliPath,
|
||||
resolveSpawnCommand,
|
||||
spawnP,
|
||||
_resetNpxResolutionWarnForTests,
|
||||
} from "./tts.mjs";
|
||||
|
||||
// Regression: on Windows, npx resolves to npx.cmd, which spawn() cannot exec
|
||||
// without shell:true — it fails ENOENT, silently swallowed as ok:false by the
|
||||
// caller. spawnP takes injectable platform/spawnFn params so this doesn't
|
||||
// need to touch the real process.platform or mock node:child_process (whose
|
||||
// ESM exports are non-configurable).
|
||||
function fakeSpawn(captured) {
|
||||
return (cmd, args, opts) => {
|
||||
captured.push({ cmd, args, opts });
|
||||
const p = new EventEmitter();
|
||||
setImmediate(() => p.emit("exit", 0));
|
||||
return p;
|
||||
};
|
||||
}
|
||||
|
||||
const envWithNpxCli = {
|
||||
npm_execpath: "/opt/node/lib/node_modules/npm/bin/npm-cli.js",
|
||||
npm_node_execpath: "/opt/node/bin/node",
|
||||
};
|
||||
const npxCliPath = "/opt/node/lib/node_modules/npm/bin/npx-cli.js";
|
||||
const pathExists = (path) => path === npxCliPath;
|
||||
|
||||
test("resolveNpxCliFromNpmExecPath finds npx-cli next to npm-cli", () => {
|
||||
assert.equal(resolveNpxCliFromNpmExecPath(envWithNpxCli.npm_execpath, pathExists), npxCliPath);
|
||||
});
|
||||
|
||||
test("resolveNpxCliPath finds npx-cli beside node when npm_execpath is unset", () => {
|
||||
const node = "C:/Program Files/nodejs/node.exe";
|
||||
const expected = "C:/Program Files/nodejs/node_modules/npm/bin/npx-cli.js";
|
||||
assert.equal(
|
||||
resolveNpxCliPath(undefined, node, (path) => path === expected),
|
||||
expected,
|
||||
);
|
||||
});
|
||||
|
||||
test("resolveSpawnCommand routes npx through node+npx-cli on win32 without shell:true", () => {
|
||||
const resolved = resolveSpawnCommand(
|
||||
"npx",
|
||||
["hyperframes", "tts", "C:\\Users\\Test User\\line.txt", "--voice", "am_michael"],
|
||||
{},
|
||||
"win32",
|
||||
envWithNpxCli,
|
||||
pathExists,
|
||||
);
|
||||
assert.ok(resolved);
|
||||
assert.equal(resolved.cmd, envWithNpxCli.npm_node_execpath);
|
||||
assert.deepEqual(resolved.args, [
|
||||
npxCliPath,
|
||||
"hyperframes",
|
||||
"tts",
|
||||
"C:\\Users\\Test User\\line.txt",
|
||||
"--voice",
|
||||
"am_michael",
|
||||
]);
|
||||
assert.equal(resolved.opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("resolveSpawnCommand preserves Windows npx shell metacharacters as argv data", () => {
|
||||
const resolved = resolveSpawnCommand(
|
||||
"npx",
|
||||
["hyperframes", "tts", "hello & calc"],
|
||||
{},
|
||||
"win32",
|
||||
envWithNpxCli,
|
||||
pathExists,
|
||||
);
|
||||
assert.ok(resolved);
|
||||
assert.deepEqual(resolved.args, [npxCliPath, "hyperframes", "tts", "hello & calc"]);
|
||||
});
|
||||
|
||||
test("spawnP uses the resolved node+npx-cli command for npx on win32", async () => {
|
||||
const captured = [];
|
||||
await spawnP(
|
||||
"npx",
|
||||
["hyperframes", "tts"],
|
||||
{},
|
||||
"win32",
|
||||
fakeSpawn(captured),
|
||||
envWithNpxCli,
|
||||
pathExists,
|
||||
);
|
||||
assert.equal(captured.length, 1);
|
||||
assert.equal(captured[0].cmd, envWithNpxCli.npm_node_execpath);
|
||||
assert.deepEqual(captured[0].args, [npxCliPath, "hyperframes", "tts"]);
|
||||
assert.equal(captured[0].opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("spawnP does not enable shell for npx on darwin/linux", async () => {
|
||||
const captured = [];
|
||||
await spawnP("npx", ["hyperframes", "tts"], {}, "darwin", fakeSpawn(captured));
|
||||
assert.equal(captured[0].cmd, "npx");
|
||||
assert.deepEqual(captured[0].args, ["hyperframes", "tts"]);
|
||||
assert.equal(captured[0].opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("spawnP does not enable shell for non-npx commands even on win32", async () => {
|
||||
const captured = [];
|
||||
await spawnP("python3", ["-c", "pass"], {}, "win32", fakeSpawn(captured));
|
||||
assert.equal(captured[0].cmd, "python3");
|
||||
assert.deepEqual(captured[0].args, ["-c", "pass"]);
|
||||
assert.equal(captured[0].opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("spawnP resolves npx beside node when npm_execpath is unset on win32", async () => {
|
||||
_resetNpxResolutionWarnForTests();
|
||||
const captured = [];
|
||||
const node = "C:/Program Files/nodejs/node.exe";
|
||||
const npxCli = "C:/Program Files/nodejs/node_modules/npm/bin/npx-cli.js";
|
||||
const result = await spawnP(
|
||||
"npx",
|
||||
["hyperframes", "tts"],
|
||||
{},
|
||||
"win32",
|
||||
fakeSpawn(captured),
|
||||
{ npm_node_execpath: node },
|
||||
(path) => path === npxCli,
|
||||
);
|
||||
assert.equal(result.status, 0);
|
||||
assert.equal(captured.length, 1);
|
||||
assert.equal(captured[0].cmd, node);
|
||||
assert.deepEqual(captured[0].args, [npxCli, "hyperframes", "tts"]);
|
||||
});
|
||||
|
||||
test("spawnP warns once with an accurate diagnostic when neither npx path exists", async () => {
|
||||
_resetNpxResolutionWarnForTests();
|
||||
const errors = [];
|
||||
const originalError = console.error;
|
||||
console.error = (message) => errors.push(String(message));
|
||||
try {
|
||||
const env = { npm_execpath: "C:/missing/npm-cli.js", npm_node_execpath: "C:/node/node.exe" };
|
||||
const missing = () => false;
|
||||
assert.equal(
|
||||
(await spawnP("npx", ["hyperframes", "tts"], {}, "win32", fakeSpawn([]), env, missing))
|
||||
.status,
|
||||
-1,
|
||||
);
|
||||
assert.equal(
|
||||
(await spawnP("npx", ["hyperframes", "tts"], {}, "win32", fakeSpawn([]), env, missing))
|
||||
.status,
|
||||
-1,
|
||||
);
|
||||
} finally {
|
||||
console.error = originalError;
|
||||
}
|
||||
assert.equal(errors.length, 1);
|
||||
assert.match(errors[0], /npm_execpath \(C:\/missing\/npm-cli\.js\)/);
|
||||
assert.doesNotMatch(errors[0], /npm_execpath is not set/);
|
||||
});
|
||||
@@ -0,0 +1,230 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
writeFileSync,
|
||||
chmodSync,
|
||||
rmSync,
|
||||
existsSync,
|
||||
} from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import {
|
||||
parseFfmpegDurationBanner,
|
||||
ffprobeDuration,
|
||||
synthesizeOne,
|
||||
synthesizeHeygen,
|
||||
synthResult,
|
||||
pickProvider,
|
||||
} from "./tts.mjs";
|
||||
|
||||
test("parseFfmpegDurationBanner reads ffmpeg's stderr Duration line", () => {
|
||||
const stderr = [
|
||||
"ffmpeg version 6.0",
|
||||
"Input #0, wav, from 'a.wav':",
|
||||
" Duration: 00:00:03.42, bitrate: 705 kb/s",
|
||||
"At least one output file must be specified",
|
||||
].join("\n");
|
||||
assert.equal(parseFfmpegDurationBanner(stderr), 3.42);
|
||||
});
|
||||
|
||||
test("parseFfmpegDurationBanner handles an hours component", () => {
|
||||
const stderr = " Duration: 01:02:03.50, start: 0.000000, bitrate: 128 kb/s";
|
||||
assert.equal(parseFfmpegDurationBanner(stderr), 3723.5);
|
||||
});
|
||||
|
||||
test("parseFfmpegDurationBanner returns NaN when there is no Duration line", () => {
|
||||
assert.ok(Number.isNaN(parseFfmpegDurationBanner("ffmpeg: command not found")));
|
||||
assert.ok(Number.isNaN(parseFfmpegDurationBanner("")));
|
||||
assert.ok(Number.isNaN(parseFfmpegDurationBanner(undefined)));
|
||||
});
|
||||
|
||||
// Regression for the actual bug: ffprobeDuration used to collapse "ffprobe
|
||||
// binary is missing" (ENOENT — the "essentials"-style Windows ffmpeg build
|
||||
// with no ffprobe.exe) and "file is genuinely unreadable" into the same NaN,
|
||||
// giving audio.mjs no way to tell "measure differently" from "give up".
|
||||
//
|
||||
// Builds an isolated PATH containing only a fake `ffmpeg` stub (no `ffprobe`
|
||||
// at all) so ffprobeDuration's spawnSync("ffprobe", ...) call ENOENTs for
|
||||
// real, then verifies it recovers the duration via the ffmpeg fallback
|
||||
// instead of returning NaN.
|
||||
test("ffprobeDuration falls back to ffmpeg when the ffprobe binary itself is missing", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-ffprobe-fallback-"));
|
||||
const fakeFfmpeg = join(dir, "ffmpeg");
|
||||
writeFileSync(
|
||||
fakeFfmpeg,
|
||||
"#!/bin/sh\necho 'Duration: 00:00:02.50, start: 0.000000, bitrate: 128 kb/s' 1>&2\nexit 1\n",
|
||||
);
|
||||
chmodSync(fakeFfmpeg, 0o755);
|
||||
const originalPath = process.env.PATH;
|
||||
try {
|
||||
process.env.PATH = dir; // only the fake ffmpeg resolves; no real ffprobe on this PATH
|
||||
assert.equal(ffprobeDuration("/does/not/matter.wav"), 2.5);
|
||||
} finally {
|
||||
process.env.PATH = originalPath;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("ffprobeDuration returns NaN when neither ffprobe nor ffmpeg resolve", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-no-binaries-"));
|
||||
const originalPath = process.env.PATH;
|
||||
try {
|
||||
process.env.PATH = dir; // empty directory — nothing resolves
|
||||
assert.ok(Number.isNaN(ffprobeDuration("/does/not/matter.wav")));
|
||||
} finally {
|
||||
process.env.PATH = originalPath;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("synthesizeOne(elevenlabs) creates the output dir before writing", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-el-mkdir-"));
|
||||
const wavAbs = join(dir, "assets", "voice", "line-0.wav"); // nested, not yet created
|
||||
const savedKey = process.env.ELEVENLABS_API_KEY;
|
||||
try {
|
||||
// Unset the key so the Python side fails fast — the mkdir must run before
|
||||
// the spawn regardless, which is what this guards.
|
||||
delete process.env.ELEVENLABS_API_KEY;
|
||||
await synthesizeOne({
|
||||
provider: "elevenlabs",
|
||||
text: "hi",
|
||||
voiceId: "v",
|
||||
wavAbs,
|
||||
hyperframesDir: dir,
|
||||
});
|
||||
assert.ok(existsSync(dirname(wavAbs)), "output directory should be created");
|
||||
} finally {
|
||||
if (savedKey === undefined) delete process.env.ELEVENLABS_API_KEY;
|
||||
else process.env.ELEVENLABS_API_KEY = savedKey;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("synthesizeHeygen surfaces a thrown HTTP error (e.g. 402) instead of swallowing it", async () => {
|
||||
const res = await synthesizeHeygen(
|
||||
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: "/tmp/x.wav" },
|
||||
{
|
||||
heygenAuthHeaders: () => ({}),
|
||||
heygenJSON: async () => {
|
||||
throw new Error("HeyGen POST /voices/speech → HTTP 402\nplan_upgrade_required");
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error, /402/);
|
||||
assert.match(res.error, /plan_upgrade_required/);
|
||||
});
|
||||
|
||||
test("synthesizeHeygen surfaces a failed audio_url fetch with its status", async () => {
|
||||
const res = await synthesizeHeygen(
|
||||
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: "/tmp/x.wav" },
|
||||
{
|
||||
heygenAuthHeaders: () => ({}),
|
||||
heygenJSON: async () => ({ data: { audio_url: "http://audio.example/x" } }),
|
||||
fetch: async () => ({ ok: false, status: 403 }),
|
||||
},
|
||||
);
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error, /HTTP 403/);
|
||||
});
|
||||
|
||||
test("synthesizeHeygen reports a missing audio_url", async () => {
|
||||
const res = await synthesizeHeygen(
|
||||
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: "/tmp/x.wav" },
|
||||
{ heygenAuthHeaders: () => ({}), heygenJSON: async () => ({}) },
|
||||
);
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error, /no audio_url/);
|
||||
});
|
||||
|
||||
test("synthesizeHeygen reports wav transcode failures", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-tts-test-"));
|
||||
try {
|
||||
const res = await synthesizeHeygen(
|
||||
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: join(dir, "voice.wav") },
|
||||
{
|
||||
heygenAuthHeaders: () => ({}),
|
||||
heygenJSON: async () => ({ data: { audio_url: "http://audio.example/x" } }),
|
||||
fetch: async () => ({ ok: true, status: 200, arrayBuffer: async () => new ArrayBuffer(0) }),
|
||||
transcodeToWav: () => false,
|
||||
},
|
||||
);
|
||||
assert.equal(res.ok, false);
|
||||
assert.equal(res.error, "wav transcode failed (ffmpeg)");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("synthResult names a non-zero subprocess exit", () => {
|
||||
const res = synthResult({ status: 2 }, "/tmp/none.wav", "kokoro (npx hyperframes tts)");
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error, /kokoro .* exited with status 2/);
|
||||
});
|
||||
|
||||
// Kokoro shells out to `npx hyperframes tts`; a fake npx on PATH records the argv
|
||||
// synthesizeOne really builds, so a dropped flag fails here. Windows runs npx via
|
||||
// node directly, so the stub never resolves there.
|
||||
async function kokoroArgv(options) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-kokoro-argv-"));
|
||||
const argvLog = join(dir, "argv.txt");
|
||||
writeFileSync(join(dir, "npx"), `#!/bin/sh\nprintf '%s\\n' "$@" > "${argvLog}"\n`);
|
||||
chmodSync(join(dir, "npx"), 0o755);
|
||||
const originalPath = process.env.PATH;
|
||||
try {
|
||||
process.env.PATH = dir;
|
||||
await synthesizeOne({
|
||||
provider: "kokoro",
|
||||
text: "hi",
|
||||
voiceId: "am_michael",
|
||||
wavAbs: join(dir, "line-0.wav"),
|
||||
hyperframesDir: dir,
|
||||
...options,
|
||||
});
|
||||
return readFileSync(argvLog, "utf8").trim().split("\n");
|
||||
} finally {
|
||||
process.env.PATH = originalPath;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test(
|
||||
"synthesizeOne(kokoro) forwards a non-default speed to the CLI",
|
||||
{ skip: process.platform === "win32" },
|
||||
async () => {
|
||||
const argv = await kokoroArgv({ speed: 0.8, lang: "es" });
|
||||
assert.deepEqual(argv.slice(-4), ["--lang", "es", "--speed", "0.8"]);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
"synthesizeOne(kokoro) leaves --speed off at the default",
|
||||
{ skip: process.platform === "win32" },
|
||||
async () => {
|
||||
assert.ok(!(await kokoroArgv({})).includes("--speed"));
|
||||
},
|
||||
);
|
||||
|
||||
test("pickProvider heygen with an unreadable credentials path says to fix that path", () => {
|
||||
const saved = ["HEYGEN_API_KEY", "HYPERFRAMES_API_KEY", "HEYGEN_CONFIG_DIR"].map((k) => [
|
||||
k,
|
||||
process.env[k],
|
||||
]);
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-cred-"));
|
||||
try {
|
||||
delete process.env.HEYGEN_API_KEY;
|
||||
delete process.env.HYPERFRAMES_API_KEY;
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
mkdirSync(join(dir, "credentials"));
|
||||
assert.throws(() => pickProvider("heygen"), /fix or remove that path/);
|
||||
} finally {
|
||||
for (const [k, v] of saved) {
|
||||
if (v === undefined) delete process.env[k];
|
||||
else process.env[k] = v;
|
||||
}
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate BGM using Google Lyria RealTime API.
|
||||
|
||||
Usage:
|
||||
python lyria-recipe.py --output <path> --duration <seconds> [tuning flags]
|
||||
|
||||
Requires:
|
||||
$GOOGLE_API_KEY or $GEMINI_API_KEY environment variable (treated as aliases).
|
||||
pip install google-genai python-dotenv. audio.mjs Step 4b installs these on
|
||||
demand when a key is set but google.genai is not importable; if that install
|
||||
fails it falls back to local MusicGen rather than leaving the video with no BGM.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
# Windows sizes stdio to the ANSI code page (cp1252). These scripts emit UTF-8 on
|
||||
# every platform; say so rather than depending on the console's code page. Carry
|
||||
# `errors` across: reconfigure() resets it to "strict", and CPython deliberately gives
|
||||
# stderr "backslashreplace" so the diagnostic path can never itself raise.
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(_stream, "reconfigure"):
|
||||
_stream.reconfigure(encoding="utf-8", errors=_stream.errors)
|
||||
|
||||
DEFAULT_PROMPT = "Uplifting corporate tech, bright and modern, gentle piano with synth pads"
|
||||
SAMPLE_RATE = 48000
|
||||
CHANNELS = 2
|
||||
SAMPLE_WIDTH = 2 # 16-bit
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description="Generate BGM via Google Lyria RealTime.")
|
||||
p.add_argument("--output", required=True, help="Output WAV path.")
|
||||
p.add_argument("--duration", type=float, required=True, help="Target duration in seconds.")
|
||||
p.add_argument("--prompt", default=DEFAULT_PROMPT, help="Mood / instrumentation prompt.")
|
||||
p.add_argument("--negative-prompt", default=None, help="Styles to exclude (optional).")
|
||||
p.add_argument("--bpm", type=int, default=110)
|
||||
p.add_argument("--brightness", type=float, default=0.8, help="0-1, higher = brighter mood.")
|
||||
p.add_argument("--density", type=float, default=0.5, help="0-1, higher = fuller mix.")
|
||||
p.add_argument(
|
||||
"--scale",
|
||||
default="MAJOR",
|
||||
help="MAJOR / MINOR / PENTATONIC / etc. — see google.genai.types.Scale. Pass empty string for none.",
|
||||
)
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
async def generate_bgm(args: argparse.Namespace) -> dict:
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
api_key = os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY") or ""
|
||||
if not api_key:
|
||||
raise RuntimeError("Neither GOOGLE_API_KEY nor GEMINI_API_KEY is set.")
|
||||
|
||||
client = genai.Client(
|
||||
api_key=api_key,
|
||||
http_options={"api_version": "v1alpha"},
|
||||
)
|
||||
|
||||
out_path = Path(args.output)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
target_bytes = int(args.duration * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)
|
||||
|
||||
cfg: dict = {"bpm": args.bpm, "temperature": 1.0}
|
||||
if args.density is not None:
|
||||
cfg["density"] = args.density
|
||||
if args.brightness is not None:
|
||||
cfg["brightness"] = args.brightness
|
||||
if args.scale:
|
||||
scale_enum = getattr(types.Scale, args.scale, None)
|
||||
if scale_enum:
|
||||
cfg["scale"] = scale_enum
|
||||
|
||||
prompts = [types.WeightedPrompt(text=args.prompt, weight=1.0)]
|
||||
if args.negative_prompt:
|
||||
prompts.append(types.WeightedPrompt(text=args.negative_prompt, weight=-1.0))
|
||||
|
||||
buf = bytearray()
|
||||
timeout = args.duration + 8
|
||||
|
||||
async with client.aio.live.music.connect(
|
||||
model="models/lyria-realtime-exp",
|
||||
) as session:
|
||||
await session.set_weighted_prompts(prompts=prompts)
|
||||
await session.set_music_generation_config(
|
||||
config=types.LiveMusicGenerationConfig(**cfg),
|
||||
)
|
||||
await session.play()
|
||||
|
||||
async def collect():
|
||||
while len(buf) < target_bytes:
|
||||
async for msg in session.receive():
|
||||
sc = msg.server_content
|
||||
if sc and sc.audio_chunks:
|
||||
for chunk in sc.audio_chunks:
|
||||
buf.extend(chunk.data)
|
||||
if len(buf) >= target_bytes:
|
||||
return
|
||||
await asyncio.sleep(1e-6)
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(collect(), timeout=timeout)
|
||||
except TimeoutError:
|
||||
print(f"Timeout after {timeout:.0f}s, collected {len(buf)} bytes", file=sys.stderr)
|
||||
|
||||
audio = bytes(buf[:target_bytes])
|
||||
with wave.open(str(out_path), "wb") as wf:
|
||||
wf.setnchannels(CHANNELS)
|
||||
wf.setsampwidth(SAMPLE_WIDTH)
|
||||
wf.setframerate(SAMPLE_RATE)
|
||||
wf.writeframes(audio)
|
||||
|
||||
actual_duration = len(audio) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)
|
||||
print(f"BGM: {out_path} ({actual_duration:.2f}s)")
|
||||
return {"file": str(out_path), "duration_sec": round(actual_duration, 2)}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
try:
|
||||
asyncio.run(generate_bgm(args))
|
||||
except RuntimeError as exc:
|
||||
print(f"BGM generation failed: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env node
|
||||
// Phase 4c pre-assemble helper — wait for detached BGM, then write status.
|
||||
//
|
||||
// audio.mjs may launch Lyria / MusicGen in a detached process so voice work can
|
||||
// keep moving. Before assemble-index.mjs decides whether to emit the BGM audio
|
||||
// track, this script gives the background renderer a bounded chance to finish
|
||||
// and converts log/process state into a small bgm_status.json file.
|
||||
//
|
||||
// Always exits 0 for normal pipeline use: missing/failed BGM should not block a
|
||||
// voice/captions/SFX render. Structural invocation errors still exit 1.
|
||||
//
|
||||
// Usage:
|
||||
// node wait-bgm.mjs --audio-meta ./audio_meta.json --hyperframes . \
|
||||
// [--timeout-ms 120000] [--interval-ms 2000] [--out ./bgm_status.json]
|
||||
|
||||
import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { recordInManifest } from "./lib/media-record.mjs";
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name, def) => {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : def;
|
||||
};
|
||||
|
||||
function die(msg) {
|
||||
console.error(`✗ wait-bgm.mjs: ${msg}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const audioMetaPath = resolve(flag("audio-meta", "./audio_meta.json"));
|
||||
const hyperframesDir = resolve(flag("hyperframes", "."));
|
||||
const outPath = resolve(flag("out", join(hyperframesDir, "bgm_status.json")));
|
||||
const timeoutMs = Math.max(0, Number(flag("timeout-ms", "120000")) || 0);
|
||||
const intervalMs = Math.max(250, Number(flag("interval-ms", "2000")) || 2000);
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
|
||||
}
|
||||
|
||||
function isProcessAlive(pid) {
|
||||
if (!pid || !Number.isFinite(Number(pid))) return false;
|
||||
try {
|
||||
process.kill(Number(pid), 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readTail(path, maxChars = 6000) {
|
||||
if (!path) return "";
|
||||
try {
|
||||
const txt = readFileSync(path, "utf8");
|
||||
return txt.slice(Math.max(0, txt.length - maxChars));
|
||||
} catch (error) {
|
||||
if (error.code === "ENOENT" || error.code === "ENOTDIR") return "";
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function detectFailure(logTail) {
|
||||
if (!logTail) return "";
|
||||
const lines = logTail.split("\n");
|
||||
// Bare "out of range" over-matched benign BGM-renderer logs (e.g. a "sample rate
|
||||
// out of range, resampling" notice), mislabelling a healthy track as failed and
|
||||
// silently dropping the music. Anchor to the actual crash strings instead:
|
||||
// Python "(list) index out of range" and torch "index … out of bounds".
|
||||
const idx = lines.findIndex((line) =>
|
||||
/(Traceback|IndexError|RuntimeError|Exception|Killed|No space left|Cannot allocate|index out of range|out of bounds)/i.test(
|
||||
line,
|
||||
),
|
||||
);
|
||||
if (idx < 0) return "";
|
||||
return lines.slice(idx).join("\n").trim();
|
||||
}
|
||||
|
||||
function writeStatus(status) {
|
||||
const payload = {
|
||||
generated_at: new Date().toISOString(),
|
||||
...status,
|
||||
};
|
||||
writeFileSync(outPath, JSON.stringify(payload, null, 2) + "\n");
|
||||
return payload;
|
||||
}
|
||||
|
||||
if (!existsSync(audioMetaPath)) die(`audio_meta.json missing at ${audioMetaPath}`);
|
||||
|
||||
const audioMeta = JSON.parse(readFileSync(audioMetaPath, "utf8"));
|
||||
const bgmPath = audioMeta.bgm?.path || "";
|
||||
const bgmAbsPath = bgmPath ? join(hyperframesDir, bgmPath) : "";
|
||||
const logPath = audioMeta.bgm_log || "";
|
||||
const pid = audioMeta.bgm_pid || null;
|
||||
|
||||
const base = {
|
||||
enabled: Boolean(audioMeta.bgm_pending && bgmPath),
|
||||
provider: audioMeta.bgm_provider || null,
|
||||
mode: audioMeta.bgm_mode || null,
|
||||
path: bgmPath || null,
|
||||
log: logPath || null,
|
||||
pid,
|
||||
target_duration_s: audioMeta.bgm_target_duration_s || null,
|
||||
seed_duration_s: audioMeta.bgm_seed_duration_s || null,
|
||||
loop_count: audioMeta.bgm_loop_count || null,
|
||||
timeout_ms: timeoutMs,
|
||||
};
|
||||
|
||||
if (!base.enabled) {
|
||||
const status = writeStatus({
|
||||
...base,
|
||||
status: "disabled",
|
||||
ready: false,
|
||||
waited_ms: 0,
|
||||
message: "BGM not requested or disabled in audio_meta.json.",
|
||||
});
|
||||
console.log(`✓ bgm: ${status.status} (${status.message})`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const started = Date.now();
|
||||
let lastFailure = "";
|
||||
let lastTail = "";
|
||||
|
||||
while (Date.now() - started <= timeoutMs) {
|
||||
if (existsSync(bgmAbsPath)) {
|
||||
const size = statSync(bgmAbsPath).size;
|
||||
writeStatus({
|
||||
...base,
|
||||
status: "ready",
|
||||
ready: true,
|
||||
waited_ms: Date.now() - started,
|
||||
size_bytes: size,
|
||||
message: `BGM ready at ${bgmPath}.`,
|
||||
});
|
||||
console.log(`✓ bgm: ready (${bgmPath}, ${size}B)`);
|
||||
const generated = { path: bgmPath, type: "bgm", source: "generated", provider: base.provider };
|
||||
for (const anomaly of recordInManifest(hyperframesDir, [generated]))
|
||||
console.error(` ${anomaly}`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
lastTail = readTail(logPath);
|
||||
lastFailure = detectFailure(lastTail);
|
||||
const alive = isProcessAlive(pid);
|
||||
if (lastFailure || (!alive && logPath && existsSync(logPath))) {
|
||||
const message = lastFailure
|
||||
? `BGM renderer failed; see ${logPath}.`
|
||||
: `BGM renderer exited without writing ${bgmPath}; see ${logPath}.`;
|
||||
const status = writeStatus({
|
||||
...base,
|
||||
status: "failed",
|
||||
ready: false,
|
||||
waited_ms: Date.now() - started,
|
||||
process_alive: alive,
|
||||
message,
|
||||
error_tail: lastFailure || lastTail.slice(-2000),
|
||||
});
|
||||
console.log(`! bgm: failed (${status.message})`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (timeoutMs === 0) break;
|
||||
await sleep(Math.min(intervalMs, Math.max(0, timeoutMs - (Date.now() - started))));
|
||||
}
|
||||
|
||||
const status = writeStatus({
|
||||
...base,
|
||||
status: "timeout",
|
||||
ready: false,
|
||||
waited_ms: Date.now() - started,
|
||||
process_alive: isProcessAlive(pid),
|
||||
message: `Timed out waiting for ${bgmPath}; assemble-index will skip BGM if still absent.`,
|
||||
log_tail: lastTail.slice(-2000),
|
||||
});
|
||||
console.log(`! bgm: timeout after ${status.waited_ms}ms (${bgmPath})`);
|
||||
@@ -0,0 +1,108 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { test } from "node:test";
|
||||
import { runInNewContext } from "node:vm";
|
||||
|
||||
const source = readFileSync(new URL("./wait-bgm.mjs", import.meta.url), "utf8");
|
||||
const start = source.indexOf("function readTail(");
|
||||
const end = source.indexOf("\nfunction detectFailure(", start);
|
||||
function load(read) {
|
||||
return runInNewContext(`${source.slice(start, end)}; readTail`, {
|
||||
readFileSync: read,
|
||||
existsSync: () => true,
|
||||
statSync: () => {
|
||||
throw new Error("pathname stat must not precede log read");
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
test("reads a log once without a separate pathname check", () => {
|
||||
let calls = 0;
|
||||
const read = load((path, encoding) => {
|
||||
calls++;
|
||||
assert.equal(path, "renderer.log");
|
||||
assert.equal(encoding, "utf8");
|
||||
return "prefix\nRuntimeError: failed";
|
||||
});
|
||||
assert.equal(read("renderer.log", 20), "RuntimeError: failed");
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
|
||||
test("preserves UTF-16 character tail boundaries and zero-length tails", () => {
|
||||
const txt = "🎵日本語".repeat(2000);
|
||||
const read = load(() => txt);
|
||||
assert.equal(read("log"), txt.slice(-6000));
|
||||
assert.equal(read("log", 0), "");
|
||||
});
|
||||
|
||||
for (const code of ["ENOENT", "ENOTDIR"]) {
|
||||
test(`a log removed during polling is absent (${code})`, () => {
|
||||
const read = load(() => {
|
||||
throw Object.assign(new Error(code), { code });
|
||||
});
|
||||
assert.equal(read("log"), "");
|
||||
});
|
||||
}
|
||||
|
||||
test("preserves unexpected read errors", () => {
|
||||
const error = Object.assign(new Error("denied"), { code: "EACCES" });
|
||||
assert.throws(
|
||||
() =>
|
||||
load(() => {
|
||||
throw error;
|
||||
})("log"),
|
||||
(caught) => caught === error,
|
||||
);
|
||||
});
|
||||
|
||||
test("empty log path does not access the filesystem", () => {
|
||||
assert.equal(
|
||||
load(() => {
|
||||
throw new Error("unexpected read");
|
||||
})(""),
|
||||
"",
|
||||
);
|
||||
});
|
||||
|
||||
test("records locally made music as generated once it is ready", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-wait-bgm-"));
|
||||
try {
|
||||
mkdirSync(join(dir, "assets/bgm"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets/bgm/track.wav"), "generated music");
|
||||
writeFileSync(
|
||||
join(dir, "audio_meta.json"),
|
||||
JSON.stringify({
|
||||
bgm: { path: "assets/bgm/track.wav" },
|
||||
bgm_pending: true,
|
||||
bgm_provider: "musicgen",
|
||||
bgm_mode: "detached-single",
|
||||
}),
|
||||
);
|
||||
const script = new URL("./wait-bgm.mjs", import.meta.url).pathname;
|
||||
const args = ["--audio-meta", join(dir, "audio_meta.json"), "--hyperframes", dir];
|
||||
|
||||
const run = spawnSync(process.execPath, [script, ...args, "--timeout-ms", "1000"], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
|
||||
assert.equal(run.status, 0, run.stderr);
|
||||
const records = readFileSync(join(dir, ".media/manifest.jsonl"), "utf8")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
assert.deepEqual(
|
||||
records.map(({ path, type, source, provenance }) => [
|
||||
path,
|
||||
type,
|
||||
source,
|
||||
provenance.provider,
|
||||
]),
|
||||
[["assets/bgm/track.wav", "bgm", "generated", "musicgen"]],
|
||||
);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user