Add HyperFrames skills (from heygen-com/hyperframes, Apache-2.0)
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FXVsD1DgdzZLiPuwtKzMSY
This commit is contained in:
300 files changed
+49175
No files matched your search
@@ -0,0 +1,310 @@
|
||||
#!/usr/bin/env node
|
||||
// audio.mjs — the shared HyperFrames audio engine. ONE implementation of TTS +
|
||||
// BGM + SFX. Callers do NOT vendor a copy: they write a neutral
|
||||
// `audio_request.json` (mapping their storyboard/scenes into it) and call:
|
||||
//
|
||||
// node <MEDIA_DIR>/scripts/audio.mjs --request ./audio_request.json --hyperframes . --out ./audio_meta.json
|
||||
//
|
||||
// TTS and BGM degrade on ONE switch — whether HeyGen is configured (credential
|
||||
// present, NOT the CLI). SFX needs no credential. This mirrors ../SKILL.md:
|
||||
//
|
||||
// TTS : HeyGen REST → ElevenLabs → Kokoro (CLI)
|
||||
// BGM : HeyGen retrieve → (no credential) Lyria/MusicGen generate
|
||||
// SFX : Openverse (CC0 / CC BY) → (offline, rate limited, no match) bundled library
|
||||
//
|
||||
// ── audio_request.json (input) ────────────────────────────────────────────────
|
||||
// {
|
||||
// "provider": "auto", // auto|heygen|elevenlabs|kokoro|gemini (override: --provider)
|
||||
// "lang": "en", "speed": 1.0,
|
||||
// "lines": [ // one TTS unit each; id joins back to the caller's model
|
||||
// { "id": "01", "text": "...", "sfx": ["whoosh", "ui click"] }
|
||||
// ],
|
||||
// "bgm": { "mode": "retrieve", // retrieve|generate|none (override: --bgm-mode / --no-bgm)
|
||||
// "query": "calm cinematic underscore", // mood for retrieval
|
||||
// "prompt": null, // full prompt for generation (else inferred)
|
||||
// "blob": "...", "archetype": "...", "arc": "..." } // optional mood-inference hints
|
||||
// }
|
||||
//
|
||||
// ── audio_meta.json (output, id-keyed) ───────────────────────────────────────
|
||||
// { tts_provider, voice_id,
|
||||
// bgm: { path, volume, mode, query?, duration_s? } | null,
|
||||
// bgm_pending, bgm_provider, bgm_pid, bgm_log, bgm_mode, bgm_target_duration_s, …,
|
||||
// voices: [ { id, path, duration_s, words: [{id,text,start,end}] } ],
|
||||
// sfx: [ { id, name, file, source, offset_s, duration_s, volume, license?, credit? } ],
|
||||
// total_duration_s }
|
||||
//
|
||||
// --only tts,bgm,sfx runs a subset and MERGES into an existing --out (so a
|
||||
// caller can do TTS+BGM early, then SFX later once cues exist). When BGM uses
|
||||
// the generate path it is spawned detached (bgm_pending:true) — run wait-bgm.mjs
|
||||
// before assembling.
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { heygenAuthHeaders, heygenCredential, loadEnvFromDir } from "./lib/heygen.mjs";
|
||||
import {
|
||||
ffprobeDuration,
|
||||
pickProvider,
|
||||
resolveVoiceId,
|
||||
synthesizeOne,
|
||||
transcribeWav,
|
||||
withWordIds,
|
||||
} from "./lib/tts.mjs";
|
||||
import { generateBgmDetached, inferBgmPrompt, retrieveBgm } from "./lib/bgm.mjs";
|
||||
import { BUNDLED_SFX_DIR, resolveSfx } from "./lib/sfx.mjs";
|
||||
import { mapWithConcurrency } from "./lib/concurrency.mjs";
|
||||
import { openAudioMeta } from "./lib/audio-meta.mjs";
|
||||
import { recordInManifest, voicePaths, writtenAssets } from "./lib/media-record.mjs";
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name, def) => {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : def;
|
||||
};
|
||||
const has = (name) => argv.includes(`--${name}`);
|
||||
const die = (m) => {
|
||||
console.error(`✗ audio engine: ${m}`);
|
||||
process.exit(1);
|
||||
};
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
|
||||
// Two independent reports of an unbounded Promise.all over TTS lines
|
||||
// overwhelming a machine: one OOM'd 12/13 concurrent Kokoro TTS +
|
||||
// whisper-transcribe lines on a resource-constrained laptop, the other saw
|
||||
// 7/8 lines fail on first run (concurrent cold-start model loads) and pass on
|
||||
// retry once the model was cached. Kokoro/Whisper each load their own local
|
||||
// model per subprocess, so firing every line at once multiplies that cost by
|
||||
// the line count. mapWithConcurrency caps how many run at once — still
|
||||
// parallel, just bounded.
|
||||
const ttsConcurrency = Math.max(1, Number(process.env.HYPERFRAMES_TTS_CONCURRENCY) || 4);
|
||||
|
||||
const hyperframesDir = resolve(flag("hyperframes", "."));
|
||||
const requestPath = resolve(flag("request", join(hyperframesDir, "audio_request.json")));
|
||||
const outPath = resolve(flag("out", join(hyperframesDir, "audio_meta.json")));
|
||||
const sfxLibDir = resolve(flag("sfx-lib", BUNDLED_SFX_DIR));
|
||||
const lyriaRecipe = resolve(flag("lyria-recipe", join(HERE, "lyria-recipe.py")));
|
||||
const onlyArg = flag("only", "tts,bgm,sfx");
|
||||
const only = new Set(
|
||||
onlyArg
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean),
|
||||
);
|
||||
const providerOverride = flag("provider", null);
|
||||
const bgmModeOverride = flag("bgm-mode", null);
|
||||
const noBgm = has("no-bgm");
|
||||
const voiceOverride = flag("voice", null);
|
||||
const speedOverride = flag("speed", null);
|
||||
const langOverride = flag("lang", null);
|
||||
const seedSeconds = Number(flag("seed-seconds", "28")) || 28;
|
||||
|
||||
if (!existsSync(requestPath)) die(`audio_request.json not found at ${requestPath}`);
|
||||
let request;
|
||||
try {
|
||||
request = JSON.parse(readFileSync(requestPath, "utf8"));
|
||||
} catch (e) {
|
||||
die(`audio_request.json parse: ${e.message}`);
|
||||
}
|
||||
const lines = Array.isArray(request.lines) ? request.lines : [];
|
||||
const lang = langOverride || request.lang || "en";
|
||||
const speedInput = speedOverride ?? request.speed ?? 1;
|
||||
const speed = Number(speedInput);
|
||||
if (!(speed > 0 && speed <= 3))
|
||||
die(`speed must be above 0 and at most 3, got ${JSON.stringify(speedInput)}`);
|
||||
|
||||
// ── env + HeyGen availability (the single switch) ─────────────────────────────
|
||||
loadEnvFromDir(hyperframesDir);
|
||||
const heygenOK = heygenCredential() !== null;
|
||||
|
||||
// ── merge base: preserve sections not selected by --only ──────────────────────
|
||||
const audioMeta = openAudioMeta(outPath);
|
||||
const prev = audioMeta.value;
|
||||
const anomalies = [];
|
||||
|
||||
// ── TTS ───────────────────────────────────────────────────────────────────────
|
||||
let voices = prev.voices ?? [];
|
||||
let ttsProvider = prev.tts_provider ?? null;
|
||||
let voiceId = prev.voice_id ?? null;
|
||||
if (only.has("tts") && lines.length) {
|
||||
try {
|
||||
ttsProvider = pickProvider(
|
||||
providerOverride || (request.provider === "auto" ? null : request.provider),
|
||||
);
|
||||
} catch (e) {
|
||||
die(e.message);
|
||||
}
|
||||
voiceId = await resolveVoiceId({
|
||||
provider: ttsProvider,
|
||||
userVoice: voiceOverride || request.voice,
|
||||
lang,
|
||||
});
|
||||
console.error(`· tts: ${ttsProvider} · voice ${voiceId} · ${lines.length} line(s)`);
|
||||
const paths = voicePaths(hyperframesDir, lines, anomalies);
|
||||
const synthLine = async (line) => {
|
||||
const id = String(line.id);
|
||||
const text = String(line.text ?? "").trim();
|
||||
if (!text) {
|
||||
anomalies.push(`line ${id}: empty text — skipped`);
|
||||
return null;
|
||||
}
|
||||
const rel = paths.get(id);
|
||||
const abs = join(hyperframesDir, rel);
|
||||
const { ok, words, error } = await synthesizeOne({
|
||||
provider: ttsProvider,
|
||||
text,
|
||||
voiceId,
|
||||
lang,
|
||||
speed,
|
||||
model: flag("tts-model", request.tts_model),
|
||||
style: flag("style", line.style ?? request.style),
|
||||
wavAbs: abs,
|
||||
hyperframesDir,
|
||||
});
|
||||
if (!ok) {
|
||||
anomalies.push(`line ${id}: TTS failed — omitted${error ? ` (${error})` : ""}`);
|
||||
return null;
|
||||
}
|
||||
let wordArr = words; // heygen: native; else transcribe
|
||||
if (!wordArr) wordArr = await transcribeWav({ wavRel: rel, lang, hyperframesDir });
|
||||
const dur = ffprobeDuration(abs);
|
||||
if (!isFinite(dur) || dur <= 0) {
|
||||
anomalies.push(`line ${id}: bad voice duration — omitted`);
|
||||
return null;
|
||||
}
|
||||
return { id, path: rel, duration_s: r3(dur), words: withWordIds(wordArr) };
|
||||
};
|
||||
const results = await mapWithConcurrency(lines, ttsConcurrency, synthLine);
|
||||
voices = results.filter(Boolean);
|
||||
for (const v of voices)
|
||||
console.error(` voice ${v.id}: ${v.path} (${v.duration_s}s, ${v.words.length} words)`);
|
||||
}
|
||||
const hasVoice = voices.length > 0;
|
||||
const totalDuration = r3(voices.reduce((a, v) => a + (v.duration_s || 0), 0));
|
||||
|
||||
// ── BGM ─────────────────────────────────────────────────────────────────────
|
||||
let bgm = prev.bgm ?? null;
|
||||
const bgmFields = {
|
||||
bgm_pending: prev.bgm_pending ?? false,
|
||||
bgm_provider: prev.bgm_provider ?? null,
|
||||
bgm_pid: prev.bgm_pid ?? null,
|
||||
bgm_log: prev.bgm_log ?? null,
|
||||
bgm_mode: prev.bgm_mode ?? null,
|
||||
bgm_target_duration_s: prev.bgm_target_duration_s ?? null,
|
||||
bgm_seed_duration_s: prev.bgm_seed_duration_s ?? null,
|
||||
bgm_loop_count: prev.bgm_loop_count ?? null,
|
||||
};
|
||||
if (only.has("bgm")) {
|
||||
bgm = null;
|
||||
Object.keys(bgmFields).forEach((k) => (bgmFields[k] = k === "bgm_pending" ? false : null));
|
||||
// Mode resolution. An EXPLICIT mode (flag or request.bgm.mode) is strict:
|
||||
// "retrieve" means retrieve-or-nothing — it never silently starts a detached
|
||||
// generate (a caller with no wait-bgm step must not get a pending job it
|
||||
// can't await). Only the UNSET/auto default picks generate
|
||||
// when HeyGen is absent.
|
||||
const explicitMode = bgmModeOverride || request.bgm?.mode || null;
|
||||
let mode = noBgm ? "none" : explicitMode || (heygenOK ? "retrieve" : "generate");
|
||||
if (mode === "retrieve" && !heygenOK) {
|
||||
anomalies.push(
|
||||
"bgm: retrieve requires a HeyGen credential — skipped (no generate fallback for an explicit retrieve)",
|
||||
);
|
||||
mode = "none";
|
||||
}
|
||||
|
||||
if (mode === "none") {
|
||||
console.error(`· bgm: disabled`);
|
||||
} else if (mode === "retrieve") {
|
||||
try {
|
||||
bgm = await retrieveBgm({
|
||||
query: request.bgm?.query,
|
||||
headers: heygenAuthHeaders(),
|
||||
hyperframesDir,
|
||||
hasVoice,
|
||||
anomalies,
|
||||
});
|
||||
if (bgm) {
|
||||
bgmFields.bgm_provider = "heygen";
|
||||
bgmFields.bgm_mode = "retrieve";
|
||||
console.error(` bgm: ${bgm.path} (retrieve "${bgm.query}")`);
|
||||
} else {
|
||||
anomalies.push(`bgm: no music match for "${request.bgm?.query ?? ""}" — skipped`);
|
||||
}
|
||||
} catch (e) {
|
||||
anomalies.push(`bgm retrieve failed: ${e.message} — skipped`);
|
||||
}
|
||||
} else {
|
||||
// generate
|
||||
const prompt = inferBgmPrompt({
|
||||
userPrompt: request.bgm?.prompt,
|
||||
blob: request.bgm?.blob || request.bgm?.query,
|
||||
archetype: request.bgm?.archetype,
|
||||
arc: request.bgm?.arc,
|
||||
});
|
||||
const gen = generateBgmDetached({
|
||||
prompt,
|
||||
durationS: totalDuration || 30,
|
||||
hyperframesDir,
|
||||
lyriaRecipe: existsSync(lyriaRecipe) ? lyriaRecipe : null,
|
||||
seedSeconds,
|
||||
hasVoice,
|
||||
anomalies,
|
||||
});
|
||||
if (gen.disabled) {
|
||||
anomalies.push(`bgm: ${gen.reason}`);
|
||||
} else {
|
||||
bgm = { path: gen.path, volume: gen.volume, mode: gen.mode, duration_s: null };
|
||||
bgmFields.bgm_pending = true;
|
||||
bgmFields.bgm_provider = gen.provider;
|
||||
bgmFields.bgm_pid = gen.pid;
|
||||
bgmFields.bgm_log = gen.log;
|
||||
bgmFields.bgm_mode = gen.mode;
|
||||
bgmFields.bgm_target_duration_s = gen.target_duration_s ?? null;
|
||||
bgmFields.bgm_seed_duration_s = gen.seed_duration_s ?? null;
|
||||
bgmFields.bgm_loop_count = gen.loop_count ?? null;
|
||||
console.error(` bgm: launched ${gen.provider} (detached, pid ${gen.pid}) → ${gen.path}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── SFX ─────────────────────────────────────────────────────────────────────
|
||||
let sfx = prev.sfx ?? [];
|
||||
if (only.has("sfx")) {
|
||||
const cues = lines.flatMap((l) =>
|
||||
(Array.isArray(l.sfx) ? l.sfx : [])
|
||||
.map((name) => ({ id: String(l.id), name: String(name).trim() }))
|
||||
.filter((c) => c.name),
|
||||
);
|
||||
const res = await resolveSfx({ cues, hyperframesDir, sfxLibDir });
|
||||
sfx = res.sfx;
|
||||
anomalies.push(...res.anomalies);
|
||||
const openverse = sfx.filter((s) => s.source === "openverse").length;
|
||||
console.error(
|
||||
`· sfx: ${sfx.length} cue(s) resolved (${openverse} Openverse, ${sfx.length - openverse} bundled library)`,
|
||||
);
|
||||
}
|
||||
|
||||
// ── write audio_meta.json ─────────────────────────────────────────────────────
|
||||
const meta = {
|
||||
tts_provider: ttsProvider,
|
||||
voice_id: voiceId,
|
||||
bgm,
|
||||
...bgmFields,
|
||||
voices,
|
||||
sfx,
|
||||
total_duration_s: totalDuration,
|
||||
};
|
||||
mkdirSync(dirname(outPath), { recursive: true });
|
||||
audioMeta.write(meta);
|
||||
const written = writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields, sfx });
|
||||
anomalies.push(...recordInManifest(hyperframesDir, written));
|
||||
|
||||
console.log(`✓ audio engine → ${outPath}`);
|
||||
console.log(` heygen: ${heygenOK ? "yes" : "no"} · ran: ${[...only].join(",")}`);
|
||||
console.log(
|
||||
` voices: ${voices.length} · bgm: ${bgm ? `${bgmFields.bgm_provider}${bgmFields.bgm_pending ? " (pending)" : ""}` : "none"} · sfx: ${sfx.length}`,
|
||||
);
|
||||
console.log(` total voice duration: ${totalDuration}s`);
|
||||
if (anomalies.length) {
|
||||
console.log(`\nanomalies (non-fatal):`);
|
||||
for (const a of anomalies) console.log(` - ${a}`);
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import { test } from "node:test";
|
||||
import { mkdtempSync, mkdirSync, readFileSync, rmSync, existsSync, writeFileSync } from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { join, dirname } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { recordInManifest } from "./lib/media-record.mjs";
|
||||
import { resolveSfx } from "./lib/sfx.mjs";
|
||||
|
||||
// Proves the relocated engine (skills/media-use/audio/) still resolves its
|
||||
// bundled SFX library from the moved location — the path most likely to break
|
||||
// on a subtree move. Openverse is unreachable unless a test serves it.
|
||||
const offline = async () => {
|
||||
throw new TypeError("fetch failed");
|
||||
};
|
||||
globalThis.fetch = offline;
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const sfxLibDir = join(HERE, "..", "assets", "sfx"); // same offset the engine uses
|
||||
|
||||
test("bundled SFX library resolves from the relocated path", async () => {
|
||||
assert.ok(existsSync(join(sfxLibDir, "manifest.json")), "moved manifest is present");
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "1", name: "whoosh" }],
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
assert.equal(sfx.length, 1, `expected 1 resolved cue, got anomalies: ${anomalies.join("; ")}`);
|
||||
assert.equal(sfx[0].source, "local");
|
||||
assert.match(sfx[0].file, /assets\/sfx\//);
|
||||
assert.ok(existsSync(join(dir, sfx[0].file)), "matched SFX copied into the project");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("an unknown cue is reported, not fatal", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "1", name: "definitely-not-a-real-sfx" }],
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
assert.equal(sfx.length, 0);
|
||||
assert.ok(anomalies.some((a) => /not in bundled library/.test(a)));
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("a person's own file under a bundled name survives, and the cue gets the next free name", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
const own = join(dir, "assets", "sfx", "whoosh.mp3");
|
||||
mkdirSync(dirname(own), { recursive: true });
|
||||
writeFileSync(own, "the person's own whoosh");
|
||||
const resolve = () =>
|
||||
resolveSfx({
|
||||
cues: [{ id: "1", name: "whoosh" }],
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
|
||||
const first = await resolve();
|
||||
const second = await resolve();
|
||||
|
||||
assert.equal(readFileSync(own, "utf8"), "the person's own whoosh");
|
||||
assert.deepEqual(
|
||||
[first, second].map(({ sfx }) => [sfx[0].file, sfx[0].source]),
|
||||
[
|
||||
["assets/sfx/whoosh-2.mp3", "local"],
|
||||
["assets/sfx/whoosh-2.mp3", "local"],
|
||||
],
|
||||
);
|
||||
assert.deepEqual(
|
||||
readFileSync(join(dir, "assets/sfx/whoosh-2.mp3")),
|
||||
readFileSync(join(sfxLibDir, "whoosh.mp3")),
|
||||
);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("two effects never share a file when one's name is taken by the person, run after run", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-audio-"));
|
||||
try {
|
||||
mkdirSync(join(dir, "assets", "sfx"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets", "sfx", "glitch.mp3"), "the person's own glitch");
|
||||
globalThis.fetch = async (url) => {
|
||||
const query = new URL(url).searchParams.get("q");
|
||||
if (query)
|
||||
return Response.json({
|
||||
results: [
|
||||
{
|
||||
license: "cc0",
|
||||
filetype: "mp3",
|
||||
duration: 500,
|
||||
url: `https://sound.test/${encodeURIComponent(query)}`,
|
||||
},
|
||||
],
|
||||
});
|
||||
return new Response(`bytes of ${new URL(url).pathname}`);
|
||||
};
|
||||
const run = async (names) => {
|
||||
const { sfx } = await resolveSfx({
|
||||
cues: names.map((name, index) => ({ id: String(index), name })),
|
||||
hyperframesDir: dir,
|
||||
sfxLibDir,
|
||||
});
|
||||
const files = sfx.map(({ file }) => file);
|
||||
recordInManifest(
|
||||
dir,
|
||||
[...new Set(files)].map((path) => ({ path, type: "sfx", source: "search" })),
|
||||
);
|
||||
return files;
|
||||
};
|
||||
try {
|
||||
assert.deepEqual(await run(["glitch"]), ["assets/sfx/glitch-2.mp3"]);
|
||||
assert.deepEqual(await run(["glitch", "glitch 2", "glitch"]), [
|
||||
"assets/sfx/glitch-2.mp3",
|
||||
"assets/sfx/glitch-2-2.mp3",
|
||||
"assets/sfx/glitch-2.mp3",
|
||||
]);
|
||||
} finally {
|
||||
globalThis.fetch = offline;
|
||||
}
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// The engine owns speed parsing; a fallback before the check would turn 0 or junk into 1.
|
||||
function runEngineWith(request, ...args) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "audio-speed-range-"));
|
||||
try {
|
||||
writeFileSync(join(dir, "audio_request.json"), JSON.stringify({ lines: [], ...request }));
|
||||
return spawnSync(process.execPath, [join(HERE, "audio.mjs"), "--hyperframes", dir, ...args], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
for (const [label, request, args, shown] of [
|
||||
["request speed 5", { speed: 5 }, [], "5"],
|
||||
["request speed 0", { speed: 0 }, [], "0"],
|
||||
["request speed -1", { speed: -1 }, [], "-1"],
|
||||
["request speed text", { speed: "fast" }, [], '"fast"'],
|
||||
["--speed 0", {}, ["--speed", "0"], '"0"'],
|
||||
["--speed text", { speed: 1 }, ["--speed", "abc"], '"abc"'],
|
||||
]) {
|
||||
test(`${label} stops the engine before any TTS runs`, () => {
|
||||
const r = runEngineWith(request, ...args);
|
||||
assert.equal(r.status, 1);
|
||||
assert.ok(r.stderr.includes(`speed must be above 0 and at most 3, got ${shown}`), r.stderr);
|
||||
});
|
||||
}
|
||||
|
||||
test("a numeric string speed, as the adapters forward it, passes the check", () => {
|
||||
assert.doesNotMatch(runEngineWith({ speed: "0.8" }, "--only", "tts").stderr, /speed must/);
|
||||
});
|
||||
@@ -0,0 +1,560 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Apply a voiceover carve to a composition, from the command line.
|
||||
*
|
||||
* The carve is an analysis: it listens to a voice track, finds the bands it
|
||||
* occupies, and writes a chain of dips into the music bed plus a level match. In
|
||||
* Studio a panel runs it. This is the same analysis for an agent that has no
|
||||
* panel to click — identical functions from `@hyperframes/core`, identical
|
||||
* output, so a composition carved here and one carved in Studio are the same
|
||||
* three attributes.
|
||||
*
|
||||
* node carve.mjs --comp index.html
|
||||
* node carve.mjs --comp index.html --bed music-bed --voice narration \
|
||||
* --voice interview-guest --strength 0.45
|
||||
*
|
||||
* With no --bed/--voice it works out the tracks itself: the bed, and every voice
|
||||
* playing over it. `--voice` may be repeated to name them instead. Every named
|
||||
* voice is analysed together, so a bed running under a narrator and an answer makes
|
||||
* room for both.
|
||||
*
|
||||
* Needs `ffmpeg` on PATH (to decode the audio) and `@hyperframes/core` resolvable
|
||||
* from the composition's project (`npm i -D @hyperframes/core`) — the CLI bundles
|
||||
* core inline rather than shipping it as a package, so it cannot be borrowed from
|
||||
* there.
|
||||
*/
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createRequire } from "node:module";
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { isMainModule } from "./lib/main-module.mjs";
|
||||
|
||||
/** Sample rate the analysis runs at. Matches Studio's own decode rate, so the
|
||||
* bands and envelopes come out the same either way. */
|
||||
const SAMPLE_RATE = 48000;
|
||||
|
||||
/** Default carve strength. 0.25 kept the bed present but still fighting the voice. */
|
||||
export const DEFAULT_STRENGTH = 0.8;
|
||||
|
||||
const usage = `carve.mjs --comp <file.html> [--bed <elementId>] [--voice <elementId> ...]
|
||||
[--strength 0..1] [--dry-run] [--core <dir>]
|
||||
|
||||
--bed id of the music track that gets carved (detected if omitted)
|
||||
--voice id of a voice to make room for; repeatable (detected if omitted)
|
||||
--strength how hard to carve, 0..1 (default ${DEFAULT_STRENGTH})
|
||||
--dry-run report what it would write, touch nothing
|
||||
--core directory to resolve @hyperframes/core from (default: the comp's)`;
|
||||
|
||||
export function parseArgs(argv) {
|
||||
const args = { strength: DEFAULT_STRENGTH, dryRun: false, voices: [] };
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const flag = argv[i];
|
||||
const next = () => {
|
||||
const value = argv[i + 1];
|
||||
if (value === undefined) fail(`${flag} needs a value`);
|
||||
i += 1;
|
||||
return value;
|
||||
};
|
||||
if (flag === "--comp") args.comp = next();
|
||||
else if (flag === "--bed") args.bed = next();
|
||||
else if (flag === "--voice") args.voices.push(next());
|
||||
else if (flag === "--strength") args.strength = Number(next());
|
||||
else if (flag === "--core") args.core = next();
|
||||
else if (flag === "--dry-run") args.dryRun = true;
|
||||
else if (flag === "-h" || flag === "--help") fail(usage, 0);
|
||||
else fail(`unknown flag: ${flag}\n\n${usage}`);
|
||||
}
|
||||
if (!args.comp) fail(`--comp is required\n\n${usage}`);
|
||||
if (!Number.isFinite(args.strength) || args.strength < 0 || args.strength > 1) {
|
||||
fail("--strength must be a number from 0 to 1");
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function fail(message, code = 1) {
|
||||
process.stderr.write(`${message}\n`);
|
||||
process.exit(code);
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the carve analysis out of `@hyperframes/core`.
|
||||
*
|
||||
* Resolved from the project rather than from this script, which lives wherever
|
||||
* the skill was installed — a sibling of the composition is what has the
|
||||
* dependency.
|
||||
*/
|
||||
export async function loadCore(fromDir) {
|
||||
const require = createRequire(pathToFileURL(resolve(fromDir, "package.json")));
|
||||
|
||||
/*
|
||||
* Two constraints at once, and satisfying either alone is broken:
|
||||
*
|
||||
* 1. Anchored at the PROJECT, not at this script. This file lives wherever
|
||||
* the skill was installed, which has no @hyperframes/core; the
|
||||
* composition's project is what holds the dependency. So a bare
|
||||
* `import("@hyperframes/core/audio-carve")` from here cannot work — bare
|
||||
* specifiers resolve relative to the importing module.
|
||||
* 2. Honouring the package's export CONDITIONS. `require.resolve` asks for
|
||||
* "require"/"node". The workspace manifest declares `node`, so this
|
||||
* resolved fine inside the monorepo — but the PUBLISHED manifest carries
|
||||
* only `import` + `types`, so every consumer of the released package got
|
||||
* ERR_PACKAGE_PATH_NOT_EXPORTED for a package that ships the file. That
|
||||
* is the audience this skill is shipped to, so the script was broken
|
||||
* everywhere except where it was developed.
|
||||
*
|
||||
* Keep the project anchor; fall back to the package's declared `import`
|
||||
* target when no require-resolvable condition exists.
|
||||
*/
|
||||
const load = async (subpath) => {
|
||||
const spec = `@hyperframes/core/${subpath}`;
|
||||
try {
|
||||
return await import(pathToFileURL(require.resolve(spec)).href);
|
||||
} catch (error) {
|
||||
if (error?.code !== "ERR_PACKAGE_PATH_NOT_EXPORTED") throw error;
|
||||
// `./package.json` is exported by every manifest, so this always resolves
|
||||
// and gives us the package root without guessing at node_modules layout.
|
||||
const pkgPath = require.resolve("@hyperframes/core/package.json");
|
||||
const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
|
||||
const entry = pkg.exports?.[`./${subpath}`];
|
||||
const target = typeof entry === "string" ? entry : (entry?.import ?? entry?.default ?? null);
|
||||
if (!target) {
|
||||
fail(
|
||||
`@hyperframes/core does not export ./${subpath}\n` +
|
||||
` found at: ${pkgPath} (version ${pkg.version})\n` +
|
||||
` update it: npm i -D @hyperframes/core`,
|
||||
);
|
||||
}
|
||||
return import(pathToFileURL(resolve(dirname(pkgPath), target)).href);
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
return {
|
||||
carve: await load("audio-carve"),
|
||||
fx: await load("audio-fx"),
|
||||
};
|
||||
} catch (error) {
|
||||
fail(
|
||||
`cannot load @hyperframes/core from ${fromDir}\n` +
|
||||
` is it installed there? npm i -D @hyperframes/core\n` +
|
||||
` or point at one: --core <dir containing node_modules/@hyperframes/core>\n` +
|
||||
` (${error.code ?? "error"}: ${error.message.split("\n")[0]})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The `sources` a carve should record for these voices, on this bed.
|
||||
*
|
||||
* SKILL.md states the invariant: "A carve against more than one clip id is
|
||||
* wrong. Group the clips and carve against the group." Naming the group lets
|
||||
* `resolveCarveSourceIds` resolve membership at analysis time, so a voice added
|
||||
* later is covered without editing `sources` — whereas a list of clip ids rots
|
||||
* silently the moment a fourth narration clip appears. The lint rule
|
||||
* `audio_carve_ungrouped_sources` enforces exactly this.
|
||||
*
|
||||
* This script was writing clip ids unconditionally, so it violated its own
|
||||
* skill's invariant and tripped its own lint rule on every run. When every
|
||||
* voice shares one group, record the group. Mixed or ungrouped voices keep
|
||||
* their ids, and the lint rule then correctly tells the author to group them.
|
||||
*
|
||||
* The bed has to be part of the decision, because the group form resolves
|
||||
* LATER and wider than it looks. If the bed is itself a member of the voices'
|
||||
* group, `resolveCarveSourceIds` expands that id to every current member on the
|
||||
* next analysis — including the bed — and the bed ends up carved against
|
||||
* itself, which SKILL.md calls a bug rather than a mix choice. This run cannot
|
||||
* see it: `main()` sums the voice list it detected and never round-trips
|
||||
* through group resolution, so the first pass is correct and only the next
|
||||
* re-analysis in Studio is wrong. So decline the group form there and fall back
|
||||
* to clip ids, which is exactly the case `audio_carve_ungrouped_sources` exists
|
||||
* to put in front of the author.
|
||||
*
|
||||
* Only an `<audio>` bed can trip it: group membership is audio-only
|
||||
* (`audioGroupOf`), so `data-audio-group` on a `<video>` bed is ignored by core
|
||||
* and expanding a group can never pull it in.
|
||||
*/
|
||||
export function carveSources(voices, bed, members) {
|
||||
const group = sharedVoiceGroup(voices);
|
||||
return group && !groupSourceRefusal(voices, bed, members) ? [group] : voices.map((v) => v.id);
|
||||
}
|
||||
|
||||
/** The one group every voice belongs to, or null if they do not share exactly one. */
|
||||
function sharedVoiceGroup(voices) {
|
||||
const groups = voices.map((v) => attrOf(v.tag, "data-audio-group"));
|
||||
const first = groups[0];
|
||||
return Boolean(first) && groups.every((g) => g === first) ? first : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Why naming the voices' shared group would persist something this run did not
|
||||
* analyse — or null when the group is safe to name.
|
||||
*
|
||||
* `members` is every `<audio>` in the composition as `{id, group, nameKind}`,
|
||||
* with `nameKind` from core's `classifyAudioName`, so this and Studio's picker
|
||||
* classify the same way.
|
||||
*
|
||||
* Required, deliberately not defaulting to `[]`. With an empty list the `mixed`
|
||||
* refusal below cannot fire, so a call that forgot the argument would return the
|
||||
* group form and restore the exact behaviour this function exists to prevent —
|
||||
* silently, because the first CLI pass is correct either way and only a later
|
||||
* Studio re-analysis is wrong. A missing argument throws on `members.filter`
|
||||
* instead.
|
||||
*
|
||||
* Two refusals, and both exist because the group form resolves LATER and WIDER
|
||||
* than the analysis: `resolveCarveSourceIds` expands a group id to every current
|
||||
* member on every analysis, and `resolveCarveVoices` keeps any audio member with
|
||||
* a src. `main()` meanwhile sums the voice list `detectTracks` returned, so the
|
||||
* first pass looks correct however wrong the persisted attribute is.
|
||||
*
|
||||
* `bed` — the bed is a member, so it would be handed to itself as a voice
|
||||
* and carved against its own content.
|
||||
* `mixed` — a member classified music or sfx is not a voice this run measured,
|
||||
* so it would enter the sidechain on the next analysis and duck the
|
||||
* bed under a whoosh.
|
||||
*
|
||||
* Deliberately NOT a refusal: a member classified `voice` or `unknown` that this
|
||||
* run left out. That is the group form working as designed — `detectTracks` only
|
||||
* takes voices that overlap the bed, and picking up a clip that starts playing
|
||||
* later without an edit to `sources` is the whole reason SKILL.md says to name
|
||||
* the group. Refusing there would collapse the group form into clip ids for
|
||||
* every ordinary narration sequence.
|
||||
*/
|
||||
export function groupSourceRefusal(voices, bed, members) {
|
||||
const group = sharedVoiceGroup(voices);
|
||||
if (!group) return null;
|
||||
if (bed?.kind === "audio" && attrOf(bed.tag, "data-audio-group") === group) {
|
||||
return { group, reason: "bed", ids: [bed.id] };
|
||||
}
|
||||
const analysed = new Set(voices.map((v) => v.id));
|
||||
const strays = members
|
||||
.filter(
|
||||
(m) =>
|
||||
m.group === group &&
|
||||
!analysed.has(m.id) &&
|
||||
(m.nameKind === "music" || m.nameKind === "sfx"),
|
||||
)
|
||||
.map((m) => m.id);
|
||||
return strays.length > 0 ? { group, reason: "mixed", ids: strays } : null;
|
||||
}
|
||||
|
||||
/** Mono float PCM for one media file, via ffmpeg. */
|
||||
function decode(path) {
|
||||
let raw;
|
||||
try {
|
||||
raw = execFileSync(
|
||||
"ffmpeg",
|
||||
[
|
||||
"-v",
|
||||
"error",
|
||||
"-i",
|
||||
path,
|
||||
"-vn",
|
||||
"-ac",
|
||||
"1",
|
||||
"-ar",
|
||||
String(SAMPLE_RATE),
|
||||
"-f",
|
||||
"f32le",
|
||||
"-",
|
||||
],
|
||||
{ maxBuffer: 1 << 30 },
|
||||
);
|
||||
} catch (error) {
|
||||
fail(`could not decode ${path}\n ${error.message.split("\n")[0]}`);
|
||||
}
|
||||
if (raw.length === 0) fail(`no audio in ${path}`);
|
||||
return new Float32Array(raw.buffer, raw.byteOffset, raw.length / 4);
|
||||
}
|
||||
|
||||
const attrOf = (tag, name) => tag.match(new RegExp(`\\s${name}="([^"]*)"`, "i"))?.[1] ?? null;
|
||||
|
||||
const unescapeAttr = (value) =>
|
||||
value
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, "&");
|
||||
|
||||
const escapeAttr = (value) => value.replace(/&/g, "&").replace(/"/g, """);
|
||||
|
||||
/** Every media element with a src, as {id, tag, kind}. */
|
||||
function mediaElements(html) {
|
||||
const found = [];
|
||||
for (const match of html.matchAll(/<(audio|video)\b[^>]*>/gi)) {
|
||||
const tag = match[0];
|
||||
// `\sid=` and not `id=`: `data-hf-id` would match first.
|
||||
const id = tag.match(/\sid="([^"]+)"/)?.[1];
|
||||
if (id && attrOf(tag, "src")) found.push({ id, tag, kind: match[1].toLowerCase() });
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/**
|
||||
* Work out which track is the bed and which tracks are its voices.
|
||||
*
|
||||
* Names first, because they are what the author already told us and the answer is
|
||||
* explainable: a track whose id or filename looks like music is the bed, ones that
|
||||
* look like speech are voices, SFX-shaped names are neither. `classifyAudioName`
|
||||
* comes from core so Studio's own picker and this cannot disagree.
|
||||
*
|
||||
* EVERY voice over the bed, not one of them. A bed usually runs under a whole
|
||||
* sequence, and they are analysed together — so there is nothing to disambiguate,
|
||||
* which is why this no longer refuses when several tracks look like speech.
|
||||
*
|
||||
* Only tracks that actually play while the bed does: one somewhere else on the
|
||||
* timeline cannot mask it. It still refuses when it cannot find a bed at all, or
|
||||
* finds no voice to make room for.
|
||||
*/
|
||||
function detectTracks(html, given, classify, overlaps) {
|
||||
const all = mediaElements(html);
|
||||
const kindOf = (el) => classify(el.id, unescapeAttr(attrOf(el.tag, "src") ?? ""));
|
||||
const spanOf = (el) => {
|
||||
const raw = attrOf(el.tag, "data-duration");
|
||||
const n = raw === null ? Number.NaN : Number(raw);
|
||||
return {
|
||||
start: startOf(el.tag),
|
||||
duration: Number.isFinite(n) ? n : null,
|
||||
};
|
||||
};
|
||||
const pick = (id, what) => {
|
||||
const found = all.find((el) => el.id === id);
|
||||
if (!found) fail(`no <audio>/<video> with id="${id}" in the composition`);
|
||||
return { ...found, why: `--${what}` };
|
||||
};
|
||||
|
||||
let bed = given.bed ? pick(given.bed, "bed") : null;
|
||||
const named = given.voices.map((id) => pick(id, "voice"));
|
||||
|
||||
if (!bed) {
|
||||
const others = all.filter((el) => !named.some((v) => v.id === el.id));
|
||||
const music = others.filter((el) => kindOf(el) === "music");
|
||||
if (music.length === 1) bed = { ...music[0], why: "name looks like music" };
|
||||
else if (music.length > 1) {
|
||||
fail(
|
||||
`several tracks look like music (${music.map((el) => el.id).join(", ")}) — name one with --bed`,
|
||||
);
|
||||
} else if (others.length === 1) {
|
||||
bed = { ...others[0], why: "only track left" };
|
||||
} else {
|
||||
fail(
|
||||
`cannot tell which track is the music bed\n` +
|
||||
` media in the composition: ${all.map((el) => el.id).join(", ") || "none"}\n` +
|
||||
` name it with --bed`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const bedSpan = spanOf(bed);
|
||||
const overlapping = (el) => overlaps(bedSpan, spanOf(el));
|
||||
const plausible = all
|
||||
.filter((el) => el.id !== bed.id && kindOf(el) !== "music" && kindOf(el) !== "sfx")
|
||||
.filter(overlapping);
|
||||
// A voiceover is normally its own <audio>. Video counts only when no audio track
|
||||
// is left to be the voice — a talking-head recut — because otherwise every B-roll
|
||||
// clip in the composition reads as somebody talking.
|
||||
const spoken = plausible.filter((el) => el.kind === "audio");
|
||||
const pool = spoken.length > 0 ? spoken : plausible;
|
||||
const voices = named.length
|
||||
? named
|
||||
: pool.map((el) => ({
|
||||
...el,
|
||||
why: kindOf(el) === "voice" ? "name looks like a voice" : "plays over the bed",
|
||||
}));
|
||||
|
||||
const usable = voices.filter((el) => attrOf(el.tag, "src"));
|
||||
if (usable.length === 0) {
|
||||
fail(
|
||||
`no voice to make room for on ${bed.id}\n` +
|
||||
` media in the composition: ${all.map((el) => el.id).join(", ") || "none"}\n` +
|
||||
` name one with --voice`,
|
||||
);
|
||||
}
|
||||
return { bed, voices: usable, all };
|
||||
}
|
||||
|
||||
const startOf = (tag) => {
|
||||
const raw = Number(attrOf(tag, "data-start"));
|
||||
return Number.isFinite(raw) ? raw : 0;
|
||||
};
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const compPath = resolve(args.comp);
|
||||
const compDir = dirname(compPath);
|
||||
const { carve: carveApi, fx: fxApi } = await loadCore(args.core ? resolve(args.core) : compDir);
|
||||
|
||||
const html = readFileSync(compPath, "utf-8");
|
||||
const {
|
||||
bed: bedEl,
|
||||
voices,
|
||||
all: media,
|
||||
} = detectTracks(html, args, carveApi.classifyAudioName, carveApi.clipsOverlap);
|
||||
// Group membership + name classification for every audio track, so the source
|
||||
// decision can see what the group will resolve to later and not just what this
|
||||
// run analysed.
|
||||
const members = media
|
||||
.filter((el) => el.kind === "audio")
|
||||
.map((el) => ({
|
||||
id: el.id,
|
||||
group: attrOf(el.tag, "data-audio-group"),
|
||||
nameKind: carveApi.classifyAudioName(el.id, unescapeAttr(attrOf(el.tag, "src") ?? "")),
|
||||
}));
|
||||
const bedTag = bedEl.tag;
|
||||
const bedSrc = attrOf(bedTag, "src");
|
||||
process.stdout.write(
|
||||
`bed ${bedEl.id} (${bedEl.why})\n` +
|
||||
voices.map((v) => `voice ${v.id} (${v.why})`).join("\n") +
|
||||
"\n",
|
||||
);
|
||||
|
||||
const profile = carveApi.carveProfile(args.strength);
|
||||
// Every voice summed onto the BED's clock before anything is measured. One
|
||||
// question — where and when is speech masking this bed — with one answer, even
|
||||
// when the answer comes from several people at different times.
|
||||
const voice = carveApi.mixCarveSources(
|
||||
voices.map((v) => ({
|
||||
samples: decode(resolve(compDir, unescapeAttr(attrOf(v.tag, "src")))),
|
||||
offsetSeconds: startOf(v.tag) - startOf(bedTag),
|
||||
})),
|
||||
SAMPLE_RATE,
|
||||
);
|
||||
if (voice.length === 0) fail("the voices do not overlap the bed, so there is nothing to carve");
|
||||
const bands = carveApi.analyseCarveBands(voice, SAMPLE_RATE, profile);
|
||||
|
||||
// The level half of the carve needs both sides: "how far over the speech is this
|
||||
// bed" cannot be answered by listening to one of them. No offset — the mix is
|
||||
// already on the bed's clock.
|
||||
const bed = profile.duckDb > 0 ? decode(resolve(compDir, unescapeAttr(bedSrc))) : null;
|
||||
const duck = bed ? carveApi.analyseCarveDuck(voice, bed, SAMPLE_RATE, profile, 0) : [];
|
||||
|
||||
// Anything the author built by hand survives a carve; only the previous
|
||||
// carve's own nodes are replaced. That is what `fromCarve` is for.
|
||||
const existingChain = attrOf(bedTag, "data-fx-chain");
|
||||
const existingNodes = existingChain
|
||||
? fxApi.parseAudioFxChain(unescapeAttr(existingChain)).nodes
|
||||
: [];
|
||||
const kept = existingNodes.filter((n) => !n.fromCarve);
|
||||
// Lanes belonging to the carve being replaced, addressed by the ids the OLD
|
||||
// nodes had. Taken before anything is minted: those ids are freed by the
|
||||
// replacement and a new node can be handed one of them, so reading them off the
|
||||
// new chain would keep exactly the stale lanes it is supposed to drop.
|
||||
const stalePrefixes = existingNodes.filter((n) => n.fromCarve && n.id).map((n) => `fx.${n.id}.`);
|
||||
|
||||
let claimed = { version: 1, nodes: kept };
|
||||
const mint = (node) => {
|
||||
const withId = { ...node, id: fxApi.mintAudioFxNodeId(claimed), fromCarve: true };
|
||||
claimed = { version: 1, nodes: [...claimed.nodes, withId] };
|
||||
return withId;
|
||||
};
|
||||
const bandNodes = bands.map((band) => mint(carveApi.carveBandsToChain([band]).nodes[0]));
|
||||
|
||||
const duckNode =
|
||||
duck.length > 0
|
||||
? mint({
|
||||
type: "gain",
|
||||
enabled: true,
|
||||
params: {
|
||||
...fxApi.defaultAudioFxParams("gain"),
|
||||
gain: 0,
|
||||
},
|
||||
})
|
||||
: null;
|
||||
const chain = {
|
||||
version: 1,
|
||||
nodes: [...bandNodes, ...(duckNode ? [duckNode] : []), ...kept],
|
||||
};
|
||||
|
||||
/**
|
||||
* One carve envelope as a lane on the BED's clock.
|
||||
*
|
||||
* Nothing to shift: the voices were summed onto that clock before the analysis
|
||||
* ran. A lane does hold its first value backwards to the start of its clip, so an
|
||||
* envelope that begins later needs an explicit "no cut" at zero or the bed starts
|
||||
* out ducked.
|
||||
*/
|
||||
const laneFor = (id, points) => {
|
||||
const timed = points
|
||||
.map((p) => ({ t: Number(p.t.toFixed(3)), v: p.v }))
|
||||
.filter((p) => p.t >= 0);
|
||||
if ((timed[0]?.t ?? 0) > 0) timed.unshift({ t: 0, v: 0 });
|
||||
return timed.length > 1 ? [{ target: `fx.${id}.gain`, points: timed }] : [];
|
||||
};
|
||||
|
||||
// Every carve follows the speech: a fixed depth thins the bed through every pause.
|
||||
const carvedLanes = [
|
||||
...carveApi
|
||||
.analyseCarveDynamics(voice, SAMPLE_RATE, bands)
|
||||
.flatMap((dyn, i) => (bandNodes[i]?.id ? laneFor(bandNodes[i].id, dyn.points) : [])),
|
||||
...(duckNode?.id && duck.length > 0 ? laneFor(duckNode.id, duck) : []),
|
||||
];
|
||||
|
||||
// Hand-drawn lanes are kept the same way hand-built nodes are: by dropping only
|
||||
// the ones that addressed the previous carve's nodes.
|
||||
const existingAutomation = attrOf(bedTag, "data-automation");
|
||||
const carriedLanes = existingAutomation
|
||||
? (JSON.parse(unescapeAttr(existingAutomation)).lanes ?? []).filter(
|
||||
(lane) => !stalePrefixes.some((prefix) => String(lane.target).startsWith(prefix)),
|
||||
)
|
||||
: [];
|
||||
const lanes = [...carriedLanes, ...carvedLanes];
|
||||
|
||||
const settings = {
|
||||
enabled: true,
|
||||
sources: carveSources(voices, bedEl, members),
|
||||
strength: args.strength,
|
||||
};
|
||||
// Say why the group form was declined, or the lint rule tells the author to
|
||||
// group clips they have already grouped.
|
||||
const refusal = groupSourceRefusal(voices, bedEl, members);
|
||||
if (refusal) {
|
||||
process.stderr.write(
|
||||
refusal.reason === "bed"
|
||||
? `note bed ${bedEl.id} is in group "${refusal.group}" with the voices, so\n` +
|
||||
` sources are clip ids: naming that group would carve the bed\n` +
|
||||
` against itself on the next analysis. Move the bed to its own group.\n`
|
||||
: `note group "${refusal.group}" also holds ${refusal.ids.join(", ")}, which this run\n` +
|
||||
` did not analyse (music/sfx by name), so sources are clip ids: naming\n` +
|
||||
` the group would pull them into the sidechain on the next analysis.\n` +
|
||||
` Move them out of the voice group.\n`,
|
||||
);
|
||||
}
|
||||
const written =
|
||||
` data-fx-carve="${escapeAttr(JSON.stringify(settings))}"` +
|
||||
` data-fx-chain="${escapeAttr(fxApi.serializeAudioFxChain(chain))}"` +
|
||||
(lanes.length > 0
|
||||
? ` data-automation="${escapeAttr(JSON.stringify({ version: 1, lanes }))}"`
|
||||
: "");
|
||||
|
||||
process.stdout.write(
|
||||
`carve strength ${args.strength}, ${voices.length} voice${voices.length === 1 ? "" : "s"}\n` +
|
||||
`bands ${bands.map((b) => `${b.freq}Hz ${b.gainDb}dB q${b.q}`).join(", ")}\n` +
|
||||
`level ${
|
||||
duckNode
|
||||
? `${duck.length}-point envelope, floor ${Math.min(...duck.map((p) => p.v))} dB`
|
||||
: "no level match at this strength"
|
||||
}\n` +
|
||||
`lanes ${carvedLanes.length} carve${carriedLanes.length ? ` + ${carriedLanes.length} kept` : ""}\n`,
|
||||
);
|
||||
if (args.dryRun) {
|
||||
process.stdout.write("dry run: nothing written\n");
|
||||
return;
|
||||
}
|
||||
|
||||
let stripped = bedTag;
|
||||
for (const attr of ["data-fx-carve", "data-fx-chain", "data-automation"]) {
|
||||
stripped = stripped.replace(new RegExp(`\\s${attr}="[^"]*"`, "i"), "");
|
||||
}
|
||||
// Inserted before the tag's own closing ">", which is the only place they can
|
||||
// go: `stripped` is the opening tag alone, so appending would land outside it.
|
||||
const nextTag = stripped.replace(/\/?>$/, (close) => `${written}${close}`);
|
||||
if (nextTag === stripped) fail("attribute write produced no change — refusing to save");
|
||||
writeFileSync(compPath, html.replace(bedTag, nextTag));
|
||||
process.stdout.write(`wrote ${args.comp} (id="${bedEl.id}")\n`);
|
||||
}
|
||||
|
||||
// Only run as a CLI. Guarded so the pure helpers above can be unit-tested by
|
||||
// importing this module (`skills/**/*.test.mjs`, run by `bun run test:skills`).
|
||||
if (isMainModule(import.meta.url)) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
DEFAULT_STRENGTH,
|
||||
carveSources,
|
||||
groupSourceRefusal,
|
||||
loadCore,
|
||||
parseArgs,
|
||||
} from "./carve.mjs";
|
||||
|
||||
const SCRIPTS_DIR = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
/** A voice as `detectTracks` yields it: an id plus its raw opening tag. */
|
||||
function voice(id, group) {
|
||||
const attr = group === undefined ? "" : ` data-audio-group="${group}"`;
|
||||
return { id, tag: `<audio id="${id}"${attr} src="${id}.wav"></audio>` };
|
||||
}
|
||||
|
||||
/** An audio member as `main` describes it for the source decision. */
|
||||
function member(id, group, nameKind) {
|
||||
return { id, group, nameKind };
|
||||
}
|
||||
|
||||
/** A bed as `mediaElements` yields it: `kind` is what decides group membership. */
|
||||
function bed(id, group, kind = "audio") {
|
||||
const attr = group === undefined ? "" : ` data-audio-group="${group}"`;
|
||||
return { id, kind, tag: `<${kind} id="${id}"${attr} src="${id}.mp3"></${kind}>` };
|
||||
}
|
||||
|
||||
async function inTempDir(run) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "carve-test-"));
|
||||
try {
|
||||
return await run(dir);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test("voices sharing one group carve against the group, not their ids", () => {
|
||||
// SKILL.md's invariant: "A carve against more than one clip id is wrong.
|
||||
// Group the clips and carve against the group." Naming the group lets
|
||||
// membership resolve at analysis time, so a voice added later is covered.
|
||||
const voices = [voice("vo1", "voiceover"), voice("vo2", "voiceover"), voice("vo3", "voiceover")];
|
||||
assert.deepEqual(carveSources(voices, undefined, []), ["voiceover"]);
|
||||
});
|
||||
|
||||
test("a single grouped voice still records the group", () => {
|
||||
assert.deepEqual(carveSources([voice("vo1", "voiceover")], undefined, []), ["voiceover"]);
|
||||
});
|
||||
|
||||
test("ungrouped voices keep their ids, so the lint rule can still say so", () => {
|
||||
// Not silently inventing a group: `audio_carve_ungrouped_sources` is the
|
||||
// right signal here, and it needs the ids to fire on.
|
||||
assert.deepEqual(carveSources([voice("vo1"), voice("vo2")], undefined, []), ["vo1", "vo2"]);
|
||||
});
|
||||
|
||||
test("voices in DIFFERENT groups keep their ids", () => {
|
||||
// One carve cannot name two groups, and picking either would silently drop
|
||||
// the other's members from the analysis.
|
||||
const voices = [voice("vo1", "narration"), voice("vo2", "interview")];
|
||||
assert.deepEqual(carveSources(voices, undefined, []), ["vo1", "vo2"]);
|
||||
});
|
||||
|
||||
test("a partially grouped set keeps its ids", () => {
|
||||
const voices = [voice("vo1", "voiceover"), voice("vo2")];
|
||||
assert.deepEqual(carveSources(voices, undefined, []), ["vo1", "vo2"]);
|
||||
});
|
||||
|
||||
test("an empty group attribute is not a group", () => {
|
||||
assert.deepEqual(carveSources([voice("vo1", ""), voice("vo2", "")], undefined, []), [
|
||||
"vo1",
|
||||
"vo2",
|
||||
]);
|
||||
});
|
||||
|
||||
// The nine cases above pass `[]` explicitly because they predate the membership
|
||||
// check and are about the bed and the group attributes alone. `members` is a
|
||||
// required argument: with `[]` the `mixed` refusal cannot fire, so a default
|
||||
// would let a call that forgot it return the group form and silently undo the
|
||||
// widening fix — see the wiring test at the bottom of this file.
|
||||
|
||||
test("a bed inside the voices' group keeps clip ids, so it is never its own source", () => {
|
||||
// `resolveCarveSourceIds` expands a group to every CURRENT member, and it gets
|
||||
// no host element to exclude — so naming a group the bed belongs to puts the
|
||||
// bed in its own voice list on the next analysis, and it is carved against
|
||||
// itself. This run cannot see it (main sums the detected voices directly), so
|
||||
// the check has to happen here.
|
||||
const voices = [voice("vo1", "mix"), voice("vo2", "mix")];
|
||||
assert.deepEqual(carveSources(voices, bed("bgm", "mix"), []), ["vo1", "vo2"]);
|
||||
});
|
||||
|
||||
test("a bed in a DIFFERENT group leaves the group form alone", () => {
|
||||
const voices = [voice("vo1", "voiceover"), voice("vo2", "voiceover")];
|
||||
assert.deepEqual(carveSources(voices, bed("bgm", "music"), []), ["voiceover"]);
|
||||
});
|
||||
|
||||
test("an ungrouped bed leaves the group form alone", () => {
|
||||
assert.deepEqual(carveSources([voice("vo1", "voiceover")], bed("bgm"), []), ["voiceover"]);
|
||||
});
|
||||
|
||||
test("a VIDEO bed is immune — group membership is audio-only", () => {
|
||||
// `data-audio-group` on a <video> is ignored by core, so expanding the group
|
||||
// can never pull this bed in and declining would be a false positive.
|
||||
const voices = [voice("vo1", "mix"), voice("vo2", "mix")];
|
||||
assert.deepEqual(carveSources(voices, bed("clip", "mix", "video"), []), ["mix"]);
|
||||
});
|
||||
|
||||
test("an sfx member of the voices' group blocks the group form", () => {
|
||||
// The group resolves wider than the analysis: `resolveCarveSourceIds` expands
|
||||
// it to every current member and `resolveCarveVoices` keeps any audio with a
|
||||
// src, so an sfx clip sharing the voice group enters the sidechain on the next
|
||||
// analysis and ducks the bed under a whoosh. This run cannot see it — it sums
|
||||
// the voices `detectTracks` returned, which correctly excluded the sfx.
|
||||
const voices = [voice("vo1", "voiceover"), voice("vo2", "voiceover")];
|
||||
const members = [
|
||||
member("vo1", "voiceover", "voice"),
|
||||
member("vo2", "voiceover", "voice"),
|
||||
member("whoosh", "voiceover", "sfx"),
|
||||
member("bgm", "music", "music"),
|
||||
];
|
||||
assert.deepEqual(carveSources(voices, bed("bgm", "music"), members), ["vo1", "vo2"]);
|
||||
});
|
||||
|
||||
test("a music member of the voices' group blocks it too", () => {
|
||||
const voices = [voice("vo1", "voiceover")];
|
||||
const members = [member("vo1", "voiceover", "voice"), member("pad", "voiceover", "music")];
|
||||
assert.deepEqual(carveSources(voices, bed("bgm", "music"), members), ["vo1"]);
|
||||
});
|
||||
|
||||
test("a voice member this run did NOT analyse keeps the group form", () => {
|
||||
// The designed case, and the one a membership check must not break: a voice
|
||||
// that does not overlap the bed is left out of the analysis on purpose, and
|
||||
// covering it on a later analysis without editing `sources` is the entire
|
||||
// reason SKILL.md says to name the group. Refusing here would collapse the
|
||||
// group form into clip ids for every ordinary narration sequence.
|
||||
const voices = [voice("vo1", "voiceover"), voice("vo2", "voiceover")];
|
||||
const members = [
|
||||
member("vo1", "voiceover", "voice"),
|
||||
member("vo2", "voiceover", "voice"),
|
||||
member("vo-outro", "voiceover", "voice"),
|
||||
];
|
||||
assert.deepEqual(carveSources(voices, bed("bgm", "music"), members), ["voiceover"]);
|
||||
});
|
||||
|
||||
test("an unclassifiable member keeps the group form", () => {
|
||||
// `unknown` is what `detectTracks` itself treats as a possible voice, so it is
|
||||
// not evidence of a non-voice member — loose in the safe direction, same as
|
||||
// detection.
|
||||
const voices = [voice("vo1", "voiceover")];
|
||||
const members = [member("vo1", "voiceover", "voice"), member("track7", "voiceover", "unknown")];
|
||||
assert.deepEqual(carveSources(voices, bed("bgm", "music"), members), ["voiceover"]);
|
||||
});
|
||||
|
||||
test("an sfx member of a DIFFERENT group is irrelevant", () => {
|
||||
const voices = [voice("vo1", "voiceover")];
|
||||
const members = [member("vo1", "voiceover", "voice"), member("whoosh", "sfx", "sfx")];
|
||||
assert.deepEqual(carveSources(voices, bed("bgm", "music"), members), ["voiceover"]);
|
||||
});
|
||||
|
||||
test("groupSourceRefusal names which member blocked the group, and why", () => {
|
||||
// The stderr note is built from this, so it has to carry the ids: "sources are
|
||||
// clip ids" plus `audio_carve_ungrouped_sources` reads as nonsense to an author
|
||||
// who did group their clips.
|
||||
const voices = [voice("vo1", "voiceover"), voice("vo2", "voiceover")];
|
||||
assert.deepEqual(
|
||||
groupSourceRefusal(voices, bed("bgm", "music"), [member("whoosh", "voiceover", "sfx")]),
|
||||
{ group: "voiceover", reason: "mixed", ids: ["whoosh"] },
|
||||
);
|
||||
assert.deepEqual(groupSourceRefusal(voices, bed("bgm", "voiceover"), []), {
|
||||
group: "voiceover",
|
||||
reason: "bed",
|
||||
ids: ["bgm"],
|
||||
});
|
||||
assert.equal(groupSourceRefusal(voices, bed("bgm", "music"), []), null);
|
||||
});
|
||||
|
||||
test("the CLI still runs when invoked through a symlinked path", () => {
|
||||
// argv[1] keeps the invoked spelling while import.meta.url is the realpath, so
|
||||
// a raw compare in the entry guard skips main() and the CLI exits 0 having
|
||||
// written nothing. macOS /tmp -> /private/tmp reaches this with no symlink of
|
||||
// one's own; so does any skill install placed behind a link.
|
||||
return inTempDir((dir) => {
|
||||
const link = join(dir, "scripts-link");
|
||||
symlinkSync(SCRIPTS_DIR, link, "junction");
|
||||
let status = 0;
|
||||
let stderr = "";
|
||||
try {
|
||||
execFileSync(process.execPath, [join(link, "carve.mjs")], { encoding: "utf-8" });
|
||||
} catch (error) {
|
||||
status = error.status;
|
||||
stderr = String(error.stderr);
|
||||
}
|
||||
// Reaching parseArgs is the proof main() ran at all: no args is an error, and
|
||||
// the broken guard's symptom is a silent exit 0 with no output.
|
||||
assert.equal(status, 1, `expected the usage error, got status ${status}`);
|
||||
assert.match(stderr, /--comp is required/);
|
||||
});
|
||||
});
|
||||
|
||||
test("loadCore honours an import-only export map, as the published core has", () => {
|
||||
// The published manifest carries only `import` + `types` for these subpaths, so
|
||||
// `require.resolve` cannot resolve them at all and the fallback has to read the
|
||||
// export map itself. A fixture pins that without depending on npm.
|
||||
return inTempDir(async (dir) => {
|
||||
const pkgDir = join(dir, "node_modules", "@hyperframes", "core");
|
||||
mkdirSync(join(pkgDir, "dist"), { recursive: true });
|
||||
writeFileSync(join(dir, "package.json"), JSON.stringify({ name: "fixture-project" }));
|
||||
writeFileSync(
|
||||
join(pkgDir, "package.json"),
|
||||
JSON.stringify({
|
||||
name: "@hyperframes/core",
|
||||
version: "0.0.0-fixture",
|
||||
type: "module",
|
||||
exports: {
|
||||
"./package.json": "./package.json",
|
||||
"./audio-carve": { import: "./dist/audioCarve.js" },
|
||||
"./audio-fx": { import: "./dist/audioFx.js" },
|
||||
},
|
||||
}),
|
||||
);
|
||||
writeFileSync(join(pkgDir, "dist", "audioCarve.js"), 'export const marker = "carve";\n');
|
||||
writeFileSync(join(pkgDir, "dist", "audioFx.js"), 'export const marker = "fx";\n');
|
||||
|
||||
const core = await loadCore(dir);
|
||||
assert.equal(core.carve.marker, "carve");
|
||||
assert.equal(core.fx.marker, "fx");
|
||||
});
|
||||
});
|
||||
|
||||
test("members is required, so dropping it cannot silently restore the group form", () => {
|
||||
// The gap this closes: `main()` is the only code that BUILDS `members`, and no
|
||||
// test runs `main()` (the symlink test stops at the usage error, a real run
|
||||
// needs ffmpeg). With `members = []` defaulted, a refactor that dropped the
|
||||
// third argument would return the group form again with the whole suite green
|
||||
// — the same signature as the bug itself: first pass correct, persisted
|
||||
// attribute wrong, nothing red. Omitting it now throws instead.
|
||||
const voices = [voice("vo1", "voiceover"), voice("vo2", "voiceover")];
|
||||
assert.throws(() => carveSources(voices, bed("bgm", "music")), TypeError);
|
||||
assert.throws(() => groupSourceRefusal(voices, bed("bgm", "music")), TypeError);
|
||||
});
|
||||
|
||||
test("the default strength is 0.8, and --strength still overrides it", () => {
|
||||
// 0.25 left the bed audibly fighting the voice on narrated builds; a carve at
|
||||
// the default has to already be a finished mix. Pinned so it cannot drift back
|
||||
// unnoticed — core pins its own DEFAULT_CARVE the same way.
|
||||
assert.equal(DEFAULT_STRENGTH, 0.8);
|
||||
assert.equal(parseArgs(["--comp", "x.html"]).strength, 0.8);
|
||||
assert.equal(parseArgs(["--comp", "x.html", "--strength", "0.25"]).strength, 0.25);
|
||||
});
|
||||
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Extract per-frame audio visualization data from an audio or video file.
|
||||
|
||||
Outputs JSON with RMS amplitude and frequency band data at the target FPS,
|
||||
ready to embed in a HyperFrames composition.
|
||||
|
||||
Usage:
|
||||
python extract-audio-data.py input.mp3 -o audio-data.json
|
||||
python extract-audio-data.py input.mp4 --fps 30 --bands 16 -o audio-data.json
|
||||
|
||||
Requirements:
|
||||
- Python 3.9+
|
||||
- ffmpeg (for decoding audio)
|
||||
- numpy (pip install numpy)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
|
||||
# Windows sizes stdio to the ANSI code page (cp1252). These scripts emit UTF-8 on
|
||||
# every platform; say so rather than depending on the console's code page. Carry
|
||||
# `errors` across: reconfigure() resets it to "strict", and CPython deliberately gives
|
||||
# stderr "backslashreplace" so the diagnostic path can never itself raise.
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(_stream, "reconfigure"):
|
||||
_stream.reconfigure(encoding="utf-8", errors=_stream.errors)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FFT parameters
|
||||
#
|
||||
# A 4096-sample window gives ~10.8 Hz per bin at 44100Hz — enough to resolve
|
||||
# low-frequency bands cleanly. The per-frame audio slice (44100/30 = 1470
|
||||
# samples at 30fps) is too small and causes low bands to map to the same bins.
|
||||
#
|
||||
# Frequency range 30Hz–16kHz covers the useful range for music. Below 30Hz is
|
||||
# sub-bass most speakers can't reproduce; above 16kHz is noise/harmonics that
|
||||
# don't contribute to perceived rhythm or melody.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SAMPLE_RATE = 44100
|
||||
FFT_SIZE = 4096
|
||||
MIN_FREQ = 30.0
|
||||
MAX_FREQ = 16000.0
|
||||
|
||||
|
||||
def decode_audio(path: str) -> np.ndarray:
|
||||
"""Decode audio to mono float32 samples via ffmpeg."""
|
||||
cmd = [
|
||||
"ffmpeg", "-i", path,
|
||||
"-vn", "-ac", "1", "-ar", str(SAMPLE_RATE),
|
||||
"-f", "s16le", "-acodec", "pcm_s16le",
|
||||
"-loglevel", "error",
|
||||
"pipe:1",
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True)
|
||||
if result.returncode != 0:
|
||||
# Decoding ffmpeg's diagnostics strictly makes the reporter the thing that
|
||||
# crashes: a Windows ffmpeg emits cp1252 bytes, and UnicodeDecodeError here
|
||||
# would bury the actual failure it was trying to report.
|
||||
print(f"ffmpeg error: {result.stderr.decode('utf-8', errors='replace')}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return np.frombuffer(result.stdout, dtype=np.int16).astype(np.float32) / 32768.0
|
||||
|
||||
|
||||
def compute_band_edges(n_bands: int) -> np.ndarray:
|
||||
"""Logarithmically-spaced frequency band edges from MIN_FREQ to MAX_FREQ."""
|
||||
return np.array([
|
||||
MIN_FREQ * (MAX_FREQ / MIN_FREQ) ** (i / n_bands)
|
||||
for i in range(n_bands + 1)
|
||||
])
|
||||
|
||||
|
||||
def compute_fft_bands(
|
||||
windowed: np.ndarray, freq_per_bin: float, n_bins: int,
|
||||
band_edges: np.ndarray, n_bands: int,
|
||||
) -> np.ndarray:
|
||||
"""Compute peak magnitude in logarithmically-spaced frequency bands."""
|
||||
magnitudes = np.abs(np.fft.rfft(windowed))
|
||||
|
||||
bands = np.zeros(n_bands)
|
||||
for b in range(n_bands):
|
||||
low_bin = max(0, int(band_edges[b] / freq_per_bin))
|
||||
high_bin = min(n_bins, int(band_edges[b + 1] / freq_per_bin))
|
||||
if high_bin <= low_bin:
|
||||
high_bin = low_bin + 1
|
||||
# Clamp to valid range to avoid empty slices
|
||||
low_bin = min(low_bin, n_bins - 1)
|
||||
high_bin = min(high_bin, n_bins)
|
||||
bands[b] = np.max(magnitudes[low_bin:high_bin])
|
||||
|
||||
return bands
|
||||
|
||||
|
||||
def extract(path: str, fps: int, n_bands: int) -> dict:
|
||||
"""Extract per-frame audio data."""
|
||||
print(f"Decoding audio from {path}...", file=sys.stderr)
|
||||
samples = decode_audio(path)
|
||||
duration = len(samples) / SAMPLE_RATE
|
||||
frame_step = SAMPLE_RATE // fps
|
||||
total_frames = int(duration * fps)
|
||||
|
||||
print(f"Duration: {duration:.1f}s, {total_frames} frames at {fps}fps", file=sys.stderr)
|
||||
print(f"FFT window: {FFT_SIZE} samples ({SAMPLE_RATE / FFT_SIZE:.1f} Hz/bin)", file=sys.stderr)
|
||||
print(f"Frequency range: {MIN_FREQ:.0f}-{MAX_FREQ:.0f} Hz, {n_bands} bands", file=sys.stderr)
|
||||
|
||||
# Precompute constants
|
||||
hann = np.hanning(FFT_SIZE)
|
||||
band_edges = compute_band_edges(n_bands)
|
||||
freq_per_bin = SAMPLE_RATE / FFT_SIZE
|
||||
n_bins = FFT_SIZE // 2 + 1
|
||||
half_fft = FFT_SIZE // 2
|
||||
|
||||
# Pass 1: extract raw values
|
||||
rms_values = np.zeros(total_frames)
|
||||
band_values = np.zeros((total_frames, n_bands))
|
||||
|
||||
for f in range(total_frames):
|
||||
# RMS from the frame's audio slice
|
||||
rms_start = f * frame_step
|
||||
rms_end = rms_start + frame_step
|
||||
frame_slice = samples[rms_start:min(rms_end, len(samples))]
|
||||
if len(frame_slice) > 0:
|
||||
rms_values[f] = np.sqrt(np.mean(frame_slice ** 2))
|
||||
|
||||
# FFT from a centered 4096-sample window
|
||||
center = rms_start + frame_step // 2
|
||||
win_start = center - half_fft
|
||||
win_end = center + half_fft
|
||||
|
||||
if win_start >= 0 and win_end <= len(samples):
|
||||
window = samples[win_start:win_end] * hann
|
||||
else:
|
||||
# Zero-pad at edges
|
||||
padded = np.zeros(FFT_SIZE)
|
||||
src_start = max(0, win_start)
|
||||
src_end = min(len(samples), win_end)
|
||||
dst_start = src_start - win_start
|
||||
dst_end = dst_start + (src_end - src_start)
|
||||
padded[dst_start:dst_end] = samples[src_start:src_end]
|
||||
window = padded * hann
|
||||
|
||||
band_values[f] = compute_fft_bands(window, freq_per_bin, n_bins, band_edges, n_bands)
|
||||
|
||||
# Pass 2: normalize
|
||||
peak_rms = rms_values.max() if total_frames > 0 else 1.0
|
||||
if peak_rms > 0:
|
||||
rms_values /= peak_rms
|
||||
|
||||
# Per-band normalization so treble is visible alongside louder bass
|
||||
band_peaks = band_values.max(axis=0)
|
||||
band_peaks[band_peaks == 0] = 1.0
|
||||
band_values /= band_peaks
|
||||
|
||||
# Build output
|
||||
frames = []
|
||||
for f in range(total_frames):
|
||||
frames.append({
|
||||
"time": round(f / fps, 4),
|
||||
"rms": round(float(rms_values[f]), 4),
|
||||
"bands": [round(float(b), 4) for b in band_values[f]],
|
||||
})
|
||||
|
||||
return {
|
||||
"duration": round(duration, 4),
|
||||
"fps": fps,
|
||||
"bands": n_bands,
|
||||
"totalFrames": total_frames,
|
||||
"frames": frames,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Extract per-frame audio visualization data")
|
||||
parser.add_argument("input", help="Audio or video file")
|
||||
parser.add_argument("-o", "--output", default="audio-data.json", help="Output JSON path")
|
||||
parser.add_argument("--fps", type=int, default=30, help="Frames per second (default: 30)")
|
||||
parser.add_argument("--bands", type=int, default=16, help="Number of frequency bands (default: 16)")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.fps < 1:
|
||||
parser.error("--fps must be at least 1")
|
||||
if args.bands < 1:
|
||||
parser.error("--bands must be at least 1")
|
||||
|
||||
data = extract(args.input, args.fps, args.bands)
|
||||
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f)
|
||||
|
||||
print(f"Wrote {args.output} ({data['totalFrames']} frames, {data['bands']} bands)", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,131 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { mkdtempSync, writeFileSync, readFileSync, mkdirSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
// Exercise the actual engine and provider together; only the paid API and
|
||||
// external audio tools are fixtures. This is not a live synthesis/render test.
|
||||
for (const expired of [false, true]) {
|
||||
for (const only of ["tts", "tts,bgm,sfx"]) {
|
||||
// With expired credentials the person already has an assets/voice/intro.wav, which the engine must keep.
|
||||
const voicePath = expired ? "assets/voice/intro-2.wav" : "assets/voice/intro.wav";
|
||||
test(`Gemini engine returns caption metadata with ${expired ? "expired" : "absent"} HeyGen credentials (${only})`, (t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-gemini-pipeline-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
if (expired) {
|
||||
mkdirSync(join(dir, "assets/voice"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets/voice/intro.wav"), "the person's own intro");
|
||||
}
|
||||
const config = join(dir, "heygen");
|
||||
mkdirSync(config);
|
||||
if (expired)
|
||||
writeFileSync(
|
||||
join(config, "credentials"),
|
||||
JSON.stringify({
|
||||
oauth: { access_token: "expired-fixture", expires_at: "2000-01-01T00:00:00Z" },
|
||||
}),
|
||||
);
|
||||
const bin = join(dir, "bin");
|
||||
mkdirSync(bin);
|
||||
const preload = join(dir, "fetch.mjs");
|
||||
writeFileSync(
|
||||
preload,
|
||||
`
|
||||
import assert from 'node:assert/strict';
|
||||
globalThis.fetch = async (url, options) => {
|
||||
assert.equal(url, 'https://generativelanguage.googleapis.com/v1beta/interactions');
|
||||
const body = JSON.parse(options.body);
|
||||
assert.equal(body.model, 'gemini-3.8-flash-lite-tts');
|
||||
assert.equal(body.input[0].content[0].annotations[0].style, 'Line direction');
|
||||
const bytes = Buffer.alloc(48);
|
||||
bytes.write('RIFF'); bytes.write('WAVE', 8);
|
||||
return Response.json({status:'completed', steps:[{type:'model_output', content:[
|
||||
{type:'audio', mime_type:'audio/wav', data:bytes.toString('base64')}
|
||||
]}]});
|
||||
};
|
||||
`,
|
||||
);
|
||||
writeFileSync(
|
||||
join(bin, "npx"),
|
||||
`#!${process.execPath}
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const args = process.argv.slice(2);
|
||||
assert.deepEqual(args.slice(0, 3), ['hyperframes', 'transcribe', '${voicePath}']);
|
||||
assert.ok(fs.existsSync(args[2]));
|
||||
assert.equal(args[args.indexOf('--model')+1], 'small.en');
|
||||
fs.writeFileSync(path.join(args[args.indexOf('--dir')+1], 'transcript.json'), JSON.stringify([
|
||||
{text:'Hello',start:0.1,end:0.4}, {text:'there.',start:0.5,end:0.9}
|
||||
]));
|
||||
`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
writeFileSync(join(bin, "ffprobe"), "#!/bin/sh\necho 1.25\n", { mode: 0o755 });
|
||||
writeFileSync(join(dir, ".env"), "");
|
||||
writeFileSync(
|
||||
join(dir, "audio_request.json"),
|
||||
JSON.stringify({
|
||||
provider: "gemini",
|
||||
tts_model: "gemini-3.8-flash-lite-tts",
|
||||
style: "Global direction",
|
||||
lines: [{ id: "intro", text: "Hello there.", style: "Line direction" }],
|
||||
bgm: { mode: "none" },
|
||||
}),
|
||||
);
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
preload,
|
||||
new URL("./audio.mjs", import.meta.url).pathname,
|
||||
"--hyperframes",
|
||||
dir,
|
||||
"--only",
|
||||
only,
|
||||
],
|
||||
{
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
HEYGEN_CONFIG_DIR: config,
|
||||
HEYGEN_API_KEY: "",
|
||||
HYPERFRAMES_API_KEY: "",
|
||||
GEMINI_API_KEY: "fixture-key",
|
||||
PATH: `${bin}:${process.env.PATH}`,
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const meta = JSON.parse(readFileSync(join(dir, "audio_meta.json"), "utf8"));
|
||||
assert.equal(meta.tts_provider, "gemini");
|
||||
assert.equal(meta.voice_id, "Kore");
|
||||
assert.deepEqual(meta.voices, [
|
||||
{
|
||||
id: "intro",
|
||||
path: voicePath,
|
||||
duration_s: 1.25,
|
||||
words: [
|
||||
{ id: "w0", text: "Hello", start: 0.1, end: 0.4 },
|
||||
{ id: "w1", text: "there.", start: 0.5, end: 0.9 },
|
||||
],
|
||||
},
|
||||
]);
|
||||
assert.equal(meta.total_duration_s, 1.25);
|
||||
const manifest = readFileSync(join(dir, ".media/manifest.jsonl"), "utf8").trim().split("\n");
|
||||
assert.deepEqual(
|
||||
manifest
|
||||
.map((line) => JSON.parse(line))
|
||||
.map(({ path, type, source }) => [path, type, source]),
|
||||
[[voicePath, "voice", "generated"]],
|
||||
);
|
||||
if (expired)
|
||||
assert.equal(
|
||||
readFileSync(join(dir, "assets/voice/intro.wav"), "utf8"),
|
||||
"the person's own intro",
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env node
|
||||
// Self-contained HeyGen TTS — single text in → one wav (+ optional words JSON)
|
||||
// out. A thin CLI over lib/tts.mjs (the same code the audio engine uses), so the
|
||||
// HeyGen REST call, starfish voice pick, mp3→wav transcode, and word-timestamp
|
||||
// filtering live in exactly one place. Bypasses the `hyperframes` CLI, which in
|
||||
// the published build is Kokoro-only.
|
||||
//
|
||||
// Usage:
|
||||
// node heygen-tts.mjs "Text to speak" -o narration.wav [--words narration.words.json]
|
||||
// node heygen-tts.mjs ./script.txt -o narration.wav --words narration.words.json
|
||||
// node heygen-tts.mjs "Bonjour" -o fr.wav --lang fr --voice <id>
|
||||
// node heygen-tts.mjs --list # list starfish voices and exit
|
||||
//
|
||||
// Flags: -o/--output (.wav → ffmpeg transcode; .mp3 → raw bytes), --words,
|
||||
// --voice (starfish id), --speed, --lang, --list.
|
||||
// Requires: $HEYGEN_API_KEY / OAuth ~/.heygen credentials and ffmpeg for .wav output.
|
||||
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { heygenAuthHeaders, heygenJSON, loadEnvFromDir } from "./lib/heygen.mjs";
|
||||
import { ffprobeDuration, resolveVoiceId, synthesizeOne, withWordIds } from "./lib/tts.mjs";
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
function flag(name, def) {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
if (i < 0) return def;
|
||||
if (i + 1 >= argv.length) return true;
|
||||
const v = argv[i + 1];
|
||||
return v.startsWith("--") ? true : v;
|
||||
}
|
||||
const die = (m) => {
|
||||
console.error(`✗ heygen-tts: ${m}`);
|
||||
process.exit(1);
|
||||
};
|
||||
|
||||
// First arg that isn't a flag or the -o value is the text / .txt path.
|
||||
const positional = (() => {
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a.startsWith("--")) {
|
||||
const next = argv[i + 1];
|
||||
if (next && !next.startsWith("--")) i++;
|
||||
continue;
|
||||
}
|
||||
if (a === "-o") {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
return a;
|
||||
}
|
||||
return null;
|
||||
})();
|
||||
|
||||
const output = resolve(
|
||||
(typeof flag("output") === "string" && flag("output")) ||
|
||||
(argv.includes("-o") && argv[argv.indexOf("-o") + 1]) ||
|
||||
"narration.wav",
|
||||
);
|
||||
const wordsPath = typeof flag("words") === "string" ? resolve(flag("words")) : null;
|
||||
const userVoice = typeof flag("voice") === "string" ? flag("voice") : null;
|
||||
const speedRaw = typeof flag("speed") === "string" ? Number(flag("speed")) : 1.0;
|
||||
const speed = isFinite(speedRaw) && speedRaw > 0 ? speedRaw : 1.0;
|
||||
const lang = typeof flag("lang") === "string" ? flag("lang") : "en";
|
||||
const listOnly = flag("list") === true;
|
||||
|
||||
loadEnvFromDir(process.cwd());
|
||||
let authHeaders;
|
||||
try {
|
||||
authHeaders = heygenAuthHeaders();
|
||||
} catch (e) {
|
||||
die(e.message);
|
||||
}
|
||||
|
||||
// ---------- --list ----------
|
||||
if (listOnly) {
|
||||
const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, {
|
||||
headers: authHeaders,
|
||||
});
|
||||
for (const v of payload.data ?? payload.voices ?? []) {
|
||||
console.log(`${v.voice_id}\t${v.name}\t${v.language ?? ""}`);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// ---------- resolve text + voice ----------
|
||||
if (!positional) die("no text given. Pass a string or a .txt path, or use --list.");
|
||||
const text =
|
||||
positional.endsWith(".txt") && existsSync(resolve(positional))
|
||||
? readFileSync(resolve(positional), "utf8").trim()
|
||||
: positional;
|
||||
if (!text) die("input text is empty");
|
||||
|
||||
const voiceId = await resolveVoiceId({ provider: "heygen", userVoice, lang });
|
||||
if (!userVoice) console.error(`· using voice ${voiceId}`);
|
||||
|
||||
// ---------- synthesize (shared engine code) ----------
|
||||
const { ok, words, error } = await synthesizeOne({
|
||||
provider: "heygen",
|
||||
text,
|
||||
voiceId,
|
||||
lang,
|
||||
speed,
|
||||
wavAbs: output,
|
||||
hyperframesDir: process.cwd(),
|
||||
});
|
||||
// HeyGen's own refusal, verbatim: a caller shows it or acts on it (a deleted voice, a limit).
|
||||
if (!ok) die(error ?? "synthesis failed (HeyGen request/transcode error)");
|
||||
|
||||
let wordCount = 0;
|
||||
if (wordsPath) {
|
||||
if (words && words.length) {
|
||||
mkdirSync(dirname(wordsPath), { recursive: true });
|
||||
writeFileSync(wordsPath, JSON.stringify(withWordIds(words), null, 2));
|
||||
wordCount = words.length;
|
||||
} else {
|
||||
console.error("⚠ no word_timestamps in response — run `hyperframes transcribe` instead");
|
||||
}
|
||||
}
|
||||
|
||||
const dur = ffprobeDuration(output);
|
||||
const durStr = isFinite(dur) ? ` (${dur.toFixed(2)}s)` : "";
|
||||
console.log(`✓ ${output}${durStr}${wordCount ? ` · ${wordsPath} (${wordCount} words)` : ""}`);
|
||||
@@ -0,0 +1,38 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const script = fileURLToPath(new URL("./heygen-tts.mjs", import.meta.url));
|
||||
// Every fetch answers as HeyGen does for a voice that no longer exists.
|
||||
const refusing = `data:text/javascript,globalThis.fetch = async () => new Response(${JSON.stringify(
|
||||
JSON.stringify({ error: { code: "voice_not_found", message: "Voice not found: vc_gone" } }),
|
||||
)}, { status: 404, headers: { "content-type": "application/json" } });`;
|
||||
|
||||
test("a refused speech prints HeyGen's own message, so a caller can act on it", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-tts-"));
|
||||
const run = spawnSync(
|
||||
process.execPath,
|
||||
["--import", refusing, script, "Hello", "--voice", "vc_gone", "-o", join(dir, "out.mp3")],
|
||||
{
|
||||
cwd: dir,
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
HEYGEN_API_KEY: "",
|
||||
HYPERFRAMES_API_KEY: "",
|
||||
HEYGEN_ACCESS_TOKEN: "t",
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(run.status, 1);
|
||||
assert.match(run.stderr, /✗ heygen-tts: Voice not found: vc_gone\n/);
|
||||
assert.doesNotMatch(
|
||||
run.stderr,
|
||||
/HTTP 404|"error"/,
|
||||
"HeyGen's message alone, not the raw response",
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env node
|
||||
// HeyGen voice clones: `clone <audio> --name <n>`, `delete <voice_id>`, `list --prefix <p>` (references/tts.md).
|
||||
// The clone id works as `heygen-tts.mjs --voice <id>`. A refusal prints HeyGen's own message and exits 1.
|
||||
// Auth: same resolver as heygen-tts.mjs (lib/heygen.mjs).
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { extname } from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import { heygenAuthHeaders, heygenJSON, heygenMessage, loadEnvFromDir } from "./lib/heygen.mjs";
|
||||
import { isMainModule } from "./lib/main-module.mjs";
|
||||
|
||||
const MEDIA_TYPES = { ".mp3": "audio/mpeg", ".wav": "audio/wav" };
|
||||
const POLL_MS = 2_000;
|
||||
const DEADLINE_MS = 120_000;
|
||||
|
||||
async function clone(file, name, headers, { sleep, now }) {
|
||||
const mediaType = MEDIA_TYPES[extname(file).toLowerCase()];
|
||||
if (!mediaType) throw new Error(`unsupported audio file ${file}: use .mp3 or .wav`);
|
||||
const data = readFileSync(file).toString("base64");
|
||||
const created = await heygenJSON("/voices/clone", {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: { voice_name: name, audio: { type: "base64", media_type: mediaType, data } },
|
||||
});
|
||||
const id = created.data.voice_clone_id;
|
||||
const deadline = now() + DEADLINE_MS;
|
||||
for (;;) {
|
||||
const { data: voice } = await heygenJSON(`/voices/${id}`, { headers });
|
||||
if (voice.status === "complete") return { voice_id: voice.voice_id ?? id };
|
||||
if (voice.status === "failed") throw new Error(voice.failure_message || "voice clone failed");
|
||||
if (now() >= deadline) throw new Error(`voice clone ${id} not ready after 120 s`);
|
||||
await sleep(POLL_MS);
|
||||
}
|
||||
}
|
||||
|
||||
async function list(prefix, headers) {
|
||||
const mine = [];
|
||||
const seen = new Set();
|
||||
let token;
|
||||
do {
|
||||
if (token && seen.has(token)) throw new Error(`HeyGen returned page token ${token} twice`);
|
||||
if (token) seen.add(token);
|
||||
const query = new URLSearchParams({ type: "private", limit: "100" });
|
||||
if (token) query.set("token", token);
|
||||
const page = await heygenJSON(`/voices?${query}`, { headers });
|
||||
mine.push(...page.data.filter((v) => (v.name ?? "").startsWith(prefix)));
|
||||
token = page.has_more ? page.next_token : null;
|
||||
} while (token);
|
||||
// ponytail: the list response has no created_at, so one GET per match; bounded by the clone limit.
|
||||
return Promise.all(
|
||||
mine.map(async ({ voice_id, name }) => {
|
||||
const { data } = await heygenJSON(`/voices/${voice_id}`, { headers });
|
||||
return { voice_id, name, created_at: data.created_at ?? null };
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function main(
|
||||
argv,
|
||||
{
|
||||
sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
|
||||
now = Date.now,
|
||||
out = (s) => process.stdout.write(`${s}\n`),
|
||||
err = (s) => process.stderr.write(`${s}\n`),
|
||||
} = {},
|
||||
) {
|
||||
try {
|
||||
const { positionals, values } = parseArgs({
|
||||
args: argv,
|
||||
allowPositionals: true,
|
||||
options: { name: { type: "string" }, prefix: { type: "string", default: "" } },
|
||||
});
|
||||
const [command, arg] = positionals;
|
||||
if (command === "clone") {
|
||||
if (!arg || !values.name) throw new Error("usage: clone <audio-file> --name <name>");
|
||||
out(JSON.stringify(await clone(arg, values.name, heygenAuthHeaders(), { sleep, now })));
|
||||
} else if (command === "delete") {
|
||||
if (!arg) throw new Error("usage: delete <voice_id>");
|
||||
await heygenJSON(`/voices/${encodeURIComponent(arg)}`, {
|
||||
method: "DELETE",
|
||||
headers: heygenAuthHeaders(),
|
||||
});
|
||||
} else if (command === "list") {
|
||||
out(JSON.stringify(await list(values.prefix, heygenAuthHeaders())));
|
||||
} else {
|
||||
throw new Error("usage: heygen-voice.mjs clone|delete|list (see the header of this file)");
|
||||
}
|
||||
return 0;
|
||||
} catch (e) {
|
||||
err(heygenMessage(e));
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (isMainModule(import.meta.url)) {
|
||||
loadEnvFromDir(process.cwd());
|
||||
process.exitCode = await main(process.argv.slice(2));
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
import { afterEach, beforeEach, test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { main } from "./heygen-voice.mjs";
|
||||
|
||||
const realFetch = globalThis.fetch;
|
||||
const savedEnv = {
|
||||
HEYGEN_ACCESS_TOKEN: process.env.HEYGEN_ACCESS_TOKEN,
|
||||
HEYGEN_API_KEY: process.env.HEYGEN_API_KEY,
|
||||
};
|
||||
let dir;
|
||||
let calls;
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), "heygen-voice-"));
|
||||
delete process.env.HEYGEN_ACCESS_TOKEN;
|
||||
process.env.HEYGEN_API_KEY = "hg_test";
|
||||
calls = [];
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = realFetch;
|
||||
for (const [k, v] of Object.entries(savedEnv)) {
|
||||
if (v === undefined) delete process.env[k];
|
||||
else process.env[k] = v;
|
||||
}
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// routes: "METHOD /path" → array of [status, body] answered in order (last one repeats).
|
||||
function mockFetch(routes) {
|
||||
globalThis.fetch = async (url, opts) => {
|
||||
const key = `${opts.method} ${String(url).replace("https://api.heygen.com/v3", "")}`;
|
||||
calls.push({ key, opts });
|
||||
const queue = routes[key];
|
||||
if (!queue) throw new Error(`unexpected request ${key}`);
|
||||
const [status, body] = queue.length > 1 ? queue.shift() : queue[0];
|
||||
return new Response(body === undefined ? null : JSON.stringify(body), { status });
|
||||
};
|
||||
}
|
||||
|
||||
async function run(argv, { sleep = async () => {}, now = () => 0 } = {}) {
|
||||
const out = [];
|
||||
const err = [];
|
||||
const code = await main(argv, {
|
||||
sleep,
|
||||
now,
|
||||
out: (s) => out.push(s),
|
||||
err: (s) => err.push(s),
|
||||
});
|
||||
return { code, out: out.join("\n"), err: err.join("\n") };
|
||||
}
|
||||
|
||||
function audioFile(name = "sample.mp3") {
|
||||
const path = join(dir, name);
|
||||
writeFileSync(path, Buffer.from("fake-audio"));
|
||||
return path;
|
||||
}
|
||||
|
||||
test("clone uploads base64 audio, polls until complete, prints the voice id", async () => {
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[200, { data: { voice_clone_id: "vc1" } }]],
|
||||
"GET /voices/vc1": [
|
||||
[200, { data: { status: "processing", voice_id: "vc1" } }],
|
||||
[200, { data: { status: "complete", voice_id: "vc1", name: "Me" } }],
|
||||
],
|
||||
});
|
||||
const sleeps = [];
|
||||
const r = await run(["clone", audioFile(), "--name", "Me"], {
|
||||
sleep: async (ms) => sleeps.push(ms),
|
||||
});
|
||||
assert.deepEqual(r, { code: 0, out: '{"voice_id":"vc1"}', err: "" });
|
||||
assert.deepEqual(sleeps, [2000]);
|
||||
const post = calls[0];
|
||||
assert.equal(post.opts.headers["X-Api-Key"], "hg_test");
|
||||
assert.deepEqual(JSON.parse(post.opts.body), {
|
||||
voice_name: "Me",
|
||||
audio: {
|
||||
type: "base64",
|
||||
media_type: "audio/mpeg",
|
||||
data: Buffer.from("fake-audio").toString("base64"),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("clone sends audio/wav for a .wav file", async () => {
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[200, { data: { voice_clone_id: "vc1" } }]],
|
||||
"GET /voices/vc1": [[200, { data: { status: "complete", voice_id: "vc1" } }]],
|
||||
});
|
||||
const r = await run(["clone", audioFile("take.WAV"), "--name", "Me"]);
|
||||
assert.equal(r.code, 0);
|
||||
assert.equal(JSON.parse(calls[0].opts.body).audio.media_type, "audio/wav");
|
||||
});
|
||||
|
||||
test("clone prints HeyGen's refusal verbatim and exits 1", async () => {
|
||||
const message =
|
||||
"Voice clone limit reached (10). Delete unused clones or contact support to increase your limit.";
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[400, { error: { code: "resource_limit_reached", message } }]],
|
||||
});
|
||||
const r = await run(["clone", audioFile(), "--name", "Me"]);
|
||||
assert.deepEqual(r, { code: 1, out: "", err: message });
|
||||
});
|
||||
|
||||
test("clone surfaces a failed clone's failure_message", async () => {
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[200, { data: { voice_clone_id: "vc1" } }]],
|
||||
"GET /voices/vc1": [[200, { data: { status: "failed", failure_message: "Audio too noisy" } }]],
|
||||
});
|
||||
const r = await run(["clone", audioFile(), "--name", "Me"]);
|
||||
assert.deepEqual(r, { code: 1, out: "", err: "Audio too noisy" });
|
||||
});
|
||||
|
||||
test("clone gives up after the 120 s deadline", async () => {
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[200, { data: { voice_clone_id: "vc1" } }]],
|
||||
"GET /voices/vc1": [[200, { data: { status: "processing" } }]],
|
||||
});
|
||||
let clock = 0;
|
||||
const r = await run(["clone", audioFile(), "--name", "Me"], {
|
||||
now: () => clock,
|
||||
sleep: async (ms) => {
|
||||
clock += ms;
|
||||
},
|
||||
});
|
||||
assert.deepEqual(r, { code: 1, out: "", err: "voice clone vc1 not ready after 120 s" });
|
||||
assert.equal(calls.filter((c) => c.key === "GET /voices/vc1").length, 61);
|
||||
});
|
||||
|
||||
test("clone rejects an audio type other than mp3 or wav without calling HeyGen", async () => {
|
||||
mockFetch({});
|
||||
const r = await run(["clone", audioFile("take.ogg"), "--name", "Me"]);
|
||||
assert.equal(r.code, 1);
|
||||
assert.match(r.err, /use \.mp3 or \.wav/);
|
||||
assert.equal(calls.length, 0);
|
||||
});
|
||||
|
||||
test("delete calls DELETE /voices/<id> and exits 0", async () => {
|
||||
mockFetch({ "DELETE /voices/vc1": [[200, { data: {} }]] });
|
||||
const r = await run(["delete", "vc1"]);
|
||||
assert.deepEqual(r, { code: 0, out: "", err: "" });
|
||||
});
|
||||
|
||||
test("delete exits 0 on a 204 with no body, and escapes the id in the path", async () => {
|
||||
mockFetch({ "DELETE /voices/a%2Fb": [[204]] });
|
||||
const r = await run(["delete", "a/b"]);
|
||||
assert.deepEqual(r, { code: 0, out: "", err: "" });
|
||||
});
|
||||
|
||||
test("clone falls back to the clone id when the finished voice has no voice_id", async () => {
|
||||
mockFetch({
|
||||
"POST /voices/clone": [[200, { data: { voice_clone_id: "vc9" } }]],
|
||||
"GET /voices/vc9": [[200, { data: { status: "complete" } }]],
|
||||
});
|
||||
const r = await run(["clone", audioFile(), "--name", "Me"]);
|
||||
assert.deepEqual(r, { code: 0, out: '{"voice_id":"vc9"}', err: "" });
|
||||
});
|
||||
|
||||
test("list skips a nameless voice and stops on a page token HeyGen repeats", async () => {
|
||||
mockFetch({
|
||||
"GET /voices?type=private&limit=100": [
|
||||
[200, { data: [{ voice_id: "n", name: null }], has_more: true, next_token: "t2" }],
|
||||
],
|
||||
"GET /voices?type=private&limit=100&token=t2": [
|
||||
[200, { data: [], has_more: true, next_token: "t2" }],
|
||||
],
|
||||
});
|
||||
// A fake answers at once, so a list that never stops would starve any timeout: cap the calls instead.
|
||||
const answer = globalThis.fetch;
|
||||
globalThis.fetch = (...args) =>
|
||||
calls.length >= 10 ? Promise.reject(new Error("paged forever")) : answer(...args);
|
||||
const r = await run(["list", "--prefix", "desk-"]);
|
||||
assert.deepEqual(r, { code: 1, out: "", err: "HeyGen returned page token t2 twice" });
|
||||
});
|
||||
|
||||
test("delete prints HeyGen's 404 message verbatim and exits 1", async () => {
|
||||
mockFetch({
|
||||
"DELETE /voices/gone": [
|
||||
[404, { error: { code: "voice_not_found", message: "Voice not found: gone" } }],
|
||||
],
|
||||
});
|
||||
const r = await run(["delete", "gone"]);
|
||||
assert.deepEqual(r, { code: 1, out: "", err: "Voice not found: gone" });
|
||||
});
|
||||
|
||||
test("list pages through private voices and returns prefix matches with created_at", async () => {
|
||||
mockFetch({
|
||||
"GET /voices?type=private&limit=100": [
|
||||
[
|
||||
200,
|
||||
{
|
||||
data: [
|
||||
{ voice_id: "a", name: "desk-1" },
|
||||
{ voice_id: "b", name: "other" },
|
||||
],
|
||||
has_more: true,
|
||||
next_token: "t2",
|
||||
},
|
||||
],
|
||||
],
|
||||
"GET /voices?type=private&limit=100&token=t2": [
|
||||
[200, { data: [{ voice_id: "c", name: "desk-2" }], has_more: false }],
|
||||
],
|
||||
"GET /voices/a": [[200, { data: { voice_id: "a", created_at: 1700000000 } }]],
|
||||
"GET /voices/c": [[200, { data: { voice_id: "c", created_at: 1700000100 } }]],
|
||||
});
|
||||
const r = await run(["list", "--prefix", "desk-"]);
|
||||
assert.equal(r.code, 0);
|
||||
assert.deepEqual(JSON.parse(r.out), [
|
||||
{ voice_id: "a", name: "desk-1", created_at: 1700000000 },
|
||||
{ voice_id: "c", name: "desk-2", created_at: 1700000100 },
|
||||
]);
|
||||
});
|
||||
|
||||
test("list prints HeyGen's refusal verbatim and exits 1", async () => {
|
||||
const message = "Invalid or expired API key. Verify your x-api-key header.";
|
||||
mockFetch({
|
||||
"GET /voices?type=private&limit=100": [[401, { error: { code: "unauthorized", message } }]],
|
||||
});
|
||||
const r = await run(["list", "--prefix", "desk-"]);
|
||||
assert.deepEqual(r, { code: 1, out: "", err: message });
|
||||
});
|
||||
|
||||
test("a host-injected HEYGEN_ACCESS_TOKEN is sent as Bearer", async () => {
|
||||
process.env.HEYGEN_ACCESS_TOKEN = "at_host";
|
||||
mockFetch({ "DELETE /voices/vc1": [[200, { data: {} }]] });
|
||||
await run(["delete", "vc1"]);
|
||||
assert.equal(calls[0].opts.headers.Authorization, "Bearer at_host");
|
||||
assert.equal(calls[0].opts.headers["X-Api-Key"], undefined);
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
import { closeSync, ftruncateSync, openSync, readFileSync, writeSync } from "node:fs";
|
||||
|
||||
// Keep the merge base and output on the same file even if its pathname changes
|
||||
// while audio generation runs. The CLI owns this handle until write or exit.
|
||||
export function openAudioMeta(path) {
|
||||
let fd;
|
||||
try {
|
||||
fd = openSync(path, "r+");
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT") throw error;
|
||||
}
|
||||
let value = {};
|
||||
if (fd !== undefined) {
|
||||
try {
|
||||
value = JSON.parse(readFileSync(fd, "utf8"));
|
||||
} catch (error) {
|
||||
closeSync(fd);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return {
|
||||
value,
|
||||
write(meta) {
|
||||
const bytes = Buffer.from(JSON.stringify(meta, null, 2));
|
||||
// Defer new-file creation until generation succeeds. Never overwrite a
|
||||
// file (or follow a link) that appeared since the missing merge base.
|
||||
if (fd === undefined) fd = openSync(path, "wx");
|
||||
try {
|
||||
let offset = 0;
|
||||
while (offset < bytes.length) {
|
||||
offset += writeSync(fd, bytes, offset, bytes.length - offset, offset);
|
||||
}
|
||||
ftruncateSync(fd, bytes.length);
|
||||
} finally {
|
||||
closeSync(fd);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import {
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
symlinkSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { openAudioMeta } from "./audio-meta.mjs";
|
||||
|
||||
function fixture(t) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "audio-meta-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
}
|
||||
|
||||
test("updates the read file from byte zero, truncates, and retains its mode", (t) => {
|
||||
const path = join(fixture(t), "meta.json");
|
||||
writeFileSync(path, JSON.stringify({ voices: ["long existing voice metadata"] }), {
|
||||
mode: 0o600,
|
||||
});
|
||||
const originalMode = statSync(path).mode;
|
||||
const handle = openAudioMeta(path);
|
||||
assert.deepEqual(handle.value, { voices: ["long existing voice metadata"] });
|
||||
handle.write({ voices: ["声"] });
|
||||
assert.equal(readFileSync(path, "utf8"), JSON.stringify({ voices: ["声"] }, null, 2));
|
||||
assert.equal(statSync(path).mode, originalMode);
|
||||
});
|
||||
|
||||
test("pathname replacement cannot redirect the existing-file write", (t) => {
|
||||
const dir = fixture(t);
|
||||
const path = join(dir, "meta.json");
|
||||
const moved = join(dir, "original.json");
|
||||
writeFileSync(path, "{}");
|
||||
const handle = openAudioMeta(path);
|
||||
renameSync(path, moved);
|
||||
writeFileSync(path, "replacement must survive");
|
||||
handle.write({ voices: [] });
|
||||
assert.equal(readFileSync(path, "utf8"), "replacement must survive");
|
||||
assert.deepEqual(JSON.parse(readFileSync(moved, "utf8")), { voices: [] });
|
||||
});
|
||||
|
||||
test("existing symlink outputs retain their original target even after retargeting", (t) => {
|
||||
const dir = fixture(t);
|
||||
const path = join(dir, "meta.json");
|
||||
const target = join(dir, "target.json");
|
||||
const victim = join(dir, "victim.json");
|
||||
writeFileSync(target, "{}");
|
||||
writeFileSync(victim, "untouched");
|
||||
symlinkSync(target, path);
|
||||
const handle = openAudioMeta(path);
|
||||
rmSync(path);
|
||||
symlinkSync(victim, path);
|
||||
handle.write({ bgm: null });
|
||||
assert.equal(readFileSync(victim, "utf8"), "untouched");
|
||||
assert.deepEqual(JSON.parse(readFileSync(target, "utf8")), { bgm: null });
|
||||
});
|
||||
|
||||
test("missing output is created only when saved", (t) => {
|
||||
const path = join(fixture(t), "meta.json");
|
||||
const handle = openAudioMeta(path);
|
||||
assert.deepEqual(handle.value, {});
|
||||
assert.throws(() => statSync(path), { code: "ENOENT" });
|
||||
handle.write({ sfx: [] });
|
||||
assert.deepEqual(JSON.parse(readFileSync(path, "utf8")), { sfx: [] });
|
||||
});
|
||||
|
||||
for (const replacement of ["file", "symlink"]) {
|
||||
test(`new output does not overwrite a racing ${replacement}`, (t) => {
|
||||
const dir = fixture(t);
|
||||
const path = join(dir, "meta.json");
|
||||
const victim = join(dir, "victim.json");
|
||||
const handle = openAudioMeta(path);
|
||||
writeFileSync(victim, "untouched");
|
||||
if (replacement === "symlink") symlinkSync(victim, path);
|
||||
else writeFileSync(path, "untouched");
|
||||
assert.throws(() => handle.write({}), { code: "EEXIST" });
|
||||
assert.equal(readFileSync(path, "utf8"), "untouched");
|
||||
assert.equal(readFileSync(victim, "utf8"), "untouched");
|
||||
});
|
||||
}
|
||||
|
||||
test("malformed merge base remains an error without changing bytes", (t) => {
|
||||
const path = join(fixture(t), "meta.json");
|
||||
writeFileSync(path, "bad json");
|
||||
assert.throws(() => openAudioMeta(path), SyntaxError);
|
||||
assert.equal(readFileSync(path, "utf8"), "bad json");
|
||||
});
|
||||
|
||||
test("CLI partial run retains unselected voices, BGM and SFX", (t) => {
|
||||
const dir = fixture(t);
|
||||
const path = join(dir, "meta.json");
|
||||
const request = join(dir, "request.json");
|
||||
const previous = {
|
||||
voices: [{ id: "a", duration_s: 2 }],
|
||||
bgm: { path: "bgm.wav" },
|
||||
sfx: [{ name: "click" }],
|
||||
tts_provider: "kokoro",
|
||||
voice_id: "voice-a",
|
||||
};
|
||||
writeFileSync(path, JSON.stringify(previous));
|
||||
writeFileSync(request, "{}");
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
fileURLToPath(new URL("../audio.mjs", import.meta.url)),
|
||||
"--request",
|
||||
request,
|
||||
"--hyperframes",
|
||||
dir,
|
||||
"--out",
|
||||
path,
|
||||
"--only",
|
||||
"none",
|
||||
],
|
||||
{
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
HEYGEN_CONFIG_DIR: dir,
|
||||
HEYGEN_API_KEY: "",
|
||||
HYPERFRAMES_API_KEY: "",
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const actual = JSON.parse(readFileSync(path, "utf8"));
|
||||
for (const key of Object.keys(previous)) assert.deepEqual(actual[key], previous[key]);
|
||||
});
|
||||
@@ -0,0 +1,4 @@
|
||||
// Music sits under narration; a silent film brings music forward.
|
||||
export const BGM_BED_VOLUME = 0.12;
|
||||
export const BGM_SILENT_VOLUME = 0.9;
|
||||
export const bgmDefaultVolume = (hasVoice) => (hasVoice ? BGM_BED_VOLUME : BGM_SILENT_VOLUME);
|
||||
@@ -0,0 +1,257 @@
|
||||
// bgm.mjs — background music for the media audio engine. Two routes, gated the
|
||||
// same way as TTS/SFX:
|
||||
//
|
||||
// retrieve (default when HeyGen is configured) — search HeyGen's music library
|
||||
// by mood, download the top track. Synchronous. assets/bgm/track.mp3.
|
||||
// generate (the alternative; the automatic choice when HeyGen is absent) —
|
||||
// Lyria (cloud, $GEMINI_API_KEY/$GOOGLE_API_KEY + google-genai) preferred,
|
||||
// else local MusicGen (facebook/musicgen-small via transformers). Spawned
|
||||
// DETACHED so the engine can return while audio renders; the caller marks
|
||||
// bgm_pending and runs wait-bgm.mjs before assembling. assets/bgm/track.wav.
|
||||
//
|
||||
// Missing/failed BGM never blocks a render.
|
||||
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, openSync, closeSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { downloadTo, searchSounds } from "./heygen.mjs";
|
||||
import { agentWritePath } from "./media-record.mjs";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
const lyriaKey = () => process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || "";
|
||||
|
||||
import { bgmDefaultVolume } from "./bgm-volume.mjs";
|
||||
export { BGM_BED_VOLUME, BGM_SILENT_VOLUME, bgmDefaultVolume } from "./bgm-volume.mjs";
|
||||
|
||||
const BGM_PY_DEPS = ["transformers", "torch", "soundfile", "numpy"];
|
||||
const BGM_PY_PROBE =
|
||||
"import transformers, soundfile, torch, numpy; from transformers import MusicgenForConditionalGeneration";
|
||||
const LYRIA_PY_DEPS = ["google-genai", "python-dotenv"];
|
||||
const LYRIA_PY_PROBE = "import google.genai";
|
||||
|
||||
function pyOk(probe) {
|
||||
const { cmd, args } = pythonInvocation(["-c", probe]);
|
||||
return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;
|
||||
}
|
||||
// `python -m pip`, not a bare `pip` binary: a Homebrew/system Python often
|
||||
// exposes only `python3`/`pip3` on PATH, so a plain `pip` spawn silently
|
||||
// no-ops (ENOENT) and the documented "auto-installed on demand" path never
|
||||
// actually installs. `-m pip` also guarantees the packages land in the SAME
|
||||
// interpreter pyOk() probes — a bare `pip`/`pip3` could resolve to a
|
||||
// different Python installation than `python3` if more than one is on PATH.
|
||||
function pipInstall(deps) {
|
||||
const { cmd, args } = pythonInvocation(["-m", "pip", "install", "-q", ...deps]);
|
||||
return spawnSync(cmd, args, { stdio: "ignore" }).status === 0;
|
||||
}
|
||||
|
||||
// ── retrieval (HeyGen music library) ──────────────────────────────────────────
|
||||
export async function retrieveBgm({ query, headers, hyperframesDir, hasVoice, anomalies }) {
|
||||
const q = query || "calm cinematic underscore";
|
||||
const results = await searchSounds(q, "music", headers, { limit: 5 });
|
||||
if (!results.length) return null;
|
||||
const top = results[0];
|
||||
const rel = agentWritePath(hyperframesDir, "assets/bgm/track.mp3", { anomalies });
|
||||
await downloadTo(top.audio_url, join(hyperframesDir, rel));
|
||||
return {
|
||||
path: rel,
|
||||
volume: bgmDefaultVolume(hasVoice),
|
||||
query: q,
|
||||
mode: "retrieve",
|
||||
duration_s: typeof top.duration === "number" ? r3(top.duration) : null,
|
||||
};
|
||||
}
|
||||
|
||||
// ── mood inference (for the generate path's prompt) ──────────────────────────
|
||||
// Industry base → archetype shape → emotional-arc tiebreaker. Exported so a
|
||||
// workflow adapter can build a rich prompt from its own narrative metadata; the
|
||||
// engine also calls it when generate has only a plain mood query.
|
||||
export function inferBgmPrompt({ blob = "", archetype = "", arc = "", userPrompt = "" } = {}) {
|
||||
if (userPrompt) return userPrompt;
|
||||
const b = String(blob).toLowerCase();
|
||||
let base;
|
||||
let bpm;
|
||||
if (/\b(crypto|nft|web3|defi|token|blockchain|exchange|wallet|dao)\b/.test(b)) {
|
||||
base = "atmospheric electronic, deep bass, futuristic synths, restrained percussion";
|
||||
bpm = 100;
|
||||
} else if (/\b(finance|fintech|bank|payment|invest|wealth|insurance|treasury)\b/.test(b)) {
|
||||
base = "calm cinematic, soft strings, subtle piano, restrained percussion";
|
||||
bpm = 92;
|
||||
} else if (/\b(creative|agency|design|studio|art|brand|marketing|content)\b/.test(b)) {
|
||||
base = "playful electronic, warm pads, light percussion";
|
||||
bpm = 115;
|
||||
} else {
|
||||
base = "uplifting corporate tech, bright modern piano with synth pads";
|
||||
bpm = 108;
|
||||
}
|
||||
const at = String(archetype).toLowerCase();
|
||||
const ar = String(arc).toLowerCase();
|
||||
if (/\bpas\b|pain.agitate|pain.+solve/.test(at))
|
||||
return `${base}, starts with subtle tension then builds to resolution, BPM ${bpm}, transitions from MINOR to MAJOR`;
|
||||
if (/\bbab\b|before.after|future.pac|vision/.test(at))
|
||||
return `${base}, cinematic and aspirational, steady build with rising energy, BPM ${bpm}, MAJOR`;
|
||||
if (/cascade|feature.benefit/.test(at))
|
||||
return `${base}, energetic and driving, consistent momentum, BPM ${Math.min(bpm + 10, 128)}, MAJOR`;
|
||||
if (/demo.loop|question.+answer/.test(at))
|
||||
return `${base}, clean and focused, minimal arrangement, BPM ${Math.max(bpm - 8, 88)}`;
|
||||
if (/frustrat|anxiety|overwhelm|tension/.test(ar) && /relief|excite|triumph/.test(ar))
|
||||
return `${base}, builds from understated tension to uplifting resolution, BPM ${bpm}, MINOR to MAJOR`;
|
||||
if (/excit|awe|power|triumph/.test(ar))
|
||||
return `${base}, energetic and confident, BPM ${bpm}, MAJOR`;
|
||||
if (/trust|ease|clarity|reassur/.test(ar))
|
||||
return `${base}, warm and reassuring, BPM ${Math.max(bpm - 5, 85)}`;
|
||||
return `${base}, BPM ${bpm}, MAJOR`;
|
||||
}
|
||||
|
||||
// ── generation (Lyria → MusicGen, detached) ──────────────────────────────────
|
||||
// Returns a bgmMeta the caller folds into audio_meta:
|
||||
// { path, mode, volume, provider, pid, log, target_duration_s, seed_duration_s,
|
||||
// loop_count, pending:true } on success, or { disabled:true, reason }.
|
||||
export function generateBgmDetached({
|
||||
prompt,
|
||||
durationS,
|
||||
hyperframesDir,
|
||||
lyriaRecipe,
|
||||
seedSeconds = 28,
|
||||
hasVoice,
|
||||
anomalies,
|
||||
}) {
|
||||
const rel = agentWritePath(hyperframesDir, "assets/bgm/track.wav", { anomalies });
|
||||
const abs = join(hyperframesDir, rel);
|
||||
mkdirSync(join(hyperframesDir, "assets", "bgm"), { recursive: true });
|
||||
const log = join(hyperframesDir, "assets", "bgm", `bgm-${Date.now()}.log`);
|
||||
const targetS = Math.max(1, durationS);
|
||||
const baseMeta = { path: rel, mode: null, volume: bgmDefaultVolume(hasVoice), pending: true };
|
||||
|
||||
const lyriaConfigured = !!lyriaKey() && !!lyriaRecipe && existsSync(lyriaRecipe);
|
||||
|
||||
// Make a backend runnable: prefer Lyria when configured (install google-genai
|
||||
// on demand), else ensure local MusicGen deps. Installs are synchronous here —
|
||||
// generation itself is detached, so the engine still returns promptly.
|
||||
if (lyriaConfigured && !pyOk(LYRIA_PY_PROBE)) pipInstall(LYRIA_PY_DEPS);
|
||||
const useLyria = lyriaConfigured && pyOk(LYRIA_PY_PROBE);
|
||||
if (!useLyria && !pyOk(BGM_PY_PROBE)) pipInstall(BGM_PY_DEPS);
|
||||
|
||||
const fd = openSync(log, "w");
|
||||
if (useLyria) {
|
||||
const { cmd, args } = pythonInvocation([
|
||||
lyriaRecipe,
|
||||
"--output",
|
||||
abs,
|
||||
"--duration",
|
||||
String(targetS),
|
||||
"--prompt",
|
||||
prompt,
|
||||
]);
|
||||
rmSync(abs, { force: true }); // wait-bgm takes any file here as the finished track
|
||||
const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] });
|
||||
proc.unref();
|
||||
closeSync(fd);
|
||||
return {
|
||||
...baseMeta,
|
||||
mode: "detached-single",
|
||||
provider: "lyria",
|
||||
pid: proc.pid,
|
||||
log,
|
||||
target_duration_s: r3(targetS),
|
||||
};
|
||||
}
|
||||
|
||||
if (pyOk(BGM_PY_PROBE)) {
|
||||
const seedS = Math.min(Math.max(seedSeconds, 10), 30);
|
||||
const loops = targetS > seedS ? Math.ceil(targetS / seedS) : 1;
|
||||
const script = musicgenScript({ prompt, abs, targetS, seedS });
|
||||
const { cmd, args } = pythonInvocation(["-c", script]);
|
||||
rmSync(abs, { force: true });
|
||||
const proc = spawn(cmd, args, { detached: true, stdio: ["ignore", fd, fd] });
|
||||
proc.unref();
|
||||
closeSync(fd);
|
||||
return {
|
||||
...baseMeta,
|
||||
mode: targetS > seedS ? "detached-seed-loop" : "detached-seed-trim",
|
||||
provider: "musicgen",
|
||||
pid: proc.pid,
|
||||
log,
|
||||
target_duration_s: r3(targetS),
|
||||
seed_duration_s: seedS,
|
||||
loop_count: loops,
|
||||
};
|
||||
}
|
||||
|
||||
closeSync(fd);
|
||||
return {
|
||||
disabled: true,
|
||||
reason: lyriaConfigured
|
||||
? `Lyria configured but google-genai uninstallable, and local MusicGen unavailable (pip install ${BGM_PY_DEPS.join(" ")})`
|
||||
: `no Lyria key/recipe and local MusicGen deps unavailable (pip install ${BGM_PY_DEPS.join(" ")})`,
|
||||
};
|
||||
}
|
||||
|
||||
// Inline MusicGen: generate ONE seed clip (≤30s to stay under the decoder's
|
||||
// positional limit), then trim it down or crossfade-loop it up to the target.
|
||||
function musicgenScript({ prompt, abs, targetS, seedS }) {
|
||||
return `
|
||||
import math, os, sys, traceback
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from transformers import MusicgenForConditionalGeneration, AutoProcessor
|
||||
|
||||
prompt = ${JSON.stringify(prompt)}
|
||||
out_path = ${JSON.stringify(abs)}
|
||||
target_s = float(${targetS.toFixed(3)})
|
||||
seed_s = float(${seedS.toFixed(3)})
|
||||
token_rate = 50
|
||||
crossfade_s = 0.3
|
||||
|
||||
def apply_fade(arr, sr, fade_in_s=0.08, fade_out_s=0.5):
|
||||
n_in = min(int(round(fade_in_s * sr)), arr.shape[0] // 2)
|
||||
n_out = min(int(round(fade_out_s * sr)), arr.shape[0] // 2)
|
||||
if n_in > 1: arr[:n_in] *= np.linspace(0.0, 1.0, n_in, dtype="float32")
|
||||
if n_out > 1: arr[-n_out:] *= np.linspace(1.0, 0.0, n_out, dtype="float32")
|
||||
return arr
|
||||
|
||||
def loop_crossfade(seed, target_len, xf):
|
||||
if seed.shape[0] >= target_len: return seed[:target_len]
|
||||
xf = min(xf, seed.shape[0] // 2)
|
||||
if xf < 1:
|
||||
reps = int(math.ceil(target_len / seed.shape[0]))
|
||||
return np.tile(seed, reps)[:target_len]
|
||||
t = np.linspace(0.0, 1.0, xf, dtype="float32")
|
||||
fade_out = np.cos(t * (math.pi / 2)); fade_in = np.sin(t * (math.pi / 2))
|
||||
out = seed.copy()
|
||||
while out.shape[0] < target_len:
|
||||
tail = out[-xf:] * fade_out; head = seed[:xf] * fade_in
|
||||
out = np.concatenate([out[:-xf], tail + head, seed[xf:]])
|
||||
return out[:target_len]
|
||||
|
||||
try:
|
||||
Path(os.path.dirname(out_path)).mkdir(parents=True, exist_ok=True)
|
||||
processor = AutoProcessor.from_pretrained("facebook/musicgen-small")
|
||||
model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small")
|
||||
model.eval()
|
||||
sr = int(model.config.audio_encoder.sampling_rate)
|
||||
gen_s = min(seed_s, target_s)
|
||||
tokens = max(1, int(math.ceil(gen_s * token_rate)))
|
||||
print(f"[musicgen] seed dur={gen_s:.2f}s tokens={tokens}", flush=True)
|
||||
inputs = processor(text=[prompt], padding=True, return_tensors="pt")
|
||||
audio = model.generate(**inputs, max_new_tokens=tokens)
|
||||
seed = audio[0, 0].detach().cpu().numpy().astype("float32")
|
||||
peak = float(np.max(np.abs(seed)))
|
||||
if peak > 1e-6: seed = seed * (0.89 / peak)
|
||||
want = max(1, int(round(target_s * sr)))
|
||||
if seed.shape[0] >= want:
|
||||
final = seed[:want].copy()
|
||||
else:
|
||||
final = loop_crossfade(seed, want, int(round(crossfade_s * sr)))
|
||||
if final.shape[0] < want: final = np.pad(final, (0, want - final.shape[0]))
|
||||
else: final = final[:want]
|
||||
final = apply_fade(final, sr)
|
||||
peak = float(np.max(np.abs(final)))
|
||||
if peak > 1.0: final = final / peak
|
||||
sf.write(out_path, final, sr)
|
||||
print(f"[musicgen] wrote {out_path} samples={final.shape[0]} sr={sr}", flush=True)
|
||||
except Exception:
|
||||
traceback.print_exc(); sys.exit(1)
|
||||
`;
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { appendRecord } from "../../../scripts/lib/manifest.mjs";
|
||||
import {
|
||||
BGM_BED_VOLUME,
|
||||
BGM_SILENT_VOLUME,
|
||||
bgmDefaultVolume,
|
||||
generateBgmDetached,
|
||||
} from "./bgm.mjs";
|
||||
|
||||
// Regression: narrated pipelines used to ship BGM at 0.8 (≈ -2 dB), ~16 dB
|
||||
// hotter than a music bed under a voice should be. The default under narration
|
||||
// must be a proper bed (≈ -18 dB); a silent film keeps the louder default.
|
||||
|
||||
const dbfs = (linear) => 20 * Math.log10(linear);
|
||||
|
||||
test("BGM under narration is a bed near -18 dB", () => {
|
||||
assert.equal(bgmDefaultVolume(true), BGM_BED_VOLUME);
|
||||
assert.equal(BGM_BED_VOLUME, 0.12);
|
||||
const db = dbfs(BGM_BED_VOLUME);
|
||||
assert.ok(db < -17 && db > -19, `bed should be ≈ -18 dB, got ${db.toFixed(1)} dB`);
|
||||
});
|
||||
|
||||
test("a silent film (no voice) keeps BGM forward", () => {
|
||||
assert.equal(bgmDefaultVolume(false), BGM_SILENT_VOLUME);
|
||||
assert.equal(BGM_SILENT_VOLUME, 0.9);
|
||||
});
|
||||
|
||||
test("the narrated default is well below the voice (≈ 0 dBFS)", () => {
|
||||
// Voice sits at data-volume="1" (0 dBFS); the bed must be ~16+ dB under it.
|
||||
const separation = dbfs(1) - dbfs(bgmDefaultVolume(true));
|
||||
assert.ok(
|
||||
separation >= 16,
|
||||
`bed should sit ≥16 dB under the voice, got ${separation.toFixed(1)} dB`,
|
||||
);
|
||||
});
|
||||
|
||||
test(
|
||||
"generating music again clears the engine's old track, so waiting cannot mistake it for the new one",
|
||||
{ skip: process.platform === "win32" && "the fake python is a shell script" },
|
||||
(t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-bgm-"));
|
||||
const path = process.env.PATH;
|
||||
t.after(() => {
|
||||
process.env.PATH = path;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
mkdirSync(join(dir, "bin"));
|
||||
for (const name of ["python3", "python"])
|
||||
writeFileSync(join(dir, "bin", name), "#!/bin/sh\nexit 0\n", { mode: 0o755 });
|
||||
process.env.PATH = `${join(dir, "bin")}:${path}`;
|
||||
mkdirSync(join(dir, "assets/bgm"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets/bgm/track.wav"), "last run's track");
|
||||
appendRecord(dir, {
|
||||
id: "bgm_001",
|
||||
type: "bgm",
|
||||
path: "assets/bgm/track.wav",
|
||||
source: "generated",
|
||||
});
|
||||
|
||||
const gen = generateBgmDetached({
|
||||
prompt: "calm",
|
||||
durationS: 5,
|
||||
hyperframesDir: dir,
|
||||
anomalies: [],
|
||||
});
|
||||
|
||||
assert.equal(gen.path, "assets/bgm/track.wav");
|
||||
assert.equal(existsSync(join(dir, "assets/bgm/track.wav")), false);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,14 @@
|
||||
// mapWithConcurrency — run `fn` over `items` with at most `limit` in flight at
|
||||
// once. Preserves input order in the result array regardless of completion order.
|
||||
export async function mapWithConcurrency(items, limit, fn) {
|
||||
const results = new Array(items.length);
|
||||
let next = 0;
|
||||
async function worker() {
|
||||
while (next < items.length) {
|
||||
const i = next++;
|
||||
results[i] = await fn(items[i], i);
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
||||
return results;
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mapWithConcurrency } from "./concurrency.mjs";
|
||||
|
||||
// Regression: audio.mjs used a bare Promise.all(lines.map(synthLine)) to
|
||||
// synthesize every TTS line at once, spawning one Kokoro/whisper model load
|
||||
// per line concurrently. Two independent reports of this overwhelming a
|
||||
// machine (OOM, and cold-start contention causing spurious failures).
|
||||
// mapWithConcurrency is the extracted cap; test it in isolation since
|
||||
// audio.mjs itself is a script (runs CLI/exit side effects on import).
|
||||
test("processes every item and preserves input order regardless of completion order", async () => {
|
||||
const order = [5, 1, 3, 2, 4];
|
||||
const results = await mapWithConcurrency(order, 2, async (n) => {
|
||||
await new Promise((r) => setTimeout(r, n));
|
||||
return n * 10;
|
||||
});
|
||||
assert.deepEqual(results, [50, 10, 30, 20, 40]);
|
||||
});
|
||||
|
||||
test("never runs more than `limit` at once", async () => {
|
||||
let inFlight = 0;
|
||||
let maxInFlight = 0;
|
||||
const items = Array.from({ length: 10 }, (_, i) => i);
|
||||
await mapWithConcurrency(items, 3, async () => {
|
||||
inFlight++;
|
||||
maxInFlight = Math.max(maxInFlight, inFlight);
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
inFlight--;
|
||||
});
|
||||
assert.equal(maxInFlight, 3);
|
||||
});
|
||||
|
||||
test("limit larger than the item count runs everything without hanging", async () => {
|
||||
const results = await mapWithConcurrency([1, 2], 10, async (n) => n * 2);
|
||||
assert.deepEqual(results, [2, 4]);
|
||||
});
|
||||
|
||||
test("empty input resolves to an empty array", async () => {
|
||||
const results = await mapWithConcurrency([], 4, async (n) => n);
|
||||
assert.deepEqual(results, []);
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
|
||||
export function geminiConfigured() {
|
||||
return Boolean(
|
||||
process.env.GEMINI_API_KEY ||
|
||||
process.env.GOOGLE_API_KEY ||
|
||||
process.env.GOOGLE_APPLICATION_CREDENTIALS ||
|
||||
process.env.GCS_CREDS,
|
||||
);
|
||||
}
|
||||
|
||||
export function geminiAuth({ run = spawnSync } = {}) {
|
||||
const key = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
|
||||
if (key) return { headers: { "x-goog-api-key": key }, secret: key };
|
||||
if (!geminiConfigured()) {
|
||||
throw new Error(
|
||||
"Gemini TTS needs GEMINI_API_KEY or GOOGLE_API_KEY, or service-account credentials (GOOGLE_APPLICATION_CREDENTIALS or GCS_CREDS)",
|
||||
);
|
||||
}
|
||||
const { cmd, args } = pythonInvocation([
|
||||
fileURLToPath(new URL("./gemini-auth.py", import.meta.url)),
|
||||
]);
|
||||
const result = run(cmd, args, { encoding: "utf8", timeout: 60_000, maxBuffer: 1024 * 1024 });
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse(result.stdout);
|
||||
} catch {
|
||||
throw new Error(
|
||||
"Gemini service-account authentication failed; check Python and install google-auth requests",
|
||||
);
|
||||
}
|
||||
if (result.status !== 0 || !data.token) {
|
||||
// Do not echo subprocess output: it could contain credentials.
|
||||
throw new Error(
|
||||
"Gemini service-account authentication failed; check service-account JSON, permissions, and Python packages google-auth requests",
|
||||
);
|
||||
}
|
||||
const headers = { Authorization: `Bearer ${data.token}` };
|
||||
if (data.project) headers["x-goog-user-project"] = data.project;
|
||||
return { headers, secret: data.token };
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Obtain Gemini OAuth headers without persisting credentials or tokens."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Keep the credential subprocess protocol UTF-8 on every platform.
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(_stream, "reconfigure"):
|
||||
_stream.reconfigure(encoding="utf-8", errors=_stream.errors)
|
||||
|
||||
|
||||
def authenticate():
|
||||
try:
|
||||
from google.auth.transport.requests import Request
|
||||
from google.oauth2 import service_account
|
||||
except ImportError:
|
||||
return {"error": "Service-account auth needs Python packages: pip install google-auth requests"}
|
||||
|
||||
try:
|
||||
path = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")
|
||||
if path:
|
||||
with open(path, encoding="utf-8") as source:
|
||||
info = json.load(source)
|
||||
else:
|
||||
info = json.loads(os.environ["GCS_CREDS"])
|
||||
if info.get("type") != "service_account":
|
||||
return {"error": "Gemini requires service-account JSON, not user ADC credentials"}
|
||||
# Only exchange credentials with Google's OAuth server.
|
||||
info["token_uri"] = "https://oauth2.googleapis.com/token"
|
||||
credentials = service_account.Credentials.from_service_account_info(
|
||||
info,
|
||||
scopes=[
|
||||
"https://www.googleapis.com/auth/generative-language.retriever",
|
||||
],
|
||||
)
|
||||
credentials.refresh(Request())
|
||||
project = (
|
||||
os.environ.get("GOOGLE_CLOUD_PROJECT")
|
||||
or os.environ.get("GCLOUD_PROJECT_ID")
|
||||
or info.get("project_id")
|
||||
)
|
||||
return {"token": credentials.token, "project": project}
|
||||
except Exception:
|
||||
# Library exceptions can contain credential data. Never forward them.
|
||||
return {"error": "Gemini service-account authentication failed; check credentials, permissions, and network access"}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
result = authenticate()
|
||||
print(json.dumps(result))
|
||||
sys.exit(1 if "error" in result else 0)
|
||||
@@ -0,0 +1,86 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { geminiAuth, geminiConfigured } from "./gemini-auth.mjs";
|
||||
import { pickProvider } from "./tts.mjs";
|
||||
|
||||
function env(t) {
|
||||
const saved = { ...process.env };
|
||||
for (const key of [
|
||||
"GEMINI_API_KEY",
|
||||
"GOOGLE_API_KEY",
|
||||
"GOOGLE_APPLICATION_CREDENTIALS",
|
||||
"GCS_CREDS",
|
||||
])
|
||||
delete process.env[key];
|
||||
t.after(() => {
|
||||
process.env = saved;
|
||||
});
|
||||
}
|
||||
|
||||
test("API keys take precedence over service accounts without invoking Python", (t) => {
|
||||
env(t);
|
||||
process.env.GEMINI_API_KEY = "gemini-key";
|
||||
process.env.GOOGLE_API_KEY = "google-key";
|
||||
process.env.GCS_CREDS = "private-json";
|
||||
const run = () => assert.fail("must not spawn Python");
|
||||
assert.deepEqual(geminiAuth({ run }).headers, { "x-goog-api-key": "gemini-key" });
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
assert.deepEqual(geminiAuth({ run }).headers, { "x-goog-api-key": "google-key" });
|
||||
});
|
||||
|
||||
test("service-account configuration selects Gemini and returns scoped helper headers", (t) => {
|
||||
env(t);
|
||||
assert.equal(geminiConfigured(), false);
|
||||
assert.throws(() => geminiAuth(), /needs GEMINI_API_KEY/);
|
||||
for (const key of ["GOOGLE_APPLICATION_CREDENTIALS", "GCS_CREDS"]) {
|
||||
process.env[key] = "credential-source";
|
||||
assert.equal(pickProvider("gemini"), "gemini");
|
||||
const auth = geminiAuth({
|
||||
run: (_, args, options) => {
|
||||
assert.ok(args.at(-1).endsWith("gemini-auth.py"));
|
||||
assert.ok(!args.includes("credential-source"));
|
||||
assert.equal(options.timeout, 60000);
|
||||
return {
|
||||
status: 0,
|
||||
stdout: JSON.stringify({ token: "access-token", project: "quota-project" }),
|
||||
};
|
||||
},
|
||||
});
|
||||
assert.deepEqual(auth.headers, {
|
||||
Authorization: "Bearer access-token",
|
||||
"x-goog-user-project": "quota-project",
|
||||
});
|
||||
delete process.env[key];
|
||||
}
|
||||
});
|
||||
|
||||
test("failed, malformed, or missing Python output cannot leak secrets or fall back", (t) => {
|
||||
env(t);
|
||||
process.env.GCS_CREDS = "private-json";
|
||||
for (const result of [
|
||||
{ status: 1, stdout: '{"error":"private-json"}', stderr: "private-json" },
|
||||
{ status: null, stdout: "private-json", error: new Error("private-json") },
|
||||
{ status: 0, stdout: "{}" },
|
||||
]) {
|
||||
assert.throws(
|
||||
() => geminiAuth({ run: () => result }),
|
||||
(error) => {
|
||||
assert.match(error.message, /authentication failed/);
|
||||
assert.ok(!error.message.includes("private-json"));
|
||||
return true;
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// Keep the Python credential-boundary suite reachable from the normal Node CI runner.
|
||||
test("Python service-account boundary validates credentials, scope, transport and errors", () => {
|
||||
const { cmd, args } = pythonInvocation([
|
||||
fileURLToPath(new URL("./gemini-auth_test.py", import.meta.url)),
|
||||
]);
|
||||
const result = spawnSync(cmd, args, { encoding: "utf8", timeout: 30_000 });
|
||||
assert.equal(result.status, 0, result.stderr || result.error?.message);
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Credential-boundary tests; stub Google's transport, never use real secrets."""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
|
||||
# Keep the credential subprocess protocol UTF-8 on every platform.
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(_stream, "reconfigure"):
|
||||
_stream.reconfigure(encoding="utf-8", errors=_stream.errors)
|
||||
|
||||
spec = importlib.util.spec_from_file_location("gemini_auth", Path(__file__).with_name("gemini-auth.py"))
|
||||
auth = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(auth)
|
||||
|
||||
|
||||
class AuthenticationTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.info = {"type": "service_account", "project_id": "account-project", "token_uri": "https://untrusted.invalid/token"}
|
||||
self.credentials = Mock(token="fixture-token")
|
||||
self.factory = Mock(return_value=self.credentials)
|
||||
self.request = Mock(return_value=object())
|
||||
transport = types.ModuleType("google.auth.transport.requests")
|
||||
transport.Request = self.request
|
||||
oauth = types.ModuleType("google.oauth2")
|
||||
oauth.service_account = types.SimpleNamespace(Credentials=types.SimpleNamespace(from_service_account_info=self.factory))
|
||||
modules = {name: types.ModuleType(name) for name in ("google", "google.auth", "google.auth.transport")}
|
||||
modules.update({"google.auth.transport.requests": transport, "google.oauth2": oauth})
|
||||
self.addCleanup(patch.stopall)
|
||||
patch.dict("sys.modules", modules).start()
|
||||
patch.dict(os.environ, {"GCS_CREDS": json.dumps(self.info)}, clear=True).start()
|
||||
|
||||
def test_scope_endpoint_and_refresh(self):
|
||||
self.assertEqual(auth.authenticate(), {"token": "fixture-token", "project": "account-project"})
|
||||
info = self.factory.call_args.args[0]
|
||||
self.assertEqual(info["token_uri"], "https://oauth2.googleapis.com/token")
|
||||
self.assertEqual(self.factory.call_args.kwargs["scopes"], ["https://www.googleapis.com/auth/generative-language.retriever"])
|
||||
self.credentials.refresh.assert_called_once_with(self.request.return_value)
|
||||
|
||||
def test_file_takes_precedence_over_json_environment(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "service.json"
|
||||
path.write_text(json.dumps({**self.info, "project_id": "file-project"}), encoding="utf-8")
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = str(path)
|
||||
os.environ["GCS_CREDS"] = "invalid-json"
|
||||
self.assertEqual(auth.authenticate()["project"], "file-project")
|
||||
|
||||
def test_quota_project_precedence(self):
|
||||
os.environ["GCLOUD_PROJECT_ID"] = "alias-project"
|
||||
self.assertEqual(auth.authenticate()["project"], "alias-project")
|
||||
os.environ["GOOGLE_CLOUD_PROJECT"] = "explicit-project"
|
||||
self.assertEqual(auth.authenticate()["project"], "explicit-project")
|
||||
|
||||
def test_user_adc_is_rejected_before_exchange(self):
|
||||
os.environ["GCS_CREDS"] = json.dumps({"type": "authorized_user"})
|
||||
self.assertIn("service-account JSON", auth.authenticate()["error"])
|
||||
self.factory.assert_not_called()
|
||||
|
||||
def test_invalid_json_or_missing_file_does_not_fall_back(self):
|
||||
os.environ["GCS_CREDS"] = "private-invalid-json"
|
||||
self.assertIn("error", auth.authenticate())
|
||||
os.environ["GCS_CREDS"] = json.dumps(self.info)
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/missing/service-account-fixture.json"
|
||||
self.assertIn("error", auth.authenticate())
|
||||
self.factory.assert_not_called()
|
||||
|
||||
def test_library_errors_are_redacted(self):
|
||||
for boundary in (self.factory, self.credentials.refresh):
|
||||
with self.subTest(boundary=boundary):
|
||||
boundary.side_effect = RuntimeError("private-key-and-token")
|
||||
result = auth.authenticate()
|
||||
self.assertIn("error", result)
|
||||
self.assertNotIn("private-key-and-token", json.dumps(result))
|
||||
boundary.side_effect = None
|
||||
|
||||
def test_missing_dependency_has_actionable_error(self):
|
||||
with patch.dict("sys.modules", {"google.auth.transport.requests": None}):
|
||||
self.assertIn("pip install google-auth requests", auth.authenticate()["error"])
|
||||
self.factory.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,122 @@
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import { geminiAuth } from "./gemini-auth.mjs";
|
||||
|
||||
export const GEMINI_TTS_MODEL = "gemini-3.8-flash-tts";
|
||||
|
||||
export const GEMINI_TTS_MODELS = [
|
||||
GEMINI_TTS_MODEL,
|
||||
"gemini-3.8-flash-lite-tts",
|
||||
"gemini-3.1-flash-tts-preview",
|
||||
"gemini-2.5-pro-preview-tts",
|
||||
"gemini-2.5-flash-preview-tts",
|
||||
];
|
||||
|
||||
// 3.8 returns WAV; older models return PCM that we wrap without resampling.
|
||||
// The shared engine transcribes the saved audio for word timings.
|
||||
export async function synthesizeGemini(
|
||||
{ text, voiceId = "Kore", model = GEMINI_TTS_MODEL, style, speed = 1, wavAbs },
|
||||
{ fetchImpl = fetch, authenticate = geminiAuth } = {},
|
||||
) {
|
||||
let secret;
|
||||
try {
|
||||
if (!GEMINI_TTS_MODELS.includes(model)) {
|
||||
throw new Error(`Unsupported Gemini TTS model: ${model}`);
|
||||
}
|
||||
if (speed !== 1) throw new Error("Gemini TTS uses style for pacing; omit speed or use 1");
|
||||
const modern = model.startsWith("gemini-3.8-");
|
||||
if (!modern && /^(voice_|voicekey_)/.test(voiceId)) {
|
||||
throw new Error(
|
||||
"Custom Gemini voices require a 3.8 TTS model; use a prebuilt voice with older models",
|
||||
);
|
||||
}
|
||||
const auth = authenticate();
|
||||
secret = auth.secret;
|
||||
const content = {
|
||||
type: "text",
|
||||
text:
|
||||
!modern && style ? `Read the following text with this delivery: ${style}\n\n${text}` : text,
|
||||
};
|
||||
if (modern && style) content.annotations = [{ type: "speech_metadata", style }];
|
||||
const response = await fetchImpl(
|
||||
"https://generativelanguage.googleapis.com/v1beta/interactions",
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", ...auth.headers },
|
||||
signal: AbortSignal.timeout(120_000),
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
input: [{ type: "user_input", content: [content] }],
|
||||
response_format: modern ? { type: "audio", mime_type: "audio/wav" } : { type: "audio" },
|
||||
generation_config: { speech_config: [{ voice: voiceId }] },
|
||||
store: false,
|
||||
}),
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
const detail = await response.text();
|
||||
throw new Error(`Gemini TTS HTTP ${response.status}: ${detail}`);
|
||||
}
|
||||
const payload = await response.json();
|
||||
if (payload.status !== "completed") {
|
||||
throw new Error(`Gemini TTS did not complete (${payload.status ?? "missing status"})`);
|
||||
}
|
||||
const audio = (payload.steps ?? [])
|
||||
.filter((step) => step.type === "model_output")
|
||||
.flatMap((step) => step.content ?? [])
|
||||
.filter((part) => part.type === "audio");
|
||||
if (audio.length !== 1 || !audio[0].data) {
|
||||
throw new Error("Gemini TTS returned no single audio block");
|
||||
}
|
||||
let bytes = Buffer.from(audio[0].data, "base64");
|
||||
if (!modern && /^audio\/l16(?:;|$)/i.test(audio[0].mime_type ?? "")) {
|
||||
bytes = pcmToWav(bytes, audio[0].mime_type);
|
||||
} else if (audio[0].mime_type !== "audio/wav") {
|
||||
throw new Error("Gemini TTS returned an unsupported audio format");
|
||||
}
|
||||
if (
|
||||
bytes.length <= 44 ||
|
||||
bytes.toString("ascii", 0, 4) !== "RIFF" ||
|
||||
bytes.toString("ascii", 8, 12) !== "WAVE"
|
||||
) {
|
||||
throw new Error("Gemini TTS returned invalid WAV audio");
|
||||
}
|
||||
mkdirSync(dirname(wavAbs), { recursive: true });
|
||||
writeFileSync(wavAbs, bytes);
|
||||
return { ok: true, words: null };
|
||||
} catch (error) {
|
||||
// Error responses must never echo the credential into logs or metadata.
|
||||
const message = String(error?.message ?? error);
|
||||
return {
|
||||
ok: false,
|
||||
words: null,
|
||||
error: secret ? message.split(secret).join("[redacted]") : message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function pcmToWav(pcm, mime) {
|
||||
const rate = Number(/(?:^|;)\s*rate=(\d+)(?:;|$)/i.exec(mime)?.[1]);
|
||||
const channels = /(?:^|;)\s*channels=([^;]+)/i.exec(mime)?.[1];
|
||||
const codec = /(?:^|;)\s*codec=([^;]+)/i.exec(mime)?.[1];
|
||||
if ((channels && channels.trim() !== "1") || (codec && codec.trim() !== "pcm")) {
|
||||
throw new Error("Gemini TTS returned unsupported PCM channels or codec");
|
||||
}
|
||||
if (!Number.isInteger(rate) || rate < 8000 || rate > 96000 || !pcm.length || pcm.length % 2) {
|
||||
throw new Error("Gemini TTS returned invalid PCM audio or sample rate");
|
||||
}
|
||||
const header = Buffer.alloc(44);
|
||||
header.write("RIFF");
|
||||
header.writeUInt32LE(36 + pcm.length, 4);
|
||||
header.write("WAVEfmt ", 8);
|
||||
header.writeUInt32LE(16, 16);
|
||||
header.writeUInt16LE(1, 20);
|
||||
header.writeUInt16LE(1, 22);
|
||||
header.writeUInt32LE(rate, 24);
|
||||
header.writeUInt32LE(rate * 2, 28);
|
||||
header.writeUInt16LE(2, 32);
|
||||
header.writeUInt16LE(16, 34);
|
||||
header.write("data", 36);
|
||||
header.writeUInt32LE(pcm.length, 40);
|
||||
return Buffer.concat([header, pcm]);
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, readFileSync, rmSync, existsSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { synthesizeGemini, GEMINI_TTS_MODEL } from "./gemini-tts.mjs";
|
||||
import { pickProvider, resolveVoiceId, synthesizeOne } from "./tts.mjs";
|
||||
|
||||
function fixture(t) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "gemini-tts-"));
|
||||
const saved = { ...process.env };
|
||||
process.env.GEMINI_API_KEY = "test-gemini-key";
|
||||
delete process.env.GOOGLE_API_KEY;
|
||||
delete process.env.GOOGLE_APPLICATION_CREDENTIALS;
|
||||
delete process.env.GCS_CREDS;
|
||||
t.after(() => {
|
||||
process.env = saved;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
const wav = Buffer.alloc(48);
|
||||
wav.write("RIFF");
|
||||
wav.writeUInt32LE(40, 4);
|
||||
wav.write("WAVEfmt ", 8);
|
||||
wav.writeUInt32LE(16, 16);
|
||||
wav.writeUInt16LE(1, 20);
|
||||
wav.writeUInt16LE(1, 22);
|
||||
wav.writeUInt32LE(24000, 24);
|
||||
wav.writeUInt32LE(48000, 28);
|
||||
wav.writeUInt16LE(2, 32);
|
||||
wav.writeUInt16LE(16, 34);
|
||||
wav.write("data", 36);
|
||||
wav.writeUInt32LE(4, 40);
|
||||
const audio = { type: "audio", mime_type: "audio/wav", data: wav.toString("base64") };
|
||||
const payload = { status: "completed", steps: [{ type: "model_output", content: [audio] }] };
|
||||
return {
|
||||
wav,
|
||||
audio,
|
||||
payload,
|
||||
args: { text: "Hello there.", voiceId: "Kore", wavAbs: join(dir, "voice", "one.wav") },
|
||||
};
|
||||
}
|
||||
|
||||
test("Gemini preserves verbatim text, directs style separately, saves WAV and requests transcription", async (t) => {
|
||||
const { wav, payload, args } = fixture(t);
|
||||
t.mock.method(globalThis, "fetch", async (url, options) => {
|
||||
assert.equal(url, "https://generativelanguage.googleapis.com/v1beta/interactions");
|
||||
assert.equal(options.headers["x-goog-api-key"], "test-gemini-key");
|
||||
const body = JSON.parse(options.body);
|
||||
assert.equal(body.model, GEMINI_TTS_MODEL);
|
||||
assert.equal(body.store, false);
|
||||
assert.deepEqual(body.input[0].content, [
|
||||
{
|
||||
type: "text",
|
||||
text: args.text,
|
||||
annotations: [{ type: "speech_metadata", style: "Warm and calm" }],
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(body.generation_config.speech_config, [{ voice: "Kore" }]);
|
||||
assert.equal(body.response_format.mime_type, "audio/wav");
|
||||
return Response.json(payload);
|
||||
});
|
||||
const result = await synthesizeOne({ ...args, provider: "gemini", style: "Warm and calm" });
|
||||
assert.deepEqual(result, { ok: true, words: null });
|
||||
assert.deepEqual(readFileSync(args.wavAbs), wav);
|
||||
});
|
||||
|
||||
test("Flash-Lite and GOOGLE_API_KEY work without a Gemini key", async (t) => {
|
||||
const { payload, args } = fixture(t);
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
process.env.GOOGLE_API_KEY = "test-google-key";
|
||||
assert.equal(pickProvider("gemini"), "gemini");
|
||||
assert.equal(await resolveVoiceId({ provider: "gemini" }), "Kore");
|
||||
const result = await synthesizeGemini(
|
||||
{ ...args, model: "gemini-3.8-flash-lite-tts" },
|
||||
{
|
||||
fetchImpl: async (_, options) => {
|
||||
assert.equal(options.headers["x-goog-api-key"], "test-google-key");
|
||||
const body = JSON.parse(options.body);
|
||||
assert.equal(body.model, "gemini-3.8-flash-lite-tts");
|
||||
assert.equal(body.input[0].content[0].annotations, undefined);
|
||||
return Response.json(payload);
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.ok, true);
|
||||
});
|
||||
|
||||
test("Gemini remains opt-in even with a key and preserves explicitly chosen voices", async (t) => {
|
||||
fixture(t);
|
||||
assert.notEqual(pickProvider(), "gemini");
|
||||
assert.equal(
|
||||
await resolveVoiceId({ provider: "gemini", userVoice: "voice_custom" }),
|
||||
"voice_custom",
|
||||
);
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
assert.throws(() => pickProvider("gemini"), /GEMINI_API_KEY or GOOGLE_API_KEY/);
|
||||
});
|
||||
|
||||
test("invalid config fails before spending a generation request", async (t) => {
|
||||
const { args } = fixture(t);
|
||||
const deps = { fetchImpl: () => assert.fail("must not call the API") };
|
||||
for (const change of [{ model: "gemini-2.5-flash" }, { speed: 1.2 }]) {
|
||||
const result = await synthesizeGemini({ ...args, ...change }, deps);
|
||||
assert.equal(result.ok, false);
|
||||
assert.ok(result.error);
|
||||
}
|
||||
delete process.env.GEMINI_API_KEY;
|
||||
assert.match((await synthesizeGemini(args, deps)).error, /needs GEMINI_API_KEY/);
|
||||
assert.equal(existsSync(args.wavAbs), false);
|
||||
});
|
||||
|
||||
test("HTTP and network failures stay actionable without leaking the key", async (t) => {
|
||||
const { args } = fixture(t);
|
||||
const result = await synthesizeGemini(args, {
|
||||
fetchImpl: async () => new Response("quota exceeded test-gemini-key", { status: 429 }),
|
||||
});
|
||||
assert.equal(result.ok, false);
|
||||
assert.match(result.error, /HTTP 429: quota exceeded/);
|
||||
assert.ok(!result.error.includes("test-gemini-key"));
|
||||
const timeout = await synthesizeGemini(args, {
|
||||
fetchImpl: async () => {
|
||||
throw new Error("timed out");
|
||||
},
|
||||
});
|
||||
assert.match(timeout.error, /timed out/);
|
||||
assert.equal(existsSync(args.wavAbs), false);
|
||||
});
|
||||
|
||||
test("incomplete, missing, raw PCM and malformed audio cannot become a successful WAV", async (t) => {
|
||||
const { args, payload, audio } = fixture(t);
|
||||
const cases = [
|
||||
{ ...payload, status: "incomplete" },
|
||||
{ status: "completed", steps: [] },
|
||||
...[{ ...audio, mime_type: "audio/l16" }, { ...audio, data: "not audio" }, audio].map(
|
||||
(part, i) => ({
|
||||
status: "completed",
|
||||
steps: [{ type: "model_output", content: i === 2 ? [part, part] : [part] }],
|
||||
}),
|
||||
),
|
||||
];
|
||||
for (const body of cases) {
|
||||
const result = await synthesizeGemini(args, { fetchImpl: async () => Response.json(body) });
|
||||
assert.equal(result.ok, false);
|
||||
assert.equal(existsSync(args.wavAbs), false);
|
||||
}
|
||||
});
|
||||
|
||||
for (const model of [
|
||||
"gemini-3.1-flash-tts-preview",
|
||||
"gemini-2.5-pro-preview-tts",
|
||||
"gemini-2.5-flash-preview-tts",
|
||||
]) {
|
||||
test(`${model} uses legacy delivery prompts and wraps PCM as mono WAV`, async (t) => {
|
||||
const { args, wav } = fixture(t);
|
||||
const pcm = wav.subarray(44);
|
||||
const result = await synthesizeGemini(
|
||||
{ ...args, model, style: "Warm and clear" },
|
||||
{
|
||||
fetchImpl: async (_, options) => {
|
||||
const body = JSON.parse(options.body);
|
||||
assert.deepEqual(body.response_format, { type: "audio" });
|
||||
assert.equal(body.input[0].content[0].annotations, undefined);
|
||||
assert.ok(body.input[0].content[0].text.endsWith(args.text));
|
||||
assert.ok(body.input[0].content[0].text.includes("Warm and clear"));
|
||||
return Response.json({
|
||||
status: "completed",
|
||||
steps: [
|
||||
{
|
||||
type: "model_output",
|
||||
content: [
|
||||
{
|
||||
type: "audio",
|
||||
mime_type: "audio/L16;codec=pcm;rate=24000",
|
||||
data: pcm.toString("base64"),
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(result.ok, true, result.error);
|
||||
assert.deepEqual(readFileSync(args.wavAbs), wav);
|
||||
});
|
||||
}
|
||||
|
||||
test("service-account bearer and quota project reach synthesis, and token errors are redacted", async (t) => {
|
||||
const { args, payload } = fixture(t);
|
||||
const authenticate = () => ({
|
||||
headers: { Authorization: "Bearer secret-token", "x-goog-user-project": "test-project" },
|
||||
secret: "secret-token",
|
||||
});
|
||||
const ok = await synthesizeGemini(args, {
|
||||
authenticate,
|
||||
fetchImpl: async (_, options) => {
|
||||
assert.equal(options.headers.Authorization, "Bearer secret-token");
|
||||
assert.equal(options.headers["x-goog-user-project"], "test-project");
|
||||
assert.equal(options.headers["x-goog-api-key"], undefined);
|
||||
return Response.json(payload);
|
||||
},
|
||||
});
|
||||
assert.equal(ok.ok, true);
|
||||
const failed = await synthesizeGemini(args, {
|
||||
authenticate,
|
||||
fetchImpl: async () => new Response("denied secret-token", { status: 403 }),
|
||||
});
|
||||
assert.match(failed.error, /HTTP 403/);
|
||||
assert.ok(!failed.error.includes("secret-token"));
|
||||
});
|
||||
|
||||
test("older models reject unsupported PCM and custom voices", async (t) => {
|
||||
const { args } = fixture(t);
|
||||
for (const mime of [
|
||||
"audio/l16",
|
||||
"audio/l16;rate=0",
|
||||
"audio/l16;rate=24000;channels=2",
|
||||
"audio/l16;rate=24000;codec=other",
|
||||
]) {
|
||||
const result = await synthesizeGemini(
|
||||
{ ...args, model: "gemini-3.1-flash-tts-preview" },
|
||||
{
|
||||
fetchImpl: async () =>
|
||||
Response.json({
|
||||
status: "completed",
|
||||
steps: [
|
||||
{
|
||||
type: "model_output",
|
||||
content: [{ type: "audio", mime_type: mime, data: "AAAAAA==" }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
);
|
||||
assert.equal(result.ok, false);
|
||||
assert.equal(existsSync(args.wavAbs), false);
|
||||
}
|
||||
const result = await synthesizeGemini(
|
||||
{ ...args, model: "gemini-2.5-pro-preview-tts", voiceId: "voice_custom" },
|
||||
{ fetchImpl: () => assert.fail("must not generate") },
|
||||
);
|
||||
assert.match(result.error, /Custom Gemini voices require/);
|
||||
});
|
||||
@@ -0,0 +1,240 @@
|
||||
import { fetchMedia } from "../../../scripts/lib/media-fetch.mjs";
|
||||
// heygen.mjs — vendored HeyGen REST helpers (auth + transport) for the audio
|
||||
// pipeline. The credential resolver matches the hyperframes CLI auth: first
|
||||
// usable source wins — a host gateway ($HEYGEN_API_BASE with its own
|
||||
// $HEYGEN_API_KEY) → a host-injected OAuth $HEYGEN_ACCESS_TOKEN (Bearer) →
|
||||
// $HEYGEN_API_KEY / $HYPERFRAMES_API_KEY → a nearby .env → ~/.heygen/
|
||||
// credentials (oauth → Bearer, else api_key → X-Api-Key; $HEYGEN_CONFIG_DIR
|
||||
// overrides the dir). $HEYGEN_API_BASE moves every request to that host, as it
|
||||
// does for the heygen CLI; plain HTTP needs $HEYGEN_ALLOW_HTTP=1, as there.
|
||||
// Vendored so the skill ships standalone. Pure node.
|
||||
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
|
||||
export const HEYGEN_BASE = "https://api.heygen.com/v3";
|
||||
|
||||
// The v3 base every request goes to: $HEYGEN_API_BASE when a host app names its own gateway (HyperFrames Desktop
|
||||
// forwards it to HeyGen with the API key saved in its Settings), else HeyGen's public API. Plain HTTP carries the key
|
||||
// in the clear, so it needs $HEYGEN_ALLOW_HTTP=1, the heygen CLI's own rule.
|
||||
export function heygenBase() {
|
||||
const host = process.env.HEYGEN_API_BASE?.trim().replace(/\/+$/, "");
|
||||
if (!host) return HEYGEN_BASE;
|
||||
if (host.startsWith("http://") && process.env.HEYGEN_ALLOW_HTTP !== "1")
|
||||
throw new Error(
|
||||
`HEYGEN_API_BASE (${host}) uses HTTP, which sends the key in plaintext. Set HEYGEN_ALLOW_HTTP=1 to allow it.`,
|
||||
);
|
||||
return `${host}/v3`;
|
||||
}
|
||||
|
||||
// No base override, or one on HeyGen's own hosts (a canary or dev API).
|
||||
function heygenOwnBase() {
|
||||
const host = process.env.HEYGEN_API_BASE?.trim();
|
||||
if (!host) return true;
|
||||
try {
|
||||
const name = new URL(host).hostname;
|
||||
return name === "heygen.com" || name.endsWith(".heygen.com");
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// A host gateway: the host named its own API base and the key that base accepts. It pays for every call, so it wins
|
||||
// over any other credential the environment carries.
|
||||
const hostGatewayKey = () =>
|
||||
process.env.HEYGEN_API_BASE?.trim() && process.env.HEYGEN_API_KEY
|
||||
? process.env.HEYGEN_API_KEY
|
||||
: null;
|
||||
export const HEYGEN_CLI_SOURCE_HEADERS = { "X-HeyGen-Source": "cli" };
|
||||
// Tool-attribution sent on EVERY media-use HeyGen call regardless of auth type, so
|
||||
// the backend can isolate media-use consumption from other free TTS / avatar video.
|
||||
// Unconditional — a paying user's media-use call is still media-use — unlike the
|
||||
// OAuth-only cli-source header above, which also gates the free allowance.
|
||||
export const HEYGEN_CLIENT_SOURCE_HEADERS = { "X-HeyGen-Client-Source": "media-use" };
|
||||
|
||||
// A missing `.env`, or a `.env` folder (some home dirs have one), is no env file: null. Read without checking first,
|
||||
// so the file cannot change between a check and the read.
|
||||
function envFileText(path) {
|
||||
try {
|
||||
return readFileSync(path, "utf8");
|
||||
} catch (error) {
|
||||
if (["ENOENT", "ENOTDIR", "EISDIR"].includes(error.code)) return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// A host app sets these in the environment it spawns, never in a project file: a project's .env naming its own base
|
||||
// would send the person's shell HEYGEN_API_KEY to that host.
|
||||
const HOST_ONLY = new Set(["HEYGEN_API_BASE", "HEYGEN_ALLOW_HTTP"]);
|
||||
|
||||
// Walk up ≤5 dirs from startDir; load the first .env (shell env always wins).
|
||||
export function loadEnvFromDir(startDir) {
|
||||
let dir = resolve(startDir);
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const text = envFileText(join(dir, ".env"));
|
||||
if (text != null) {
|
||||
for (const raw of text.split("\n")) {
|
||||
let line = raw.trim();
|
||||
if (!line || line.startsWith("#")) continue;
|
||||
if (line.startsWith("export ")) line = line.slice(7).trim();
|
||||
const eq = line.indexOf("=");
|
||||
if (eq < 1) continue;
|
||||
const key = line.slice(0, eq).trim();
|
||||
let val = line.slice(eq + 1).trim();
|
||||
if (val.startsWith('"') || val.startsWith("'")) {
|
||||
const q = val[0];
|
||||
const end = val.indexOf(q, 1);
|
||||
val = end > 0 ? val.slice(1, end) : val.slice(1);
|
||||
}
|
||||
if (!HOST_ONLY.has(key) && !(key in process.env)) process.env[key] = val;
|
||||
}
|
||||
return;
|
||||
}
|
||||
const parent = dirname(dir);
|
||||
if (parent === dir) break;
|
||||
dir = parent;
|
||||
}
|
||||
}
|
||||
|
||||
// → { headers } | { expired: true } | null. Never throws.
|
||||
export function heygenCredential() {
|
||||
const cred = resolveCredential();
|
||||
return cred?.unreadable ? null : cred;
|
||||
}
|
||||
|
||||
// heygenCredential's answer, or { unreadable: { file, code } } when the credentials path exists but cannot be read
|
||||
// (a folder, a locked ~/.heygen), so heygenAuthHeaders can say to fix that path: logging in again would fail there too.
|
||||
// Read without checking first, so the file cannot change between a check and the read.
|
||||
function resolveCredential() {
|
||||
const gatewayKey = hostGatewayKey();
|
||||
if (gatewayKey) return { headers: { "X-Api-Key": gatewayKey } };
|
||||
// Every other credential belongs to HeyGen: a base on any other host gets none of them.
|
||||
if (!heygenOwnBase()) return null;
|
||||
const accessToken = process.env.HEYGEN_ACCESS_TOKEN;
|
||||
if (accessToken) return { headers: { Authorization: `Bearer ${accessToken}` } };
|
||||
const envKey = process.env.HEYGEN_API_KEY || process.env.HYPERFRAMES_API_KEY;
|
||||
if (envKey) return { headers: { "X-Api-Key": envKey } };
|
||||
|
||||
const file = join(process.env.HEYGEN_CONFIG_DIR || join(homedir(), ".heygen"), "credentials");
|
||||
let raw;
|
||||
try {
|
||||
raw = readFileSync(file, "utf8").trim();
|
||||
} catch (error) {
|
||||
return error.code === "ENOENT" ? null : { unreadable: { file, code: error.code } };
|
||||
}
|
||||
if (!raw) return null;
|
||||
if (!raw.startsWith("{")) return { headers: { "X-Api-Key": raw } };
|
||||
|
||||
// A malformed credentials file (partial write / wrong shape) must degrade to
|
||||
// "no credential", not crash the engine at startup.
|
||||
let cred;
|
||||
try {
|
||||
cred = JSON.parse(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const oauth = cred.oauth;
|
||||
if (oauth?.access_token) {
|
||||
const expired = oauth.expires_at && new Date(oauth.expires_at).getTime() - 60_000 < Date.now();
|
||||
if (!expired) return { headers: { Authorization: `Bearer ${oauth.access_token}` } };
|
||||
if (!cred.api_key) return { expired: true };
|
||||
}
|
||||
if (cred.api_key) return { headers: { "X-Api-Key": cred.api_key } };
|
||||
return null;
|
||||
}
|
||||
|
||||
// → "oauth" | "api_key" | null. Same oauth-vs-api-key check heygenAuthHeaders()
|
||||
// makes internally, exposed on its own so callers that only need to *tag* the
|
||||
// auth path (telemetry) don't have to parse headers back apart. Never throws:
|
||||
// no credential (or an expired one) is just `null`, same as a fresh resolve
|
||||
// with nothing to tag.
|
||||
export function heygenAuthMethod() {
|
||||
const cred = heygenCredential();
|
||||
if (!cred?.headers) return null;
|
||||
return "Authorization" in cred.headers ? "oauth" : "api_key";
|
||||
}
|
||||
|
||||
// → auth headers object, or throw with a fix hint.
|
||||
export function heygenAuthHeaders() {
|
||||
const cred = resolveCredential();
|
||||
if (cred?.headers) {
|
||||
// Only tag OAuth (Bearer) traffic as cli-source — the backend uses it to
|
||||
// grant the free allowance for OAuth requests and ignores it for API-key
|
||||
// (X-Api-Key) traffic, where it's dead metadata.
|
||||
const isOauth = "Authorization" in cred.headers;
|
||||
return isOauth
|
||||
? { ...cred.headers, ...HEYGEN_CLI_SOURCE_HEADERS, ...HEYGEN_CLIENT_SOURCE_HEADERS }
|
||||
: { ...cred.headers, ...HEYGEN_CLIENT_SOURCE_HEADERS };
|
||||
}
|
||||
if (cred?.unreadable)
|
||||
throw new Error(
|
||||
`HeyGen credentials at ${cred.unreadable.file} can't be read (${cred.unreadable.code}) — fix or remove that path, then run \`npx hyperframes auth login\``,
|
||||
);
|
||||
if (cred?.expired)
|
||||
throw new Error(
|
||||
"HeyGen OAuth token expired — run `npx hyperframes auth refresh` (or `npx hyperframes auth login`)",
|
||||
);
|
||||
throw new Error(
|
||||
"no HeyGen credentials — set $HEYGEN_API_KEY, or run `npx hyperframes auth login` (writes ~/.heygen/credentials)",
|
||||
);
|
||||
}
|
||||
|
||||
// Authed JSON request against the v3 API; throws on a non-OK status.
|
||||
export async function heygenJSON(path, { method = "GET", headers = {}, body } = {}) {
|
||||
const opts = { method, headers: { ...HEYGEN_CLIENT_SOURCE_HEADERS, ...headers } };
|
||||
if (body !== undefined) {
|
||||
opts.headers["Content-Type"] = "application/json";
|
||||
opts.body = JSON.stringify(body);
|
||||
}
|
||||
const res = await fetch(`${heygenBase()}${path}`, opts);
|
||||
if (!res.ok) {
|
||||
const detail = await res.text().catch(() => "");
|
||||
const message = `HeyGen ${method} ${path} → HTTP ${res.status}${detail ? `\n${detail.slice(0, 300)}` : ""}`;
|
||||
throw Object.assign(new Error(message), { status: res.status, body: detail });
|
||||
}
|
||||
// A DELETE may answer 204 with no body.
|
||||
const text = await res.text();
|
||||
return text ? JSON.parse(text) : {};
|
||||
}
|
||||
|
||||
// HeyGen's own words for a failed call: its {"error":{"message"}}, else the raw body, else the error's message.
|
||||
export function heygenMessage(e) {
|
||||
if (!e?.body) return e?.message ? String(e.message) : String(e);
|
||||
try {
|
||||
return JSON.parse(e.body).error?.message ?? e.body;
|
||||
} catch {
|
||||
return e.body;
|
||||
}
|
||||
}
|
||||
|
||||
// Download a (presigned) URL to destPath; returns byte length.
|
||||
export async function downloadTo(url, destPath) {
|
||||
const res = await fetchMedia(url);
|
||||
if (!res.ok) throw new Error(`download HTTP ${res.status}: ${String(url).slice(0, 80)}`);
|
||||
const bytes = Buffer.from(await res.arrayBuffer());
|
||||
mkdirSync(dirname(destPath), { recursive: true });
|
||||
writeFileSync(destPath, bytes);
|
||||
return bytes.length;
|
||||
}
|
||||
|
||||
// Retrieval search over HeyGen's audio catalog (NOT generation). type =
|
||||
// "music" | "sound_effects". Returns the ranked results array (best first); each
|
||||
// item has a presigned `audio_url` (+ `duration`, `description`, `name`, `score`).
|
||||
// `query` is required (≥1 char, empty → HTTP 400) and `limit` is capped at 50.
|
||||
// `minScore`: omit to use the server default (0.7). That default is TOO HIGH for
|
||||
// sound_effects — good SFX hits score ~0.5–0.67, so callers wanting SFX should
|
||||
// pass a lower floor (~0.4); music scores high and is fine at the default.
|
||||
export async function searchSounds(query, type, headers, { limit = 5, minScore } = {}) {
|
||||
const params = new URLSearchParams({ query, type, limit: String(limit) });
|
||||
if (minScore != null) params.set("min_score", String(minScore));
|
||||
const payload = await heygenJSON(`/audio/sounds?${params.toString()}`, { headers });
|
||||
// `data` comes back as a ranked array (best first). Older responses keyed it by
|
||||
// numeric index ("0","1",…); normalize both shapes to an array (empty → []).
|
||||
const data = payload?.data ?? payload;
|
||||
if (Array.isArray(data)) return data;
|
||||
if (data && typeof data === "object") return Object.values(data);
|
||||
throw new Error(
|
||||
`unexpected /audio/sounds shape — top keys: ${Object.keys(payload ?? {}).join(", ")}`,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { createServer } from "node:http";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
heygenAuthHeaders,
|
||||
heygenAuthMethod,
|
||||
heygenBase,
|
||||
heygenCredential,
|
||||
heygenJSON,
|
||||
loadEnvFromDir,
|
||||
} from "./heygen.mjs";
|
||||
|
||||
const HEYGEN_ENV = [
|
||||
"HEYGEN_ACCESS_TOKEN",
|
||||
"HEYGEN_API_KEY",
|
||||
"HYPERFRAMES_API_KEY",
|
||||
"HEYGEN_CONFIG_DIR",
|
||||
"HEYGEN_API_BASE",
|
||||
"HEYGEN_ALLOW_HTTP",
|
||||
];
|
||||
|
||||
// Runs fn with every HeyGen variable unset, then puts them back; an async fn restores once it settles.
|
||||
function withCleanHeygenEnv(fn) {
|
||||
const previous = Object.fromEntries(HEYGEN_ENV.map((name) => [name, process.env[name]]));
|
||||
const restore = () => {
|
||||
for (const [name, value] of Object.entries(previous)) {
|
||||
if (value === undefined) delete process.env[name];
|
||||
else process.env[name] = value;
|
||||
}
|
||||
};
|
||||
for (const name of HEYGEN_ENV) delete process.env[name];
|
||||
let result;
|
||||
try {
|
||||
result = fn();
|
||||
} catch (error) {
|
||||
restore();
|
||||
throw error;
|
||||
}
|
||||
if (result && typeof result.then === "function") return result.finally(restore);
|
||||
restore();
|
||||
return result;
|
||||
}
|
||||
|
||||
test("heygenAuthHeaders does not tag API-key requests as CLI traffic, but still carries the media-use tool tag", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_API_KEY = "hg_test";
|
||||
// API-key requests use normal billing; the backend ignores the cli-source
|
||||
// header for them, so it's not sent. The tool-attribution header IS sent on
|
||||
// every media-use call (any auth type) so the backend can isolate media-use.
|
||||
assert.deepEqual(heygenAuthHeaders(), {
|
||||
"X-Api-Key": "hg_test",
|
||||
"X-HeyGen-Client-Source": "media-use",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthHeaders tags OAuth requests as CLI traffic and with the media-use tool tag", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
writeFileSync(
|
||||
join(dir, "credentials"),
|
||||
JSON.stringify({
|
||||
oauth: {
|
||||
access_token: "at_test",
|
||||
expires_at: "2099-01-01T00:00:00Z",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.deepEqual(heygenAuthHeaders(), {
|
||||
Authorization: "Bearer at_test",
|
||||
"X-HeyGen-Source": "cli",
|
||||
"X-HeyGen-Client-Source": "media-use",
|
||||
});
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("a host-injected HEYGEN_ACCESS_TOKEN wins over an API key and is treated as OAuth", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_ACCESS_TOKEN = "at_host";
|
||||
process.env.HEYGEN_API_KEY = "hg_test";
|
||||
assert.deepEqual(heygenAuthHeaders(), {
|
||||
Authorization: "Bearer at_host",
|
||||
"X-HeyGen-Source": "cli",
|
||||
"X-HeyGen-Client-Source": "media-use",
|
||||
});
|
||||
assert.equal(heygenAuthMethod(), "oauth");
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns api_key for an env API key, without tagging headers", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_API_KEY = "hg_test";
|
||||
assert.equal(heygenAuthMethod(), "api_key");
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns oauth for a live OAuth credential", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
writeFileSync(
|
||||
join(dir, "credentials"),
|
||||
JSON.stringify({
|
||||
oauth: {
|
||||
access_token: "at_test",
|
||||
expires_at: "2099-01-01T00:00:00Z",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.equal(heygenAuthMethod(), "oauth");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns null with no credential at all", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir; // no credentials file written
|
||||
assert.equal(heygenAuthMethod(), null);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("loadEnvFromDir skips a .env folder and loads the .env file above it", () => {
|
||||
const root = mkdtempSync(join(tmpdir(), "heygen-env-"));
|
||||
const project = join(root, "project");
|
||||
mkdirSync(join(project, ".env"), { recursive: true });
|
||||
writeFileSync(join(root, ".env"), "MEDIA_USE_ENV_DIR_TEST=from-parent\n");
|
||||
try {
|
||||
loadEnvFromDir(project);
|
||||
assert.equal(process.env.MEDIA_USE_ENV_DIR_TEST, "from-parent");
|
||||
} finally {
|
||||
delete process.env.MEDIA_USE_ENV_DIR_TEST;
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("a project's .env cannot name the HeyGen base, so a shell key never leaves for its host", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const project = mkdtempSync(join(tmpdir(), "heygen-env-"));
|
||||
writeFileSync(
|
||||
join(project, ".env"),
|
||||
"HEYGEN_API_BASE=https://proxy.example.com\nHEYGEN_ALLOW_HTTP=1\nMEDIA_USE_ENV_BASE_TEST=loaded\n",
|
||||
);
|
||||
try {
|
||||
process.env.HEYGEN_API_KEY = "hg_shell_real";
|
||||
loadEnvFromDir(project);
|
||||
assert.equal(process.env.MEDIA_USE_ENV_BASE_TEST, "loaded");
|
||||
assert.equal(process.env.HEYGEN_API_BASE, undefined);
|
||||
assert.equal(process.env.HEYGEN_ALLOW_HTTP, undefined);
|
||||
assert.equal(heygenBase(), "https://api.heygen.com/v3");
|
||||
} finally {
|
||||
delete process.env.MEDIA_USE_ENV_BASE_TEST;
|
||||
rmSync(project, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns null when the credentials path is a folder", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
mkdirSync(join(dir, "credentials"));
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
assert.equal(heygenAuthMethod(), null);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthMethod returns null when the credentials path is a symlink loop", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
symlinkSync(join(dir, "credentials"), join(dir, "credentials"));
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
assert.equal(heygenAuthMethod(), null);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenAuthHeaders says to fix an unreadable credentials path, and to log in when there is none", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
assert.throws(() => heygenAuthHeaders(), /no HeyGen credentials/);
|
||||
mkdirSync(join(dir, "credentials"));
|
||||
assert.equal(heygenCredential(), null);
|
||||
assert.throws(
|
||||
() => heygenAuthHeaders(),
|
||||
(error) =>
|
||||
error.message.includes(join(dir, "credentials")) &&
|
||||
/fix or remove that path/.test(error.message),
|
||||
);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenBase is HeyGen's public API unless a host names another", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
assert.equal(heygenBase(), "https://api.heygen.com/v3");
|
||||
process.env.HEYGEN_API_BASE = "https://api-canary.heygen.com/";
|
||||
assert.equal(heygenBase(), "https://api-canary.heygen.com/v3");
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenBase refuses a plain-HTTP host unless HEYGEN_ALLOW_HTTP is set, as the heygen CLI does", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_API_BASE = "http://127.0.0.1:4100";
|
||||
assert.throws(() => heygenBase(), /HEYGEN_ALLOW_HTTP=1/);
|
||||
process.env.HEYGEN_ALLOW_HTTP = "1";
|
||||
assert.equal(heygenBase(), "http://127.0.0.1:4100/v3");
|
||||
});
|
||||
});
|
||||
|
||||
test("a host gateway (HEYGEN_API_BASE with its own HEYGEN_API_KEY) wins over a host OAuth token", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
process.env.HEYGEN_API_BASE = "http://127.0.0.1:4100";
|
||||
process.env.HEYGEN_ALLOW_HTTP = "1";
|
||||
process.env.HEYGEN_API_KEY = "gateway-token";
|
||||
process.env.HEYGEN_ACCESS_TOKEN = "at_host";
|
||||
assert.deepEqual(heygenAuthHeaders(), {
|
||||
"X-Api-Key": "gateway-token",
|
||||
"X-HeyGen-Client-Source": "media-use",
|
||||
});
|
||||
assert.equal(heygenAuthMethod(), "api_key");
|
||||
});
|
||||
});
|
||||
|
||||
test("heygenJSON sends its request to the host's HEYGEN_API_BASE with the host's key", async () => {
|
||||
/** @type {{ url?: string, key?: string | string[] }} */
|
||||
const seen = {};
|
||||
const server = createServer((req, res) => {
|
||||
seen.url = req.url;
|
||||
seen.key = req.headers["x-api-key"];
|
||||
res.writeHead(200, { "content-type": "application/json" }).end('{"data":[]}');
|
||||
});
|
||||
await new Promise((done) => server.listen(0, "127.0.0.1", done));
|
||||
const { port } = /** @type {import("node:net").AddressInfo} */ (server.address());
|
||||
try {
|
||||
await withCleanHeygenEnv(async () => {
|
||||
process.env.HEYGEN_API_BASE = `http://127.0.0.1:${port}`;
|
||||
process.env.HEYGEN_ALLOW_HTTP = "1";
|
||||
process.env.HEYGEN_API_KEY = "gateway-token";
|
||||
const reply = await heygenJSON("/voices?limit=1", { headers: heygenAuthHeaders() });
|
||||
assert.deepEqual(reply, { data: [] });
|
||||
});
|
||||
assert.equal(seen.url, "/v3/voices?limit=1");
|
||||
assert.equal(seen.key, "gateway-token");
|
||||
} finally {
|
||||
server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("a base that isn't HeyGen's gets no stored or host OAuth credential, only a key named for it", () => {
|
||||
withCleanHeygenEnv(() => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "heygen-cred-"));
|
||||
try {
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
writeFileSync(join(dir, "credentials"), JSON.stringify({ api_key: "hg_stored" }));
|
||||
process.env.HEYGEN_ACCESS_TOKEN = "at_host";
|
||||
process.env.HEYGEN_API_BASE = "https://proxy.example.com";
|
||||
assert.equal(heygenCredential(), null);
|
||||
assert.throws(() => heygenAuthHeaders(), /no HeyGen credentials/);
|
||||
// HeyGen's own hosts keep every credential source.
|
||||
process.env.HEYGEN_API_BASE = "https://api-canary.heygen.com";
|
||||
assert.equal(heygenAuthMethod(), "oauth");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,13 @@
|
||||
import { realpathSync } from "node:fs";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
// Node resolves symlinks in the main module's import.meta.url but argv[1] keeps the invoked
|
||||
// spelling (a symlinked skills dir, or /tmp vs /private/tmp on macOS), so compare real paths.
|
||||
export function isMainModule(importMetaUrl) {
|
||||
if (!process.argv[1]) return false;
|
||||
try {
|
||||
return pathToFileURL(realpathSync(process.argv[1])).href === importMetaUrl;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { extname, join } from "node:path";
|
||||
import { AGENT_SOURCES, latestRecordFor, recordInPlace } from "../../../scripts/lib/manifest.mjs";
|
||||
import { regenerateIndex } from "../../../scripts/lib/index-gen.mjs";
|
||||
|
||||
/**
|
||||
* Where the engine may write `rel`: there, unless another file of this run has `taken` it or the file there is
|
||||
* the person's (not `reusable` and not recorded as agent-made). Then the first free `name-2.ext`, with an anomaly.
|
||||
*/
|
||||
export function agentWritePath(
|
||||
hyperframesDir,
|
||||
rel,
|
||||
{ anomalies, taken = new Set(), reusable = () => false },
|
||||
) {
|
||||
const free = (path) =>
|
||||
!taken.has(path) &&
|
||||
(!existsSync(join(hyperframesDir, path)) ||
|
||||
reusable(path) ||
|
||||
AGENT_SOURCES.includes(latestRecordFor(hyperframesDir, path)?.source));
|
||||
if (free(rel)) return rel;
|
||||
const ext = extname(rel);
|
||||
const stem = rel.slice(0, rel.length - ext.length);
|
||||
let n = 2;
|
||||
while (!free(`${stem}-${n}${ext}`)) n++;
|
||||
const path = `${stem}-${n}${ext}`;
|
||||
const why = taken.has(rel)
|
||||
? "another file of this run goes there"
|
||||
: "the file there is yours (the media manifest does not record it as made by the engine)";
|
||||
const note = `${rel}: kept, because ${why}; writing ${path} instead (audio_meta.json has the path used)`;
|
||||
if (!anomalies.includes(note)) anomalies.push(note);
|
||||
return path;
|
||||
}
|
||||
|
||||
/** Each spoken line's file, picked before lines synthesize concurrently so two never land on one free name. */
|
||||
export function voicePaths(hyperframesDir, lines, anomalies) {
|
||||
const taken = new Set();
|
||||
const paths = new Map();
|
||||
for (const line of lines.filter((l) => String(l.text ?? "").trim())) {
|
||||
const rel = agentWritePath(hyperframesDir, `assets/voice/${line.id}.wav`, { anomalies, taken });
|
||||
taken.add(rel);
|
||||
paths.set(String(line.id), rel);
|
||||
}
|
||||
return paths;
|
||||
}
|
||||
|
||||
// A cue's source as the manifest records it, and who provided it. A source missing here records as the person's
|
||||
// file, so every rerun would write a numbered copy beside it.
|
||||
const SFX_SOURCES = {
|
||||
openverse: { source: "search", provider: "openverse.audio" },
|
||||
local: { source: "bundled", provider: "bundled.sfx" },
|
||||
};
|
||||
|
||||
// The files one engine run wrote, each with how it was made, for the project's media manifest.
|
||||
export function writtenAssets({ only, lines, voices, ttsProvider, bgm, bgmFields, sfx }) {
|
||||
const textById = new Map(lines.map((line) => [String(line.id), String(line.text ?? "").trim()]));
|
||||
const assets = [];
|
||||
if (only.has("tts")) {
|
||||
for (const voice of voices) {
|
||||
assets.push({
|
||||
path: voice.path,
|
||||
type: "voice",
|
||||
source: "generated",
|
||||
intent: textById.get(voice.id),
|
||||
duration: voice.duration_s,
|
||||
provider: ttsProvider,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (only.has("bgm") && bgm && !bgmFields.bgm_pending) {
|
||||
assets.push({
|
||||
path: bgm.path,
|
||||
type: "bgm",
|
||||
source: bgmFields.bgm_mode === "retrieve" ? "search" : "generated",
|
||||
intent: bgm.query,
|
||||
duration: bgm.duration_s,
|
||||
provider: bgmFields.bgm_provider,
|
||||
});
|
||||
}
|
||||
if (only.has("sfx")) {
|
||||
for (const cue of new Map(sfx.map((entry) => [entry.file, entry])).values()) {
|
||||
const { source, provider } = SFX_SOURCES[cue.source];
|
||||
assets.push({
|
||||
path: cue.file,
|
||||
type: "sfx",
|
||||
source,
|
||||
intent: cue.name,
|
||||
duration: cue.duration_s,
|
||||
provider,
|
||||
});
|
||||
}
|
||||
}
|
||||
return assets;
|
||||
}
|
||||
|
||||
/** Records each asset where it lies; returns one anomaly per file left unrecorded. */
|
||||
export function recordInManifest(hyperframesDir, assets) {
|
||||
const anomalies = [];
|
||||
for (const { path, type, source, intent, duration, provider } of assets) {
|
||||
try {
|
||||
recordInPlace(hyperframesDir, {
|
||||
type,
|
||||
path,
|
||||
source,
|
||||
description: intent,
|
||||
duration,
|
||||
provenance: { provider: provider || "local", ...(intent && { prompt: intent }) },
|
||||
});
|
||||
} catch (error) {
|
||||
anomalies.push(`${path}: not recorded in the media manifest (${error.message})`);
|
||||
}
|
||||
}
|
||||
try {
|
||||
if (anomalies.length < assets.length) regenerateIndex(hyperframesDir);
|
||||
} catch (error) {
|
||||
anomalies.push(`.media/index.md: not refreshed (${error.message})`);
|
||||
}
|
||||
return anomalies;
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
import { strict as assert } from "node:assert";
|
||||
import { test } from "node:test";
|
||||
import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { findByPrompt, readManifest } from "../../../scripts/lib/manifest.mjs";
|
||||
import { agentWritePath, recordInManifest, voicePaths, writtenAssets } from "./media-record.mjs";
|
||||
|
||||
const noBgm = { bgm: null, bgmFields: { bgm_pending: false } };
|
||||
|
||||
function project(t) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-record-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
}
|
||||
|
||||
test("a voice run leaves a manifest entry marked generated", (t) => {
|
||||
const dir = project(t);
|
||||
const assets = writtenAssets({
|
||||
only: new Set(["tts"]),
|
||||
lines: [{ id: "01", text: "Welcome to the launch" }],
|
||||
voices: [{ id: "01", path: "assets/voice/01.wav", duration_s: 2.34 }],
|
||||
ttsProvider: "kokoro",
|
||||
sfx: [],
|
||||
...noBgm,
|
||||
});
|
||||
|
||||
assert.deepEqual(recordInManifest(dir, assets), []);
|
||||
|
||||
assert.deepEqual(
|
||||
readManifest(dir).map(({ path, type, source, description, duration, provenance }) => ({
|
||||
path,
|
||||
type,
|
||||
source,
|
||||
description,
|
||||
duration,
|
||||
provenance,
|
||||
})),
|
||||
[
|
||||
{
|
||||
path: "assets/voice/01.wav",
|
||||
type: "voice",
|
||||
source: "generated",
|
||||
description: "Welcome to the launch",
|
||||
duration: 2.3,
|
||||
provenance: { provider: "kokoro", prompt: "Welcome to the launch" },
|
||||
},
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
const voiceLine = (intent, duration) => [
|
||||
{ path: "assets/voice/01.wav", type: "voice", source: "generated", intent, duration },
|
||||
];
|
||||
|
||||
test("a rerun that writes the same take keeps one record, and a new length is a new take", (t) => {
|
||||
const dir = project(t);
|
||||
|
||||
recordInManifest(dir, voiceLine("Hello world", 1.25));
|
||||
recordInManifest(dir, voiceLine("Hello world", 1.25));
|
||||
assert.equal(readManifest(dir).length, 1);
|
||||
|
||||
recordInManifest(dir, voiceLine("Hello world", 2));
|
||||
assert.deepEqual(
|
||||
readManifest(dir).map(({ duration }) => duration),
|
||||
[1.3, 2],
|
||||
);
|
||||
});
|
||||
|
||||
test("a rerun with new text records the new take, and the old text no longer finds the file", (t) => {
|
||||
const dir = project(t);
|
||||
|
||||
recordInManifest(dir, voiceLine("Hello world", 1.25));
|
||||
recordInManifest(dir, voiceLine("Welcome back to the show", 3.5));
|
||||
|
||||
assert.deepEqual(
|
||||
readManifest(dir).map(({ description, duration }) => [description, duration]),
|
||||
[
|
||||
["Hello world", 1.3],
|
||||
["Welcome back to the show", 3.5],
|
||||
],
|
||||
);
|
||||
assert.equal(findByPrompt(dir, "Hello world", "voice"), null);
|
||||
const index = readFileSync(join(dir, ".media/index.md"), "utf8");
|
||||
assert.match(index, /Welcome back to the show/);
|
||||
assert.doesNotMatch(index, /Hello world/);
|
||||
assert.equal(findByPrompt(dir, "Welcome back to the show", "voice")?.path, "assets/voice/01.wav");
|
||||
});
|
||||
|
||||
test("music and sound effects are marked by where they came from", () => {
|
||||
const assets = writtenAssets({
|
||||
only: new Set(["bgm", "sfx"]),
|
||||
lines: [],
|
||||
voices: [{ id: "01", path: "assets/voice/01.wav" }],
|
||||
bgm: { path: "assets/bgm/track.mp3", query: "calm" },
|
||||
bgmFields: { bgm_pending: false, bgm_mode: "retrieve" },
|
||||
sfx: [
|
||||
{ file: "assets/sfx/whoosh.mp3", name: "whoosh", source: "local" },
|
||||
{ file: "assets/sfx/whoosh.mp3", name: "whoosh", source: "local" },
|
||||
{ file: "assets/sfx/glass.mp3", name: "glass", source: "openverse" },
|
||||
],
|
||||
});
|
||||
|
||||
assert.deepEqual(
|
||||
assets.map(({ path, source, provider }) => [path, source, provider]),
|
||||
[
|
||||
["assets/bgm/track.mp3", "search", undefined],
|
||||
["assets/sfx/whoosh.mp3", "bundled", "bundled.sfx"],
|
||||
["assets/sfx/glass.mp3", "search", "openverse.audio"],
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test("a rerun of an Openverse sound effect writes over its own file, not a numbered copy", (t) => {
|
||||
const dir = project(t);
|
||||
mkdirSync(join(dir, "assets/sfx"), { recursive: true });
|
||||
const anomalies = [];
|
||||
for (let run = 0; run < 3; run++) {
|
||||
const file = agentWritePath(dir, "assets/sfx/door-knock.mp3", { anomalies });
|
||||
writeFileSync(join(dir, file), "knock");
|
||||
const sfx = [{ file, name: "door knock", source: "openverse", duration_s: 1 }];
|
||||
recordInManifest(
|
||||
dir,
|
||||
writtenAssets({ only: new Set(["sfx"]), lines: [], voices: [], sfx, ...noBgm }),
|
||||
);
|
||||
}
|
||||
|
||||
assert.deepEqual(readdirSync(join(dir, "assets/sfx")), ["door-knock.mp3"]);
|
||||
assert.deepEqual(anomalies, []);
|
||||
assert.equal(readManifest(dir).length, 1);
|
||||
});
|
||||
|
||||
test("music made locally is marked generated once it is ready, not while pending", () => {
|
||||
const written = (bgm_pending) =>
|
||||
writtenAssets({
|
||||
only: new Set(["bgm"]),
|
||||
lines: [],
|
||||
voices: [],
|
||||
sfx: [],
|
||||
bgm: { path: "assets/bgm/track.wav" },
|
||||
bgmFields: { bgm_pending, bgm_mode: "detached-single" },
|
||||
}).map(({ source }) => source);
|
||||
|
||||
assert.deepEqual(written(true), []);
|
||||
assert.deepEqual(written(false), ["generated"]);
|
||||
});
|
||||
|
||||
test("a file that cannot be recorded becomes an anomaly, not a failure", (t) => {
|
||||
const dir = project(t);
|
||||
writeFileSync(join(dir, ".media"), "a file where the media folder should be");
|
||||
|
||||
const anomalies = recordInManifest(dir, [
|
||||
{ path: "assets/voice/01.wav", type: "voice", source: "generated" },
|
||||
]);
|
||||
|
||||
assert.equal(anomalies.length, 1);
|
||||
assert.match(anomalies[0], /^assets\/voice\/01\.wav: not recorded in the media manifest/);
|
||||
});
|
||||
|
||||
test("the engine writes over only its own files, else the next free name", (t) => {
|
||||
const dir = project(t);
|
||||
mkdirSync(join(dir, "assets/sfx"), { recursive: true });
|
||||
const names = ["mine", "made", "adopted", "twice", "twice-2", "kept", "kept-2"];
|
||||
for (const name of names) writeFileSync(join(dir, `assets/sfx/${name}.mp3`), name);
|
||||
for (const path of ["assets/sfx/made.mp3", "assets/sfx/kept-2.mp3"])
|
||||
recordInManifest(dir, [{ path, type: "sfx", source: "search" }]);
|
||||
recordInManifest(dir, [{ path: "assets/sfx/adopted.mp3", type: "sfx", source: "existing" }]);
|
||||
const anomalies = [];
|
||||
const at = (name, reusable) =>
|
||||
agentWritePath(dir, `assets/sfx/${name}.mp3`, { anomalies, reusable });
|
||||
|
||||
assert.equal(at("new"), "assets/sfx/new.mp3");
|
||||
assert.equal(at("made"), "assets/sfx/made.mp3");
|
||||
assert.equal(
|
||||
at("mine", (path) => path === "assets/sfx/mine.mp3"),
|
||||
"assets/sfx/mine.mp3",
|
||||
);
|
||||
assert.deepEqual(anomalies, []);
|
||||
assert.equal(at("mine"), "assets/sfx/mine-2.mp3");
|
||||
assert.equal(at("adopted"), "assets/sfx/adopted-2.mp3");
|
||||
assert.equal(at("twice"), "assets/sfx/twice-3.mp3");
|
||||
assert.equal(at("kept"), "assets/sfx/kept-2.mp3");
|
||||
assert.equal(anomalies.length, 4);
|
||||
assert.match(
|
||||
anomalies[0],
|
||||
/^assets\/sfx\/mine\.mp3: kept, .* writing assets\/sfx\/mine-2\.mp3 instead/,
|
||||
);
|
||||
});
|
||||
|
||||
test("two spoken lines never share a file when one's name is taken by the person", (t) => {
|
||||
const dir = project(t);
|
||||
mkdirSync(join(dir, "assets/voice"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets/voice/hook.wav"), "the person's own hook");
|
||||
const anomalies = [];
|
||||
|
||||
const paths = voicePaths(
|
||||
dir,
|
||||
[
|
||||
{ id: "hook", text: "First" },
|
||||
{ id: "hook-2", text: "Second" },
|
||||
{ id: "blank", text: " " },
|
||||
],
|
||||
anomalies,
|
||||
);
|
||||
|
||||
assert.deepEqual(Object.fromEntries(paths), {
|
||||
hook: "assets/voice/hook-2.wav",
|
||||
"hook-2": "assets/voice/hook-2-2.wav",
|
||||
});
|
||||
assert.equal(anomalies.length, 2);
|
||||
});
|
||||
|
||||
test("a record never takes the id of a download still in flight", (t) => {
|
||||
const dir = project(t);
|
||||
mkdirSync(join(dir, ".media/audio/bgm"), { recursive: true });
|
||||
writeFileSync(join(dir, ".media/audio/bgm/bgm_001.mp3"), "");
|
||||
|
||||
recordInManifest(dir, [{ path: "assets/bgm/track.wav", type: "bgm", source: "generated" }]);
|
||||
|
||||
assert.deepEqual(
|
||||
readManifest(dir).map(({ id }) => id),
|
||||
["bgm_002"],
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,134 @@
|
||||
// Sound effects from Openverse (https://api.openverse.org): free, no key.
|
||||
// Anonymous use is rate limited (20 requests a minute, 200 a day); callers fall
|
||||
// back to the bundled library when it is unavailable.
|
||||
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
|
||||
const OPENVERSE_AUDIO_URL = "https://api.openverse.org/v1/audio/";
|
||||
// The only licences media-use accepts: no NonCommercial, ShareAlike or NoDerivatives.
|
||||
export const OPENVERSE_SFX_LICENSES = new Set(["cc0", "by"]);
|
||||
const PLAYABLE_FILETYPES = new Set(["mp3", "wav", "ogg", "flac"]);
|
||||
const MAX_SFX_MS = 15000;
|
||||
const IDEAL_SFX_MS = 4000;
|
||||
const BY_PENALTY = 2;
|
||||
const SEARCH_TIMEOUT_MS = 10000;
|
||||
|
||||
// Openverse text lands in markdown and JSON: one line, no control or markdown-structure characters, bounded.
|
||||
const oneLine = (text) =>
|
||||
String(text ?? "")
|
||||
.replace(/[\p{Cc}`[\]<>\\]+/gu, " ")
|
||||
.trim()
|
||||
.slice(0, 300);
|
||||
// Openverse's audio providers (api.openverse.org/v1/audio/stats/); a new provider's link is dropped until listed.
|
||||
const SOURCE_HOSTS = new Set(["freesound.org", "www.jamendo.com", "commons.wikimedia.org"]);
|
||||
function httpsUrl(url, hosts) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return parsed.protocol === "https:" && (!hosts || hosts.has(parsed.hostname))
|
||||
? oneLine(parsed.href)
|
||||
: "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
/** Openverse could not answer (offline, rate limited, server error): stop asking it this run. */
|
||||
export class OpenverseUnavailableError extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
this.name = "OpenverseUnavailableError";
|
||||
}
|
||||
}
|
||||
|
||||
function usable(result) {
|
||||
return (
|
||||
OPENVERSE_SFX_LICENSES.has(result?.license) &&
|
||||
PLAYABLE_FILETYPES.has(result.filetype) &&
|
||||
typeof result.url === "string" &&
|
||||
result.url.startsWith("https://") &&
|
||||
Number.isFinite(result.duration) &&
|
||||
result.duration > 0 &&
|
||||
result.duration <= MAX_SFX_MS
|
||||
);
|
||||
}
|
||||
|
||||
// Lower is better: relevance position, then seconds past a typical cue length, then CC0 over CC BY.
|
||||
function penalty(result, position) {
|
||||
const longBy = Math.max(0, result.duration - IDEAL_SFX_MS) / 1000;
|
||||
return position + longBy + (result.license === "by" ? BY_PENALTY : 0);
|
||||
}
|
||||
|
||||
/** Best usable sound in Openverse's relevance-ordered results, or null. */
|
||||
export function pickOpenverseSfx(results) {
|
||||
let best = null;
|
||||
results.forEach((result, position) => {
|
||||
if (!usable(result)) return;
|
||||
const score = penalty(result, position);
|
||||
if (!best || score < best.score) best = { result, score };
|
||||
});
|
||||
if (!best) return null;
|
||||
const r = best.result;
|
||||
return {
|
||||
id: r.id,
|
||||
url: r.url,
|
||||
ext: `.${r.filetype}`,
|
||||
title: oneLine(r.title),
|
||||
creator: oneLine(r.creator),
|
||||
license: r.license,
|
||||
licenseUrl: httpsUrl(r.license_url),
|
||||
sourceUrl: httpsUrl(r.foreign_landing_url, SOURCE_HOSTS),
|
||||
credit: oneLine(r.attribution),
|
||||
durationS: Math.round(r.duration) / 1000,
|
||||
};
|
||||
}
|
||||
|
||||
/** Best licensed sound effect for `query`, null on no match; throws OpenverseUnavailableError. */
|
||||
export async function searchOpenverseSfx(query, { fetchImpl = fetch } = {}) {
|
||||
const params = new URLSearchParams({
|
||||
q: query,
|
||||
page_size: "20",
|
||||
license: [...OPENVERSE_SFX_LICENSES].join(","),
|
||||
});
|
||||
let response;
|
||||
try {
|
||||
response = await fetchImpl(`${OPENVERSE_AUDIO_URL}?${params}`, {
|
||||
headers: { Accept: "application/json" },
|
||||
signal: AbortSignal.timeout(SEARCH_TIMEOUT_MS),
|
||||
});
|
||||
} catch (error) {
|
||||
throw new OpenverseUnavailableError(`Openverse unreachable (${error.message})`);
|
||||
}
|
||||
if (response.status === 429)
|
||||
throw new OpenverseUnavailableError("Openverse rate limit reached (HTTP 429)");
|
||||
if (!response.ok) throw new OpenverseUnavailableError(`Openverse HTTP ${response.status}`);
|
||||
let body;
|
||||
try {
|
||||
body = await response.json();
|
||||
} catch {
|
||||
throw new OpenverseUnavailableError("Openverse returned non-JSON");
|
||||
}
|
||||
return pickOpenverseSfx(Array.isArray(body?.results) ? body.results : []);
|
||||
}
|
||||
|
||||
/** Record a CC BY sound's credit for `file` in the project's credits file (one line per file). */
|
||||
export function writeSfxCredit(creditsPath, file, sound) {
|
||||
if (sound.license !== "by") return;
|
||||
// The sound may come from a saved record, not this run's pick: clean it again here.
|
||||
const sourceUrl = httpsUrl(sound.sourceUrl, SOURCE_HOSTS);
|
||||
const source = sourceUrl ? ` Source: ${sourceUrl}` : "";
|
||||
const line = `- \`${file}\`: ${oneLine(sound.credit)}${source}`;
|
||||
let prior = "";
|
||||
try {
|
||||
prior = readFileSync(creditsPath, "utf8");
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT") throw error;
|
||||
}
|
||||
const kept = prior
|
||||
.split("\n")
|
||||
.filter((l) => l.startsWith("- ") && !l.startsWith(`- \`${file}\`:`));
|
||||
const header =
|
||||
"# Sound effect credits\n\nThese sounds are licensed CC BY: credit them wherever the video is published.\n\n";
|
||||
mkdirSync(dirname(creditsPath), { recursive: true });
|
||||
writeFileSync(creditsPath, `${header}${[...kept, line].join("\n")}\n`);
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
// python.mjs — resolve which Python 3 executable to spawn, per platform.
|
||||
//
|
||||
// The audio engine (tts.mjs, bgm.mjs) shells out to `python3` for ElevenLabs
|
||||
// TTS and the local Lyria/MusicGen BGM paths. `python3` is the right name on
|
||||
// macOS/Linux, but on Windows the python.org installer only creates
|
||||
// `python.exe` plus the `py` launcher — there is no `python3.exe` (only the
|
||||
// Microsoft Store build adds one). So a bare `spawn("python3", …)` ENOENTs on a
|
||||
// standard Windows Python install, silently disabling every Python-backed audio
|
||||
// feature until the user hand-creates a `python3.exe` shim (reported twice).
|
||||
//
|
||||
// Resolve once, per process: probe the platform's candidates in order and take
|
||||
// the first that actually runs. `py` is the launcher, so it needs a `-3` arg to
|
||||
// select Python 3 — hence candidates are argv PREFIXES, not bare names.
|
||||
//
|
||||
// `HYPERFRAMES_PYTHON` (the documented PEP 668 venv setup) wins over the PATH probe, matching
|
||||
// the CLI's `findPython()`; ignoring it made BGM silently disable itself (#4614).
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
// Accept only an interpreter whose `--version` reports Python 3, as the CLI's
|
||||
// `validatePythonOverride()` does. Exit status alone would take a Python 2
|
||||
// `python` (or any executable that exits 0) that `doctor` rejects. Python 2
|
||||
// prints its version to stderr, so read both streams.
|
||||
export function defaultProbe(cmd, args) {
|
||||
try {
|
||||
const result = spawnSync(cmd, args, { encoding: "utf-8", timeout: 5000 });
|
||||
return result.status === 0 && /Python 3/.test(`${result.stdout}${result.stderr}`);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the argv prefix that launches Python 3 on this platform.
|
||||
* Returns e.g. `["python3"]`, `["python"]`, or `["py", "-3"]`.
|
||||
*
|
||||
* Pure except for `probe` (which runs `<cmd> … --version`); `platform`, `probe`
|
||||
* and `env` are injectable so every branch is unit-testable without spawning.
|
||||
* If nothing probes OK, falls back to the canonical name for the platform so
|
||||
* the eventual spawn fails loudly exactly as it did before — never worse.
|
||||
*/
|
||||
export function resolvePythonCommand(
|
||||
platform = process.platform,
|
||||
probe = defaultProbe,
|
||||
env = process.env,
|
||||
) {
|
||||
const override = env.HYPERFRAMES_PYTHON;
|
||||
if (override && probe(override, ["--version"])) return [override];
|
||||
const candidates =
|
||||
platform === "win32" ? [["python3"], ["python"], ["py", "-3"]] : [["python3"], ["python"]];
|
||||
for (const prefix of candidates) {
|
||||
if (probe(prefix[0], [...prefix.slice(1), "--version"])) return prefix;
|
||||
}
|
||||
return candidates[0];
|
||||
}
|
||||
|
||||
let cached = null;
|
||||
|
||||
/** Cached `resolvePythonCommand()` — probing spawns, so resolve at most once. */
|
||||
export function pythonCommand() {
|
||||
if (!cached) cached = resolvePythonCommand();
|
||||
return cached;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a `{ cmd, args }` for running Python 3 with `extraArgs`, using the
|
||||
* resolved (or supplied) prefix. Keeps the launcher's `-3` (and any future
|
||||
* prefix args) ahead of the caller's own arguments.
|
||||
*/
|
||||
export function pythonInvocation(extraArgs, prefix = pythonCommand()) {
|
||||
return { cmd: prefix[0], args: [...prefix.slice(1), ...extraArgs] };
|
||||
}
|
||||
|
||||
/** Test-only: clear the cached resolution so a test can re-probe. */
|
||||
export function _resetPythonCommandCacheForTests() {
|
||||
cached = null;
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { defaultProbe, resolvePythonCommand, pythonInvocation } from "./python.mjs";
|
||||
|
||||
// Regression: on Windows a standard python.org install has no `python3.exe`
|
||||
// (only `python.exe` + the `py` launcher), so `spawn("python3", …)` ENOENTs and
|
||||
// every Python-backed audio feature silently no-ops. resolvePythonCommand takes
|
||||
// injectable platform/probe params so all branches are testable without
|
||||
// spawning a real interpreter.
|
||||
|
||||
// probeFor(names): a probe that reports success only for the given argv-0 names.
|
||||
function probeFor(...names) {
|
||||
const ok = new Set(names);
|
||||
return (cmd) => ok.has(cmd);
|
||||
}
|
||||
|
||||
test("non-win32 uses python3 when it runs", () => {
|
||||
assert.deepEqual(resolvePythonCommand("linux", probeFor("python3")), ["python3"]);
|
||||
assert.deepEqual(resolvePythonCommand("darwin", probeFor("python3")), ["python3"]);
|
||||
});
|
||||
|
||||
test("win32 prefers python3 when the Microsoft Store build provides it", () => {
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor("python3", "python", "py")), ["python3"]);
|
||||
});
|
||||
|
||||
test("win32 falls back to python.exe when python3 is absent (python.org install)", () => {
|
||||
// The exact reported scenario: no python3, but `python` exists.
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor("python", "py")), ["python"]);
|
||||
});
|
||||
|
||||
test("win32 falls back to the py launcher with -3 when only py exists", () => {
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor("py")), ["py", "-3"]);
|
||||
});
|
||||
|
||||
test("py launcher is probed as `py -3 --version`, not bare `py`", () => {
|
||||
const seen = [];
|
||||
const probe = (cmd, args) => {
|
||||
seen.push([cmd, ...args]);
|
||||
return cmd === "py";
|
||||
};
|
||||
resolvePythonCommand("win32", probe);
|
||||
assert.deepEqual(seen.at(-1), ["py", "-3", "--version"]);
|
||||
});
|
||||
|
||||
test("falls back to the canonical name (loud failure, unchanged) when nothing runs", () => {
|
||||
// No interpreter anywhere — must not throw, and must return python3 so the
|
||||
// eventual spawn fails exactly as it did before this fix, never worse.
|
||||
assert.deepEqual(
|
||||
resolvePythonCommand("win32", () => false),
|
||||
["python3"],
|
||||
);
|
||||
assert.deepEqual(
|
||||
resolvePythonCommand("linux", () => false),
|
||||
["python3"],
|
||||
);
|
||||
});
|
||||
|
||||
// Regression (#4614): on PEP 668 systems the documented setup is a venv named
|
||||
// by HYPERFRAMES_PYTHON; the audio engine ignored it and probed the bare
|
||||
// system python3, so MusicGen BGM was skipped while `doctor` said installed.
|
||||
const VENV = "/home/u/.venvs/hf/bin/python";
|
||||
|
||||
test("HYPERFRAMES_PYTHON wins over python3 on PATH when it runs", () => {
|
||||
const env = { HYPERFRAMES_PYTHON: VENV };
|
||||
assert.deepEqual(resolvePythonCommand("darwin", probeFor(VENV, "python3"), env), [VENV]);
|
||||
assert.deepEqual(resolvePythonCommand("win32", probeFor(VENV, "python", "py"), env), [VENV]);
|
||||
});
|
||||
|
||||
test("a HYPERFRAMES_PYTHON that doesn't run falls through to the PATH probe", () => {
|
||||
const env = { HYPERFRAMES_PYTHON: "/nope/python" };
|
||||
assert.deepEqual(resolvePythonCommand("linux", probeFor("python3"), env), ["python3"]);
|
||||
});
|
||||
|
||||
test("an empty HYPERFRAMES_PYTHON is ignored without being probed", () => {
|
||||
const seen = [];
|
||||
const probe = (cmd) => {
|
||||
seen.push(cmd);
|
||||
return cmd === "python3";
|
||||
};
|
||||
assert.deepEqual(resolvePythonCommand("linux", probe, { HYPERFRAMES_PYTHON: "" }), ["python3"]);
|
||||
assert.deepEqual(seen, ["python3"]);
|
||||
});
|
||||
|
||||
// defaultProbe must match the CLI's validatePythonOverride(): exiting 0 is not
|
||||
// enough, `--version` has to report Python 3. Stubs are shell scripts, so
|
||||
// these run on POSIX only.
|
||||
function withStub(script, fn) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-python-probe-"));
|
||||
try {
|
||||
const stub = join(dir, "python");
|
||||
writeFileSync(stub, `#!/bin/sh\n${script}\n`);
|
||||
chmodSync(stub, 0o755);
|
||||
fn(stub);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
const posixOnly = { skip: process.platform === "win32" && "stub scripts need a POSIX shell" };
|
||||
|
||||
test("defaultProbe accepts an interpreter reporting Python 3", posixOnly, () => {
|
||||
withStub('echo "Python 3.12.4"', (stub) => assert.equal(defaultProbe(stub, ["--version"]), true));
|
||||
});
|
||||
|
||||
test("defaultProbe rejects Python 2, which prints its version to stderr", posixOnly, () => {
|
||||
withStub('echo "Python 2.7.18" >&2', (stub) =>
|
||||
assert.equal(defaultProbe(stub, ["--version"]), false),
|
||||
);
|
||||
});
|
||||
|
||||
test(
|
||||
"defaultProbe rejects an executable that exits 0 without a Python 3 version",
|
||||
posixOnly,
|
||||
() => {
|
||||
withStub("exit 0", (stub) => assert.equal(defaultProbe(stub, ["--version"]), false));
|
||||
},
|
||||
);
|
||||
|
||||
test("defaultProbe rejects a command that doesn't exist", () => {
|
||||
assert.equal(defaultProbe("/nonexistent/hf-python", ["--version"]), false);
|
||||
});
|
||||
|
||||
test("pythonInvocation prepends the resolved prefix ahead of caller args", () => {
|
||||
assert.deepEqual(pythonInvocation(["-c", "import x"], ["python"]), {
|
||||
cmd: "python",
|
||||
args: ["-c", "import x"],
|
||||
});
|
||||
// The py launcher's -3 must stay ahead of the caller's own arguments.
|
||||
assert.deepEqual(pythonInvocation(["-c", "import x"], ["py", "-3"]), {
|
||||
cmd: "py",
|
||||
args: ["-3", "-c", "import x"],
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,196 @@
|
||||
// sfx.mjs — sound effects for the media audio engine. Each cue is searched on
|
||||
// Openverse (CC0 / CC BY only, see openverse.mjs). A cue Openverse cannot answer,
|
||||
// and every cue after Openverse is unreachable or rate limited, resolves against
|
||||
// the bundled 21-file library (assets/sfx/manifest.json) instead.
|
||||
//
|
||||
// A cue that matches nothing is skipped (recorded as an anomaly); SFX never
|
||||
// blocks a render. Every cue sits at volume ~0.35, under voice + BGM.
|
||||
|
||||
import { copyFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { downloadTo } from "./heygen.mjs";
|
||||
import { OpenverseUnavailableError, searchOpenverseSfx, writeSfxCredit } from "./openverse.mjs";
|
||||
import { latestRecordFor } from "../../../scripts/lib/manifest.mjs";
|
||||
import { agentWritePath } from "./media-record.mjs";
|
||||
|
||||
export const BUNDLED_SFX_DIR = join(
|
||||
dirname(fileURLToPath(import.meta.url)),
|
||||
"..",
|
||||
"..",
|
||||
"assets",
|
||||
"sfx",
|
||||
);
|
||||
const SFX_VOLUME = 0.35;
|
||||
const slug = (s) =>
|
||||
s
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.slice(0, 40) || "x";
|
||||
const r3 = (x) => Number(x.toFixed(3));
|
||||
|
||||
// Lookups by manifest key, file basename, and slug of either, so a cue can name
|
||||
// "whoosh", "whoosh.mp3", or "ui click" (→ slug match). { error } when unusable.
|
||||
function loadBundledLibrary(sfxLibDir) {
|
||||
const manifestPath = join(sfxLibDir, "manifest.json");
|
||||
if (!existsSync(manifestPath)) return { error: `no bundled SFX library at ${sfxLibDir}` };
|
||||
let manifest;
|
||||
try {
|
||||
manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
|
||||
} catch (e) {
|
||||
return { error: `bundled SFX manifest parse failed (${e.message})` };
|
||||
}
|
||||
const byKey = new Map();
|
||||
for (const [key, entry] of Object.entries(manifest)) {
|
||||
if (!entry?.file || !isFinite(entry.duration)) continue;
|
||||
const rec = { key, file: entry.file, duration: entry.duration };
|
||||
byKey.set(key, rec);
|
||||
byKey.set(entry.file, rec);
|
||||
byKey.set(slug(key), rec);
|
||||
byKey.set(slug(entry.file.replace(/\.\w+$/, "")), rec);
|
||||
}
|
||||
return { byKey };
|
||||
}
|
||||
|
||||
// cues: [{ id, name }] (id = the line/frame/scene the cue fires in). Returns
|
||||
// { sfx: [{ id, name, file, source, offset_s, duration_s, volume, license?, credit? }], anomalies }.
|
||||
export async function resolveSfx({ cues, hyperframesDir, sfxLibDir = BUNDLED_SFX_DIR }) {
|
||||
const sfx = [];
|
||||
const anomalies = [];
|
||||
const destDir = join(hyperframesDir, "assets", "sfx");
|
||||
// Each effect's file this run: one effect named twice gets one file, and two effects never share one.
|
||||
const fileFor = new Map();
|
||||
const taken = () => new Set(fileFor.values());
|
||||
// One Openverse sound per effect name this run, so a repeated cue costs no second request.
|
||||
const openverseFor = new Map();
|
||||
let openverseDown = false;
|
||||
let library;
|
||||
|
||||
// Dedupe identical (id,name) cues — the same effect named twice in one line
|
||||
// downloads/copies once.
|
||||
const seen = new Set();
|
||||
const uniq = cues.filter((c) => {
|
||||
const k = `${c.id}:${c.name}`;
|
||||
if (seen.has(k)) return false;
|
||||
seen.add(k);
|
||||
return true;
|
||||
});
|
||||
|
||||
async function fromOpenverse({ id, name }) {
|
||||
const key = `openverse:${slug(name)}`;
|
||||
let entry = openverseFor.get(key);
|
||||
if (!entry) {
|
||||
const sound = await searchOpenverseSfx(name);
|
||||
if (!sound) return null;
|
||||
const file = agentWritePath(hyperframesDir, `assets/sfx/${slug(name)}${sound.ext}`, {
|
||||
anomalies,
|
||||
taken: taken(),
|
||||
});
|
||||
try {
|
||||
await downloadTo(sound.url, join(hyperframesDir, file));
|
||||
} catch (e) {
|
||||
anomalies.push(`sfx "${name}" (id ${id}): Openverse download failed — ${e.message}`);
|
||||
return null;
|
||||
}
|
||||
fileFor.set(key, file);
|
||||
writeSfxCredit(join(destDir, "CREDITS.md"), file, sound);
|
||||
entry = {
|
||||
file,
|
||||
source: "openverse",
|
||||
duration_s: r3(sound.durationS),
|
||||
license: sound.license,
|
||||
credit: sound.credit,
|
||||
};
|
||||
openverseFor.set(key, entry);
|
||||
}
|
||||
return { id, name, ...entry, offset_s: 0, volume: SFX_VOLUME };
|
||||
}
|
||||
|
||||
function fromBundled({ id, name }) {
|
||||
library ??= loadBundledLibrary(sfxLibDir);
|
||||
if (library.error) {
|
||||
anomalies.push(`sfx "${name}" (id ${id}): ${library.error} — skipped`);
|
||||
return null;
|
||||
}
|
||||
const { byKey } = library;
|
||||
const hit = byKey.get(name) ?? byKey.get(slug(name));
|
||||
if (!hit) {
|
||||
const known = [...new Set([...byKey.values()].map((v) => v.key))].slice(0, 8).join(", ");
|
||||
anomalies.push(
|
||||
`sfx "${name}" (id ${id}): not in bundled library — skipped (have: ${known}…)`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
mkdirSync(destDir, { recursive: true });
|
||||
const src = join(sfxLibDir, hit.file);
|
||||
const bytes = existsSync(src) ? readFileSync(src) : null;
|
||||
const isLibraryCopy = (rel) =>
|
||||
bytes &&
|
||||
existsSync(join(hyperframesDir, rel)) &&
|
||||
readFileSync(join(hyperframesDir, rel)).equals(bytes);
|
||||
const destRel =
|
||||
fileFor.get(hit.file) ??
|
||||
agentWritePath(hyperframesDir, `assets/sfx/${hit.file}`, {
|
||||
anomalies,
|
||||
taken: taken(),
|
||||
// An unrecorded copy of the library file is one an engine run made before the manifest had it.
|
||||
reusable: (rel) => !latestRecordFor(hyperframesDir, rel) && isLibraryCopy(rel),
|
||||
});
|
||||
const dest = join(hyperframesDir, destRel);
|
||||
// Some installs of the skill ship manifest.json without the mp3s. An entry
|
||||
// pointing at a file we never copied would silently drop downstream, so
|
||||
// skip the cue with a loud anomaly instead.
|
||||
if (bytes) {
|
||||
if (!isLibraryCopy(destRel)) copyFileSync(src, dest);
|
||||
} else if (!existsSync(dest)) {
|
||||
anomalies.push(
|
||||
`sfx "${name}" (id ${id}): bundled file ${hit.file} missing from the offline ` +
|
||||
`library (${sfxLibDir}) — skipped. Reinstall the media-use skill to ` +
|
||||
`restore assets/sfx/*.mp3.`,
|
||||
);
|
||||
return null;
|
||||
} // else the engine's earlier copy at dest stands in for the file this install lacks
|
||||
fileFor.set(hit.file, destRel);
|
||||
return {
|
||||
id,
|
||||
name,
|
||||
file: destRel,
|
||||
source: "local",
|
||||
offset_s: 0,
|
||||
duration_s: r3(hit.duration),
|
||||
volume: SFX_VOLUME,
|
||||
};
|
||||
}
|
||||
|
||||
for (const cue of uniq) {
|
||||
let entry = null;
|
||||
if (!openverseDown) {
|
||||
try {
|
||||
entry = await fromOpenverse(cue);
|
||||
} catch (e) {
|
||||
if (!(e instanceof OpenverseUnavailableError)) throw e;
|
||||
openverseDown = true;
|
||||
anomalies.push(`${e.message}: the remaining SFX cues use the bundled library`);
|
||||
}
|
||||
}
|
||||
entry ??= fromBundled(cue);
|
||||
if (entry) sfx.push(entry);
|
||||
}
|
||||
return { sfx, anomalies };
|
||||
}
|
||||
|
||||
/**
|
||||
* One sound effect for `query`, saved under the project's assets/sfx/ (the entry point for hosts).
|
||||
* { sound: { file, source, duration_s, license?, credit? } | null, anomalies }.
|
||||
*/
|
||||
export async function findSoundEffect(query, { hyperframesDir, sfxLibDir = BUNDLED_SFX_DIR }) {
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "1", name: query }],
|
||||
hyperframesDir,
|
||||
sfxLibDir,
|
||||
});
|
||||
if (!sfx.length) return { sound: null, anomalies };
|
||||
const { file, source, duration_s, license, credit } = sfx[0];
|
||||
return { sound: { file, source, duration_s, ...(license && { license, credit }) }, anomalies };
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, existsSync, readFileSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { appendRecord } from "../../../scripts/lib/manifest.mjs";
|
||||
import { BUNDLED_SFX_DIR, findSoundEffect, resolveSfx } from "./sfx.mjs";
|
||||
|
||||
// No real network: each test serves Openverse's answers; by default it is unreachable.
|
||||
const offline = async () => {
|
||||
throw new TypeError("fetch failed");
|
||||
};
|
||||
let serve = offline;
|
||||
const UNREACHABLE =
|
||||
"Openverse unreachable (fetch failed): the remaining SFX cues use the bundled library";
|
||||
globalThis.fetch = (url, init) => serve(String(url), init);
|
||||
|
||||
// Offline SFX resolution: the bundled library may ship manifest.json
|
||||
// without the actual mp3s. The old code copied only when the source existed but
|
||||
// pushed the sfx entry unconditionally — producing a dangling reference that
|
||||
// silently dropped downstream ("not on disk"). These tests lock in the loud
|
||||
// behavior: a present file is copied + referenced; a missing file yields an
|
||||
// anomaly and NO dangling entry.
|
||||
|
||||
async function withDirs(fn) {
|
||||
const root = mkdtempSync(join(tmpdir(), "hf-sfx-"));
|
||||
const libDir = join(root, "lib");
|
||||
const projDir = join(root, "proj");
|
||||
mkdirSync(libDir, { recursive: true });
|
||||
mkdirSync(projDir, { recursive: true });
|
||||
try {
|
||||
// `await` is load-bearing: without it the finally cleanup runs before the
|
||||
// async test body resolves, deleting the temp dir mid-assertion.
|
||||
return await fn({ libDir, projDir });
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test("offline: copies and references a present bundled file", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
writeFileSync(
|
||||
join(libDir, "manifest.json"),
|
||||
JSON.stringify({ whoosh: { file: "whoosh.mp3", duration: 0.8 } }),
|
||||
);
|
||||
writeFileSync(join(libDir, "whoosh.mp3"), "ID3-fake-bytes");
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
assert.equal(sfx.length, 1);
|
||||
assert.equal(sfx[0].file, "assets/sfx/whoosh.mp3");
|
||||
assert.equal(sfx[0].source, "local");
|
||||
assert.ok(existsSync(join(projDir, "assets/sfx/whoosh.mp3")), "mp3 copied into project");
|
||||
assert.deepEqual(anomalies, [UNREACHABLE]);
|
||||
});
|
||||
});
|
||||
|
||||
test("offline: a matched-but-missing bundled file yields an anomaly and NO dangling entry", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
// Manifest names whoosh.mp3, but the mp3 was never shipped (the reported bug).
|
||||
writeFileSync(
|
||||
join(libDir, "manifest.json"),
|
||||
JSON.stringify({ whoosh: { file: "whoosh.mp3", duration: 0.8 } }),
|
||||
);
|
||||
const { sfx, anomalies } = await resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
assert.equal(sfx.length, 0, "no dangling entry for a file that was never copied");
|
||||
assert.equal(anomalies.length, 2);
|
||||
assert.match(anomalies[1], /missing from the offline library/);
|
||||
assert.ok(!existsSync(join(projDir, "assets/sfx/whoosh.mp3")), "nothing copied");
|
||||
});
|
||||
});
|
||||
|
||||
const whooshLibrary = (libDir, withFile) => {
|
||||
writeFileSync(
|
||||
join(libDir, "manifest.json"),
|
||||
JSON.stringify({ whoosh: { file: "whoosh.mp3", duration: 0.8 } }),
|
||||
);
|
||||
if (withFile) writeFileSync(join(libDir, "whoosh.mp3"), "ID3-library-bytes");
|
||||
};
|
||||
const record = (projDir, source) =>
|
||||
appendRecord(projDir, { id: "sfx_001", type: "sfx", path: "assets/sfx/whoosh.mp3", source });
|
||||
const bundledOnly = (libDir, projDir) =>
|
||||
resolveSfx({
|
||||
cues: [{ id: "s1", name: "whoosh" }],
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
|
||||
test("offline: the engine's earlier copy stands in for a library file this install lacks", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
whooshLibrary(libDir, false);
|
||||
mkdirSync(join(projDir, "assets/sfx"), { recursive: true });
|
||||
writeFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "engine copy");
|
||||
record(projDir, "bundled");
|
||||
|
||||
const { sfx, anomalies } = await bundledOnly(libDir, projDir);
|
||||
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file }) => file),
|
||||
["assets/sfx/whoosh.mp3"],
|
||||
);
|
||||
assert.deepEqual(anomalies, [UNREACHABLE]);
|
||||
});
|
||||
});
|
||||
|
||||
test("offline: a person's file with the library's bytes stays theirs once recorded as their own", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
whooshLibrary(libDir, true);
|
||||
mkdirSync(join(projDir, "assets/sfx"), { recursive: true });
|
||||
writeFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "ID3-library-bytes");
|
||||
record(projDir, "existing");
|
||||
|
||||
const { sfx } = await bundledOnly(libDir, projDir);
|
||||
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file }) => file),
|
||||
["assets/sfx/whoosh-2.mp3"],
|
||||
);
|
||||
assert.equal(readFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "utf8"), "ID3-library-bytes");
|
||||
});
|
||||
});
|
||||
|
||||
const libraryWith = (libDir, names) => {
|
||||
const manifest = {};
|
||||
for (const name of names) {
|
||||
manifest[name] = { file: `${name}.mp3`, duration: 0.5 };
|
||||
writeFileSync(join(libDir, `${name}.mp3`), `library ${name}`);
|
||||
}
|
||||
writeFileSync(join(libDir, "manifest.json"), JSON.stringify(manifest));
|
||||
};
|
||||
const sound = (overrides = {}) => ({
|
||||
id: "ov-1",
|
||||
title: "Swoosh",
|
||||
creator: "Ana",
|
||||
license: "by",
|
||||
foreign_landing_url: "https://freesound.org/s/1",
|
||||
url: "https://cdn.freesound.test/swoosh.mp3",
|
||||
filetype: "mp3",
|
||||
duration: 820,
|
||||
attribution: '"Swoosh" by Ana is licensed under CC BY 4.0.',
|
||||
...overrides,
|
||||
});
|
||||
// Openverse stand-in: `answer(query)` gives a search response; any other URL is the sound file.
|
||||
function openverse(answer) {
|
||||
const searches = [];
|
||||
serve = async (url) => {
|
||||
const query = new URL(url).searchParams.get("q");
|
||||
if (!url.startsWith("https://api.openverse.org/")) return new Response(`bytes of ${url}`);
|
||||
searches.push(query);
|
||||
return answer(query);
|
||||
};
|
||||
return searches;
|
||||
}
|
||||
const run = (libDir, projDir, names) =>
|
||||
resolveSfx({
|
||||
cues: names.map((name, index) => ({ id: String(index + 1), name })),
|
||||
hyperframesDir: projDir,
|
||||
sfxLibDir: libDir,
|
||||
});
|
||||
|
||||
test("openverse: each cue gets the best licensed sound, with its licence and credit", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
libraryWith(libDir, ["whoosh"]);
|
||||
openverse(() => Response.json({ results: [sound()] }));
|
||||
try {
|
||||
const { sfx, anomalies } = await run(libDir, projDir, ["whoosh"]);
|
||||
assert.deepEqual(sfx, [
|
||||
{
|
||||
id: "1",
|
||||
name: "whoosh",
|
||||
file: "assets/sfx/whoosh.mp3",
|
||||
source: "openverse",
|
||||
duration_s: 0.82,
|
||||
license: "by",
|
||||
credit: '"Swoosh" by Ana is licensed under CC BY 4.0.',
|
||||
offset_s: 0,
|
||||
volume: 0.35,
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(anomalies, []);
|
||||
assert.equal(
|
||||
readFileSync(join(projDir, "assets/sfx/whoosh.mp3"), "utf8"),
|
||||
"bytes of https://cdn.freesound.test/swoosh.mp3",
|
||||
);
|
||||
assert.match(
|
||||
readFileSync(join(projDir, "assets/sfx/CREDITS.md"), "utf8"),
|
||||
/- `assets\/sfx\/whoosh\.mp3`: "Swoosh" by Ana is licensed under CC BY 4\.0\. Source: https:\/\/freesound\.org\/s\/1/,
|
||||
);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("openverse: a rate limit sends this and every later cue to the bundled library, asking once", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
libraryWith(libDir, ["whoosh", "pop"]);
|
||||
const searches = openverse(() => new Response("slow down", { status: 429 }));
|
||||
try {
|
||||
const { sfx, anomalies } = await run(libDir, projDir, ["whoosh", "pop"]);
|
||||
assert.deepEqual(searches, ["whoosh"]);
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file, source }) => [file, source]),
|
||||
[
|
||||
["assets/sfx/whoosh.mp3", "local"],
|
||||
["assets/sfx/pop.mp3", "local"],
|
||||
],
|
||||
);
|
||||
assert.equal(readFileSync(join(projDir, "assets/sfx/pop.mp3"), "utf8"), "library pop");
|
||||
assert.deepEqual(anomalies, [
|
||||
"Openverse rate limit reached (HTTP 429): the remaining SFX cues use the bundled library",
|
||||
]);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("openverse: a cue with no licensed match falls back to the bundled library alone", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
libraryWith(libDir, ["pop"]);
|
||||
const searches = openverse((query) =>
|
||||
Response.json({ results: query === "pop" ? [sound({ license: "by-nc" })] : [sound()] }),
|
||||
);
|
||||
try {
|
||||
const { sfx } = await run(libDir, projDir, ["pop", "swoosh"]);
|
||||
assert.deepEqual(searches, ["pop", "swoosh"]);
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file, source }) => [file, source]),
|
||||
[
|
||||
["assets/sfx/pop.mp3", "local"],
|
||||
["assets/sfx/swoosh.mp3", "openverse"],
|
||||
],
|
||||
);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("openverse: a failed download falls back to the bundled file with an anomaly", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
libraryWith(libDir, ["whoosh"]);
|
||||
serve = async (url) =>
|
||||
url.startsWith("https://api.openverse.org/")
|
||||
? Response.json({ results: [sound()] })
|
||||
: new Response("gone", { status: 404 });
|
||||
try {
|
||||
const { sfx, anomalies } = await run(libDir, projDir, ["whoosh"]);
|
||||
assert.deepEqual(
|
||||
sfx.map(({ file, source }) => [file, source]),
|
||||
[["assets/sfx/whoosh.mp3", "local"]],
|
||||
);
|
||||
assert.match(anomalies[0], /Openverse download failed — download HTTP 404/);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("openverse: one effect named in two lines is searched and downloaded once", async () => {
|
||||
await withDirs(async ({ libDir, projDir }) => {
|
||||
libraryWith(libDir, []);
|
||||
const searches = openverse(() => Response.json({ results: [sound({ license: "cc0" })] }));
|
||||
try {
|
||||
const { sfx } = await run(libDir, projDir, ["whoosh", "whoosh"]);
|
||||
assert.deepEqual(searches, ["whoosh"]);
|
||||
assert.deepEqual(
|
||||
sfx.map(({ id, file }) => [id, file]),
|
||||
[
|
||||
["1", "assets/sfx/whoosh.mp3"],
|
||||
["2", "assets/sfx/whoosh.mp3"],
|
||||
],
|
||||
);
|
||||
assert.equal(
|
||||
existsSync(join(projDir, "assets/sfx/CREDITS.md")),
|
||||
false,
|
||||
"CC0 needs no credit",
|
||||
);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("findSoundEffect: a host gets the saved file, licence, credit and duration for one query", async () => {
|
||||
await withDirs(async ({ projDir }) => {
|
||||
const searches = openverse(() => Response.json({ results: [sound()] }));
|
||||
try {
|
||||
assert.deepEqual(await findSoundEffect("whoosh", { hyperframesDir: projDir }), {
|
||||
sound: {
|
||||
file: "assets/sfx/whoosh.mp3",
|
||||
source: "openverse",
|
||||
duration_s: 0.82,
|
||||
license: "by",
|
||||
credit: '"Swoosh" by Ana is licensed under CC BY 4.0.',
|
||||
},
|
||||
anomalies: [],
|
||||
});
|
||||
assert.deepEqual(searches, ["whoosh"]);
|
||||
} finally {
|
||||
serve = offline;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("findSoundEffect: offline, the skill's own bundled library answers", async () => {
|
||||
await withDirs(async ({ projDir }) => {
|
||||
const { sound, anomalies } = await findSoundEffect("whoosh", { hyperframesDir: projDir });
|
||||
assert.deepEqual(sound, {
|
||||
file: "assets/sfx/whoosh.mp3",
|
||||
source: "local",
|
||||
duration_s: sound.duration_s,
|
||||
});
|
||||
assert.deepEqual(
|
||||
readFileSync(join(projDir, sound.file)),
|
||||
readFileSync(join(BUNDLED_SFX_DIR, "whoosh.mp3")),
|
||||
);
|
||||
assert.deepEqual(anomalies, [UNREACHABLE]);
|
||||
assert.equal(
|
||||
(await findSoundEffect("no such sound at all", { hyperframesDir: projDir })).sound,
|
||||
null,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,175 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
|
||||
const ttsUrl = new URL("./tts.mjs", import.meta.url).href;
|
||||
const stub = `#!${process.execPath}
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const args = process.argv.slice(2);
|
||||
if (args[0] === "--version") {
|
||||
console.log("Python 3.12.0");
|
||||
process.exit(0);
|
||||
}
|
||||
const input = args[2];
|
||||
const output = args[0] === "-c" ? args[4] : args[args.indexOf("--output") + 1];
|
||||
const text = fs.readFileSync(input, "utf8");
|
||||
fs.writeFileSync(output + ".started", input);
|
||||
const finish = () => {
|
||||
if (text === "slow" && !fs.existsSync(process.env.RELEASE_FILE)) {
|
||||
setTimeout(finish, 5);
|
||||
return;
|
||||
}
|
||||
const consumed = fs.readFileSync(input, "utf8");
|
||||
fs.writeFileSync(output + ".json", JSON.stringify({ input, text, consumed, args }));
|
||||
if (process.env.CASE_MODE === "nonzero") process.exit(7);
|
||||
if (process.env.CASE_MODE !== "no-output") fs.writeFileSync(output, consumed);
|
||||
};
|
||||
setTimeout(finish, 10);
|
||||
`;
|
||||
|
||||
const exercise = `
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import childProcess from "node:child_process";
|
||||
import { syncBuiltinESMExports } from "node:module";
|
||||
import { join, dirname } from "node:path";
|
||||
const { CASE_ROOT: root, CASE_MODE: mode, CASE_PROVIDER: provider } = process.env;
|
||||
const temp = join(root, "tmp");
|
||||
const preexisting = join(temp, "hf-txt-preexisting");
|
||||
fs.mkdirSync(preexisting);
|
||||
fs.writeFileSync(join(preexisting, "marker"), "keep");
|
||||
const injected = Object.assign(new Error("synthetic " + mode), { code: "EACCES" });
|
||||
const originalMkdtemp = fs.mkdtempSync;
|
||||
const originalWrite = fs.writeFileSync;
|
||||
const originalRm = fs.rmSync;
|
||||
if (mode === "allocation") fs.mkdtempSync = (prefix, ...args) => {
|
||||
if (prefix.startsWith(join(temp, "hf-txt-"))) throw injected;
|
||||
return originalMkdtemp(prefix, ...args);
|
||||
};
|
||||
if (mode === "write") fs.writeFileSync = (file, ...args) => {
|
||||
if (String(file).endsWith("line.txt")) {
|
||||
originalWrite(file, "partial");
|
||||
throw injected;
|
||||
}
|
||||
return originalWrite(file, ...args);
|
||||
};
|
||||
if (mode === "spawn-throw") childProcess.spawn = () => { throw injected; };
|
||||
if (mode === "cleanup") fs.rmSync = (dir, ...args) => {
|
||||
if (String(dir).startsWith(join(temp, "hf-txt-"))) throw injected;
|
||||
return originalRm(dir, ...args);
|
||||
};
|
||||
syncBuiltinESMExports();
|
||||
const { synthesizeOne } = await import(${JSON.stringify(ttsUrl)});
|
||||
const project = join(root, "project");
|
||||
const wavAbs = join(project, "line.wav");
|
||||
const request = (chosen = provider, text = "synthetic café", output = wavAbs) =>
|
||||
synthesizeOne({ provider: chosen, text, voiceId: "synthetic-voice", lang: "es",
|
||||
speed: 0.8, wavAbs: output, hyperframesDir: project });
|
||||
const owned = () => fs.readdirSync(temp).filter(name => name !== "hf-txt-preexisting");
|
||||
if (["allocation", "write", "spawn-throw"].includes(mode)) {
|
||||
await assert.rejects(request(), error => error === injected);
|
||||
assert.equal(fs.existsSync(wavAbs), false);
|
||||
} else if (mode === "concurrent") {
|
||||
const slowWav = join(project, "slow.wav");
|
||||
const slow = request("elevenlabs", "slow", slowWav);
|
||||
try {
|
||||
const deadline = Date.now() + 5000;
|
||||
while (!fs.existsSync(slowWav + ".started")) {
|
||||
assert.ok(Date.now() < deadline, "slow subprocess must start");
|
||||
await new Promise(resolve => setTimeout(resolve, 5));
|
||||
}
|
||||
const slowInput = fs.readFileSync(slowWav + ".started", "utf8");
|
||||
assert.deepEqual(await request("kokoro"), { ok: true, words: null });
|
||||
assert.deepEqual(owned(), [dirname(slowInput).split(/[\\\\/]/).pop()]);
|
||||
assert.equal(fs.readFileSync(slowInput, "utf8"), "slow");
|
||||
} finally {
|
||||
fs.writeFileSync(process.env.RELEASE_FILE, "release");
|
||||
await slow;
|
||||
}
|
||||
assert.equal(fs.readFileSync(slowWav, "utf8"), "slow");
|
||||
} else {
|
||||
const result = await request();
|
||||
const label = provider === "elevenlabs" ? "elevenlabs (python)" : "kokoro (npx hyperframes tts)";
|
||||
if (["nonzero", "enoent", "no-output"].includes(mode)) {
|
||||
const error = mode === "no-output" ? label + " produced no wav file" :
|
||||
label + " exited with status " + (mode === "enoent" ? -1 : 7);
|
||||
assert.deepEqual(result, { ok: false, words: null, error });
|
||||
assert.equal(fs.existsSync(wavAbs), false);
|
||||
} else {
|
||||
assert.deepEqual(result, { ok: true, words: null });
|
||||
assert.equal(fs.readFileSync(wavAbs, "utf8"), "synthetic café");
|
||||
}
|
||||
if (mode !== "enoent") {
|
||||
const log = JSON.parse(fs.readFileSync(wavAbs + ".json", "utf8"));
|
||||
assert.equal(log.text, "synthetic café");
|
||||
assert.equal(log.consumed, log.text);
|
||||
if (provider === "kokoro") assert.deepEqual(log.args.slice(-4), ["--lang", "es", "--speed", "0.8"]);
|
||||
}
|
||||
}
|
||||
if (mode === "cleanup") assert.equal(owned().length, 1);
|
||||
else assert.deepEqual(owned(), [], "owned text directories must be removed");
|
||||
assert.equal(fs.readFileSync(join(preexisting, "marker"), "utf8"), "keep");
|
||||
`;
|
||||
|
||||
function runCase(provider, mode) {
|
||||
const root = mkdtempSync(join(tmpdir(), "tts-cleanup-test-"));
|
||||
try {
|
||||
for (const dir of ["home", "tmp", "bin", "project"]) mkdirSync(join(root, dir));
|
||||
if (mode !== "enoent") {
|
||||
for (const name of ["npx", "python3"]) {
|
||||
const executable = join(root, "bin", name);
|
||||
writeFileSync(executable, stub);
|
||||
chmodSync(executable, 0o755);
|
||||
}
|
||||
}
|
||||
const result = spawnSync(process.execPath, ["--input-type=module", "-e", exercise], {
|
||||
encoding: "utf8",
|
||||
timeout: 15000,
|
||||
env: {
|
||||
HOME: join(root, "home"),
|
||||
USERPROFILE: join(root, "home"),
|
||||
TMPDIR: join(root, "tmp"),
|
||||
TMP: join(root, "tmp"),
|
||||
TEMP: join(root, "tmp"),
|
||||
PATH: join(root, "bin"),
|
||||
CASE_ROOT: root,
|
||||
CASE_PROVIDER: provider,
|
||||
CASE_MODE: mode,
|
||||
RELEASE_FILE: join(root, "release"),
|
||||
},
|
||||
});
|
||||
assert.equal(result.error, undefined);
|
||||
assert.equal(result.status, 0, result.stdout + result.stderr);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
for (const provider of ["elevenlabs", "kokoro"]) {
|
||||
for (const mode of [
|
||||
"success",
|
||||
"nonzero",
|
||||
"no-output",
|
||||
"enoent",
|
||||
"spawn-throw",
|
||||
"allocation",
|
||||
"write",
|
||||
"cleanup",
|
||||
]) {
|
||||
test(
|
||||
`synthesizeOne(${provider}) preserves ${mode} behavior and cleans only its text directory`,
|
||||
{ skip: process.platform === "win32" },
|
||||
() => runCase(provider, mode),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
test(
|
||||
"concurrent providers retain each other's text until their subprocess finishes",
|
||||
{ skip: process.platform === "win32" },
|
||||
() => runCase("kokoro", "concurrent"),
|
||||
);
|
||||
@@ -0,0 +1,394 @@
|
||||
import { fetchMedia } from "../../../scripts/lib/media-fetch.mjs";
|
||||
// tts.mjs — multi-provider TTS for the media audio engine. The provider chain,
|
||||
// auto-detected from env, is the one documented in ../SKILL.md:
|
||||
//
|
||||
// 1. HeyGen (Starfish) — $HEYGEN_API_KEY / $HYPERFRAMES_API_KEY / ~/.heygen.
|
||||
// Direct v3 REST (NOT `hyperframes tts`, which in the published build is
|
||||
// Kokoro-only and silently ignores a HeyGen key). Returns word_timestamps
|
||||
// in the same call, so no separate transcribe pass.
|
||||
// 2. ElevenLabs — $ELEVENLABS_API_KEY + `pip install elevenlabs`. No
|
||||
// word timings → caller chains transcribeWav().
|
||||
// 3. Kokoro-82M (local) — always available, via the published `hyperframes tts`
|
||||
// CLI. No word timings → caller chains transcribeWav().
|
||||
//
|
||||
// "HeyGen available" is decided by CREDENTIAL presence (heygenCredential), never
|
||||
// by the CLI — see the note above.
|
||||
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { heygenAuthHeaders, heygenCredential, heygenJSON, heygenMessage } from "./heygen.mjs";
|
||||
import { pythonInvocation } from "./python.mjs";
|
||||
import { synthesizeGemini } from "./gemini-tts.mjs";
|
||||
import { geminiConfigured } from "./gemini-auth.mjs";
|
||||
|
||||
// ── provider detection ────────────────────────────────────────────────────────
|
||||
export function heygenAvailable() {
|
||||
return heygenCredential() !== null;
|
||||
}
|
||||
export function elevenlabsAvailable() {
|
||||
if (!process.env.ELEVENLABS_API_KEY) return false;
|
||||
const { cmd, args } = pythonInvocation(["-c", "import elevenlabs"]);
|
||||
const r = spawnSync(cmd, args, {
|
||||
stdio: "ignore",
|
||||
});
|
||||
return r.status === 0;
|
||||
}
|
||||
|
||||
// First available provider wins; an explicit choice is honored (and validated).
|
||||
export function pickProvider(userProvider) {
|
||||
if (userProvider) {
|
||||
if (!["heygen", "elevenlabs", "kokoro", "gemini"].includes(userProvider))
|
||||
throw new Error(`invalid provider "${userProvider}" (heygen | elevenlabs | kokoro | gemini)`);
|
||||
if (userProvider === "gemini" && !geminiConfigured())
|
||||
throw new Error(
|
||||
"provider=gemini needs GEMINI_API_KEY or GOOGLE_API_KEY, or service-account credentials (GOOGLE_APPLICATION_CREDENTIALS or GCS_CREDS)",
|
||||
);
|
||||
// heygenAuthHeaders owns the reason and its fix: no credential, or a credentials path that cannot be read.
|
||||
if (userProvider === "heygen" && !heygenAvailable()) heygenAuthHeaders();
|
||||
if (userProvider === "elevenlabs" && !process.env.ELEVENLABS_API_KEY)
|
||||
throw new Error("provider=elevenlabs but $ELEVENLABS_API_KEY is not set");
|
||||
return userProvider;
|
||||
}
|
||||
return heygenAvailable() ? "heygen" : elevenlabsAvailable() ? "elevenlabs" : "kokoro";
|
||||
}
|
||||
|
||||
// ── voice resolution ──────────────────────────────────────────────────────────
|
||||
// HeyGen /v3/voices/speech only accepts STARFISH voice_ids; auto-pick the first
|
||||
// English public starfish voice when none is pinned. ElevenLabs/Kokoro have
|
||||
// their own defaults.
|
||||
export async function resolveVoiceId({ provider, userVoice, lang = "en" }) {
|
||||
if (userVoice) return userVoice;
|
||||
if (provider === "gemini") return "Kore";
|
||||
if (provider === "elevenlabs") return "21m00Tcm4TlvDq8ikWAM"; // Rachel
|
||||
if (provider === "kokoro") {
|
||||
if (lang === "en") return "am_michael";
|
||||
throw new Error("Kokoro non-English needs an explicit --voice (see references/tts.md)");
|
||||
}
|
||||
// heygen — pin a fixed English default so the choice is deterministic. The old
|
||||
// "first English voice the API returns" drifts whenever HeyGen re-sorts the
|
||||
// public catalog. Marcia (mature, low female). Override with --voice / request.voice.
|
||||
if (lang === "en") return "05f19352e8f74b0392a8f411eba40de1"; // Marcia · English · female
|
||||
// Non-English: no fixed default — fall back to the first matching catalog voice.
|
||||
const payload = await heygenJSON(`/voices?engine=starfish&type=public&limit=50`, {
|
||||
headers: heygenAuthHeaders(),
|
||||
});
|
||||
const voices = payload.data ?? payload.voices ?? [];
|
||||
const pick = voices.find((v) => v.language === "English") ?? voices[0];
|
||||
if (!pick) throw new Error("no public starfish voice to default to — pass --voice");
|
||||
return pick.voice_id;
|
||||
}
|
||||
|
||||
// ── helpers ─────────────────────────────────────────────────────────────────
|
||||
export function withWordIds(words) {
|
||||
return (words ?? []).map((w, i) => ({
|
||||
id: `w${i}`,
|
||||
text: w.text,
|
||||
start: w.start,
|
||||
end: w.end,
|
||||
}));
|
||||
}
|
||||
|
||||
// `ffmpeg -i <file>` prints a `Duration: HH:MM:SS.ms` line to stderr even
|
||||
// though it exits non-zero with no output requested. Parsing pulled out as
|
||||
// a pure function so the ENOENT fallback below can be tested without
|
||||
// depending on whether ffprobe/ffmpeg are actually installed on the
|
||||
// machine running the tests.
|
||||
export function parseFfmpegDurationBanner(stderrText) {
|
||||
const match = /Duration:\s*(\d+):(\d+):(\d+(?:\.\d+)?)/.exec(stderrText ?? "");
|
||||
if (!match) return NaN;
|
||||
const [, hours, minutes, seconds] = match;
|
||||
return Number(hours) * 3600 + Number(minutes) * 60 + Number(seconds);
|
||||
}
|
||||
|
||||
// Some "essentials"-style ffmpeg distributions (common on Windows) ship
|
||||
// ffmpeg.exe without ffprobe.exe. ffprobeDuration's caller (audio.mjs)
|
||||
// otherwise reads a spurious NaN as "the WAV file is corrupt" and drops an
|
||||
// already-successfully-synthesized TTS line, rather than "the tool for
|
||||
// measuring it is missing".
|
||||
function ffmpegDurationFallback(absPath) {
|
||||
const r = spawnSync("ffmpeg", ["-i", absPath], { encoding: "utf8" });
|
||||
return parseFfmpegDurationBanner(r.stderr);
|
||||
}
|
||||
|
||||
export function ffprobeDuration(absPath) {
|
||||
const r = spawnSync(
|
||||
"ffprobe",
|
||||
["-v", "error", "-show_entries", "format=duration", "-of", "default=nw=1:nk=1", "--", absPath],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
if (r.error?.code === "ENOENT") return ffmpegDurationFallback(absPath);
|
||||
if (r.status !== 0) return NaN;
|
||||
return parseFloat(String(r.stdout).trim());
|
||||
}
|
||||
|
||||
export function resolveNpxCliFromNpmExecPath(
|
||||
npmExecPath = process.env.npm_execpath,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
if (!npmExecPath) return null;
|
||||
const fileName = npmExecPath.replace(/\\/g, "/").split("/").pop()?.toLowerCase();
|
||||
const npxCliPath =
|
||||
fileName === "npx-cli.js" ? npmExecPath : join(dirname(npmExecPath), "npx-cli.js");
|
||||
return pathExists(npxCliPath) ? npxCliPath : null;
|
||||
}
|
||||
|
||||
export function resolveNpxCliPath(
|
||||
npmExecPath = process.env.npm_execpath,
|
||||
nodeExecPath = process.env.npm_node_execpath || process.execPath,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
const fromNpm = resolveNpxCliFromNpmExecPath(npmExecPath, pathExists);
|
||||
if (fromNpm) return fromNpm;
|
||||
const besideNode = join(dirname(nodeExecPath), "node_modules", "npm", "bin", "npx-cli.js");
|
||||
return pathExists(besideNode) ? besideNode : null;
|
||||
}
|
||||
|
||||
export function resolveSpawnCommand(
|
||||
cmd,
|
||||
args,
|
||||
opts = {},
|
||||
platform = process.platform,
|
||||
env = process.env,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
if (cmd !== "npx" || platform !== "win32") {
|
||||
return { cmd, args, opts: { stdio: "ignore", ...opts } };
|
||||
}
|
||||
|
||||
// On Windows, npx resolves to npx.cmd, which Node cannot execute directly.
|
||||
// Avoid `shell:true` and the .cmd shim entirely by invoking npm's JS CLI with
|
||||
// node, preserving request-provided values as argv data instead of shell text.
|
||||
const nodeExecPath = env.npm_node_execpath || process.execPath;
|
||||
const npxCliPath = resolveNpxCliPath(env.npm_execpath, nodeExecPath, pathExists);
|
||||
if (!npxCliPath) return null;
|
||||
return {
|
||||
cmd: nodeExecPath,
|
||||
args: [npxCliPath, ...args.map((arg) => String(arg))],
|
||||
opts: { stdio: "ignore", windowsHide: true, ...opts },
|
||||
};
|
||||
}
|
||||
|
||||
// `platform`/`spawnFn` params (default process.platform / the real spawn)
|
||||
// exist so tests can exercise the win32 branch without mocking node:child_process
|
||||
// (its ESM exports are non-configurable, so mock.method can't patch it).
|
||||
// One-shot so a whole batch of TTS lines doesn't repeat the same diagnostic.
|
||||
let _warnedNpxResolution = false;
|
||||
/** Test-only: reset the one-shot npx-resolution warning latch. */
|
||||
export function _resetNpxResolutionWarnForTests() {
|
||||
_warnedNpxResolution = false;
|
||||
}
|
||||
|
||||
export function spawnP(
|
||||
cmd,
|
||||
args,
|
||||
opts = {},
|
||||
platform = process.platform,
|
||||
spawnFn = spawn,
|
||||
env = process.env,
|
||||
pathExists = existsSync,
|
||||
) {
|
||||
const resolved = resolveSpawnCommand(cmd, args, opts, platform, env, pathExists);
|
||||
if (!resolved) {
|
||||
// resolveSpawnCommand only returns null for the npx-on-win32 case where
|
||||
// neither npm's configured CLI nor the beside-node fallback exists. Without
|
||||
// this, every call silently returns status:-1 and stdio:"ignore" hides why.
|
||||
if (!_warnedNpxResolution) {
|
||||
_warnedNpxResolution = true;
|
||||
const reason = env.npm_execpath
|
||||
? `npm_execpath (${env.npm_execpath}) and the beside-node npm fallback could not be found`
|
||||
: "npm_execpath is unset and the beside-node npm fallback could not be found";
|
||||
console.error(
|
||||
`[media-use] Cannot run "${cmd}" on Windows: ${reason}. ` +
|
||||
`Every "${cmd}" call is being skipped. Install npm with Node, or run via ` +
|
||||
`\`npx\`/\`npm run\` with a valid npm_execpath.`,
|
||||
);
|
||||
}
|
||||
return Promise.resolve({ status: -1 });
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const p = spawnFn(resolved.cmd, resolved.args, resolved.opts);
|
||||
p.on("exit", (code) => resolve({ status: code ?? -1 }));
|
||||
p.on("error", () => resolve({ status: -1 }));
|
||||
});
|
||||
}
|
||||
|
||||
// mp3/whatever bytes → wav 44.1k mono at destWav (ffmpeg detects true format).
|
||||
function transcodeToWav(bytes, destWav) {
|
||||
const td = mkdtempSync(join(tmpdir(), "hf-tts-"));
|
||||
const tmp = join(td, "a.mp3");
|
||||
writeFileSync(tmp, bytes);
|
||||
mkdirSync(dirname(destWav), { recursive: true });
|
||||
const ff = spawnSync(
|
||||
"ffmpeg",
|
||||
["-y", "-loglevel", "error", "-i", tmp, "-ar", "44100", "-ac", "1", destWav],
|
||||
{ stdio: "ignore" },
|
||||
);
|
||||
rmSync(td, { recursive: true, force: true });
|
||||
return ff.status === 0 && existsSync(destWav);
|
||||
}
|
||||
|
||||
const ELEVENLABS_PY = `
|
||||
import os, sys
|
||||
from elevenlabs.client import ElevenLabs
|
||||
from elevenlabs import save
|
||||
client = ElevenLabs(api_key=os.environ["ELEVENLABS_API_KEY"])
|
||||
text = open(sys.argv[1], encoding="utf-8").read()
|
||||
audio = client.text_to_speech.convert(
|
||||
text=text, voice_id=sys.argv[2],
|
||||
model_id="eleven_multilingual_v2", output_format="mp3_44100_128",
|
||||
)
|
||||
save(audio, sys.argv[3])
|
||||
`;
|
||||
|
||||
// ── synthesize one line ───────────────────────────────────────────────────────
|
||||
// Writes wav at wavAbs. Returns { ok, words, error } — words is the raw
|
||||
// [{text,start,end}] array for HeyGen (native), or null for ElevenLabs/Kokoro
|
||||
// (caller must transcribeWav). Never throws; failures return { ok:false, error }
|
||||
// where `error` states WHY (so the caller can surface it, not a bare "TTS failed").
|
||||
export async function synthesizeOne({
|
||||
provider,
|
||||
text,
|
||||
voiceId,
|
||||
lang = "en",
|
||||
speed = 1.0,
|
||||
model,
|
||||
style,
|
||||
wavAbs,
|
||||
hyperframesDir,
|
||||
}) {
|
||||
if (provider === "heygen") return synthesizeHeygen({ text, voiceId, lang, speed, wavAbs });
|
||||
if (provider === "gemini")
|
||||
return synthesizeGemini({ text, voiceId, model, style, speed, wavAbs });
|
||||
if (provider === "elevenlabs") {
|
||||
// The Python helper writes straight to wavAbs; unlike heygen (transcodeToWav)
|
||||
// and kokoro (the `hyperframes tts` CLI), it does NOT create the parent dir,
|
||||
// so on a fresh project (no assets/voice/ yet) the save fails and the line is
|
||||
// silently dropped as "TTS failed - omitted". Create it first, like the other
|
||||
// providers do. Guarded so a mkdir failure (EACCES/EROFS) returns
|
||||
// { ok:false } like the rest of this branch rather than throwing (the
|
||||
// function's contract is "never throws; failures return { ok:false }").
|
||||
try {
|
||||
mkdirSync(dirname(wavAbs), { recursive: true });
|
||||
} catch {
|
||||
return { ok: false, words: null };
|
||||
}
|
||||
return withTmpText(text, async (textPath) => {
|
||||
const { cmd, args } = pythonInvocation(["-c", ELEVENLABS_PY, textPath, voiceId, wavAbs]);
|
||||
const r = await spawnP(cmd, args, {});
|
||||
return synthResult(r, wavAbs, "elevenlabs (python)");
|
||||
});
|
||||
}
|
||||
// kokoro — via the published CLI; --output is relative to the project dir.
|
||||
const wavRel = relTo(hyperframesDir, wavAbs);
|
||||
return withTmpText(text, async (textPath) => {
|
||||
const args = ["hyperframes", "tts", textPath, "--voice", voiceId, "--output", wavRel];
|
||||
if (lang !== "en") args.push("--lang", lang);
|
||||
if (speed !== 1) args.push("--speed", String(speed));
|
||||
const r = await spawnP("npx", args, { cwd: hyperframesDir });
|
||||
return synthResult(r, wavAbs, "kokoro (npx hyperframes tts)");
|
||||
});
|
||||
}
|
||||
|
||||
// Shape a spawn result into { ok, words, error }, naming why on failure so the
|
||||
// caller surfaces it instead of a bare "TTS failed".
|
||||
export function synthResult(r, wavAbs, label) {
|
||||
if (r.status === 0 && existsSync(wavAbs)) return { ok: true, words: null };
|
||||
const why =
|
||||
r.status !== 0 ? `${label} exited with status ${r.status}` : `${label} produced no wav file`;
|
||||
return { ok: false, words: null, error: why };
|
||||
}
|
||||
|
||||
// `deps` is injectable for tests; production uses the real network/ffmpeg impls.
|
||||
// Every failure path returns an `error` string so the caller can surface WHY a
|
||||
// line was dropped instead of the bare "TTS failed" that hid the real cause
|
||||
// (e.g. an HTTP 402 plan_upgrade_required thrown by heygenJSON was swallowed).
|
||||
export async function synthesizeHeygen({ text, voiceId, lang, speed, wavAbs }, deps = {}) {
|
||||
const requestJSON = deps.heygenJSON ?? heygenJSON;
|
||||
const authHeaders = deps.heygenAuthHeaders ?? heygenAuthHeaders;
|
||||
const fetchImpl = deps.fetch ?? fetch;
|
||||
const transcode = deps.transcodeToWav ?? transcodeToWav;
|
||||
try {
|
||||
const body = { text, voice_id: voiceId, speed };
|
||||
if (lang !== "en") body.language = lang;
|
||||
const payload = await requestJSON(`/voices/speech`, {
|
||||
method: "POST",
|
||||
headers: authHeaders(),
|
||||
body,
|
||||
});
|
||||
const inner = payload.data ?? payload;
|
||||
if (!inner.audio_url) {
|
||||
return { ok: false, words: null, error: "HeyGen /voices/speech returned no audio_url" };
|
||||
}
|
||||
const res = await fetchMedia(inner.audio_url, { fetchImpl });
|
||||
if (!res.ok) {
|
||||
return { ok: false, words: null, error: `audio_url fetch failed: HTTP ${res.status}` };
|
||||
}
|
||||
const bytes = Buffer.from(await res.arrayBuffer());
|
||||
// .wav output → transcode to 44.1k mono; .mp3 → raw bytes (no ffmpeg). The
|
||||
// engine always asks for .wav; the standalone heygen-tts CLI may ask for .mp3.
|
||||
if (wavAbs.endsWith(".wav")) {
|
||||
if (!transcode(bytes, wavAbs)) {
|
||||
return {
|
||||
ok: false,
|
||||
words: null,
|
||||
error: "wav transcode failed (ffmpeg)",
|
||||
};
|
||||
}
|
||||
} else {
|
||||
mkdirSync(dirname(wavAbs), { recursive: true });
|
||||
writeFileSync(wavAbs, bytes);
|
||||
}
|
||||
const words = Array.isArray(inner.word_timestamps)
|
||||
? inner.word_timestamps
|
||||
.filter((w) => w && typeof w.word === "string" && isFinite(w.start) && isFinite(w.end))
|
||||
.filter((w) => !/^<.*>$/.test(w.word.trim())) // drop <start>/<end> sentinels
|
||||
.map((w) => ({ text: w.word, start: w.start, end: w.end }))
|
||||
: [];
|
||||
return { ok: true, words };
|
||||
} catch (e) {
|
||||
return { ok: false, words: null, error: heygenMessage(e) };
|
||||
}
|
||||
}
|
||||
|
||||
// ElevenLabs/Kokoro have no word timings — run Whisper over the wav. Returns the
|
||||
// flat [{id,text,start,end}] word array, or null. Each call uses a throwaway
|
||||
// --dir so parallel scenes don't collide on transcript.json.
|
||||
export async function transcribeWav({ wavRel, lang = "en", hyperframesDir }) {
|
||||
const model = lang === "en" ? "small.en" : "small";
|
||||
const td = mkdtempSync(join(tmpdir(), "hf-trans-"));
|
||||
const args = ["hyperframes", "transcribe", wavRel, "--model", model, "--dir", td];
|
||||
if (lang !== "en") args.push("--language", lang);
|
||||
const r = await spawnP("npx", args, { cwd: hyperframesDir });
|
||||
let words = null;
|
||||
if (r.status === 0) {
|
||||
const src = join(td, "transcript.json");
|
||||
if (existsSync(src)) {
|
||||
try {
|
||||
const arr = JSON.parse(readFileSync(src, "utf8"));
|
||||
if (Array.isArray(arr) && arr.length) words = arr;
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
rmSync(td, { recursive: true, force: true });
|
||||
return words;
|
||||
}
|
||||
|
||||
// ── tiny local utils ──────────────────────────────────────────────────────────
|
||||
async function withTmpText(text, run) {
|
||||
const td = mkdtempSync(join(tmpdir(), "hf-txt-"));
|
||||
try {
|
||||
const p = join(td, "line.txt");
|
||||
writeFileSync(p, text);
|
||||
return await run(p);
|
||||
} finally {
|
||||
// Cleanup must preserve the synthesis result or the original write/spawn error.
|
||||
try {
|
||||
rmSync(td, { recursive: true, force: true });
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
function relTo(base, abs) {
|
||||
return abs.startsWith(base + "/") ? abs.slice(base.length + 1) : abs;
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { EventEmitter } from "node:events";
|
||||
import {
|
||||
resolveNpxCliFromNpmExecPath,
|
||||
resolveNpxCliPath,
|
||||
resolveSpawnCommand,
|
||||
spawnP,
|
||||
_resetNpxResolutionWarnForTests,
|
||||
} from "./tts.mjs";
|
||||
|
||||
// Regression: on Windows, npx resolves to npx.cmd, which spawn() cannot exec
|
||||
// without shell:true — it fails ENOENT, silently swallowed as ok:false by the
|
||||
// caller. spawnP takes injectable platform/spawnFn params so this doesn't
|
||||
// need to touch the real process.platform or mock node:child_process (whose
|
||||
// ESM exports are non-configurable).
|
||||
function fakeSpawn(captured) {
|
||||
return (cmd, args, opts) => {
|
||||
captured.push({ cmd, args, opts });
|
||||
const p = new EventEmitter();
|
||||
setImmediate(() => p.emit("exit", 0));
|
||||
return p;
|
||||
};
|
||||
}
|
||||
|
||||
const envWithNpxCli = {
|
||||
npm_execpath: "/opt/node/lib/node_modules/npm/bin/npm-cli.js",
|
||||
npm_node_execpath: "/opt/node/bin/node",
|
||||
};
|
||||
const npxCliPath = "/opt/node/lib/node_modules/npm/bin/npx-cli.js";
|
||||
const pathExists = (path) => path === npxCliPath;
|
||||
|
||||
test("resolveNpxCliFromNpmExecPath finds npx-cli next to npm-cli", () => {
|
||||
assert.equal(resolveNpxCliFromNpmExecPath(envWithNpxCli.npm_execpath, pathExists), npxCliPath);
|
||||
});
|
||||
|
||||
test("resolveNpxCliPath finds npx-cli beside node when npm_execpath is unset", () => {
|
||||
const node = "C:/Program Files/nodejs/node.exe";
|
||||
const expected = "C:/Program Files/nodejs/node_modules/npm/bin/npx-cli.js";
|
||||
assert.equal(
|
||||
resolveNpxCliPath(undefined, node, (path) => path === expected),
|
||||
expected,
|
||||
);
|
||||
});
|
||||
|
||||
test("resolveSpawnCommand routes npx through node+npx-cli on win32 without shell:true", () => {
|
||||
const resolved = resolveSpawnCommand(
|
||||
"npx",
|
||||
["hyperframes", "tts", "C:\\Users\\Test User\\line.txt", "--voice", "am_michael"],
|
||||
{},
|
||||
"win32",
|
||||
envWithNpxCli,
|
||||
pathExists,
|
||||
);
|
||||
assert.ok(resolved);
|
||||
assert.equal(resolved.cmd, envWithNpxCli.npm_node_execpath);
|
||||
assert.deepEqual(resolved.args, [
|
||||
npxCliPath,
|
||||
"hyperframes",
|
||||
"tts",
|
||||
"C:\\Users\\Test User\\line.txt",
|
||||
"--voice",
|
||||
"am_michael",
|
||||
]);
|
||||
assert.equal(resolved.opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("resolveSpawnCommand preserves Windows npx shell metacharacters as argv data", () => {
|
||||
const resolved = resolveSpawnCommand(
|
||||
"npx",
|
||||
["hyperframes", "tts", "hello & calc"],
|
||||
{},
|
||||
"win32",
|
||||
envWithNpxCli,
|
||||
pathExists,
|
||||
);
|
||||
assert.ok(resolved);
|
||||
assert.deepEqual(resolved.args, [npxCliPath, "hyperframes", "tts", "hello & calc"]);
|
||||
});
|
||||
|
||||
test("spawnP uses the resolved node+npx-cli command for npx on win32", async () => {
|
||||
const captured = [];
|
||||
await spawnP(
|
||||
"npx",
|
||||
["hyperframes", "tts"],
|
||||
{},
|
||||
"win32",
|
||||
fakeSpawn(captured),
|
||||
envWithNpxCli,
|
||||
pathExists,
|
||||
);
|
||||
assert.equal(captured.length, 1);
|
||||
assert.equal(captured[0].cmd, envWithNpxCli.npm_node_execpath);
|
||||
assert.deepEqual(captured[0].args, [npxCliPath, "hyperframes", "tts"]);
|
||||
assert.equal(captured[0].opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("spawnP does not enable shell for npx on darwin/linux", async () => {
|
||||
const captured = [];
|
||||
await spawnP("npx", ["hyperframes", "tts"], {}, "darwin", fakeSpawn(captured));
|
||||
assert.equal(captured[0].cmd, "npx");
|
||||
assert.deepEqual(captured[0].args, ["hyperframes", "tts"]);
|
||||
assert.equal(captured[0].opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("spawnP does not enable shell for non-npx commands even on win32", async () => {
|
||||
const captured = [];
|
||||
await spawnP("python3", ["-c", "pass"], {}, "win32", fakeSpawn(captured));
|
||||
assert.equal(captured[0].cmd, "python3");
|
||||
assert.deepEqual(captured[0].args, ["-c", "pass"]);
|
||||
assert.equal(captured[0].opts.shell, undefined);
|
||||
});
|
||||
|
||||
test("spawnP resolves npx beside node when npm_execpath is unset on win32", async () => {
|
||||
_resetNpxResolutionWarnForTests();
|
||||
const captured = [];
|
||||
const node = "C:/Program Files/nodejs/node.exe";
|
||||
const npxCli = "C:/Program Files/nodejs/node_modules/npm/bin/npx-cli.js";
|
||||
const result = await spawnP(
|
||||
"npx",
|
||||
["hyperframes", "tts"],
|
||||
{},
|
||||
"win32",
|
||||
fakeSpawn(captured),
|
||||
{ npm_node_execpath: node },
|
||||
(path) => path === npxCli,
|
||||
);
|
||||
assert.equal(result.status, 0);
|
||||
assert.equal(captured.length, 1);
|
||||
assert.equal(captured[0].cmd, node);
|
||||
assert.deepEqual(captured[0].args, [npxCli, "hyperframes", "tts"]);
|
||||
});
|
||||
|
||||
test("spawnP warns once with an accurate diagnostic when neither npx path exists", async () => {
|
||||
_resetNpxResolutionWarnForTests();
|
||||
const errors = [];
|
||||
const originalError = console.error;
|
||||
console.error = (message) => errors.push(String(message));
|
||||
try {
|
||||
const env = { npm_execpath: "C:/missing/npm-cli.js", npm_node_execpath: "C:/node/node.exe" };
|
||||
const missing = () => false;
|
||||
assert.equal(
|
||||
(await spawnP("npx", ["hyperframes", "tts"], {}, "win32", fakeSpawn([]), env, missing))
|
||||
.status,
|
||||
-1,
|
||||
);
|
||||
assert.equal(
|
||||
(await spawnP("npx", ["hyperframes", "tts"], {}, "win32", fakeSpawn([]), env, missing))
|
||||
.status,
|
||||
-1,
|
||||
);
|
||||
} finally {
|
||||
console.error = originalError;
|
||||
}
|
||||
assert.equal(errors.length, 1);
|
||||
assert.match(errors[0], /npm_execpath \(C:\/missing\/npm-cli\.js\)/);
|
||||
assert.doesNotMatch(errors[0], /npm_execpath is not set/);
|
||||
});
|
||||
@@ -0,0 +1,230 @@
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
writeFileSync,
|
||||
chmodSync,
|
||||
rmSync,
|
||||
existsSync,
|
||||
} from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import {
|
||||
parseFfmpegDurationBanner,
|
||||
ffprobeDuration,
|
||||
synthesizeOne,
|
||||
synthesizeHeygen,
|
||||
synthResult,
|
||||
pickProvider,
|
||||
} from "./tts.mjs";
|
||||
|
||||
test("parseFfmpegDurationBanner reads ffmpeg's stderr Duration line", () => {
|
||||
const stderr = [
|
||||
"ffmpeg version 6.0",
|
||||
"Input #0, wav, from 'a.wav':",
|
||||
" Duration: 00:00:03.42, bitrate: 705 kb/s",
|
||||
"At least one output file must be specified",
|
||||
].join("\n");
|
||||
assert.equal(parseFfmpegDurationBanner(stderr), 3.42);
|
||||
});
|
||||
|
||||
test("parseFfmpegDurationBanner handles an hours component", () => {
|
||||
const stderr = " Duration: 01:02:03.50, start: 0.000000, bitrate: 128 kb/s";
|
||||
assert.equal(parseFfmpegDurationBanner(stderr), 3723.5);
|
||||
});
|
||||
|
||||
test("parseFfmpegDurationBanner returns NaN when there is no Duration line", () => {
|
||||
assert.ok(Number.isNaN(parseFfmpegDurationBanner("ffmpeg: command not found")));
|
||||
assert.ok(Number.isNaN(parseFfmpegDurationBanner("")));
|
||||
assert.ok(Number.isNaN(parseFfmpegDurationBanner(undefined)));
|
||||
});
|
||||
|
||||
// Regression for the actual bug: ffprobeDuration used to collapse "ffprobe
|
||||
// binary is missing" (ENOENT — the "essentials"-style Windows ffmpeg build
|
||||
// with no ffprobe.exe) and "file is genuinely unreadable" into the same NaN,
|
||||
// giving audio.mjs no way to tell "measure differently" from "give up".
|
||||
//
|
||||
// Builds an isolated PATH containing only a fake `ffmpeg` stub (no `ffprobe`
|
||||
// at all) so ffprobeDuration's spawnSync("ffprobe", ...) call ENOENTs for
|
||||
// real, then verifies it recovers the duration via the ffmpeg fallback
|
||||
// instead of returning NaN.
|
||||
test("ffprobeDuration falls back to ffmpeg when the ffprobe binary itself is missing", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-ffprobe-fallback-"));
|
||||
const fakeFfmpeg = join(dir, "ffmpeg");
|
||||
writeFileSync(
|
||||
fakeFfmpeg,
|
||||
"#!/bin/sh\necho 'Duration: 00:00:02.50, start: 0.000000, bitrate: 128 kb/s' 1>&2\nexit 1\n",
|
||||
);
|
||||
chmodSync(fakeFfmpeg, 0o755);
|
||||
const originalPath = process.env.PATH;
|
||||
try {
|
||||
process.env.PATH = dir; // only the fake ffmpeg resolves; no real ffprobe on this PATH
|
||||
assert.equal(ffprobeDuration("/does/not/matter.wav"), 2.5);
|
||||
} finally {
|
||||
process.env.PATH = originalPath;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("ffprobeDuration returns NaN when neither ffprobe nor ffmpeg resolve", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-no-binaries-"));
|
||||
const originalPath = process.env.PATH;
|
||||
try {
|
||||
process.env.PATH = dir; // empty directory — nothing resolves
|
||||
assert.ok(Number.isNaN(ffprobeDuration("/does/not/matter.wav")));
|
||||
} finally {
|
||||
process.env.PATH = originalPath;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("synthesizeOne(elevenlabs) creates the output dir before writing", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-el-mkdir-"));
|
||||
const wavAbs = join(dir, "assets", "voice", "line-0.wav"); // nested, not yet created
|
||||
const savedKey = process.env.ELEVENLABS_API_KEY;
|
||||
try {
|
||||
// Unset the key so the Python side fails fast — the mkdir must run before
|
||||
// the spawn regardless, which is what this guards.
|
||||
delete process.env.ELEVENLABS_API_KEY;
|
||||
await synthesizeOne({
|
||||
provider: "elevenlabs",
|
||||
text: "hi",
|
||||
voiceId: "v",
|
||||
wavAbs,
|
||||
hyperframesDir: dir,
|
||||
});
|
||||
assert.ok(existsSync(dirname(wavAbs)), "output directory should be created");
|
||||
} finally {
|
||||
if (savedKey === undefined) delete process.env.ELEVENLABS_API_KEY;
|
||||
else process.env.ELEVENLABS_API_KEY = savedKey;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("synthesizeHeygen surfaces a thrown HTTP error (e.g. 402) instead of swallowing it", async () => {
|
||||
const res = await synthesizeHeygen(
|
||||
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: "/tmp/x.wav" },
|
||||
{
|
||||
heygenAuthHeaders: () => ({}),
|
||||
heygenJSON: async () => {
|
||||
throw new Error("HeyGen POST /voices/speech → HTTP 402\nplan_upgrade_required");
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error, /402/);
|
||||
assert.match(res.error, /plan_upgrade_required/);
|
||||
});
|
||||
|
||||
test("synthesizeHeygen surfaces a failed audio_url fetch with its status", async () => {
|
||||
const res = await synthesizeHeygen(
|
||||
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: "/tmp/x.wav" },
|
||||
{
|
||||
heygenAuthHeaders: () => ({}),
|
||||
heygenJSON: async () => ({ data: { audio_url: "http://audio.example/x" } }),
|
||||
fetch: async () => ({ ok: false, status: 403 }),
|
||||
},
|
||||
);
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error, /HTTP 403/);
|
||||
});
|
||||
|
||||
test("synthesizeHeygen reports a missing audio_url", async () => {
|
||||
const res = await synthesizeHeygen(
|
||||
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: "/tmp/x.wav" },
|
||||
{ heygenAuthHeaders: () => ({}), heygenJSON: async () => ({}) },
|
||||
);
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error, /no audio_url/);
|
||||
});
|
||||
|
||||
test("synthesizeHeygen reports wav transcode failures", async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hf-tts-test-"));
|
||||
try {
|
||||
const res = await synthesizeHeygen(
|
||||
{ text: "hi", voiceId: "v1", lang: "en", speed: 1, wavAbs: join(dir, "voice.wav") },
|
||||
{
|
||||
heygenAuthHeaders: () => ({}),
|
||||
heygenJSON: async () => ({ data: { audio_url: "http://audio.example/x" } }),
|
||||
fetch: async () => ({ ok: true, status: 200, arrayBuffer: async () => new ArrayBuffer(0) }),
|
||||
transcodeToWav: () => false,
|
||||
},
|
||||
);
|
||||
assert.equal(res.ok, false);
|
||||
assert.equal(res.error, "wav transcode failed (ffmpeg)");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("synthResult names a non-zero subprocess exit", () => {
|
||||
const res = synthResult({ status: 2 }, "/tmp/none.wav", "kokoro (npx hyperframes tts)");
|
||||
assert.equal(res.ok, false);
|
||||
assert.match(res.error, /kokoro .* exited with status 2/);
|
||||
});
|
||||
|
||||
// Kokoro shells out to `npx hyperframes tts`; a fake npx on PATH records the argv
|
||||
// synthesizeOne really builds, so a dropped flag fails here. Windows runs npx via
|
||||
// node directly, so the stub never resolves there.
|
||||
async function kokoroArgv(options) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-kokoro-argv-"));
|
||||
const argvLog = join(dir, "argv.txt");
|
||||
writeFileSync(join(dir, "npx"), `#!/bin/sh\nprintf '%s\\n' "$@" > "${argvLog}"\n`);
|
||||
chmodSync(join(dir, "npx"), 0o755);
|
||||
const originalPath = process.env.PATH;
|
||||
try {
|
||||
process.env.PATH = dir;
|
||||
await synthesizeOne({
|
||||
provider: "kokoro",
|
||||
text: "hi",
|
||||
voiceId: "am_michael",
|
||||
wavAbs: join(dir, "line-0.wav"),
|
||||
hyperframesDir: dir,
|
||||
...options,
|
||||
});
|
||||
return readFileSync(argvLog, "utf8").trim().split("\n");
|
||||
} finally {
|
||||
process.env.PATH = originalPath;
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test(
|
||||
"synthesizeOne(kokoro) forwards a non-default speed to the CLI",
|
||||
{ skip: process.platform === "win32" },
|
||||
async () => {
|
||||
const argv = await kokoroArgv({ speed: 0.8, lang: "es" });
|
||||
assert.deepEqual(argv.slice(-4), ["--lang", "es", "--speed", "0.8"]);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
"synthesizeOne(kokoro) leaves --speed off at the default",
|
||||
{ skip: process.platform === "win32" },
|
||||
async () => {
|
||||
assert.ok(!(await kokoroArgv({})).includes("--speed"));
|
||||
},
|
||||
);
|
||||
|
||||
test("pickProvider heygen with an unreadable credentials path says to fix that path", () => {
|
||||
const saved = ["HEYGEN_API_KEY", "HYPERFRAMES_API_KEY", "HEYGEN_CONFIG_DIR"].map((k) => [
|
||||
k,
|
||||
process.env[k],
|
||||
]);
|
||||
const dir = mkdtempSync(join(tmpdir(), "tts-cred-"));
|
||||
try {
|
||||
delete process.env.HEYGEN_API_KEY;
|
||||
delete process.env.HYPERFRAMES_API_KEY;
|
||||
process.env.HEYGEN_CONFIG_DIR = dir;
|
||||
mkdirSync(join(dir, "credentials"));
|
||||
assert.throws(() => pickProvider("heygen"), /fix or remove that path/);
|
||||
} finally {
|
||||
for (const [k, v] of saved) {
|
||||
if (v === undefined) delete process.env[k];
|
||||
else process.env[k] = v;
|
||||
}
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate BGM using Google Lyria RealTime API.
|
||||
|
||||
Usage:
|
||||
python lyria-recipe.py --output <path> --duration <seconds> [tuning flags]
|
||||
|
||||
Requires:
|
||||
$GOOGLE_API_KEY or $GEMINI_API_KEY environment variable (treated as aliases).
|
||||
pip install google-genai python-dotenv. audio.mjs Step 4b installs these on
|
||||
demand when a key is set but google.genai is not importable; if that install
|
||||
fails it falls back to local MusicGen rather than leaving the video with no BGM.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
# Windows sizes stdio to the ANSI code page (cp1252). These scripts emit UTF-8 on
|
||||
# every platform; say so rather than depending on the console's code page. Carry
|
||||
# `errors` across: reconfigure() resets it to "strict", and CPython deliberately gives
|
||||
# stderr "backslashreplace" so the diagnostic path can never itself raise.
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(_stream, "reconfigure"):
|
||||
_stream.reconfigure(encoding="utf-8", errors=_stream.errors)
|
||||
|
||||
DEFAULT_PROMPT = "Uplifting corporate tech, bright and modern, gentle piano with synth pads"
|
||||
SAMPLE_RATE = 48000
|
||||
CHANNELS = 2
|
||||
SAMPLE_WIDTH = 2 # 16-bit
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description="Generate BGM via Google Lyria RealTime.")
|
||||
p.add_argument("--output", required=True, help="Output WAV path.")
|
||||
p.add_argument("--duration", type=float, required=True, help="Target duration in seconds.")
|
||||
p.add_argument("--prompt", default=DEFAULT_PROMPT, help="Mood / instrumentation prompt.")
|
||||
p.add_argument("--negative-prompt", default=None, help="Styles to exclude (optional).")
|
||||
p.add_argument("--bpm", type=int, default=110)
|
||||
p.add_argument("--brightness", type=float, default=0.8, help="0-1, higher = brighter mood.")
|
||||
p.add_argument("--density", type=float, default=0.5, help="0-1, higher = fuller mix.")
|
||||
p.add_argument(
|
||||
"--scale",
|
||||
default="MAJOR",
|
||||
help="MAJOR / MINOR / PENTATONIC / etc. — see google.genai.types.Scale. Pass empty string for none.",
|
||||
)
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
async def generate_bgm(args: argparse.Namespace) -> dict:
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
api_key = os.environ.get("GOOGLE_API_KEY") or os.environ.get("GEMINI_API_KEY") or ""
|
||||
if not api_key:
|
||||
raise RuntimeError("Neither GOOGLE_API_KEY nor GEMINI_API_KEY is set.")
|
||||
|
||||
client = genai.Client(
|
||||
api_key=api_key,
|
||||
http_options={"api_version": "v1alpha"},
|
||||
)
|
||||
|
||||
out_path = Path(args.output)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
target_bytes = int(args.duration * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)
|
||||
|
||||
cfg: dict = {"bpm": args.bpm, "temperature": 1.0}
|
||||
if args.density is not None:
|
||||
cfg["density"] = args.density
|
||||
if args.brightness is not None:
|
||||
cfg["brightness"] = args.brightness
|
||||
if args.scale:
|
||||
scale_enum = getattr(types.Scale, args.scale, None)
|
||||
if scale_enum:
|
||||
cfg["scale"] = scale_enum
|
||||
|
||||
prompts = [types.WeightedPrompt(text=args.prompt, weight=1.0)]
|
||||
if args.negative_prompt:
|
||||
prompts.append(types.WeightedPrompt(text=args.negative_prompt, weight=-1.0))
|
||||
|
||||
buf = bytearray()
|
||||
timeout = args.duration + 8
|
||||
|
||||
async with client.aio.live.music.connect(
|
||||
model="models/lyria-realtime-exp",
|
||||
) as session:
|
||||
await session.set_weighted_prompts(prompts=prompts)
|
||||
await session.set_music_generation_config(
|
||||
config=types.LiveMusicGenerationConfig(**cfg),
|
||||
)
|
||||
await session.play()
|
||||
|
||||
async def collect():
|
||||
while len(buf) < target_bytes:
|
||||
async for msg in session.receive():
|
||||
sc = msg.server_content
|
||||
if sc and sc.audio_chunks:
|
||||
for chunk in sc.audio_chunks:
|
||||
buf.extend(chunk.data)
|
||||
if len(buf) >= target_bytes:
|
||||
return
|
||||
await asyncio.sleep(1e-6)
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(collect(), timeout=timeout)
|
||||
except TimeoutError:
|
||||
print(f"Timeout after {timeout:.0f}s, collected {len(buf)} bytes", file=sys.stderr)
|
||||
|
||||
audio = bytes(buf[:target_bytes])
|
||||
with wave.open(str(out_path), "wb") as wf:
|
||||
wf.setnchannels(CHANNELS)
|
||||
wf.setsampwidth(SAMPLE_WIDTH)
|
||||
wf.setframerate(SAMPLE_RATE)
|
||||
wf.writeframes(audio)
|
||||
|
||||
actual_duration = len(audio) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)
|
||||
print(f"BGM: {out_path} ({actual_duration:.2f}s)")
|
||||
return {"file": str(out_path), "duration_sec": round(actual_duration, 2)}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
try:
|
||||
asyncio.run(generate_bgm(args))
|
||||
except RuntimeError as exc:
|
||||
print(f"BGM generation failed: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env node
|
||||
// Phase 4c pre-assemble helper — wait for detached BGM, then write status.
|
||||
//
|
||||
// audio.mjs may launch Lyria / MusicGen in a detached process so voice work can
|
||||
// keep moving. Before assemble-index.mjs decides whether to emit the BGM audio
|
||||
// track, this script gives the background renderer a bounded chance to finish
|
||||
// and converts log/process state into a small bgm_status.json file.
|
||||
//
|
||||
// Always exits 0 for normal pipeline use: missing/failed BGM should not block a
|
||||
// voice/captions/SFX render. Structural invocation errors still exit 1.
|
||||
//
|
||||
// Usage:
|
||||
// node wait-bgm.mjs --audio-meta ./audio_meta.json --hyperframes . \
|
||||
// [--timeout-ms 120000] [--interval-ms 2000] [--out ./bgm_status.json]
|
||||
|
||||
import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { recordInManifest } from "./lib/media-record.mjs";
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const flag = (name, def) => {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
return i >= 0 && i + 1 < argv.length ? argv[i + 1] : def;
|
||||
};
|
||||
|
||||
function die(msg) {
|
||||
console.error(`✗ wait-bgm.mjs: ${msg}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const audioMetaPath = resolve(flag("audio-meta", "./audio_meta.json"));
|
||||
const hyperframesDir = resolve(flag("hyperframes", "."));
|
||||
const outPath = resolve(flag("out", join(hyperframesDir, "bgm_status.json")));
|
||||
const timeoutMs = Math.max(0, Number(flag("timeout-ms", "120000")) || 0);
|
||||
const intervalMs = Math.max(250, Number(flag("interval-ms", "2000")) || 2000);
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
|
||||
}
|
||||
|
||||
function isProcessAlive(pid) {
|
||||
if (!pid || !Number.isFinite(Number(pid))) return false;
|
||||
try {
|
||||
process.kill(Number(pid), 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readTail(path, maxChars = 6000) {
|
||||
if (!path) return "";
|
||||
try {
|
||||
const txt = readFileSync(path, "utf8");
|
||||
return txt.slice(Math.max(0, txt.length - maxChars));
|
||||
} catch (error) {
|
||||
if (error.code === "ENOENT" || error.code === "ENOTDIR") return "";
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function detectFailure(logTail) {
|
||||
if (!logTail) return "";
|
||||
const lines = logTail.split("\n");
|
||||
// Bare "out of range" over-matched benign BGM-renderer logs (e.g. a "sample rate
|
||||
// out of range, resampling" notice), mislabelling a healthy track as failed and
|
||||
// silently dropping the music. Anchor to the actual crash strings instead:
|
||||
// Python "(list) index out of range" and torch "index … out of bounds".
|
||||
const idx = lines.findIndex((line) =>
|
||||
/(Traceback|IndexError|RuntimeError|Exception|Killed|No space left|Cannot allocate|index out of range|out of bounds)/i.test(
|
||||
line,
|
||||
),
|
||||
);
|
||||
if (idx < 0) return "";
|
||||
return lines.slice(idx).join("\n").trim();
|
||||
}
|
||||
|
||||
function writeStatus(status) {
|
||||
const payload = {
|
||||
generated_at: new Date().toISOString(),
|
||||
...status,
|
||||
};
|
||||
writeFileSync(outPath, JSON.stringify(payload, null, 2) + "\n");
|
||||
return payload;
|
||||
}
|
||||
|
||||
if (!existsSync(audioMetaPath)) die(`audio_meta.json missing at ${audioMetaPath}`);
|
||||
|
||||
const audioMeta = JSON.parse(readFileSync(audioMetaPath, "utf8"));
|
||||
const bgmPath = audioMeta.bgm?.path || "";
|
||||
const bgmAbsPath = bgmPath ? join(hyperframesDir, bgmPath) : "";
|
||||
const logPath = audioMeta.bgm_log || "";
|
||||
const pid = audioMeta.bgm_pid || null;
|
||||
|
||||
const base = {
|
||||
enabled: Boolean(audioMeta.bgm_pending && bgmPath),
|
||||
provider: audioMeta.bgm_provider || null,
|
||||
mode: audioMeta.bgm_mode || null,
|
||||
path: bgmPath || null,
|
||||
log: logPath || null,
|
||||
pid,
|
||||
target_duration_s: audioMeta.bgm_target_duration_s || null,
|
||||
seed_duration_s: audioMeta.bgm_seed_duration_s || null,
|
||||
loop_count: audioMeta.bgm_loop_count || null,
|
||||
timeout_ms: timeoutMs,
|
||||
};
|
||||
|
||||
if (!base.enabled) {
|
||||
const status = writeStatus({
|
||||
...base,
|
||||
status: "disabled",
|
||||
ready: false,
|
||||
waited_ms: 0,
|
||||
message: "BGM not requested or disabled in audio_meta.json.",
|
||||
});
|
||||
console.log(`✓ bgm: ${status.status} (${status.message})`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const started = Date.now();
|
||||
let lastFailure = "";
|
||||
let lastTail = "";
|
||||
|
||||
while (Date.now() - started <= timeoutMs) {
|
||||
if (existsSync(bgmAbsPath)) {
|
||||
const size = statSync(bgmAbsPath).size;
|
||||
writeStatus({
|
||||
...base,
|
||||
status: "ready",
|
||||
ready: true,
|
||||
waited_ms: Date.now() - started,
|
||||
size_bytes: size,
|
||||
message: `BGM ready at ${bgmPath}.`,
|
||||
});
|
||||
console.log(`✓ bgm: ready (${bgmPath}, ${size}B)`);
|
||||
const generated = { path: bgmPath, type: "bgm", source: "generated", provider: base.provider };
|
||||
for (const anomaly of recordInManifest(hyperframesDir, [generated]))
|
||||
console.error(` ${anomaly}`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
lastTail = readTail(logPath);
|
||||
lastFailure = detectFailure(lastTail);
|
||||
const alive = isProcessAlive(pid);
|
||||
if (lastFailure || (!alive && logPath && existsSync(logPath))) {
|
||||
const message = lastFailure
|
||||
? `BGM renderer failed; see ${logPath}.`
|
||||
: `BGM renderer exited without writing ${bgmPath}; see ${logPath}.`;
|
||||
const status = writeStatus({
|
||||
...base,
|
||||
status: "failed",
|
||||
ready: false,
|
||||
waited_ms: Date.now() - started,
|
||||
process_alive: alive,
|
||||
message,
|
||||
error_tail: lastFailure || lastTail.slice(-2000),
|
||||
});
|
||||
console.log(`! bgm: failed (${status.message})`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (timeoutMs === 0) break;
|
||||
await sleep(Math.min(intervalMs, Math.max(0, timeoutMs - (Date.now() - started))));
|
||||
}
|
||||
|
||||
const status = writeStatus({
|
||||
...base,
|
||||
status: "timeout",
|
||||
ready: false,
|
||||
waited_ms: Date.now() - started,
|
||||
process_alive: isProcessAlive(pid),
|
||||
message: `Timed out waiting for ${bgmPath}; assemble-index will skip BGM if still absent.`,
|
||||
log_tail: lastTail.slice(-2000),
|
||||
});
|
||||
console.log(`! bgm: timeout after ${status.waited_ms}ms (${bgmPath})`);
|
||||
@@ -0,0 +1,108 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { test } from "node:test";
|
||||
import { runInNewContext } from "node:vm";
|
||||
|
||||
const source = readFileSync(new URL("./wait-bgm.mjs", import.meta.url), "utf8");
|
||||
const start = source.indexOf("function readTail(");
|
||||
const end = source.indexOf("\nfunction detectFailure(", start);
|
||||
function load(read) {
|
||||
return runInNewContext(`${source.slice(start, end)}; readTail`, {
|
||||
readFileSync: read,
|
||||
existsSync: () => true,
|
||||
statSync: () => {
|
||||
throw new Error("pathname stat must not precede log read");
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
test("reads a log once without a separate pathname check", () => {
|
||||
let calls = 0;
|
||||
const read = load((path, encoding) => {
|
||||
calls++;
|
||||
assert.equal(path, "renderer.log");
|
||||
assert.equal(encoding, "utf8");
|
||||
return "prefix\nRuntimeError: failed";
|
||||
});
|
||||
assert.equal(read("renderer.log", 20), "RuntimeError: failed");
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
|
||||
test("preserves UTF-16 character tail boundaries and zero-length tails", () => {
|
||||
const txt = "🎵日本語".repeat(2000);
|
||||
const read = load(() => txt);
|
||||
assert.equal(read("log"), txt.slice(-6000));
|
||||
assert.equal(read("log", 0), "");
|
||||
});
|
||||
|
||||
for (const code of ["ENOENT", "ENOTDIR"]) {
|
||||
test(`a log removed during polling is absent (${code})`, () => {
|
||||
const read = load(() => {
|
||||
throw Object.assign(new Error(code), { code });
|
||||
});
|
||||
assert.equal(read("log"), "");
|
||||
});
|
||||
}
|
||||
|
||||
test("preserves unexpected read errors", () => {
|
||||
const error = Object.assign(new Error("denied"), { code: "EACCES" });
|
||||
assert.throws(
|
||||
() =>
|
||||
load(() => {
|
||||
throw error;
|
||||
})("log"),
|
||||
(caught) => caught === error,
|
||||
);
|
||||
});
|
||||
|
||||
test("empty log path does not access the filesystem", () => {
|
||||
assert.equal(
|
||||
load(() => {
|
||||
throw new Error("unexpected read");
|
||||
})(""),
|
||||
"",
|
||||
);
|
||||
});
|
||||
|
||||
test("records locally made music as generated once it is ready", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mu-wait-bgm-"));
|
||||
try {
|
||||
mkdirSync(join(dir, "assets/bgm"), { recursive: true });
|
||||
writeFileSync(join(dir, "assets/bgm/track.wav"), "generated music");
|
||||
writeFileSync(
|
||||
join(dir, "audio_meta.json"),
|
||||
JSON.stringify({
|
||||
bgm: { path: "assets/bgm/track.wav" },
|
||||
bgm_pending: true,
|
||||
bgm_provider: "musicgen",
|
||||
bgm_mode: "detached-single",
|
||||
}),
|
||||
);
|
||||
const script = new URL("./wait-bgm.mjs", import.meta.url).pathname;
|
||||
const args = ["--audio-meta", join(dir, "audio_meta.json"), "--hyperframes", dir];
|
||||
|
||||
const run = spawnSync(process.execPath, [script, ...args, "--timeout-ms", "1000"], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
|
||||
assert.equal(run.status, 0, run.stderr);
|
||||
const records = readFileSync(join(dir, ".media/manifest.jsonl"), "utf8")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
assert.deepEqual(
|
||||
records.map(({ path, type, source, provenance }) => [
|
||||
path,
|
||||
type,
|
||||
source,
|
||||
provenance.provider,
|
||||
]),
|
||||
[["assets/bgm/track.wav", "bgm", "generated", "musicgen"]],
|
||||
);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user