From ed66325490f8153d1a6560125deaf34d05e7952a Mon Sep 17 00:00:00 2001 From: JamesFranc Date: Mon, 19 Oct 2026 22:11:23 -0600 Subject: [PATCH 1/3] feat: narrate walkthrough videos with a local Kokoro voice A free, offline voice beside ElevenLabs, chosen with WALKTHROUGH_VOICE=kokoro. Co-Authored-By: Claude Opus 5.5 --- README.md | 9 +- skills/architecture/references/walkthrough.md | 64 +++--- tools/cli/src/check-tools/main.ts | 7 +- tools/cli/src/check-tools/specs.ts | 9 + tools/cli/src/check-tools/types.ts | 1 + tools/cli/src/commands/walkthrough.test.ts | 186 ++++++++++++++++- tools/cli/src/commands/walkthrough.ts | 21 +- tools/cli/src/walkthrough/compose.test.ts | 2 +- tools/cli/src/walkthrough/compose.ts | 4 +- .../walkthrough/fixtures/kokoro-words.json | 135 ++++++++++++ tools/cli/src/walkthrough/kokoro-speak.py | 47 +++++ tools/cli/src/walkthrough/kokoro.test.ts | 193 ++++++++++++++++++ tools/cli/src/walkthrough/kokoro.ts | 176 ++++++++++++++++ tools/cli/src/walkthrough/main.ts | 143 ++++++++----- tools/cli/src/walkthrough/prepare.ts | 2 +- tools/cli/src/walkthrough/silent.ts | 6 +- 16 files changed, 898 insertions(+), 107 deletions(-) create mode 100644 tools/cli/src/walkthrough/fixtures/kokoro-words.json create mode 100644 tools/cli/src/walkthrough/kokoro-speak.py create mode 100644 tools/cli/src/walkthrough/kokoro.test.ts create mode 100644 tools/cli/src/walkthrough/kokoro.ts diff --git a/README.md b/README.md index 0ca21b2..6a62e2a 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,9 @@ None of these is required. Each one changes what the workflow does when it is pr |---|---|---| | Jira (an Atlassian MCP server, plus any `JIRA_*` or `ATLASSIAN_*` variable) | Work is tracked on a ticket, and branches are named after it | No ticket; branches are named from a short description | | Notion (a Notion MCP server, plus `NOTION_PAGE_URL`) | Design docs are published to Notion | Design docs stay as local files | -| ElevenLabs (`ELEVEN_LABS_API_KEY`) | Each architecture doc's walkthrough video is narrated | The video is silent, with the narration as subtitles | -| A macOS voice (`WALKTHROUGH_MAC_VOICE`) | The walkthrough video is narrated by that voice, when ElevenLabs isn't configured | With `ELEVEN_LABS_API_KEY` also unset, the video is silent, with the narration as subtitles | +| ElevenLabs (`ELEVEN_LABS_API_KEY`) | Each architecture doc's walkthrough video is narrated | A macOS voice narrates when set, else the video is silent, with the narration as subtitles | +| Kokoro (`WALKTHROUGH_VOICE=kokoro`, needs `uv`) | Each architecture doc's walkthrough video is narrated locally, and takes precedence over ElevenLabs and a macOS voice | ElevenLabs narrates when configured, else a macOS voice, else the video is silent, with the narration as subtitles | +| A macOS voice (`WALKTHROUGH_MAC_VOICE`) | The walkthrough video is narrated by that voice, when neither Kokoro nor ElevenLabs is configured | With `ELEVEN_LABS_API_KEY` also unset, the video is silent, with the narration as subtitles | | The board (`ai-workflow-app`) | Questions and approvals go to the board | The workflow asks in the terminal | | CodeRabbit | Its review is part of the pull request loop | Only the agent review runs | @@ -92,7 +93,9 @@ These are Claude Code's own settings. The workflow depends on what they do. | `NOTION_PAGE_URL` | The Notion page design docs are published under. Unset: design docs stay as local files. | | `ELEVEN_LABS_API_KEY` | ElevenLabs API key used to narrate an architecture doc's walkthrough video. Unset: a macOS voice narrates it if `WALKTHROUGH_MAC_VOICE` is set; otherwise the video is silent, with the narration as subtitles. | | `ELEVEN_LABS_VOICE_ID` | The ElevenLabs voice for that narration. Default: the voice the CLI pins. | -| `WALKTHROUGH_MAC_VOICE` | A macOS voice, named as `say -v '?'` lists it, that narrates the walkthrough video when `ELEVEN_LABS_API_KEY` is unset. Unset (and `ELEVEN_LABS_API_KEY` unset too): the video is silent, with the narration as subtitles. Premium voices (System Settings → Accessibility → Read & Speak → System voice → Manage Voices…; Spoken Content before macOS 26) sound far better than the built-in ones. | +| `WALKTHROUGH_VOICE` | `kokoro` narrates walkthrough videos locally with Kokoro through `uv`, ahead of ElevenLabs and a macOS voice. Any other non-empty value fails the render. Unset: ElevenLabs when its key is set, else a macOS voice when `WALKTHROUGH_MAC_VOICE` is set, else silent. | +| `KOKORO_VOICE` | The Kokoro voice for that narration. It must be an American or British English voice (prefix `a` or `b`). Default: `af_heart`. | +| `WALKTHROUGH_MAC_VOICE` | A macOS voice, named as `say -v '?'` lists it, that narrates the walkthrough video when `ELEVEN_LABS_API_KEY` is unset and Kokoro is not selected. Unset (and `ELEVEN_LABS_API_KEY` unset too): the video is silent, with the narration as subtitles. Premium voices (System Settings → Accessibility → Read & Speak → System voice → Manage Voices…; Spoken Content before macOS 26) sound far better than the built-in ones. | | `AI_WORKFLOW_APP_BIN` | Path to the board's `ai-workflow-app` binary. Default: `ai-workflow-app` found on `PATH`. If the path you give does not exist, the board is treated as absent; `PATH` is not searched instead. | | `PERMISSION_HOLD_DEADLINE_S` | When a tool call needs permission, the prompt is also posted to the board, and whichever answer comes first wins. This is how many seconds the board side waits. Default: `570`. A value that is not a whole number below `600` uses the default. | | `PERMISSION_POLL_INTERVAL_S` | Seconds between checks for the board's answer to that prompt. Default: `3`. | diff --git a/skills/architecture/references/walkthrough.md b/skills/architecture/references/walkthrough.md index 9b32ca6..ca81001 100644 --- a/skills/architecture/references/walkthrough.md +++ b/skills/architecture/references/walkthrough.md @@ -1,11 +1,11 @@ # Walkthrough Videos The Architecture skill gives each architecture doc a walkthrough video at the top of its hosted -Notion page: narrated by ElevenLabs or a macOS voice when one is configured, and silent with the -narration as subtitles when neither is. The video tells the story of the work in a few minutes, so -the user can review the design by watching it. The skill writes the script after the doc and -renders it before hosting. Whenever a revision changes what the video says, it re-renders before -re-hosting. +Notion page: narrated by Kokoro, ElevenLabs or a macOS voice when one is configured, and silent +with the narration as subtitles when none is. The video tells the story of the work in a few +minutes, so the user can review the design by watching it. The skill writes the script after the +doc and renders it before hosting. Whenever a revision changes what the video says, it re-renders +before re-hosting. ## Files @@ -15,7 +15,7 @@ All of these sit beside the doc in `docs/plans/`. `` is the doc's | File | What it is | |---|---| | `.walkthrough.json` | The script. You write it. | -| `.walkthrough.cache/` | Narration audio, cached by text and voice, and by the model and speed the CLI pins for ElevenLabs or by the scene's cues for a macOS voice. `walkthrough render` writes it; keep it between renders. | +| `.walkthrough.cache/` | Narration audio, cached by text, voice, and the speed the CLI pins, plus the ElevenLabs model it pins, the Kokoro package versions it pins (Kokoro's model is not pinned), or the scene's cues for a macOS voice. `walkthrough render` writes it; keep it between renders. | | `.walkthrough.mp4` | The video. | The doc's first line under its title embeds the video: `![Walkthrough](.walkthrough.mp4)`. @@ -24,20 +24,27 @@ Hosting turns that line into a Notion video (`skills/workflow-guide/references/n ## Choosing the voice -Run `check-tools --require elevenlabs,mac-voice` first. A `failed` result for either one: stop -and print the reason. Otherwise the first configured voice narrates the video: -- `elevenlabs` configured: ElevenLabs narrates it. `ELEVEN_LABS_VOICE_ID` overrides the voice the - CLI pins, per project. -- else `mac-voice` configured: the macOS voice `WALKTHROUGH_MAC_VOICE` names narrates it, free and - local. The name is as `say -v '?'` lists it. Premium voices, downloaded under System Settings → - Accessibility → Read & Speak → System voice → Manage Voices… (Spoken Content before macOS 26), - sound far better than the built-in compact ones. -- neither: `walkthrough render` makes the video without a voice: each scene lasts as long as - reading its narration takes, and the narration shows as subtitles. Tell the user in one line - that the walkthrough is silent and subtitled because no voice is configured. - -With both configured, ElevenLabs wins and `walkthrough render` says on stderr that the macOS voice -is ignored. +Run `check-tools --require kokoro,elevenlabs,mac-voice` first. +- any `failed`: stop and print the reason. Any non-empty `WALKTHROUGH_VOICE` other than exactly + `kokoro` is a `failed` `kokoro`, not a missing one, and `walkthrough render` exits 1 on it. +- `kokoro` `configured` (`WALKTHROUGH_VOICE=kokoro`): go on. Kokoro narrates locally through `uv`, + and neither ElevenLabs nor a macOS voice is used even when configured. The first run downloads + the model and a Python environment with PyTorch. +- else `elevenlabs` `configured`: go on. ElevenLabs narrates. +- else `mac-voice` `configured`: go on. The macOS voice `WALKTHROUGH_MAC_VOICE` names narrates it, + free and local. The name is as `say -v '?'` lists it. Premium voices, downloaded under System + Settings → Accessibility → Read & Speak → System voice → Manage Voices… (Spoken Content before + macOS 26), sound far better than the built-in compact ones. +- none configured: go on. `walkthrough render` makes the video without a voice: each scene lasts as + long as reading its narration takes, and the narration shows as subtitles. Tell the user in one + line that the walkthrough is silent and subtitled because no voice is configured. + +Each voice defaults to the one the CLI pins: `KOKORO_VOICE` overrides Kokoro's, and +`ELEVEN_LABS_VOICE_ID` overrides ElevenLabs'. `KOKORO_VOICE` must be an American or British English +voice (prefix `a` or `b`); any other voice gives no word timings, and the render exits 1. + +With ElevenLabs and a macOS voice both configured, and Kokoro not selected, ElevenLabs wins and +`walkthrough render` says on stderr that the macOS voice is ignored. ## Write the script @@ -150,17 +157,18 @@ walkthrough render --script "docs/plans/.walkthrough.json" --out " problems means fix the script and re-run. - Exit 2 means a tool it needs isn't usable: - `ffmpeg -version` doesn't run cleanly, whether `ffmpeg` is missing or broken; - - `ELEVEN_LABS_API_KEY` is unset, `WALKTHROUGH_MAC_VOICE` is set, and `say` or `ffprobe` doesn't - run cleanly, whether it is missing or broken (`say` is macOS-only); + - `ELEVEN_LABS_API_KEY` is unset, Kokoro is not selected, `WALKTHROUGH_MAC_VOICE` is set, and + `say` or `ffprobe` doesn't run cleanly, whether it is missing or broken (`say` is macOS-only); - `hyperframes browser ensure`, run through `npx`, doesn't exit 0. For example: `npx` is missing, the HyperFrames package can't be fetched, or no Chrome can be found or downloaded. -- An ElevenLabs error exits 1 with its status. Stop and report ElevenLabs as broken, the same as - `check-tools`' `onFailure`. Scenes already voiced stay cached. -- With `ELEVEN_LABS_API_KEY` unset, a `WALKTHROUGH_MAC_VOICE` naming a voice that isn't installed - exits 1 before anything renders. - Stop and report it, the same as `check-tools`' `onFailure` for `mac-voice`. +- A voice error exits 1: ElevenLabs with its status, Kokoro with its exit. Stop and report that + voice as broken, the same as `check-tools`' `onFailure`. Scenes already voiced stay cached. With + Kokoro selected, a missing `uv` exits 2 like any other missing tool. +- With neither Kokoro selected nor `ELEVEN_LABS_API_KEY` set, a `WALKTHROUGH_MAC_VOICE` naming a + voice that isn't installed exits 1 before anything renders. Stop and report it, the same as + `check-tools`' `onFailure` for `mac-voice`. - stdout is a JSON summary. `narrated: false` means the video was rendered silent, with subtitles. - `voice` says which voice narrated it: `elevenlabs`, `mac`, or `null`. + `voice` says which voice narrated it: `elevenlabs`, `kokoro`, `mac`, or `null`. `overNotionLimit: true` means the video is over 20 MiB and won't upload. Host the doc without the video and tell the user. diff --git a/tools/cli/src/check-tools/main.ts b/tools/cli/src/check-tools/main.ts index 6884da5..01f9d73 100644 --- a/tools/cli/src/check-tools/main.ts +++ b/tools/cli/src/check-tools/main.ts @@ -17,7 +17,7 @@ export const runCheckTools = (tools: readonly string[]): void => { return } -const classifyTool = (tool: string, env: NodeJS.ProcessEnv): ToolResult => { +export const classifyTool = (tool: string, env: NodeJS.ProcessEnv): ToolResult => { const spec = TOOL_SPECS[tool] if (!spec) return unknownTool(tool) @@ -25,6 +25,11 @@ const classifyTool = (tool: string, env: NodeJS.ProcessEnv): ToolResult => { const missing = findMissingEnv(spec, env) const accepted = acceptedEnv(spec) + const problem = spec.envProblem?.(env) + if (problem !== undefined) { + return result(spec, 'failed', present, missing, accepted, problem) + } + if (missing.length > 0 && present.length > 0) { return result( spec, diff --git a/tools/cli/src/check-tools/specs.ts b/tools/cli/src/check-tools/specs.ts index 2c0fc6e..165c5ad 100644 --- a/tools/cli/src/check-tools/specs.ts +++ b/tools/cli/src/check-tools/specs.ts @@ -1,3 +1,4 @@ +import { unknownEngineProblem } from '../walkthrough/kokoro.ts' import type { ToolSpec } from './types.ts' export const TOOL_SPECS: Readonly> = { @@ -22,6 +23,14 @@ export const TOOL_SPECS: Readonly> = { 'STOP and tell the user ElevenLabs is configured but walkthrough render cannot voice the narration. Do NOT fall back.', requiresAll: ['ELEVEN_LABS_API_KEY'], }, + kokoro: { + tool: 'kokoro', + use: 'Local Kokoro narration for walkthrough videos', + onFailure: + 'STOP and tell the user walkthrough render cannot voice the narration, with the reason given. Do NOT fall back.', + requiresAll: ['WALKTHROUGH_VOICE'], + envProblem: (env) => unknownEngineProblem(env.WALKTHROUGH_VOICE), + }, 'mac-voice': { tool: 'mac-voice', use: 'A macOS voice narrating walkthrough videos', diff --git a/tools/cli/src/check-tools/types.ts b/tools/cli/src/check-tools/types.ts index 52d77de..af3075e 100644 --- a/tools/cli/src/check-tools/types.ts +++ b/tools/cli/src/check-tools/types.ts @@ -6,6 +6,7 @@ export type ToolSpec = { readonly onFailure: string readonly requiresAll?: readonly string[] readonly acceptsAnyPrefix?: readonly string[] + readonly envProblem?: (env: NodeJS.ProcessEnv) => string | undefined } export type EnvSummary = { diff --git a/tools/cli/src/commands/walkthrough.test.ts b/tools/cli/src/commands/walkthrough.test.ts index f60f95f..8d9ee6d 100644 --- a/tools/cli/src/commands/walkthrough.test.ts +++ b/tools/cli/src/commands/walkthrough.test.ts @@ -10,6 +10,7 @@ import { fileURLToPath } from 'node:url' import { getRequestedExitCode, setRequestedExitCode } from '../runtime-exit-code.ts' import { stderrBeyondNodeWarnings } from '../testing/subprocess-stderr.ts' import { SCENE_TAIL } from '../walkthrough/compose.ts' +import { classifyTool } from '../check-tools/main.ts' import { runWalkthroughRender } from '../walkthrough/main.ts' import { readingAlignment } from '../walkthrough/silent.ts' import { DEFAULT_VOICE_ID } from '../walkthrough/tts.ts' @@ -356,7 +357,7 @@ test('walkthrough render: with no ElevenLabs key it renders silent, subtitled an const stdout: string[] = [] const stderr = mock.method(console, 'error', () => undefined) try { - for (const env of [{}, { ELEVEN_LABS_API_KEY: '' }]) { + for (const env of [{}, { ELEVEN_LABS_API_KEY: '' }, { WALKTHROUGH_VOICE: '' }]) { setRequestedExitCode(0) stdout.length = 0 stderr.mock.resetCalls() @@ -369,7 +370,8 @@ test('walkthrough render: with no ElevenLabs key it renders silent, subtitled an stderr.mock.calls.map((call) => call.arguments), [ [ - 'walkthrough: neither ELEVEN_LABS_API_KEY nor WALKTHROUGH_MAC_VOICE is set, so the ' + + 'walkthrough: none of WALKTHROUGH_VOICE=kokoro, ELEVEN_LABS_API_KEY or ' + + 'WALKTHROUGH_MAC_VOICE is set, so the ' + 'walkthrough is rendered without a voice, with the narration as subtitles.', ], ] @@ -429,8 +431,8 @@ test('CLI: walkthrough render with no ElevenLabs key exits 0 and prints the sile assert.equal(result.status, 0) assert.equal( stderrBeyondNodeWarnings(result.stderr), - 'walkthrough: neither ELEVEN_LABS_API_KEY nor WALKTHROUGH_MAC_VOICE is set, so the ' + - 'walkthrough is rendered without a voice, with the narration as subtitles.' + 'walkthrough: none of WALKTHROUGH_VOICE=kokoro, ELEVEN_LABS_API_KEY or ' + + 'WALKTHROUGH_MAC_VOICE is set, so the walkthrough is rendered without a voice, with the narration as subtitles.' ) assert.equal( result.stdout, @@ -640,6 +642,182 @@ test('walkthrough render: an ElevenLabs key wins over WALKTHROUGH_MAC_VOICE', as } }) +test('walkthrough render: a WALKTHROUGH_VOICE other than kokoro exits 1 before any tool runs', async () => { + const dir = mkdtempSync(join(tmpdir(), 'walkthrough-cli-test-')) + writeFileSync(join(dir, 'ffmpeg'), `#!/bin/sh\necho ran > "${dir}/ffmpeg-ran"\nexit 0\n`) + chmodSync(join(dir, 'ffmpeg'), 0o755) + const path = process.env.PATH + process.env.PATH = `${dir}:${path ?? ''}` + const stdout: string[] = [] + const stderr = mock.method(console, 'error', () => undefined) + try { + await runWalkthroughRender(join(dir, 's.json'), join(dir, 'o.mp4'), { + env: { WALKTHROUGH_VOICE: 'elevenlabs', ELEVEN_LABS_API_KEY: 'k' }, + stdout: (chunk) => stdout.push(chunk), + }) + assert.equal(getRequestedExitCode(), 1) + assert.deepEqual(stdout, []) + assert.deepEqual( + stderr.mock.calls.map((call) => call.arguments), + [['walkthrough: WALKTHROUGH_VOICE is "elevenlabs"; the only voice it can name is kokoro.']] + ) + assert.throws(() => readFileSync(join(dir, 'ffmpeg-ran'))) + } finally { + stderr.mock.restore() + process.env.PATH = path + setRequestedExitCode(0) + rmSync(dir, { recursive: true, force: true }) + } +}) + +test('walkthrough render: WALKTHROUGH_VOICE=kokoro without uv exits 2 before Chrome or narration', async () => { + const dir = mkdtempSync(join(tmpdir(), 'walkthrough-cli-test-')) + writeFileSync(join(dir, 'ffmpeg'), '#!/bin/sh\nexit 0\n') + writeFileSync(join(dir, 'npx'), `#!/bin/sh\necho ran > "${dir}/npx-ran"\nexit 0\n`) + for (const stub of ['ffmpeg', 'npx']) { + chmodSync(join(dir, stub), 0o755) + } + const path = process.env.PATH + process.env.PATH = dir + const script = join(dir, 's.json') + writeFileSync( + script, + JSON.stringify({ + title: 'T', + palette: 'classic', + scenes: [{ heading: 'A', narration: 'Hi.', visual: { kind: 'title' } }], + }) + ) + const stderr = mock.method(console, 'error', () => undefined) + try { + await runWalkthroughRender(script, join(dir, 'o.mp4'), { + env: { WALKTHROUGH_VOICE: 'kokoro' }, + stdout: () => undefined, + }) + assert.equal(getRequestedExitCode(), 2) + assert.deepEqual( + stderr.mock.calls.map((call) => call.arguments), + [['walkthrough: WALKTHROUGH_VOICE=kokoro needs uv on PATH; install it (brew install uv).']] + ) + assert.throws(() => readFileSync(join(dir, 'npx-ran'))) + } finally { + stderr.mock.restore() + process.env.PATH = path + setRequestedExitCode(0) + rmSync(dir, { recursive: true, force: true }) + } +}) + +test('walkthrough render: WALKTHROUGH_VOICE=kokoro narrates with Kokoro even when an ElevenLabs key is set', async () => { + const dir = mkdtempSync(join(tmpdir(), 'walkthrough-cli-test-')) + let requests = 0 + const server = createServer((_, res) => { + requests++ + res.writeHead(500).end() + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const address = server.address() + const port = typeof address === 'object' && address !== null ? address.port : 0 + writeFileSync( + join(dir, 'npx'), + `#!/bin/sh\n[ "$3" = render ] && /bin/cp "$4/index.html" "${dir}/composed.html"\nexit 0\n` + ) + writeFileSync( + join(dir, 'ffmpeg'), + `#!/bin/sh\n[ "$1" = -version ] && exit 0\nprintf '%s\\n' "$@" > "${dir}/ffmpeg-args"\nfor last; do :; done\nprintf mp3 > "$last"\n` + ) + writeFileSync( + join(dir, 'uv'), + [ + '#!/bin/sh', + '[ "$1" = --version ] && exit 0', + 'while [ $# -gt 0 ]; do', + ' case "$1" in', + ' --audio) audio="$2"; shift ;;', + ' --words) words="$2"; shift ;;', + ' esac', + ' shift', + 'done', + `cat > "${dir}/uv-stdin"`, + ': > "$audio"', + `printf '%s' '[{"text":"The","start":0,"end":0.2},{"text":"client","start":0.2,"end":0.6},{"text":"calls","start":0.6,"end":1},{"text":".","start":1,"end":1.1}]' > "$words"`, + '', + ].join('\n') + ) + for (const stub of ['ffmpeg', 'npx', 'uv']) { + chmodSync(join(dir, stub), 0o755) + } + const path = process.env.PATH + process.env.PATH = `${dir}:${path ?? ''}` + const script = join(dir, 's.json') + writeFileSync( + script, + JSON.stringify({ + title: 'T', + palette: 'classic', + scenes: [ + { + heading: 'B', + narration: 'The client calls.', + visual: { kind: 'diagram', d2: '*: {class: tool}\nclient -> server' }, + steps: [{ cue: 'client calls', show: ['server'] }], + }, + ], + }) + ) + const stdout: string[] = [] + const stderr = mock.method(console, 'error', () => undefined) + try { + setRequestedExitCode(0) + await runWalkthroughRender(script, join(dir, 'o.mp4'), { + env: { WALKTHROUGH_VOICE: 'kokoro', ELEVEN_LABS_API_KEY: 'k' }, + apiBase: `http://127.0.0.1:${port}`, + stdout: (chunk) => stdout.push(chunk), + }) + assert.equal(getRequestedExitCode(), 0) + assert.equal(requests, 0) + assert.deepEqual(stderr.mock.calls, []) + assert.equal(readFileSync(join(dir, 'uv-stdin'), 'utf-8'), 'The client calls.') + const summary: unknown = JSON.parse(stdout.join('')) + assert.ok(typeof summary === 'object' && summary !== null) + assert.equal('narrated' in summary && summary.narrated, true) + assert.equal('voice' in summary && summary.voice, 'kokoro') + assert.equal('synthesizedScenes' in summary && summary.synthesizedScenes, 1) + assert.match(readFileSync(join(dir, 'composed.html'), 'utf-8'), /