commit 5bf4de59f0c839c6c39b9ce4074daf1f75a9bc64
parent 1ffdaeb52ae1b2907bf540707f9128c1982fa341
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sun, 4 Oct 2026 18:38:43 -0400
report-to-video: an encode test for --audio-fallback; README and the [Unreleased] bullet on --no-network's picture rule
audio-fallback.test.mjs: a corpus clip whose record has a page and whose
only local media is audio.mp3 is refused by --no-network (with the note that
its sound is on disk) and, with --audio-fallback, builds a 320x180 poster
segment whose fetch event's source is "audio-fallback".
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
3 files changed, 116 insertions(+), 9 deletions(-)
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,7 +1,7 @@
# Changelog
## [Unreleased]
-- **A report video can play a clip that has only sound, and a clip can be a file beside the manifest.** When a clip's source has no picture, `build-video.mjs` plays it under a poster: a card with the clip's channel, title and date, the size of the picture area, with the sound's waveform moving along its foot (`render.audioPoster.waveform: false` keeps it still). The segment matches every other one in size, frame rate and sound, and the header, footer and on-screen deck are drawn over it as over footage. A video's saved sound (`audio.mp3` and the like in its folder) is now a source the build can cut from, after every saved picture: before any download when the clip has no picture to fetch (`"audioOnly": true` on the clip, `"preferLocalAudio": true` in `render`, or a podcast or feed record), and otherwise only when nothing can be downloaded (`--no-network`, `--skip-fetch`, or a record with no page); `--no-network` lists such clips as playing from audio only instead of refusing them. A clip may also give `"src"` (a video or audio file) and `"cues"` (its transcript, either a `transcript.cues.json` or a `parakeet-stitch` transcript), both relative to the manifest, instead of a channel and video: it plays the whole file unless `start`/`end` cut inside it, `resolve-windows.mjs` widens it with those cues, it gets no QR unless it has a `citeUrl`, and a path that leaves the manifest's folder (or an absolute one, without `"allowAbsoluteSrc": true` in `render`), a missing file or an unreadable transcript stops the build before anything runs, naming the clip.
+- **A report video can play a clip that has only sound, and a clip can be a file beside the manifest.** When a clip's source has no picture, `build-video.mjs` plays it under a poster: a card with the clip's channel, title and date, the size of the picture area, with the sound's waveform moving along its foot (`render.audioPoster.waveform: false` keeps it still). The segment matches every other one in size, frame rate and sound, and the header, footer and on-screen deck are drawn over it as over footage. A video's saved sound (`audio.mp3` and the like in its folder) is now a source the build can cut from, after every saved picture: before any download when the clip has no picture to fetch (`"audioOnly": true` on the clip, `"preferLocalAudio": true` in `render`, a podcast or feed record, or a record with no page). A clip that should have a picture is not quietly played from its sound: with `--no-network` or `--skip-fetch`, one whose picture is missing still stops the build, listed as needing a download with a note that its sound is on disk, so `--no-network` still proves every picture is there. Add `--audio-fallback` to play such clips from their sound under the poster instead; they are listed apart (and logged as `audio-fallback`), and clips that have no picture to fetch are listed as playing from audio only rather than refused. A clip may also give `"src"` (a video or audio file) and `"cues"` (its transcript, either a `transcript.cues.json` or a `parakeet-stitch` transcript), both relative to the manifest, instead of a channel and video: it plays the whole file unless `start`/`end` cut inside it, `resolve-windows.mjs` widens it with those cues, it gets no QR unless it has a `citeUrl`, and a path that leaves the manifest's folder (or an absolute one, without `"allowAbsoluteSrc": true` in `render`), a missing file or an unreadable transcript stops the build before anything runs, naming the clip.
- **A report build cuts from media already on disk before it downloads anything, and `--no-network` makes sure it never does.** For each clip, `build-video.mjs` now looks, in order, in the project's own `out/clips-raw`, in the clip windows the editor fetched into the channel (`channels/<slug>/data/<id>/clips/`), and in a saved whole source video (through the saved-video store's pointer, or a `source-media` file still in the video's folder), and cuts from the first that holds the clip plus its fetch pad; only when none does is the window downloaded. A file that is a link to a drive that is not mounted counts as not there, and the next place is tried. The build prints one line per clip naming where its source came from (`raw-cache`, `corpus-window`, `saved-video`, or a network fetch). With `--no-network`, every clip's source is found before anything is rendered, and if any clip would need a download the build stops at once and lists each one (its position in the timeline, channel, video and the span it needs). umtool's clip bench reads the same three places, so a clip it shows as fetched is one the build cuts from without downloading.
- **umtool's report videos can show a highlighted sentence from a saved article.** `node umtool/report-to-video/shoot-page.mjs --page <saved page.html> --quote "<sentence>" --out <shot.png>` opens a web page saved to disk, finds the sentence in its text, highlights it and saves a PNG of the paragraph that holds it, ready to be a report manifest's `image` entry. `--batch <items.json> --out <dir>` does a list of `{ id, page, quote, context? }` at once and writes `<id>.png` for each plus a `results.json` recording each shot's crop, the matched text and the block it shot. The page is opened offline: nothing is fetched except files saved beside it, and its own scripts do not run unless `--js` is given. The sentence is found whether its quotes and apostrophes are curly or straight, across links and emphasis, and through non-breaking spaces, soft hyphens and line breaks in the page's source. A sentence that is not on the page is listed in `results.json` and on the terminal, and the run ends with an error rather than leaving it out. `--color` sets the highlight; `context` picks one occurrence of a sentence that appears more than once. On a page where a whole post is one block of paragraphs separated by line breaks, `--crop mark` (or an item's `"crop": "mark"`) shoots only the sentence's own lines and one whole line above and below (`--context-lines` sets how many) instead of the whole post; `results.json` records which crop each shot used.
- **A clip or whole-recording fetch can name the tallest source video it wants.** The MCP's `fetch_clip` takes `maxHeight`, `fetch-via-editor.mjs` takes `--max-height`, and the editor's fetch endpoint takes `maxHeight`: a whole number of pixels from 144 to 2160; anything else is refused before anything is fetched. A window is fetched at or under that height (720 when none is given, as before). A whole recording asked for at 720 or less is saved as the **Video 720p** quality, and above 720 at the original quality; with no height it follows the channel's, else the global, source video quality, as before. umtool's whole-source fetch from the clip bench now asks at the report's `render.maxHeightSource`. A file already on disk is returned as it is and never fetched again for a different height; the answer now gives its height (a window's is read from the file, a whole recording's from what its persist recorded) and says when it is taller than the height asked for.
diff --git a/umtool/report-to-video/README.md b/umtool/report-to-video/README.md
@@ -163,17 +163,27 @@ order, and the first place holding the clip's **padded** span (its extent plus
4. **`audio`** — the recording's sound alone, `data/<id>/audio.<ext>` (`mp3`,
then `m4a`, then `opus`, then any other: `aac`, `ogg`, `wav`, `flac`, an
audio `webm` or `mp4`), the window `[0, duration]`. Only when allowed, by one
- rule (`sources.mjs` `audioAllowed`):
- - **before the network** when the clip has no picture to fetch: the entry
- says `audioOnly: true`, the manifest says `render.preferLocalAudio: true`,
- or the record's `platform` is a feed (`podcast`, `feed`, `rss`);
- - **otherwise only when nothing could be fetched anyway**: `--no-network`,
- `--skip-fetch`, or a record with no page (`webpageUrl`) to fetch from.
+ rule (`sources.mjs` `audioUse`), which also names the use:
+ - **`audio-only`, before the network**, when the clip has no picture to
+ fetch: the entry says `audioOnly: true`, the manifest says
+ `render.preferLocalAudio: true`, the record's `platform` is a feed
+ (`podcast`, `feed`, `rss`), or the record has no page (`webpageUrl`) to
+ fetch from;
+ - **`audio-fallback`, only when asked**: a clip that SHOULD have a picture,
+ in a build that may not fetch one (`--no-network` or `--skip-fetch`),
+ with `--audio-fallback` given.
Being last, it never beats a picture already on disk. Without the rule a cut
would quietly lose its pictures to it: many transcribed videos keep an
- `audio.mp3`. Under `--no-network` an audio-only clip is satisfied — listed up
- front as playing from audio only, not as a miss.
+ `audio.mp3`. **`--no-network` still proves every picture is on disk**: a clip
+ that should have one and has only its sound is listed as needing a fetch,
+ with a note that its sound is there (`--audio-fallback` would play it). An
+ audio-only clip is satisfied — listed up front as playing from audio only,
+ not as a miss — and with `--audio-fallback` the fallbacks are listed
+ separately, and their `fetch` event's `source` is `"audio-fallback"`, not
+ `"audio"`. `planLocalSources` returns the three as `{missing, audio,
+ audioFallback}`; it reads a clip's record only when its sound is all that is
+ on disk.
Only a miss in all of them fetches. The channels tree is the one the clip bench
reads for the project: `provenance.channelsDir`, else a `.shadow-channels/`
diff --git a/umtool/report-to-video/audio-fallback.test.mjs b/umtool/report-to-video/audio-fallback.test.mjs
@@ -0,0 +1,97 @@
+// A corpus clip that SHOULD have a picture, with only its sound on disk:
+// `--no-network` refuses it, and `--no-network --audio-fallback` builds it
+// from the sound under a poster, logged as `audio-fallback`.
+//
+// A real encode, kept tiny (a three-second lavfi mp3, a 320x180 cut). Its own
+// file because the build's cue reader takes its corpus from CHANNELS_DIR when
+// the module loads, so the variable is set before the import.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+const have = ["ffmpeg", "ffprobe", "magick"].every((b) => spawnSync(b, ["-version"]).status === 0);
+const SLUG = "demo-channel";
+const VIDEO = "abc123";
+
+const root = await mkdtemp(path.join(os.tmpdir(), "rtv-audio-fallback-"));
+const channelsDir = path.join(root, "channels");
+process.env.CHANNELS_DIR = channelsDir;
+const { buildVideo, setProgressMode } = await import("./build-video.mjs");
+const { MONO_FONT_FILE } = await import("./brand.mjs");
+
+const PALETTE = { bg: "#15121c", fg: "#ece8f4", muted: "#9a93ad", accent: "#7c5cff", amber: "#f2b84b" };
+
+async function project() {
+ const videoDir = path.join(channelsDir, SLUG, "data", VIDEO);
+ await mkdir(videoDir, { recursive: true });
+ const r = spawnSync("ffmpeg", [
+ "-v", "error", "-y", "-f", "lavfi", "-i", "sine=f=220:r=44100:d=3",
+ "-c:a", "libmp3lame", "-b:a", "64k", path.join(videoDir, "audio.mp3"),
+ ]);
+ assert.equal(r.status, 0, String(r.stderr));
+ // A record WITH a page: this recording has a picture somewhere.
+ await writeFile(path.join(videoDir, "transcript.cues.json"), JSON.stringify({
+ id: VIDEO, title: "A recording with a picture", channel: "Demo Channel", uploadDate: "20260102",
+ duration: 3, platform: "youtube", webpageUrl: "https://example.invalid/watch?v=abc123",
+ cues: [{ start: 0.2, end: 2.8, text: "A tone, held." }],
+ }));
+ const manifestPath = path.join(root, "video.manifest.json");
+ await writeFile(manifestPath, JSON.stringify({
+ slug: "demo",
+ title: "Demo",
+ provenance: { channelSlug: SLUG },
+ render: {
+ width: 320, height: 180, fps: 30, audioRate: 48000, audioChannels: 2, preset: "ultrafast",
+ transition: 0, headerHeight: 24, palette: PALETTE, fontRegular: MONO_FONT_FILE, fetchPad: 0.2, qr: false,
+ },
+ timeline: [{ id: "c1", type: "clip", video: VIDEO, start: 0.5, end: 2.5 }],
+ }));
+ return manifestPath;
+}
+
+test("a video clip with only its sound: refused by --no-network, a poster with --audio-fallback", { skip: !have && "needs ffmpeg, ffprobe and magick" }, async (t) => {
+ t.after(() => rm(root, { recursive: true, force: true }));
+ const manifestPath = await project();
+
+ await assert.rejects(
+ buildVideo({ manifestPath, only: "c1", opts: { noNetwork: true } }),
+ /timeline\[0\] c1 .*only its sound, audio\.mp3, is on disk: --audio-fallback plays it/,
+ );
+
+ // The events, as umtool reads them.
+ const events = [];
+ const write = process.stdout.write.bind(process.stdout);
+ setProgressMode("ndjson");
+ process.stdout.write = (chunk, ...rest) => {
+ for (const line of String(chunk).split("\n")) {
+ if (line.startsWith("{")) events.push(JSON.parse(line));
+ }
+ return true;
+ };
+ let r;
+ try {
+ r = await buildVideo({ manifestPath, only: "c1", opts: { noNetwork: true, audioFallback: true } });
+ } finally {
+ process.stdout.write = write;
+ setProgressMode("human");
+ }
+ assert.equal(r.failures.length, 0);
+ const fetch = events.find((e) => e.ev === "fetch" && e.id === "c1");
+ assert.equal(fetch.source, "audio-fallback");
+ assert.equal(fetch.reuse, "audio.mp3");
+ assert.ok(events.some((e) => e.ev === "note" && /--audio-fallback: 1 clip\(s\)/.test(e.message)));
+
+ const p = spawnSync("ffprobe", ["-v", "error", "-show_streams", "-of", "json", r.out]);
+ const streams = JSON.parse(String(p.stdout)).streams;
+ const v = streams.find((s) => s.codec_type === "video");
+ assert.ok(v, "the segment has a picture: the poster");
+ assert.equal(v.width, 320);
+ assert.equal(v.height, 180);
+ assert.equal(v.r_frame_rate, "30/1");
+ assert.ok(streams.some((s) => s.codec_type === "audio"));
+});
