commit 151ca982e565ef49dd2d4fb4123a449136dde3b3
parent 0c3f1a34e16afc39ca20a8359c500f20e9d3ff80
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sun, 4 Oct 2026 17:20:17 -0400
Merge render-audio-src (report-to-video: an audio-only source tier with a poster frame where there is no picture; clips with their own src/cues relative to the manifest)
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
14 files changed, 1162 insertions(+), 78 deletions(-)
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,7 @@
# Changelog
## [Unreleased]
+- **A report video can play a clip that has only sound, and a clip can be a file beside the manifest.** When a clip's source has no picture, `build-video.mjs` plays it under a poster: a card with the clip's channel, title and date, the size of the picture area, with the sound's waveform moving along its foot (`render.audioPoster.waveform: false` keeps it still). The segment matches every other one in size, frame rate and sound, and the header, footer and on-screen deck are drawn over it as over footage. A video's saved sound (`audio.mp3` and the like in its folder) is now a source the build can cut from, after every saved picture: before any download when the clip has no picture to fetch (`"audioOnly": true` on the clip, `"preferLocalAudio": true` in `render`, or a podcast or feed record), and otherwise only when nothing can be downloaded (`--no-network`, `--skip-fetch`, or a record with no page); `--no-network` lists such clips as playing from audio only instead of refusing them. A clip may also give `"src"` (a video or audio file) and `"cues"` (its transcript, either a `transcript.cues.json` or a `parakeet-stitch` transcript), both relative to the manifest, instead of a channel and video: it plays the whole file unless `start`/`end` cut inside it, `resolve-windows.mjs` widens it with those cues, it gets no QR unless it has a `citeUrl`, and a path that leaves the manifest's folder (or an absolute one, without `"allowAbsoluteSrc": true` in `render`), a missing file or an unreadable transcript stops the build before anything runs, naming the clip.
- **A report build cuts from media already on disk before it downloads anything, and `--no-network` makes sure it never does.** For each clip, `build-video.mjs` now looks, in order, in the project's own `out/clips-raw`, in the clip windows the editor fetched into the channel (`channels/<slug>/data/<id>/clips/`), and in a saved whole source video (through the saved-video store's pointer, or a `source-media` file still in the video's folder), and cuts from the first that holds the clip plus its fetch pad; only when none does is the window downloaded. A file that is a link to a drive that is not mounted counts as not there, and the next place is tried. The build prints one line per clip naming where its source came from (`raw-cache`, `corpus-window`, `saved-video`, or a network fetch). With `--no-network`, every clip's source is found before anything is rendered, and if any clip would need a download the build stops at once and lists each one (its position in the timeline, channel, video and the span it needs). umtool's clip bench reads the same three places, so a clip it shows as fetched is one the build cuts from without downloading.
- **umtool's report videos can show a highlighted sentence from a saved article.** `node umtool/report-to-video/shoot-page.mjs --page <saved page.html> --quote "<sentence>" --out <shot.png>` opens a web page saved to disk, finds the sentence in its text, highlights it and saves a PNG of the paragraph that holds it, ready to be a report manifest's `image` entry. `--batch <items.json> --out <dir>` does a list of `{ id, page, quote, context? }` at once and writes `<id>.png` for each plus a `results.json` recording each shot's crop, the matched text and the block it shot. The page is opened offline: nothing is fetched except files saved beside it, and its own scripts do not run unless `--js` is given. The sentence is found whether its quotes and apostrophes are curly or straight, across links and emphasis, and through non-breaking spaces, soft hyphens and line breaks in the page's source. A sentence that is not on the page is listed in `results.json` and on the terminal, and the run ends with an error rather than leaving it out. `--color` sets the highlight; `context` picks one occurrence of a sentence that appears more than once.
- **A clip or whole-recording fetch can name the tallest source video it wants.** The MCP's `fetch_clip` takes `maxHeight`, `fetch-via-editor.mjs` takes `--max-height`, and the editor's fetch endpoint takes `maxHeight`: a whole number of pixels from 144 to 2160; anything else is refused before anything is fetched. A window is fetched at or under that height (720 when none is given, as before). A whole recording asked for at 720 or less is saved as the **Video 720p** quality, and above 720 at the original quality; with no height it follows the channel's, else the global, source video quality, as before. umtool's whole-source fetch from the clip bench now asks at the report's `render.maxHeightSource`. A file already on disk is returned as it is and never fetched again for a different height; the answer now gives its height (a window's is read from the file, a whole recording's from what its persist recorded) and says when it is taller than the height asked for.
diff --git a/umtool/lib/projects/report.mjs b/umtool/lib/projects/report.mjs
@@ -869,6 +869,9 @@ export async function reportDecisions(ctx, summary) {
// --- per clip, against the cues -----------------------------------------
const byVideo = new Map();
for (const e of clips) {
+ // A clip with its own media (`src`, `cues`) has no corpus cue file: the
+ // build checks its paths and its window itself.
+ if (e.src != null) continue;
const key = `${channelFor(m, e)}/${e.video}`;
if (!byVideo.has(key)) byVideo.set(key, []);
byVideo.get(key).push(e);
diff --git a/umtool/report-to-video/README.md b/umtool/report-to-video/README.md
@@ -160,7 +160,22 @@ order, and the first place holding the clip's **padded** span (its extent plus
(and picture height) read by one `ffprobe` — run only when tiers 1 and 2
missed.
-Only a miss in all three fetches. The channels tree is the one the clip bench
+4. **`audio`** — the recording's sound alone, `data/<id>/audio.<ext>` (`mp3`,
+ then `m4a`, then `opus`, then any other: `aac`, `ogg`, `wav`, `flac`, an
+ audio `webm` or `mp4`), the window `[0, duration]`. Only when allowed, by one
+ rule (`sources.mjs` `audioAllowed`):
+ - **before the network** when the clip has no picture to fetch: the entry
+ says `audioOnly: true`, the manifest says `render.preferLocalAudio: true`,
+ or the record's `platform` is a feed (`podcast`, `feed`, `rss`);
+ - **otherwise only when nothing could be fetched anyway**: `--no-network`,
+ `--skip-fetch`, or a record with no page (`webpageUrl`) to fetch from.
+
+ Being last, it never beats a picture already on disk. Without the rule a cut
+ would quietly lose its pictures to it: many transcribed videos keep an
+ `audio.mp3`. Under `--no-network` an audio-only clip is satisfied — listed up
+ front as playing from audio only, not as a miss.
+
+Only a miss in all of them fetches. The channels tree is the one the clip bench
reads for the project: `provenance.channelsDir`, else a `.shadow-channels/`
beside the manifest, else `CHANNELS_DIR` / the checkout's `transcripts/channels`.
A file reached through a link into `media/` (perhaps on another drive) is
@@ -175,7 +190,20 @@ fetch would have produced — rather than decoding a fifteen-minute window or a
three-hour container. A raw-cache window is measured whole, as before, so no
cached cut moves. umtool's bench lists tiers 2 and 3 through the same module
(`clipWindowDirs`), with a container's span from its cue doc rather than an
-ffprobe, so what the bench calls fetched is what the build cuts from.
+ffprobe, so what the bench calls fetched is what the build cuts from. (Not the
+audio tier: the bench would call every video that kept its sound fetched.)
+
+**A source with no picture plays under a poster.** Whatever a clip is cut from
+— the audio tier, a `src` that is an mp3 (below), a window with no video
+stream — the build `ffprobe`s it, and when it holds no video (an mp3's cover
+art does not count) the segment's picture is a poster: the clip's own card —
+channel, title and date, resolved as the header resolves them — rendered by
+`renderCard` at exactly the picture box's size, with the sound's waveform
+moving along its foot in the accent colour (`render.audioPoster.waveform`,
+default `true`; `false` keeps the still). It replaces `[0:v]` and nothing
+else, so the letterbox, the header, the footer and the deck's framing all
+apply, and the encode is the same: size, fps, pixel format, SAR and audio
+layout match every other segment's, and the concat cannot tell it from one.
## Driven from umtool
@@ -253,6 +281,43 @@ window:
Only the played window is in the segment, so a mark past `cutEnd` (inside the
extent, so it validates) mutes nothing; the build says it will not be heard.
+### A clip with its own media: `src` and `cues`
+
+A clip may name a FILE instead of a corpus recording — a podcast episode, a
+recording no channel holds — with no stub channel built for it:
+
+```jsonc
+{ "type": "clip", "id": "p01",
+ "src": "media/episode-12.mp3", // relative to the MANIFEST; video or audio-only
+ "cues": "media/episode-12.json", // optional: its cues, relative to the manifest
+ "start": 61.2, "end": 74.9, // optional: default the whole file
+ "title": "Episode 12", "channelTitle": "The Feed", "date": "2026-01-02",
+ "citeUrl": "https://…" } // optional: the ONLY thing that draws a QR
+```
+
+- **The window is the file**, `[0, its ffprobe duration]`; `start`/`end` cut
+ inside it and default to its two ends. Nothing is looked up or fetched, and
+ `--fetch-only` on one is a no-op. The cut still snaps to silence.
+- **`cues`** is read in either shape — the corpus's `transcript.cues.json`
+ (`cues: [{start, end, text}]`, with `title`, `channel`, `uploadDate`,
+ `platform`) or the transcript `scripts/parakeet-stitch.mjs` writes
+ (`chunk_data: [{start_time, end_time, text}]`, `duration_seconds`) — through
+ one normaliser (`local-media.mjs` `normaliseCueDoc`), and feeds
+ `resolve-windows.mjs`'s widening and `--cut-to-quote` as a record's cues do.
+ Without `cues`, or without `start`/`end`, resolve-windows leaves the clip as
+ it is. The header, the chapter and the deck read the cue file's `title`,
+ `channel` and `uploadDate` (the file's name is the title when it has none);
+ the entry's own `title`, `channelTitle` and `date` win as on any clip.
+- **No QR is derived**: the file has no page on the archive. `citeUrl` draws
+ one. (The legacy rail draws a code on every tile, so there a `src` clip
+ without `citeUrl` gets a card's — the sweep's link.)
+- **Paths may not climb out of the manifest's directory**, and an absolute one
+ is refused unless the manifest sets `render.allowAbsoluteSrc: true` — media
+ too big to keep beside the manifest is a decision written down. A `src` beside
+ a `video`, a `cues` without a `src`, a file that is not there, a cue file in
+ neither shape, or an `end` past the file's end refuses the build before
+ anything runs, naming each entry (`timeline[<i>] <id>: …`).
+
`render.endFade` (seconds, default 0 = off, at most 10) fades the cut's LAST
segment — whatever it is — picture to `palette.bg` and sound to silence over its
final `endFade` seconds (all of it, when the segment is shorter), reaching both
diff --git a/umtool/report-to-video/audio-poster.test.mjs b/umtool/report-to-video/audio-poster.test.mjs
@@ -0,0 +1,159 @@
+// A clip whose source is SOUND ALONE -- here a `src` mp3 beside the manifest --
+// builds a segment with a picture: a poster at the picture box's size, the
+// waveform moving along its foot, and a stream the concat cannot tell from any
+// other segment's (size, fps, pixel format, SAR, audio layout).
+//
+// Real encodes, kept tiny: a three-second lavfi mp3 and a 320x180 cut.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { audioPosterParts, buildVideo, deckFraming, posterWaveGeometry } from "./build-video.mjs";
+import { MONO_FONT_FILE } from "./brand.mjs";
+
+const have = ["ffmpeg", "ffprobe", "magick"].every((b) => spawnSync(b, ["-version"]).status === 0);
+const PALETTE = { bg: "#15121c", fg: "#ece8f4", muted: "#9a93ad", accent: "#7c5cff", amber: "#f2b84b" };
+const RENDER = {
+ width: 320, height: 180, fps: 30, audioRate: 48000, audioChannels: 2, preset: "ultrafast",
+ transition: 0, headerHeight: 24, palette: PALETTE, fontRegular: MONO_FONT_FILE,
+};
+
+function probe(file) {
+ const r = spawnSync("ffprobe", ["-v", "error", "-show_streams", "-show_format", "-of", "json", file]);
+ assert.equal(r.status, 0, String(r.stderr));
+ const doc = JSON.parse(String(r.stdout));
+ const v = doc.streams.find((s) => s.codec_type === "video");
+ const a = doc.streams.find((s) => s.codec_type === "audio");
+ return { v, a, duration: Number(doc.format.duration) };
+}
+
+/** What a stream-copy concat needs to be equal between two segments. */
+const concatKey = ({ v, a }) => ({
+ vcodec: v.codec_name, width: v.width, height: v.height, pix_fmt: v.pix_fmt,
+ fps: v.r_frame_rate, sar: v.sample_aspect_ratio ?? "1:1", tb: v.time_base,
+ acodec: a.codec_name, rate: a.sample_rate, channels: a.channels, layout: a.channel_layout,
+});
+
+async function project(render = RENDER) {
+ const root = await mkdtemp(path.join(os.tmpdir(), "rtv-audio-poster-"));
+ await mkdir(path.join(root, "media"), { recursive: true });
+ // Three seconds of a tone whose loudness swells: a waveform that moves.
+ const mp3 = path.join(root, "media", "episode.mp3");
+ const r = spawnSync("ffmpeg", [
+ "-v", "error", "-y", "-f", "lavfi", "-i", "sine=f=220:r=44100:d=3",
+ "-af", "volume='0.2+0.8*t/3':eval=frame", "-c:a", "libmp3lame", "-b:a", "64k", mp3,
+ ]);
+ assert.equal(r.status, 0, String(r.stderr));
+ await writeFile(path.join(root, "media", "episode.json"), JSON.stringify({
+ duration_seconds: 3,
+ chunk_data: [{ start_time: 0.2, end_time: 2.8, text: "A tone, held." }],
+ }));
+ const manifestPath = path.join(root, "video.manifest.json");
+ await writeFile(manifestPath, JSON.stringify({
+ slug: "demo",
+ title: "Demo",
+ render,
+ timeline: [
+ { id: "t0", type: "card", style: "chapter", heading: "A card", seconds: 1 },
+ {
+ id: "p1", type: "clip", src: "media/episode.mp3", cues: "media/episode.json",
+ start: 0.5, end: 2.5, title: "Episode twelve", channelTitle: "Demo Feed", date: "2026-01-02",
+ },
+ ],
+ }));
+ return { root, manifestPath };
+}
+
+test("posterWaveGeometry: even, inside the card's margins, along the foot", () => {
+ const g = posterWaveGeometry(1574, 886);
+ assert.equal(g.width % 2, 0);
+ assert.equal(g.height % 2, 0);
+ assert.ok(g.x > 0 && g.x + g.width <= 1574);
+ assert.ok(g.y > 886 / 2 && g.y + g.height < 886);
+});
+
+test("audioPosterParts: the still alone, or the sound's waveform over it", () => {
+ const box = { width: 320, height: 156 };
+ assert.deepEqual(
+ audioPosterParts({ posterIdx: 3, box, render: { ...RENDER, audioPoster: { waveform: false } } }),
+ ["[3:v]null[pic]"],
+ );
+ const parts = audioPosterParts({ posterIdx: 1, box, render: RENDER });
+ const g = posterWaveGeometry(320, 156);
+ assert.equal(parts.length, 2);
+ assert.match(parts[0], /^\[0:a\]aformat=channel_layouts=mono,showwaves=s=\d+x\d+:mode=cline:draw=full:rate=30:colors=#7c5cff\[wave\]$/);
+ assert.equal(parts[1], `[1:v][wave]overlay=x=${g.x}:y=${g.y}:eof_action=pass[pic]`);
+});
+
+test("an audio-only src clip builds a poster segment the concat cannot tell from a card's", { skip: !have && "needs ffmpeg, ffprobe and magick" }, async () => {
+ const { root, manifestPath } = await project();
+ try {
+ const clip = await buildVideo({ manifestPath, only: "p1", opts: { noNetwork: true } });
+ assert.equal(clip.failures.length, 0);
+ const card = await buildVideo({ manifestPath, only: "t0", opts: { noNetwork: true } });
+
+ const c = probe(clip.out);
+ assert.ok(c.v, "the segment has a picture");
+ assert.equal(c.v.width, 320);
+ assert.equal(c.v.height, 180);
+ assert.equal(c.v.r_frame_rate, "30/1");
+ assert.equal(c.v.pix_fmt, "yuv420p");
+ assert.equal(c.a.channels, 2);
+ assert.equal(c.a.sample_rate, "48000");
+ assert.deepEqual(concatKey(c), concatKey(probe(card.out)));
+ // About the played window, snapped (start 0.5 → 2.5 of a 3 s file), and the
+ // picture is no longer than the sound.
+ assert.ok(c.duration > 1 && c.duration <= 3.1, `duration ${c.duration}`);
+ assert.ok(Number(c.v.duration) <= Number(c.a.duration) + 1 / 30 + 0.03, `${c.v.duration} vs ${c.a.duration}`);
+
+ // The poster was drawn at the picture box's size: the frame less the header.
+ const still = probe(path.join(root, "out", "sourced", "cards", "p1.poster.png"));
+ assert.equal(still.v.width, 320);
+ assert.equal(still.v.height, 180 - 24);
+
+ // The waveform moves: the frames are not one still.
+ const md5 = spawnSync("ffmpeg", ["-v", "error", "-i", clip.out, "-map", "0:v", "-f", "framemd5", "-"]);
+ assert.equal(md5.status, 0, String(md5.stderr));
+ const hashes = new Set(String(md5.stdout).split("\n").filter((l) => l && !l.startsWith("#")).map((l) => l.split(",").pop().trim()));
+ assert.ok(hashes.size > 10, `${hashes.size} distinct frames`);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("under the deck, the poster is framed into the footage box like any footage", { skip: !have && "needs ffmpeg, ffprobe and magick" }, async () => {
+ const { root, manifestPath } = await project({
+ ...RENDER, width: 1920, height: 1080, headerHeight: undefined,
+ chrome: { engine: "hyperframes", layout: "deck", deck: {} },
+ audioPoster: { waveform: false },
+ });
+ try {
+ const clip = await buildVideo({ manifestPath, only: "p1", opts: { noNetwork: true } });
+ const c = probe(clip.out);
+ assert.equal(c.v.width, 1920);
+ assert.equal(c.v.height, 1080);
+ assert.equal(c.v.r_frame_rate, "30/1");
+ const still = probe(path.join(root, "out", "sourced", "cards", "p1.poster.png"));
+ const box = deckFraming({ ...RENDER, width: 1920, height: 1080, chrome: { engine: "hyperframes", layout: "deck", deck: {} } }).box;
+ assert.equal(still.v.width, box.width);
+ assert.equal(still.v.height, box.height);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("--fetch-only on a src clip has nothing to fetch", async () => {
+ if (!have) return;
+ const { root, manifestPath } = await project();
+ try {
+ const r = await buildVideo({ manifestPath, fetchOnly: "p1", opts: {} });
+ assert.equal(r.out, null);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
diff --git a/umtool/report-to-video/build-video.mjs b/umtool/report-to-video/build-video.mjs
@@ -28,9 +28,11 @@
//
// Before any fetch, a clip's source is looked for ON DISK (sources.mjs): this
// build's out/clips-raw, then the corpus's channels/<slug>/data/<id>/clips/
-// windows the editor fetched, then a saved whole source. Only a miss in all
-// three goes to the network, and --no-network refuses the build up front if
-// any clip would.
+// windows the editor fetched, then a saved whole source -- and, by
+// sources.mjs's audioAllowed rule, the recording's sound alone, which plays
+// under a poster. Only a miss in all of them goes to the network, and
+// --no-network refuses the build up front if any clip would. A clip with its
+// own `src` (a file beside the manifest, local-media.mjs) is its own source.
//
// In the app: not used. On the CLI:
// node umtool/report-to-video/build-video.mjs <manifest.json> [options]
@@ -89,8 +91,11 @@ import { DEFAULT_CHANNELS_DIR, createCueSource, siteOriginFromManifest } from ".
// Where a clip's media is ALREADY on disk -- the build's raw cache, the
// editor's corpus windows, the saved source -- asked before anything fetches.
import {
- SHADOW_CHANNELS, channelsDirFor, findContainingWindow, rawWindowName, resolveLocalSource,
+ SHADOW_CHANNELS, WIN_EPS as SRC_EPS, audioAllowed, channelsDirFor, findContainingWindow, rawWindowName,
+ resolveLocalSource,
} from "./sources.mjs";
+// A clip whose media is a file beside the manifest (`src`, `cues`).
+import { clipLabel, createLocalMedia, hasLocalMedia, validateLocalMedia } from "./local-media.mjs";
import { createPostChannelResolver, resolvePostLinks } from "./post-links.mjs";
import { ensureWriteDir } from "../lib/report/storage.mjs";
// The deck (`render.chrome`): its geometry, validation and schedule are pure
@@ -110,7 +115,7 @@ import { platformArgsForUrl } from "yt-dlp-transcript-common/ytdlp/platformArgs.
// the clip bench imports too -- a preview that shows a line this renderer
// would never draw is worse than no preview.
import {
- attributionLine, channelName, hms, imageAttributionLine, uploadDateToIso,
+ attributionLine, attributionParts, channelName, hms, imageAttributionLine, uploadDateToIso,
} from "./attribution.mjs";
// An opt-in brand preset (`render.brand`). Every hook below branches on the
// brand BEFORE building an argument, so a manifest without one renders exactly
@@ -136,6 +141,8 @@ export { FFMPEG as FFMPEG_BIN, FFPROBE as FFPROBE_BIN };
// `transcripts/` directory. Built once main() has the manifest (it carries the
// archive origin); see cues.mjs.
let CUES = null;
+// The manifest's own media (`src` clips): set by buildVideo() beside CUES.
+let LOCAL = null;
const exists = (p) => access(p).then(() => true, () => false);
@@ -359,7 +366,8 @@ const HUMAN = {
fetch: (e) =>
e.cached
? ` source ${e.id}: ${e.source ?? "raw-cache"} ${e.reuse ?? "(exact window)"} covers ` +
- `${hms(e.from)}–${hms(e.to)} — no download` + (e.height ? ` (${e.height}p)` : "")
+ `${hms(e.from)}–${hms(e.to)} — no download` + (e.height ? ` (${e.height}p)` : "") +
+ (e.source === "audio" ? " — audio only, under a poster" : "")
: ` fetch ${e.id}: ${e.video} ${hms(e.from)}–${hms(e.to)} (network)`,
snap: (e) =>
` snap ${e.id}: ${e.start ? "start✓" : "start–"} ${e.end ? "end✓" : "end–"} ` +
@@ -438,10 +446,26 @@ export async function videoMeta(videoId, channelSlug, hints = {}) {
// this record is loaded for the title and the upload date anyway.
return {
title: d.title, uploadDate: d.uploadDate, webpageUrl: d.webpageUrl,
- duration: d.duration, channel: d.channel,
+ duration: d.duration, channel: d.channel, platform: d.platform ?? null,
};
}
+/**
+ * A clip's metadata, wherever it lives: a `src` clip's from its own cue file
+ * (local-media.mjs), every other clip's from its corpus record. Every reader
+ * of a clip's title, date and channel asks this, so a `src` clip is named the
+ * same in its header, its chapter and the deck.
+ */
+export async function clipMeta(entry, provenance = {}) {
+ if (hasLocalMedia(entry)) {
+ if (!LOCAL) throw new Error("clipMeta: no local media reader — it is set up by buildVideo()");
+ return LOCAL.meta(entry);
+ }
+ return videoMeta(entry.video, entry.channel ?? provenance?.channelSlug, {
+ siteChannel: entry.siteChannel, siteVideo: entry.siteVideo,
+ });
+}
+
// The CONTAINER's duration is max(video, audio), and the audio is longer: the
// AAC encoder pads the front with ~21 ms of decoder delay, and a video duration
// is rarely an exact multiple of the frame interval. Either way the excess is
@@ -469,6 +493,87 @@ export async function probeDuration(file, fps) {
return Number(stdout.trim());
}
+/**
+ * What a clip's source file holds: its duration, and whether it has a PICTURE
+ * -- a video stream that is not a cover image (an mp3's attached picture is
+ * one frame of album art, not footage).
+ *
+ * @returns {Promise<{ hasVideo: boolean, duration: number|null }>}
+ */
+export async function probeMedia(file) {
+ const { stdout } = await execFileP(FFPROBE, [
+ "-v", "error",
+ "-show_entries", "stream=codec_type:stream_disposition=attached_pic:format=duration",
+ "-of", "json", file,
+ ]);
+ const doc = JSON.parse(stdout);
+ const hasVideo = (doc.streams ?? []).some(
+ (st) => st.codec_type === "video" && !Number(st.disposition?.attached_pic),
+ );
+ const duration = Number(doc.format?.duration);
+ return { hasVideo, duration: Number.isFinite(duration) && duration > 0 ? duration : null };
+}
+
+// ---- the audio poster ------------------------------------------------------
+// A clip whose source has no picture -- a podcast episode, a channel that kept
+// only its sound (sources.mjs's audio tier), a `src` that is an mp3 -- plays
+// under a POSTER where the footage would be: the clip's own card (channel,
+// title, date: attributionParts, the header's resolver) drawn by renderCard at
+// exactly the picture box's size, with the sound's waveform moving along its
+// foot (`render.audioPoster.waveform`, default on).
+//
+// It stands in for `[0:v]` and nothing else: every filter after it -- the
+// letterbox, the header, the footer, the deck's framing -- runs on it as on
+// footage, and the encode is the same, so the segment's size, fps, pixel
+// format, SAR and audio layout are every other segment's and the concat
+// cannot tell it from one.
+
+/** Where the waveform sits in a WxH poster: along the foot, inside the card's margins. */
+export function posterWaveGeometry(W, H) {
+ const even = (n) => Math.max(2, 2 * Math.round(n / 2));
+ const x = Math.round(W * 0.09);
+ const width = even(W - 2 * x);
+ const height = even(H * 0.16);
+ return { x, y: H - height - Math.round(H * 0.07), width, height };
+}
+
+/**
+ * The filter parts that make the poster's picture, labelled `[pic]`: the
+ * still, then the waveform of input 0's sound over it.
+ *
+ * @param {{ posterIdx: number, box: { width: number, height: number }, render: any }} args
+ */
+export function audioPosterParts({ posterIdx, box, render }) {
+ if (render.audioPoster?.waveform === false) return [`[${posterIdx}:v]null[pic]`];
+ const g = posterWaveGeometry(box.width, box.height);
+ return [
+ `[0:a]aformat=channel_layouts=mono,` +
+ `showwaves=s=${g.width}x${g.height}:mode=cline:draw=full:rate=${render.fps}:colors=${render.palette.accent}[wave]`,
+ // `eof_action=pass`: the still runs on bare if the waveform ends a frame
+ // early. The default (repeat) hands the cut's `fps` a frame stamped far in
+ // the future, and it fills the gap -- an hour of poster behind 2 s of sound.
+ `[${posterIdx}:v][wave]overlay=x=${g.x}:y=${g.y}:eof_action=pass[pic]`,
+ ];
+}
+
+/**
+ * Render the poster still and return its ffmpeg input: a `seconds`-long loop
+ * at the cut's fps, so the picture ends where the sound does.
+ */
+async function audioPosterInput({ entry, meta, provenance, render, outDir, box, seconds }) {
+ const parts = attributionParts(entry, meta ?? {}, provenance ?? {});
+ const card = {
+ id: `${entry.id}.poster`,
+ style: "chapter",
+ ...(parts.channel ? { kicker: parts.channel } : {}),
+ heading: parts.title || clipLabel(entry),
+ ...(parts.date ? { sub: parts.date } : {}),
+ };
+ // At the box's size, and with no rail: the card IS the picture box.
+ const png = await renderCard(card, { ...render, width: box.width, height: box.height, rail: undefined }, outDir, null);
+ return ["-loop", "1", "-framerate", String(render.fps), "-t", seconds.toFixed(3), "-i", png];
+}
+
// yt-dlp exits 101 on a clean early stop (break-on-existing / max-downloads).
// The repo treats that as success everywhere else; do the same here.
const ytdlpOk = (err) => err?.code === 101;
@@ -574,13 +679,14 @@ export function localSources({ rawDir, channelsDir, channelSlug = null, probe }
rawDir,
channelsDir,
slugOf,
- async resolve(entry, span, { exact = false } = {}) {
+ // `audio` admits the audio tier: sources.mjs `audioAllowed` decides it.
+ async resolve(entry, span, { exact = false, audio = false } = {}) {
const slug = slugOf(entry);
- const key = `${slug}/${entry.video}/${span.from}/${span.to}/${exact}`;
+ const key = `${slug}/${entry.video}/${span.from}/${span.to}/${exact}/${audio}`;
if (memo.has(key)) return memo.get(key);
const hit = await resolveLocalSource(
{ video: entry.video, slug, from: span.from, to: span.to },
- { rawDir, channelsDir, exact, probe },
+ { rawDir, channelsDir, exact, probe, audio },
);
// Hits only: a miss is about to be fetched into the raw cache, and the
// next clip asking for the same span must find that file.
@@ -590,19 +696,27 @@ export function localSources({ rawDir, channelsDir, channelSlug = null, probe }
};
}
+/** May this build reach the network at all? */
+const networkOn = (opts) => !opts.noNetwork && !opts.skipFetch;
+
/**
- * The clips of `entries` that no local source can serve, i.e. the ones that
- * would need a fetch: `--no-network` refuses the build on any of them before
- * a frame is rendered. `index` is the entry's position in the manifest's
- * timeline.
+ * Where each clip of `entries` would come from without the network: `missing`
+ * are the ones no local source serves -- `--no-network` refuses the build on
+ * any of them before a frame is rendered -- and `audio` the ones only their
+ * sound serves, which play under a poster. `index` is the entry's position in
+ * the manifest's timeline. A `src` clip is its own source and is in neither.
*/
-export async function clipsNeedingFetch(entries, timeline, render, opts, local) {
- const out = [];
+export async function planLocalSources(entries, timeline, render, opts, local) {
+ const missing = [];
+ const audio = [];
for (const entry of entries) {
- if (!isClipEntry(entry)) continue;
+ if (!isClipEntry(entry) || hasLocalMedia(entry)) continue;
const span = fetchSpan(entry, render, opts);
- if (await local.resolve(entry, span, { exact: !!opts.noReuse })) continue;
- out.push({
+ const hit = await local.resolve(entry, span, {
+ exact: !!opts.noReuse,
+ audio: audioAllowed({ entry, render, network: networkOn(opts) }),
+ });
+ const row = {
// By id: a variant's view may be a copy of the manifest's entry.
index: timeline.findIndex((e) => e === entry || (entry.id != null && e?.id === entry.id)),
id: entry.id,
@@ -610,9 +724,26 @@ export async function clipsNeedingFetch(entries, timeline, render, opts, local)
video: entry.video,
from: span.from,
to: span.to,
- });
+ };
+ if (!hit) missing.push(row);
+ else if (hit.kind === "audio") audio.push({ ...row, local: hit.name });
}
- return out;
+ return { missing, audio };
+}
+
+/** planLocalSources' `missing`: the clips that would need a fetch. */
+export async function clipsNeedingFetch(entries, timeline, render, opts, local) {
+ return (await planLocalSources(entries, timeline, render, opts, local)).missing;
+}
+
+/** One line per clip that plays from its sound alone, for the log. */
+export function audioOnlyMessage(audio) {
+ return (
+ `${audio.length} clip(s) play from audio only (a poster where the picture would be):\n` +
+ audio
+ .map((m) => ` timeline[${m.index}] ${m.id} ${m.slug ?? "(no channel)"}/${m.video} ${m.local}`)
+ .join("\n")
+ );
}
/** One line per clip the network would have to serve, for the refusal. */
@@ -636,6 +767,22 @@ async function fetchClip(entry, meta, render, local, opts) {
const { rawDir } = local;
const { from, to } = fetchSpan(entry, render, opts);
+ // A `src` clip IS its source: the whole file, [0, duration], beside the
+ // manifest. Nothing is looked up and nothing is fetched.
+ if (hasLocalMedia(entry)) {
+ const file = LOCAL.file(entry);
+ const duration = entry.srcDuration ?? (await probeMedia(file)).duration;
+ EMIT("fetch", {
+ id: entry.id, video: clipLabel(entry), from, to: Number.isFinite(duration) ? Math.min(to, duration) : to,
+ cached: true, source: "src",
+ reuse: path.basename(file), local: file, window: [0, duration],
+ });
+ return {
+ path: file, fetchStart: 0, cached: true, source: "src",
+ scan: { from: Math.max(0, from), to: Number.isFinite(duration) ? Math.min(to, duration) : to },
+ };
+ }
+
// Shared across variants, and deliberately so: this is the only expensive
// thing in a build, and the two cuts overlap almost entirely.
const name = rawWindowName(entry.video, from, to);
@@ -646,7 +793,10 @@ async function fetchClip(entry, meta, render, local, opts) {
// named for exactly this span. fetchStart is the SOURCE's start, not the
// requested one -- every cut downstream is relative to it, so where the
// bytes came from is transparent.
- const hit = await local.resolve(entry, { from, to }, { exact: !!opts.noReuse });
+ // The audio tier only by audioAllowed's rule: before the network for a clip
+ // with no picture to fetch, otherwise only when there is no network to ask.
+ const audio = audioAllowed({ entry, meta, render, network: networkOn(opts) });
+ const hit = await local.resolve(entry, { from, to }, { exact: !!opts.noReuse, audio });
if (hit) {
EMIT("fetch", {
id: entry.id, video: entry.video, from, to, cached: true, source: hit.kind,
@@ -903,14 +1053,29 @@ async function buildClipSegment(entry, meta, render, dirs, opts, chrome, nodes,
// and the segment is the picture alone, framed into the box above the deck.
// The composition is overlaid on the whole concat later; nothing here knows
// about it. Same cut, same audio map, same encode as every other segment.
+ // No picture in the source: a poster stands in for `[0:v]` (see above).
+ const media = await probeMedia(raw);
+ const posterSeconds = Math.max(1 / render.fps, (media.duration != null ? Math.min(cutB, media.duration) : cutB) - cutA);
+ if (!media.hasVideo) EMIT("note", { message: `${entry.id}: no picture in ${path.basename(raw)} — a poster plays under its sound` });
+ const poster = (box, posterIdx) =>
+ media.hasVideo
+ ? null
+ : audioPosterInput({ entry, meta, provenance, render, outDir, box, seconds: posterSeconds })
+ .then((input) => ({ input, parts: audioPosterParts({ posterIdx, box, render }) }));
+
if (deckOn(render)) {
const fr = framing ?? segmentFraming(render);
+ const pst = await poster(fr.box, 1);
await execFileP(
FFMPEG,
[
"-nostdin", "-v", "error", "-y",
...cutArgs(raw, cutA, cutB),
- "-filter_complex", `[0:v]${deckFramingFilter(render, { feed: fr.layout === "feed" })}[v]`,
+ ...(pst ? pst.input : []),
+ "-filter_complex", [
+ ...(pst ? pst.parts : []),
+ `${pst ? "[pic]" : "[0:v]"}${deckFramingFilter(render, { feed: fr.layout === "feed" })}[v]`,
+ ].join(";"),
"-map", "[v]", "-map", "0:a",
...encodeArgs(render),
seg,
@@ -1014,16 +1179,24 @@ async function buildClipSegment(entry, meta, render, dirs, opts, chrome, nodes,
markIdx = nextIdx++;
inputs.push("-i", await markPng(render, brandMark.mark, path.join(outDir, "cards")));
}
+ // The poster, when the source has no picture: after everything else.
+ const pst = await poster({ width: VW, height: VH }, nextIdx);
+ if (pst) {
+ nextIdx += 1;
+ inputs.push(...pst.input);
+ }
+ const pic = pst ? "[pic]" : "[0:v]";
const parts = hasFooter
? [
- `[0:v]${base}[b]`,
+ ...(pst ? pst.parts : []),
+ `${pic}${base}[b]`,
`[b][${footerIdx}:v]overlay=0:${height - FH}[f]`,
`[${barIdx}:v]crop=w=${chrome.trackLen}:h=3:x='${chrome.trackLen}-(${fillExpr})':y=0[bar]`,
`[f][bar]overlay=x=${chrome.x0}:y=${trackAbsY - 1}[g]`,
`[g][${markerIdx}:v]overlay=x=${markX}:y=${trackAbsY - chrome.markerRadius}[q]`,
]
- : [`[0:v]${base}[q]`];
+ : [...(pst ? pst.parts : []), `${pic}${base}[q]`];
let q = "q";
if (brandMark) {
@@ -1075,6 +1248,8 @@ export async function readCutRecord(seg) {
// not scan) and it must keep its quiet zone (the white border is part of the
// symbol, not decoration).
async function qrForEntry(entry, provenance, render, outDir) {
+ // A `src` clip's file has no page on the archive: only its own citeUrl.
+ if (hasLocalMedia(entry) && !entry.citeUrl) return null;
const q = render.qr ?? {};
// A mirror's LOCAL slug is not the id the site serves, and a clip taken from a
// copy whose archived transcript is broken should point at the copy that reads —
@@ -3249,7 +3424,7 @@ export async function chapterTitle(entry, index, provenance, { deck = false } =
}
if (entry.type !== "clip") return entry.title ?? entry.heading ?? `Card ${index + 1}`;
try {
- const meta = await videoMeta(entry.video, entry.channel ?? provenance.channelSlug, { siteChannel: entry.siteChannel, siteVideo: entry.siteVideo });
+ const meta = await clipMeta(entry, provenance);
// The SAME overrides the header honours, resolved by the SAME functions. A
// chapter list that says 2019 for a clip whose burned-in line says 2016 is
// the mp4 disagreeing with itself, and only one of the two is on screen
@@ -3258,11 +3433,11 @@ export async function chapterTitle(entry, index, provenance, { deck = false } =
// lands in.
const who = channelName(entry, meta, provenance);
const date = entry.date ?? uploadDateToIso(meta.uploadDate);
- const title = String(entry.title ?? meta.title ?? entry.video);
+ const title = String(entry.title ?? meta.title ?? clipLabel(entry));
const tail = `${date} — ${title.length > 60 ? `${title.slice(0, 57)}…` : title}`.trim();
return who ? `${who} · ${tail}` : tail;
} catch {
- return `${index + 1}. ${entry.video}`;
+ return `${index + 1}. ${clipLabel(entry)}`;
}
}
@@ -3296,11 +3471,7 @@ export async function measureChromeSchedule({ manifest, entries, segments, D })
metas.push(null);
continue;
}
- metas.push(
- await videoMeta(e.video, e.channel ?? provenance.channelSlug, {
- siteChannel: e.siteChannel, siteVideo: e.siteVideo,
- }).catch(() => null),
- );
+ metas.push(await clipMeta(e, provenance).catch(() => null));
}
// The variant's posts, placed on the clips this cut plays with their real
// upload dates. A manifest without posts writes the schedule it always did.
@@ -3429,11 +3600,48 @@ async function assertConcatLength(file, expected, fps, what) {
* ffmpeg inside a request handler has no cancellation story, and a runaway
* grandchild would outlive the request that started it.
*/
+/**
+ * A cut's `src` clips (local-media.mjs), made ready before anything else runs:
+ * every path checked -- a refusal names each entry, before a fetch or a frame
+ * -- the reader set, and each clip's window filled in on this run's copy:
+ * `start`/`end` default to the file's two ends and must lie inside it.
+ */
+export async function prepareLocalMedia(timeline, manifestDir, render = {}) {
+ const allowAbsolute = render?.allowAbsoluteSrc === true;
+ const errors = await validateLocalMedia(timeline, manifestDir, { allowAbsolute, isClip: isClipEntry });
+ if (errors.length) throw new Error(`manifest: ${errors.join("; ")}`);
+ LOCAL = createLocalMedia({ baseDir: manifestDir, allowAbsolute });
+ for (const [i, e] of (timeline ?? []).entries()) {
+ if (!e || !isClipEntry(e) || !hasLocalMedia(e)) continue;
+ const who = `timeline[${i}] ${e.id}`;
+ const { duration } = await probeMedia(LOCAL.file(e)).catch(() => ({ duration: null }));
+ if (duration == null) {
+ errors.push(`${who}: ffprobe cannot read a duration from ${e.src}`);
+ continue;
+ }
+ e.srcDuration = duration;
+ e.start ??= 0;
+ e.end ??= duration;
+ if (!(Number.isFinite(e.start) && Number.isFinite(e.end) && e.start >= 0 && e.end > e.start)) {
+ errors.push(`${who}: start/end must be seconds with 0 ≤ start < end (got ${e.start}–${e.end})`);
+ } else if (e.end > duration + SRC_EPS) {
+ errors.push(`${who}: end ${e.end} is past the end of ${e.src} (${duration.toFixed(2)} s)`);
+ }
+ }
+ if (errors.length) throw new Error(`manifest: ${errors.join("; ")}`);
+}
+
export async function buildVideo({ manifestPath, opts = {}, out, only, fetchOnly } = {}) {
const variant = opts.variant ?? "sourced";
const whole = JSON.parse(await readFile(manifestPath, "utf8"));
const manifest = selectVariant(whole, variant);
const { render, provenance } = manifest;
+ // An `image` entry's `src` is relative to the MANIFEST, which is checked in
+ // beside the pictures it cites -- not to the cwd the build was started from.
+ // So is a clip's.
+ const manifestDir = path.dirname(path.resolve(manifestPath));
+ // A `src` clip's file and cues, checked and measured before anything else.
+ await prepareLocalMedia(manifest.timeline, manifestDir, render);
// A `render.chrome` that cannot be built is refused here, before a single
// fetch is spent. Absent, validateChrome has nothing to say.
if (render.chrome !== undefined && render.chrome !== null) assertChrome(render.chrome, render);
@@ -3450,9 +3658,6 @@ export async function buildVideo({ manifestPath, opts = {}, out, only, fetchOnly
const errors = validatePosts(whole.posts, whole.timeline ?? [], whole.render);
if (errors.length) throw new Error(`posts: ${errors.join("; ")}`);
}
- // An `image` entry's `src` is relative to the MANIFEST, which is checked in
- // beside the pictures it cites -- not to the cwd the build was started from.
- const manifestDir = path.dirname(path.resolve(manifestPath));
// Where each post's QR goes: its page on the archive (`posts.links`
// "archive", the default), which needs the archive channel that keeps it.
@@ -3517,8 +3722,9 @@ export async function buildVideo({ manifestPath, opts = {}, out, only, fetchOnly
// A still has nothing to fetch and is already on disk, so this is a no-op
// rather than an error: a bench that walks the timeline asking for each
// entry's window should not have to know which kinds have one.
- if (entry?.type === "image" || entry?.type === "teaser") {
- EMIT("note", { message: `${fetchOnly} is ${entry.type === "image" ? "an image" : "a teaser"} entry — nothing to fetch` });
+ if (entry?.type === "image" || entry?.type === "teaser" || (entry?.type === "clip" && hasLocalMedia(entry))) {
+ const what = entry.type === "image" ? "an image" : entry.type === "teaser" ? "a teaser" : "a clip with its own `src`";
+ EMIT("note", { message: `${fetchOnly} is ${what} entry — nothing to fetch` });
EMIT("done", { out: null, nothingToFetch: true });
return { out: null, failures: [] };
}
@@ -3569,8 +3775,11 @@ export async function buildVideo({ manifestPath, opts = {}, out, only, fetchOnly
// rebuild no segment fetch nothing, so they are not asked.)
if (opts.noNetwork && !opts.chaptersOnly && !opts.railOnly && !opts.chromeOnly && !opts.chromePreview) {
const want = manifest.timeline.filter((e) => !only || e.id === only);
- const missing = await clipsNeedingFetch(want, whole.timeline ?? [], render, opts, dirs.local);
+ const { missing, audio } = await planLocalSources(want, whole.timeline ?? [], render, opts, dirs.local);
if (missing.length) throw new Error(needsFetchMessage(missing));
+ // Satisfied, not missed: said once, up front, so a cut that will show a
+ // poster instead of a face is never a surprise at the end.
+ if (audio.length) EMIT("note", { message: audioOnlyMessage(audio) });
}
// Footer chrome is shared by every clip, so build it once up front. The deck
@@ -3784,9 +3993,9 @@ export async function buildVideo({ manifestPath, opts = {}, out, only, fetchOnly
: await buildLedgerSegment(entry, render, outDir, manifest.ledger, availability),
);
} else {
- const meta = await videoMeta(entry.video, entry.channel ?? provenance.channelSlug, { siteChannel: entry.siteChannel, siteVideo: entry.siteVideo });
+ const meta = await clipMeta(entry, provenance);
EMIT("clip", {
- id: entry.id, i, n: entries.length, video: entry.video,
+ id: entry.id, i, n: entries.length, video: clipLabel(entry),
section: entry.section, sectionEnter: !!entry.sectionEnter,
});
segments.push(
diff --git a/umtool/report-to-video/check-availability.mjs b/umtool/report-to-video/check-availability.mjs
@@ -87,7 +87,9 @@ export async function checkAvailability(manifestPath, { outDir, maxAgeDays = 0 }
return wanted.get(key);
};
for (const e of manifest.timeline ?? []) {
- if (e.type !== "clip") continue;
+ // A clip with its own `src` is a file beside the manifest, on no platform:
+ // the build checks it is there.
+ if (e.type !== "clip" || e.src != null) continue;
want(e.channel ?? slug, e.video).clips.push(e.id);
}
// LEDGER SOURCES TOO, not just the clipped ones. A cut that stacks its
diff --git a/umtool/report-to-video/deck.mjs b/umtool/report-to-video/deck.mjs
@@ -546,6 +546,8 @@ export function deckText(entry, meta, provenance, deck, multiChannel) {
export function deckQrUrl(entry, provenance = {}, { links = DECK_DEFAULTS.qr.links, meta = null } = {}) {
if (entry.type === "clip") {
if (entry.citeUrl) return entry.citeUrl;
+ // A clip from a file beside the manifest (`src`) has no page anywhere.
+ if (entry.src != null) return null;
const original = links === "original" ? originalUrlAt(meta?.webpageUrl, entry.start) : null;
return original ??
`${provenance.siteOrigin}/?v=${encodeURIComponent(
diff --git a/umtool/report-to-video/local-media.mjs b/umtool/report-to-video/local-media.mjs
@@ -0,0 +1,203 @@
+// local-media.mjs — a clip whose media is a FILE BESIDE THE MANIFEST.
+//
+// Most clips name a recording in the corpus (`video`, `channel`), and the build
+// finds its cues, its metadata and its media through that. A clip may instead
+// carry its own:
+//
+// { "type": "clip", "id": "p01",
+// "src": "media/episode-12.mp3", // video or audio-only
+// "cues": "media/episode-12.json", // optional
+// "start": 61.2, "end": 74.9 } // optional: default the whole file
+//
+// Both paths are RELATIVE TO THE MANIFEST, like an image entry's `src`, and may
+// not climb out of the manifest's directory: a manifest is checked in beside
+// the media it cites, and a `../../` that happens to resolve on one machine is
+// a build that breaks on the next. An absolute path is refused too, unless the
+// manifest says `render.allowAbsoluteSrc: true` -- media too big to keep beside
+// the manifest (a podcast archive on another drive) is a decision to write
+// down, not an accident to discover.
+//
+// The window is the whole file, [0, its probed duration]; `start`/`end` cut
+// inside it and default to its two ends. No channel, no corpus lookup, no QR to
+// a site (the file has no page there) unless the entry carries `citeUrl`.
+//
+// `cues` is read in either of two shapes and normalised by ONE function, so the
+// sentence-widening and the cut-to-quote passes read a manifest-local cue file
+// exactly as they read the corpus's:
+//
+// - the corpus's `transcript.cues.json`: `{ title, channel, uploadDate,
+// duration, platform, cues: [{ start, end, text }] }`
+// - the transcript `scripts/parakeet-stitch.mjs` writes: `{ duration_seconds,
+// chunks, text, chunk_data: [{ start_time, end_time, text }] }`
+//
+// A LEAF, like sources.mjs: no imports from build-video.mjs, and no path
+// computed from the cwd -- the manifest's directory arrives as an argument.
+import { readFile, stat } from "node:fs/promises";
+import path from "node:path";
+
+/** Does this clip carry its own media (rather than a corpus `video`)? Ask it of clips only. */
+export const hasLocalMedia = (entry) => entry != null && entry.src != null;
+
+/** What a clip is called in a log line: its corpus video, else its file. */
+export const clipLabel = (entry) => entry?.video ?? entry?.src ?? entry?.id ?? "?";
+
+/**
+ * One cue document in either shape, as the corpus's: `{ cues: [{start, end,
+ * text}], duration?, title?, channel?, uploadDate?, platform?, webpageUrl? }`.
+ * A cue with no finite start or end is dropped, never guessed at; a document in
+ * neither shape throws.
+ *
+ * @param {any} doc parsed JSON
+ * @param {string} [label] what the error names
+ */
+export function normaliseCueDoc(doc, label = "cues") {
+ if (!doc || typeof doc !== "object") throw new Error(`${label}: not a cue document`);
+ let raw;
+ let duration;
+ if (Array.isArray(doc.cues)) {
+ raw = doc.cues.map((c) => ({ start: c?.start, end: c?.end, text: c?.text }));
+ duration = doc.duration;
+ } else if (Array.isArray(doc.chunk_data)) {
+ raw = doc.chunk_data.map((c) => ({ start: c?.start_time, end: c?.end_time, text: c?.text }));
+ duration = doc.duration_seconds;
+ } else {
+ throw new Error(
+ `${label}: neither a transcript.cues.json (\`cues\`) nor a parakeet transcript (\`chunk_data\`)`,
+ );
+ }
+ const cues = [];
+ for (const c of raw) {
+ const start = Number(c.start);
+ const end = Number(c.end);
+ if (!Number.isFinite(start) || !Number.isFinite(end)) continue;
+ cues.push({ start, end, text: String(c.text ?? "") });
+ }
+ cues.sort((a, b) => a.start - b.start);
+ const d = Number(duration);
+ const out = { cues };
+ if (Number.isFinite(d) && d > 0) out.duration = d;
+ for (const k of ["title", "channel", "uploadDate", "platform", "webpageUrl"]) {
+ if (typeof doc[k] === "string" && doc[k]) out[k] = doc[k];
+ }
+ return out;
+}
+
+/** Read and normalise one cue file. */
+export async function readCueFile(file, label = file) {
+ return normaliseCueDoc(JSON.parse(await readFile(file, "utf8")), label);
+}
+
+/**
+ * A manifest-relative path, resolved -- or why it is refused. Relative paths
+ * must stay inside `baseDir`; an absolute one needs `allowAbsolute`.
+ *
+ * @returns {{ file: string } | { error: string }}
+ */
+export function resolveManifestPath(baseDir, p, { allowAbsolute = false } = {}) {
+ if (typeof p !== "string" || !p.trim()) return { error: "must be a non-empty path" };
+ if (path.isAbsolute(p)) {
+ if (!allowAbsolute) {
+ return { error: `${p} is absolute; set render.allowAbsoluteSrc to allow it` };
+ }
+ return { file: path.normalize(p) };
+ }
+ const file = path.resolve(/* turbopackIgnore: true */ baseDir, p);
+ const rel = path.relative(/* turbopackIgnore: true */ baseDir, file);
+ if (rel === ".." || rel.startsWith(`..${path.sep}`) || path.isAbsolute(rel)) {
+ return { error: `${p} climbs out of the manifest's directory` };
+ }
+ return { file };
+}
+
+const isFile = (p) => stat(p).then((s) => s.isFile(), () => false);
+
+/**
+ * Every problem with the timeline's manifest-local media, as sentences naming
+ * the entry: a path that escapes or is absolute without leave, a file that is
+ * not there, a cue file in no known shape, `src` beside a `video`, `cues`
+ * without `src`. Empty when there is nothing wrong (or nothing local).
+ *
+ * @param {any[]} timeline
+ * @param {string} baseDir the manifest's directory
+ * @param {{ allowAbsolute?: boolean, isClip?: (e: any) => boolean }} [opts]
+ * `isClip` is the build's own predicate (its vocabulary is open: anything it
+ * has no other branch for is a clip); other entries' `src` is not this one.
+ */
+export async function validateLocalMedia(timeline, baseDir, { allowAbsolute = false, isClip = (e) => e?.type === "clip" } = {}) {
+ const errors = [];
+ for (const [i, e] of (timeline ?? []).entries()) {
+ if (!e || !isClip(e)) continue;
+ const who = `timeline[${i}] ${e.id ?? "(no id)"}`;
+ if (e.src == null) {
+ if (e.cues != null) errors.push(`${who}: \`cues\` belongs to a clip with \`src\``);
+ continue;
+ }
+ if (e.video != null) errors.push(`${who}: a clip takes \`src\` or \`video\`, not both`);
+ const src = resolveManifestPath(baseDir, e.src, { allowAbsolute });
+ if (src.error) errors.push(`${who}: src ${src.error}`);
+ else if (!(await isFile(src.file))) errors.push(`${who}: no media at ${src.file} (src: ${e.src})`);
+ if (e.cues == null) continue;
+ const cues = resolveManifestPath(baseDir, e.cues, { allowAbsolute });
+ if (cues.error) {
+ errors.push(`${who}: cues ${cues.error}`);
+ } else if (!(await isFile(cues.file))) {
+ errors.push(`${who}: no cue file at ${cues.file} (cues: ${e.cues})`);
+ } else {
+ try {
+ await readCueFile(cues.file, e.cues);
+ } catch (err) {
+ errors.push(`${who}: ${err?.message ?? err}`);
+ }
+ }
+ }
+ return errors;
+}
+
+/**
+ * The reader a build (or resolve-windows) holds for one manifest's local
+ * media: each entry's resolved file, its cue document (read once) and the
+ * metadata the header, the chapters and the deck read in place of a corpus
+ * record's. Assumes validateLocalMedia passed.
+ *
+ * @param {{ baseDir: string, allowAbsolute?: boolean }} config
+ */
+export function createLocalMedia({ baseDir, allowAbsolute = false }) {
+ const docs = new Map();
+ const resolveOrThrow = (entry, key) => {
+ const r = resolveManifestPath(baseDir, entry[key], { allowAbsolute });
+ if (r.error) throw new Error(`${entry.id}: ${key} ${r.error}`);
+ return r.file;
+ };
+ return {
+ baseDir,
+ /** The entry's media file, absolute. */
+ file: (entry) => resolveOrThrow(entry, "src"),
+ /** Its normalised cue document, or null when it names none. */
+ async doc(entry) {
+ if (entry.cues == null) return null;
+ const file = resolveOrThrow(entry, "cues");
+ if (!docs.has(file)) docs.set(file, readCueFile(file, entry.cues));
+ return docs.get(file);
+ },
+ /** Its cues alone ([] without a cue file). */
+ async cues(entry) {
+ return (await this.doc(entry))?.cues ?? [];
+ },
+ /**
+ * The metadata a corpus record would have given: the cue document's when it
+ * carries any, the file's name as the title otherwise. The entry's own
+ * `title`, `date` and `channelTitle` still win where the header reads them.
+ */
+ async meta(entry) {
+ const d = await this.doc(entry);
+ return {
+ title: d?.title ?? path.basename(entry.src, path.extname(entry.src)),
+ uploadDate: d?.uploadDate ?? null,
+ webpageUrl: null,
+ duration: d?.duration ?? null,
+ channel: d?.channel ?? null,
+ platform: d?.platform ?? null,
+ };
+ },
+ };
+}
diff --git a/umtool/report-to-video/local-media.test.mjs b/umtool/report-to-video/local-media.test.mjs
@@ -0,0 +1,170 @@
+// Tests for local-media.mjs: a clip whose media and cues are files beside the
+// manifest (`src`, `cues`) -- the two cue shapes, the path rules, validation,
+// and the reader the build and resolve-windows hold.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import test from "node:test";
+import path from "node:path";
+import os from "node:os";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+
+import {
+ clipLabel, createLocalMedia, hasLocalMedia, normaliseCueDoc, resolveManifestPath, validateLocalMedia,
+} from "./local-media.mjs";
+import { unresolvable } from "./resolve-windows.mjs";
+import { buildVideo } from "./build-video.mjs";
+
+// The same three cues, in each shape.
+const CORPUS = {
+ title: "Episode 12", channel: "Demo Feed", uploadDate: "20260102", duration: 12.5, platform: "podcast",
+ cues: [
+ { start: 4, end: 7.25, text: "and then the second." },
+ { start: 0.5, end: 3.9, text: "First thing." },
+ { start: 7.5, end: 12, text: "The third one?" },
+ ],
+};
+const PARAKEET = {
+ duration_seconds: 12.5,
+ chunks: 3,
+ text: "First thing. and then the second. The third one?",
+ chunk_data: [
+ { start_time: 0.5, end_time: 3.9, text: "First thing." },
+ { start_time: 4, end_time: 7.25, text: "and then the second." },
+ { start_time: 7.5, end_time: 12, text: "The third one?" },
+ ],
+};
+
+async function dir() {
+ const root = await mkdtemp(path.join(os.tmpdir(), "rtv-local-media-"));
+ await mkdir(path.join(root, "media"), { recursive: true });
+ return root;
+}
+
+test("both cue shapes normalise to the same cues, in time order", () => {
+ const a = normaliseCueDoc(CORPUS);
+ const b = normaliseCueDoc(PARAKEET);
+ assert.deepEqual(a.cues, b.cues);
+ assert.deepEqual(a.cues.map((c) => c.start), [0.5, 4, 7.5]);
+ assert.equal(a.duration, 12.5);
+ assert.equal(b.duration, 12.5);
+ // The corpus shape carries the record's metadata; the parakeet one has none.
+ assert.equal(a.title, "Episode 12");
+ assert.equal(a.platform, "podcast");
+ assert.equal(b.title, undefined);
+});
+
+test("a cue with no finite start or end is dropped; neither shape is an error", () => {
+ const got = normaliseCueDoc({ cues: [{ start: 1, end: 2, text: "a" }, { start: "x", end: 3, text: "b" }, { start: 4 }] });
+ assert.deepEqual(got.cues, [{ start: 1, end: 2, text: "a" }]);
+ assert.throws(() => normaliseCueDoc({ segments: [] }, "m/ep.json"), /m\/ep\.json: neither a transcript\.cues\.json/);
+ assert.throws(() => normaliseCueDoc(null), /not a cue document/);
+});
+
+test("resolveManifestPath: inside the manifest's directory, or absolute only with leave", () => {
+ const base = path.resolve("/srv/proj");
+ assert.deepEqual(resolveManifestPath(base, "media/ep.mp3"), { file: path.join(base, "media", "ep.mp3") });
+ assert.deepEqual(resolveManifestPath(base, "media/../ep.mp3"), { file: path.join(base, "ep.mp3") });
+ assert.match(resolveManifestPath(base, "../other/ep.mp3").error, /climbs out of the manifest's directory/);
+ assert.match(resolveManifestPath(base, "media/../../ep.mp3").error, /climbs out/);
+ assert.match(resolveManifestPath(base, "..").error, /climbs out/);
+ // A name that merely starts with two dots is inside.
+ assert.deepEqual(resolveManifestPath(base, "..ep.mp3"), { file: path.join(base, "..ep.mp3") });
+ assert.match(resolveManifestPath(base, "/mnt/feed/ep.mp3").error, /absolute; set render\.allowAbsoluteSrc/);
+ assert.deepEqual(resolveManifestPath(base, "/mnt/feed/ep.mp3", { allowAbsolute: true }), { file: "/mnt/feed/ep.mp3" });
+ assert.match(resolveManifestPath(base, "").error, /non-empty/);
+});
+
+test("validateLocalMedia: every problem, each naming its entry", async () => {
+ const root = await dir();
+ try {
+ await writeFile(path.join(root, "media", "ep.mp3"), "x");
+ await writeFile(path.join(root, "media", "ep.json"), JSON.stringify(PARAKEET));
+ await writeFile(path.join(root, "media", "bad.json"), JSON.stringify({ words: [] }));
+ const timeline = [
+ { id: "ok", type: "clip", src: "media/ep.mp3", cues: "media/ep.json" },
+ { id: "nofile", type: "clip", src: "media/missing.mp3" },
+ { id: "up", type: "clip", src: "../ep.mp3" },
+ { id: "abs", type: "clip", src: path.join(root, "media", "ep.mp3") },
+ { id: "both", type: "clip", src: "media/ep.mp3", video: "abc123" },
+ { id: "nocues", type: "clip", src: "media/ep.mp3", cues: "media/none.json" },
+ { id: "badcues", type: "clip", src: "media/ep.mp3", cues: "media/bad.json" },
+ { id: "orphan", type: "clip", video: "abc123", cues: "media/ep.json" },
+ // Not this field: an image's `src` is a picture, and a card is no clip.
+ { id: "img", type: "image", src: "media/missing.png" },
+ { id: "card", type: "card", src: "whatever" },
+ ];
+ const errors = await validateLocalMedia(timeline, root);
+ assert.equal(errors.length, 7, errors.join("\n"));
+ assert.match(errors[0], /^timeline\[1\] nofile: no media at .*missing\.mp3 \(src: media\/missing\.mp3\)$/);
+ assert.match(errors[1], /^timeline\[2\] up: src \.\.\/ep\.mp3 climbs out/);
+ assert.match(errors[2], /^timeline\[3\] abs: src .* is absolute/);
+ assert.match(errors[3], /^timeline\[4\] both: a clip takes `src` or `video`, not both$/);
+ assert.match(errors[4], /^timeline\[5\] nocues: no cue file at .*none\.json/);
+ assert.match(errors[5], /^timeline\[6\] badcues: media\/bad\.json: neither/);
+ assert.match(errors[6], /^timeline\[7\] orphan: `cues` belongs to a clip with `src`$/);
+ // With leave, the absolute one passes.
+ const abs = await validateLocalMedia([timeline[3]], root, { allowAbsolute: true });
+ assert.deepEqual(abs, []);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("createLocalMedia: the file, the cues and the metadata a record would have given", async () => {
+ const root = await dir();
+ try {
+ await writeFile(path.join(root, "media", "a.json"), JSON.stringify(CORPUS));
+ await writeFile(path.join(root, "media", "b.json"), JSON.stringify(PARAKEET));
+ const local = createLocalMedia({ baseDir: root });
+ const a = { id: "a", type: "clip", src: "media/ep.mp3", cues: "media/a.json" };
+ const b = { id: "b", type: "clip", src: "media/ep.mp3", cues: "media/b.json" };
+ const bare = { id: "c", type: "clip", src: "media/episode-12.mp3" };
+ assert.equal(local.file(a), path.join(root, "media", "ep.mp3"));
+ assert.deepEqual(await local.cues(a), await local.cues(b));
+ assert.deepEqual(await local.cues(bare), []);
+ assert.deepEqual(await local.meta(a), {
+ title: "Episode 12", uploadDate: "20260102", webpageUrl: null, duration: 12.5,
+ channel: "Demo Feed", platform: "podcast",
+ });
+ // No record: the file's own name is the title, and there is no page.
+ const m = await local.meta(bare);
+ assert.equal(m.title, "episode-12");
+ assert.equal(m.webpageUrl, null);
+ assert.equal(hasLocalMedia(bare), true);
+ assert.equal(hasLocalMedia({ id: "v", type: "clip", video: "abc123" }), false);
+ assert.equal(clipLabel(bare), "media/episode-12.mp3");
+ assert.equal(clipLabel({ video: "abc123", src: "x" }), "abc123");
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
+
+test("resolve-windows leaves a src clip alone without cues or a window", () => {
+ assert.equal(unresolvable({ type: "clip", video: "abc123", start: 1, end: 2 }), null);
+ assert.equal(unresolvable({ type: "clip", src: "m.mp3", start: 1, end: 2 }), "no cues file");
+ assert.equal(unresolvable({ type: "clip", src: "m.mp3", cues: "m.json" }), "no start/end (the whole file)");
+ assert.equal(unresolvable({ type: "clip", src: "m.mp3", cues: "m.json", start: 1, end: 2 }), null);
+});
+
+test("buildVideo refuses a missing src before anything runs, naming the entry", async () => {
+ const root = await dir();
+ try {
+ const manifestPath = path.join(root, "video.manifest.json");
+ await writeFile(manifestPath, JSON.stringify({
+ slug: "demo",
+ title: "Demo",
+ render: { width: 320, height: 180, fps: 30 },
+ timeline: [
+ { id: "t0", type: "card", style: "chapter", heading: "Hi" },
+ { id: "p1", type: "clip", src: "media/gone.mp3" },
+ ],
+ }));
+ await assert.rejects(
+ buildVideo({ manifestPath, opts: { noNetwork: true } }),
+ /^Error: manifest: timeline\[1\] p1: no media at .*gone\.mp3 \(src: media\/gone\.mp3\)$/,
+ );
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});
diff --git a/umtool/report-to-video/package.json b/umtool/report-to-video/package.json
@@ -26,6 +26,7 @@
"./deck": "./deck.mjs",
"./factcheck": "./factcheck.mjs",
"./ledger-totals": "./ledger-totals.mjs",
+ "./local-media": "./local-media.mjs",
"./mute": "./mute.mjs",
"./package.json": "./package.json",
"./post-links": "./post-links.mjs",
diff --git a/umtool/report-to-video/render-cards.mjs b/umtool/report-to-video/render-cards.mjs
@@ -727,7 +727,11 @@ function rosterCellSvg(cell, x, y, g, pal, face) {
// will not scan) and it must keep its quiet zone (the white border is part of
// the symbol, not decoration).
export function qrUrlFor(entry, provenance) {
- if (entry.type === "clip") {
+ // A clip from a file beside the manifest (`src`) has no moment on the archive
+ // to link: its own citeUrl, else -- the rail draws a code on every tile --
+ // the sweep's, as a card's.
+ const momentless = entry.type === "clip" && entry.src != null && !entry.citeUrl;
+ if (entry.type === "clip" && !momentless) {
// A mirror's LOCAL slug is not the id the site serves, so an explicit
// per-clip citeUrl always wins over the derived one.
return (
@@ -779,7 +783,7 @@ async function qrTileStrip(entries, provenance, render, g, outDir) {
const images = [];
entries.forEach((e, i) => {
const y = i * TH;
- const isClip = e.type === "clip";
+ const isClip = e.type === "clip" && !(e.src != null && !e.citeUrl);
body.push(
`<rect x="0.5" y="${y + 0.5}" width="${g.TILEW - 1}" height="${TH - 1}" fill="${pal.bg}" ` +
`stroke="${pal.accent}" stroke-width="1"/>`,
diff --git a/umtool/report-to-video/resolve-windows.mjs b/umtool/report-to-video/resolve-windows.mjs
@@ -40,10 +40,16 @@
// once a corpus moves past its last publish — see cues.mjs.
//
// A clip entry may set `lockStart` / `lockEnd` to pin that edge exactly.
+//
+// A clip with its own media (`src`) reads its own `cues` file, relative to the
+// manifest, in either shape local-media.mjs normalises; one with no `cues`, or
+// with no `start`/`end` (it plays the whole file), is left as it is.
import { readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
import { createCueSource, siteOriginFromManifest } from "./cues.mjs";
+import { clipLabel, createLocalMedia, hasLocalMedia } from "./local-media.mjs";
const ENDS_SENTENCE = /[.!?]["'”’)\]]*\s*$/;
@@ -305,19 +311,36 @@ function retargetCiteUrl(entry, cite, channelSlug) {
* worth having; a flag that silently did both would make one of those two
* decisions on their behalf.
*/
-async function cutPass(manifest, { loadCues, slug, opts, write, force }) {
+/**
+ * Why a clip cannot be resolved against cues, or null when it can: a `src`
+ * clip with no `cues` file, or one with no window (it plays the whole file).
+ */
+export function unresolvable(e) {
+ if (!hasLocalMedia(e)) return null;
+ if (e.cues == null) return "no cues file";
+ if (!Number.isFinite(e.start) || !Number.isFinite(e.end)) return "no start/end (the whole file)";
+ return null;
+}
+
+async function cutPass(manifest, { cuesOf, slug, opts, write, force }) {
let changed = 0;
let unmatched = 0;
let cites = 0;
for (const e of manifest.timeline) {
if (e.type !== "clip") continue;
const id = String(e.id).padEnd(4);
+ const label = String(clipLabel(e)).padEnd(12);
+ const why = unresolvable(e);
+ if (why) {
+ console.log(`${id} ${label} ${why} — left as is`);
+ continue;
+ }
if (e.lockCut) {
- console.log(`${id} ${String(e.video).padEnd(12)} lockCut — left at ${e.cutStart}–${e.cutEnd}`);
+ console.log(`${id} ${label} lockCut — left at ${e.cutStart}–${e.cutEnd}`);
continue;
}
if (!force && (e.cutStart != null || e.cutEnd != null)) {
- console.log(`${id} ${String(e.video).padEnd(12)} already cut ${e.cutStart}–${e.cutEnd} (--force-cut to redo)`);
+ console.log(`${id} ${label} already cut ${e.cutStart}–${e.cutEnd} (--force-cut to redo)`);
// The cut stands, but a cite left behind by an EARLIER run of this pass
// still names a second the clip does not play. Aligning it needs no
// cues, no network and no re-derivation, so it is not worth a --force.
@@ -334,20 +357,19 @@ async function cutPass(manifest, { loadCues, slug, opts, write, force }) {
}
continue;
}
- const chan = e.channel ?? slug;
- const cues = await loadCues(e.video, chan, { siteChannel: e.siteChannel, siteVideo: e.siteVideo });
+ const cues = await cuesOf(e);
const r = cutToQuote(cues, e.quote, { start: e.start, end: e.end }, opts);
const extent = `${e.start.toFixed(1)}–${e.end.toFixed(1)}`;
if (!r.ok) {
unmatched += 1;
console.log(
- `${id} ${String(e.video).padEnd(12)} ${extent} UNMATCHED (${r.score.toFixed(2)}) — ${r.why}`,
+ `${id} ${label} ${extent} UNMATCHED (${r.score.toFixed(2)}) — ${r.why}`,
);
if (r.matched) console.log(` best partial: ${r.matched.slice(0, 120)}`);
continue;
}
console.log(
- `${id} ${String(e.video).padEnd(12)} ${extent} -> cut ${r.cutStart.toFixed(1)}–${r.cutEnd.toFixed(1)} ` +
+ `${id} ${label} ${extent} -> cut ${r.cutStart.toFixed(1)}–${r.cutEnd.toFixed(1)} ` +
`(${(r.cutEnd - r.cutStart).toFixed(1)}s, match ${r.score.toFixed(2)})`,
);
console.log(` ${r.matched.slice(0, 120)}`);
@@ -389,7 +411,7 @@ async function main() {
const opts = { maxLead: num("--max-lead", 8), maxTail: num("--max-tail", 12) };
const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
- const slug = manifest.provenance.channelSlug;
+ const slug = manifest.provenance?.channelSlug;
const cache = new Map();
// Cues come from a local corpus when there is one, and from the published
@@ -405,13 +427,23 @@ async function main() {
prefer: flag("--cue-source") ?? "auto",
log: (m) => console.error(` · ${m}`),
});
- const loadCues = (videoId, channelSlug, hints) =>
- cues.load(channelSlug, videoId, hints).then((r) => r.cues);
+ // A `src` clip's cues are its own file, beside the manifest; every other
+ // clip's are its corpus record's.
+ const local = createLocalMedia({
+ baseDir: path.dirname(path.resolve(manifestPath)),
+ allowAbsolute: manifest.render?.allowAbsoluteSrc === true,
+ });
+ const cuesOf = (e) =>
+ hasLocalMedia(e)
+ ? local.cues(e)
+ : cues
+ .load(e.channel ?? slug, e.video, { siteChannel: e.siteChannel, siteVideo: e.siteVideo })
+ .then((r) => r.cues);
// ---- the cut pass, which is a different question ----
if (argv.includes("--cut-to-quote")) {
const r = await cutPass(manifest, {
- loadCues,
+ cuesOf,
slug,
opts: { ...opts, minMatch: num("--min-match", 0.6) },
write: argv.includes("--write"),
@@ -441,18 +473,18 @@ async function main() {
// An author can trim a clip to land mid-cue on purpose — a cue often carries
// a whole paragraph, and cutting a quote short is an editorial decision.
// Widening would undo exactly that, so `lock` opts the clip out.
- if (e.lock) {
- console.log(`${e.id.padEnd(4)} ${e.video.padEnd(12)} locked, left at ${e.start.toFixed(1)}–${e.end.toFixed(1)}`);
+ const label = String(clipLabel(e)).padEnd(12);
+ const why = unresolvable(e);
+ if (why) {
+ console.log(`${e.id.padEnd(4)} ${label} ${why} — left as is`);
continue;
}
- const chan = e.channel ?? slug;
- const key = `${chan}/${e.video}`;
- if (!cache.has(key)) {
- cache.set(
- key,
- await loadCues(e.video, chan, { siteChannel: e.siteChannel, siteVideo: e.siteVideo }),
- );
+ if (e.lock) {
+ console.log(`${e.id.padEnd(4)} ${label} locked, left at ${e.start.toFixed(1)}–${e.end.toFixed(1)}`);
+ continue;
}
+ const key = hasLocalMedia(e) ? `src:${e.cues}` : `${e.channel ?? slug}/${e.video}`;
+ if (!cache.has(key)) cache.set(key, await cuesOf(e));
const cues = cache.get(key);
const before = { start: e.start, end: e.end };
@@ -474,7 +506,7 @@ async function main() {
Math.abs(w.start - before.start) > 0.05 || Math.abs(w.end - before.end) > 0.05;
if (moved) changed += 1;
console.log(
- `${e.id.padEnd(4)} ${e.video.padEnd(12)} ` +
+ `${e.id.padEnd(4)} ${label} ` +
`${before.start.toFixed(1)}–${before.end.toFixed(1)} -> ` +
`${w.start.toFixed(1)}–${w.end.toFixed(1)} (+${dLead}s lead, +${dTail}s tail, ${dur}s)`,
);
@@ -493,9 +525,11 @@ async function main() {
// The later clip's start is the deliberate one, so trim the earlier clip's tail.
const byVideo = new Map();
for (const e of manifest.timeline) {
- if (e.type !== "clip") continue;
- if (!byVideo.has(e.video)) byVideo.set(e.video, []);
- byVideo.get(e.video).push(e);
+ if (e.type !== "clip" || unresolvable(e)) continue;
+ // A `src` clip's recording is its file.
+ const rec = hasLocalMedia(e) ? `src:${e.src}` : e.video;
+ if (!byVideo.has(rec)) byVideo.set(rec, []);
+ byVideo.get(rec).push(e);
}
for (const [video, list] of byVideo) {
if (list.length < 2) continue;
diff --git a/umtool/report-to-video/sources.mjs b/umtool/report-to-video/sources.mjs
@@ -12,6 +12,18 @@
// pointer (`data/<id>/saved-video.json` -> `<dir>/<file>`),
// else a `data/<id>/source-media.<ext>` not yet moved there.
// It is the window [0, its probed duration].
+// 4. audio `data/<id>/audio.<ext>`: the recording's SOUND alone, the
+// window [0, its probed duration]. The build draws a poster
+// where the picture would be.
+//
+// THE AUDIO TIER IS ASKED ONLY WHEN THE CALLER SAYS SO (`audio: true`), and
+// `audioAllowed` below is the one rule for when: a clip with no picture to
+// fetch (`audioOnly: true`, `render.preferLocalAudio`, or a record whose
+// platform carries none), or a build that could not fetch one anyway
+// (`--no-network`, `--skip-fetch`, a record with no page to fetch from). Being
+// last, it never beats a picture already on disk; asked before the network, it
+// is what plays a podcast. Many transcribed videos keep an `audio.mp3`, so
+// without the rule a cut would quietly lose its pictures to it.
//
// A window qualifies only if it holds the REQUESTED span whole -- the clip plus
// the build's fetch pad -- to WIN_EPS. Within a tier the tightest wins (the
@@ -42,7 +54,29 @@ import { promisify } from "node:util";
const execFileP = promisify(execFile);
/** The source kinds, in the order they are consulted. */
-export const SOURCE_KINDS = ["raw-cache", "corpus-window", "saved-video"];
+export const SOURCE_KINDS = ["raw-cache", "corpus-window", "saved-video", "audio"];
+
+/**
+ * Platforms whose records have no picture: a feed of episodes. A clip from one
+ * reads its local audio BEFORE the network, since a fetch has no video to get.
+ */
+export const AUDIO_ONLY_PLATFORMS = new Set(["podcast", "feed", "rss"]);
+
+/**
+ * Should the audio tier be consulted for this clip? (See the note at the top.)
+ *
+ * @param {{ entry?: any, meta?: { platform?: string|null, webpageUrl?: string|null }|null,
+ * render?: any, network?: boolean }} args
+ * `network` is whether this build may fetch at all; `meta` is the record,
+ * when it has been read (without it, only the entry and render can say).
+ */
+export function audioAllowed({ entry = null, meta = null, render = null, network = true } = {}) {
+ if (entry?.audioOnly === true || render?.preferLocalAudio === true) return true;
+ if (AUDIO_ONLY_PLATFORMS.has(String(meta?.platform ?? "").toLowerCase())) return true;
+ if (!network) return true;
+ // A record with no page has nothing a fetch could get.
+ return meta != null && !meta.webpageUrl;
+}
// A window read back from a 2 dp name can sit a hair outside the request that
// produced it; the same tolerance resolve-windows.mjs and common's
@@ -196,6 +230,24 @@ export async function wholeContainersOf(videoDir) {
return out;
}
+// The sound files a video dir may hold: common/lib/mediaFiles.ts's anchored
+// `audio.<ext>` over AUDIO_EXTS, plus the `.webm`/`.mp4` an extract-to-mp3 that
+// failed leaves behind (sound only). Read in AUDIO_READ_PREFERENCE's order --
+// the file the transcribe path reads -- then by name.
+const AUDIO_FILE_RE = /^audio\.(?:mp3|m4a|aac|ogg|oga|opus|wav|flac|webm|mp4)$/i;
+export const AUDIO_PREFERENCE = ["audio.mp3", "audio.m4a", "audio.opus"];
+
+/** A video dir's audio files, best first. Not yet checked for existence. */
+export async function audioFilesOf(videoDir) {
+ const names = (await listRawNames(videoDir)).filter((n) => AUDIO_FILE_RE.test(n));
+ const rank = (n) => {
+ const i = AUDIO_PREFERENCE.indexOf(n);
+ return i < 0 ? AUDIO_PREFERENCE.length : i;
+ };
+ names.sort((a, b) => rank(a) - rank(b) || a.localeCompare(b));
+ return names.map((name) => ({ name, path: path.join(/* turbopackIgnore: true */ videoDir, name) }));
+}
+
/**
* Is there a readable file at `p`, through every link on the way? A dangling
* link, a missing file and an unanswering drive are all "no", never a throw.
@@ -263,10 +315,25 @@ async function savedVideoWindows({ video, slug, channelsDir, probe = ffprobeSour
return out;
}
+// The ONE best audio file, not every one: they are all the same recording, and
+// probing three formats of it would buy nothing.
+async function audioWindows({ video, slug, channelsDir, probe = ffprobeSource }) {
+ if (!channelsDir || !slug) return [];
+ for (const f of await audioFilesOf(videoDirOf(channelsDir, slug, video))) {
+ if (!(await present(f.path))) continue;
+ const duration = Number((await probe(f.path))?.duration);
+ if (!Number.isFinite(duration) || duration <= 0) continue;
+ return [{ name: f.name, path: f.path, from: 0, to: duration }];
+ }
+ return [];
+}
+
+// `audio: true` marks a tier consulted only when the caller allows it.
export const TIERS = [
{ kind: "raw-cache", windows: rawCacheWindows },
{ kind: "corpus-window", windows: corpusWindows },
{ kind: "saved-video", windows: savedVideoWindows },
+ { kind: "audio", windows: audioWindows, audio: true },
];
const asSource = (kind, w) => ({
@@ -289,24 +356,29 @@ const asSource = (kind, w) => ({
* kinds?: string[] }} [config]
* `exact` (`--no-reuse`) accepts only the raw-cache file named for exactly
* this span. `kinds` narrows the tiers consulted (default: all, in order).
+ * `audio` admits the audio tier (default false; see `audioAllowed`).
* @returns {Promise<null | { kind: string, path: string, name: string,
* windowStart: number, windowEnd: number, height?: number }>}
*/
export async function resolveLocalSource(want, config = {}) {
const { video, slug = null, from, to } = want;
- const { rawDir = null, channelsDir = null, exact = false, probe, kinds = null } = config;
+ const { rawDir = null, channelsDir = null, exact = false, probe, kinds = null, audio = false } = config;
if (!video || !Number.isFinite(from) || !Number.isFinite(to)) return null;
if (exact) {
- if (!rawDir) return null;
- const name = rawWindowName(video, from, to);
- const p = path.join(/* turbopackIgnore: true */ rawDir, name);
- return (await present(p)) ? asSource("raw-cache", { name, path: p, from, to }) : null;
+ if (rawDir) {
+ const name = rawWindowName(video, from, to);
+ const p = path.join(/* turbopackIgnore: true */ rawDir, name);
+ if (await present(p)) return asSource("raw-cache", { name, path: p, from, to });
+ }
+ // `--no-reuse` re-cuts no cached window, but the sound is not one.
+ if (!audio) return null;
}
const ctx = { video, slug, rawDir, channelsDir, probe };
for (const tier of TIERS) {
if (kinds && !kinds.includes(tier.kind)) continue;
+ if (tier.audio ? !audio : exact) continue;
const windows = (await tier.windows(ctx)).filter((w) => windowContains(w, from, to));
// The raw cache's EXACT name first, as it always was; then tightest.
const exactName = tier.kind === "raw-cache" ? rawWindowName(video, from, to) : null;
@@ -327,12 +399,15 @@ export async function resolveLocalSource(want, config = {}) {
*
* `probe` decides a whole container's span. The bench passes a cheap one (the
* cue doc's duration) and is asked only for a video that has a container.
+ *
+ * Not the audio tier: many transcribed videos keep their sound, and a
+ * bench that called each of those "fetched" would never offer a fetch.
*/
export async function corpusWindowsOf({ video, slug, channelsDir, probe = ffprobeSource }) {
const ctx = { video, slug, channelsDir, probe };
const out = [];
for (const tier of TIERS) {
- if (tier.kind === "raw-cache") continue;
+ if (tier.kind === "raw-cache" || tier.audio) continue;
for (const w of await tier.windows(ctx)) {
if (await present(w.path)) out.push({ ...w, kind: tier.kind });
}
diff --git a/umtool/report-to-video/sources.test.mjs b/umtool/report-to-video/sources.test.mjs
@@ -11,6 +11,7 @@ import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
import {
SOURCE_KINDS,
TIERS,
+ audioAllowed,
channelsDirFor,
corpusWindowsOf,
rawWindowName,
@@ -19,12 +20,14 @@ import {
windowsFromBareNames,
} from "./sources.mjs";
import {
+ audioOnlyMessage,
buildVideo,
clipsNeedingFetch,
fetchSpan,
findContainingWindow,
localSources,
needsFetchMessage,
+ planLocalSources,
} from "./build-video.mjs";
const SLUG = "demo-channel";
@@ -62,7 +65,8 @@ async function pointTo(videoDir, storeDir, file, extra = {}) {
test("the tiers are listed in the order they are consulted", () => {
assert.deepEqual(TIERS.map((t) => t.kind), SOURCE_KINDS);
- assert.deepEqual(SOURCE_KINDS, ["raw-cache", "corpus-window", "saved-video"]);
+ assert.deepEqual(SOURCE_KINDS, ["raw-cache", "corpus-window", "saved-video", "audio"]);
+ assert.deepEqual(TIERS.filter((t) => t.audio).map((t) => t.kind), ["audio"]);
});
test("rawWindowName is the build's cache key, two decimals always", () => {
@@ -355,3 +359,155 @@ test("buildVideo --no-network refuses before rendering, naming every clip that w
await rm(t.root, { recursive: true, force: true });
}
});
+
+// ---- the audio tier ------------------------------------------------------------
+
+test("audio: asked only when allowed, the whole file, the preferred format", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.videoDir, "audio.m4a"));
+ await touch(path.join(t.videoDir, "audio.mp3"));
+ // Not sound files: a subtitle, a partial, the transcribe scratch.
+ await touch(path.join(t.videoDir, "audio.en-orig.vtt"));
+ await touch(path.join(t.videoDir, "audio.mp3.part"));
+ await touch(path.join(t.videoDir, "audio.tmp-123.mp3"));
+ const { probe, calls } = fakeProbe(900);
+ const want = { video: VIDEO, slug: SLUG, from: 100, to: 110 };
+ const cfg = { channelsDir: t.channelsDir, probe };
+ assert.equal(await resolveLocalSource(want, cfg), null, "not without audio: true");
+ assert.equal(calls.length, 0, "and nothing probed for it");
+ const hit = await resolveLocalSource(want, { ...cfg, audio: true });
+ assert.deepEqual(hit, {
+ kind: "audio", path: path.join(t.videoDir, "audio.mp3"), name: "audio.mp3", windowStart: 0, windowEnd: 900,
+ });
+ assert.deepEqual(calls, [path.join(t.videoDir, "audio.mp3")], "one file probed, not every format");
+ // Past the recording's end is not covered.
+ assert.equal(await resolveLocalSource({ ...want, from: 895, to: 905 }, { ...cfg, audio: true }), null);
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("audio: every picture on disk wins over the sound, even when the sound is allowed", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.videoDir, "audio.mp3"));
+ await touch(path.join(t.storeDir, "source-media.mp4"));
+ await pointTo(t.videoDir, t.storeDir, "source-media.mp4");
+ const cfg = { rawDir: t.rawDir, channelsDir: t.channelsDir, probe: fakeProbe(600).probe, audio: true };
+ const want = { video: VIDEO, slug: SLUG, from: 100, to: 110 };
+ assert.equal((await resolveLocalSource(want, cfg)).kind, "saved-video");
+ await touch(path.join(t.clipsDir, "90.00-120.00.mp4"));
+ assert.equal((await resolveLocalSource(want, cfg)).kind, "corpus-window");
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("audio: a dangling audio link is not there; --no-reuse still reaches the sound", async () => {
+ const t = await tree();
+ try {
+ await symlink(path.join("..", "..", "media", VIDEO, "audio.mp3"), path.join(t.videoDir, "audio.mp3"));
+ await touch(path.join(t.videoDir, "audio.opus"));
+ const { probe, calls } = fakeProbe(300);
+ const want = { video: VIDEO, slug: SLUG, from: 10, to: 20 };
+ const hit = await resolveLocalSource(want, { channelsDir: t.channelsDir, probe, audio: true });
+ assert.equal(hit.name, "audio.opus");
+ assert.deepEqual(calls, [path.join(t.videoDir, "audio.opus")]);
+ // exact: the raw file named for the span, else (allowed) the sound -- never a corpus window.
+ await touch(path.join(t.clipsDir, "0.00-60.00.mp4"));
+ const cfg = { rawDir: t.rawDir, channelsDir: t.channelsDir, probe, exact: true };
+ assert.equal(await resolveLocalSource(want, cfg), null);
+ assert.equal((await resolveLocalSource(want, { ...cfg, audio: true })).kind, "audio");
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("audio: the bench's corpus windows leave the sound out", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.videoDir, "audio.mp3"));
+ const got = await corpusWindowsOf({ video: VIDEO, slug: SLUG, channelsDir: t.channelsDir, probe: fakeProbe(90).probe });
+ assert.deepEqual(got, []);
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("audioAllowed: no picture to fetch, or no fetch to make", () => {
+ const web = { platform: "youtube", webpageUrl: "https://example.invalid/v" };
+ // A clip with a picture somewhere, and a network to get it: never the sound.
+ assert.equal(audioAllowed({ entry: {}, meta: web, render: {} }), false);
+ assert.equal(audioAllowed({ entry: {}, render: {} }), false, "no record read yet");
+ // No picture to fetch.
+ assert.equal(audioAllowed({ entry: { audioOnly: true }, meta: web }), true);
+ assert.equal(audioAllowed({ entry: {}, meta: web, render: { preferLocalAudio: true } }), true);
+ assert.equal(audioAllowed({ entry: {}, meta: { ...web, platform: "podcast" } }), true);
+ assert.equal(audioAllowed({ entry: {}, meta: { ...web, platform: "RSS" } }), true);
+ // No fetch to make: --no-network / --skip-fetch, or a record with no page.
+ assert.equal(audioAllowed({ entry: {}, meta: web, network: false }), true);
+ assert.equal(audioAllowed({ entry: {}, meta: { platform: "youtube", webpageUrl: null } }), true);
+});
+
+test("planLocalSources: an audio-only clip is satisfied, and named as such, never a miss", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.videoDir, "audio.mp3"));
+ const timeline = [
+ { id: "c1", type: "clip", video: VIDEO, start: 10, end: 20 },
+ { id: "c2", type: "clip", video: "def456", start: 10, end: 20 },
+ { id: "c3", type: "clip", video: VIDEO, start: 30, end: 40, audioOnly: true },
+ ];
+ const local = localSources({ rawDir: t.rawDir, channelsDir: t.channelsDir, channelSlug: SLUG, probe: fakeProbe(600).probe });
+ const render = { fetchPad: 3 };
+
+ // --no-network: the sound serves c1 and c3; c2 has nothing.
+ let plan = await planLocalSources(timeline, timeline, render, { noNetwork: true }, local);
+ assert.deepEqual(plan.missing.map((m) => m.id), ["c2"]);
+ assert.deepEqual(plan.audio.map((m) => [m.index, m.id, m.local]), [[0, "c1", "audio.mp3"], [2, "c3", "audio.mp3"]]);
+ assert.deepEqual(await clipsNeedingFetch(timeline, timeline, render, { noNetwork: true }, local), plan.missing);
+ const msg = audioOnlyMessage(plan.audio);
+ assert.match(msg, /2 clip\(s\) play from audio only/);
+ assert.match(msg, /timeline\[0\] c1 {2}demo-channel\/abc123 {2}audio\.mp3/);
+
+ // With the network: only the clip that asked for its sound gets it before a fetch.
+ plan = await planLocalSources(timeline, timeline, render, {}, local);
+ assert.deepEqual(plan.missing.map((m) => m.id), ["c1", "c2"]);
+ assert.deepEqual(plan.audio.map((m) => m.id), ["c3"]);
+ // ...and every clip of a cut that prefers local sound.
+ plan = await planLocalSources(timeline, timeline, { ...render, preferLocalAudio: true }, {}, local);
+ assert.deepEqual(plan.audio.map((m) => m.id), ["c1", "c3"]);
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("buildVideo --no-network: an audio-only clip is not in the refusal", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.videoDir, "audio.mp3"));
+ const manifestPath = path.join(t.root, "video.manifest.json");
+ await writeFile(manifestPath, JSON.stringify({
+ slug: "demo",
+ title: "Demo",
+ provenance: { channelSlug: SLUG, channelsDir: "channels", siteOrigin: "https://example.invalid" },
+ render: { width: 1280, height: 720, fps: 30, fetchPad: 3 },
+ timeline: [
+ { id: "c1", type: "clip", video: VIDEO, start: 10, end: 20 },
+ { id: "c2", type: "clip", video: "def456", start: 1, end: 4 },
+ ],
+ }));
+ await assert.rejects(
+ buildVideo({ manifestPath, opts: { noNetwork: true, probe: fakeProbe(600).probe } }),
+ (err) => {
+ assert.match(err.message, /1 clip\(s\) have no local source/);
+ assert.match(err.message, /c2/);
+ assert.doesNotMatch(err.message, /c1/);
+ return true;
+ },
+ );
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
