commit 320eda0583d00580cbde2e13d52fb20d50777ac4
parent 0c3f1a34e16afc39ca20a8359c500f20e9d3ff80
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sun, 4 Oct 2026 16:54:50 -0400
report-to-video: an audio tier, a poster where a source has no picture, and clips with their own src/cues
sources.mjs gains a fourth tier, `audio` (data/<id>/audio.<ext>, the window
[0, duration]), consulted only when audioAllowed says so: a clip with no
picture to fetch (audioOnly, render.preferLocalAudio, a feed platform) reads
it before the network; any other clip only when it cannot fetch (--no-network,
--skip-fetch, a record with no page). Video tiers always win. --no-network
lists audio-only clips as satisfied.
A clip whose source has no video stream plays under a poster: its card
(channel, title, date) rendered at the picture box's size, with the sound's
waveform (render.audioPoster.waveform, default on), standing in for [0:v]
so every later filter and the encode are unchanged.
A clip may carry `src` and `cues`, paths relative to the manifest
(local-media.mjs): no corpus lookup, the window is the whole file, start/end
default to its ends, cues in either the transcript.cues.json or the
parakeet-stitch shape, no derived QR. resolve-windows reads them too.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
8 files changed, 752 insertions(+), 72 deletions(-)
diff --git a/umtool/lib/projects/report.mjs b/umtool/lib/projects/report.mjs
@@ -869,6 +869,9 @@ export async function reportDecisions(ctx, summary) {
// --- per clip, against the cues -----------------------------------------
const byVideo = new Map();
for (const e of clips) {
+ // A clip with its own media (`src`, `cues`) has no corpus cue file: the
+ // build checks its paths and its window itself.
+ if (e.src != null) continue;
const key = `${channelFor(m, e)}/${e.video}`;
if (!byVideo.has(key)) byVideo.set(key, []);
byVideo.get(key).push(e);
diff --git a/umtool/report-to-video/build-video.mjs b/umtool/report-to-video/build-video.mjs
@@ -89,8 +89,11 @@ import { DEFAULT_CHANNELS_DIR, createCueSource, siteOriginFromManifest } from ".
// Where a clip's media is ALREADY on disk -- the build's raw cache, the
// editor's corpus windows, the saved source -- asked before anything fetches.
import {
- SHADOW_CHANNELS, channelsDirFor, findContainingWindow, rawWindowName, resolveLocalSource,
+ SHADOW_CHANNELS, WIN_EPS as SRC_EPS, audioAllowed, channelsDirFor, findContainingWindow, rawWindowName,
+ resolveLocalSource,
} from "./sources.mjs";
+// A clip whose media is a file beside the manifest (`src`, `cues`).
+import { clipLabel, createLocalMedia, hasLocalMedia, validateLocalMedia } from "./local-media.mjs";
import { createPostChannelResolver, resolvePostLinks } from "./post-links.mjs";
import { ensureWriteDir } from "../lib/report/storage.mjs";
// The deck (`render.chrome`): its geometry, validation and schedule are pure
@@ -110,7 +113,7 @@ import { platformArgsForUrl } from "yt-dlp-transcript-common/ytdlp/platformArgs.
// the clip bench imports too -- a preview that shows a line this renderer
// would never draw is worse than no preview.
import {
- attributionLine, channelName, hms, imageAttributionLine, uploadDateToIso,
+ attributionLine, attributionParts, channelName, hms, imageAttributionLine, uploadDateToIso,
} from "./attribution.mjs";
// An opt-in brand preset (`render.brand`). Every hook below branches on the
// brand BEFORE building an argument, so a manifest without one renders exactly
@@ -136,6 +139,8 @@ export { FFMPEG as FFMPEG_BIN, FFPROBE as FFPROBE_BIN };
// `transcripts/` directory. Built once main() has the manifest (it carries the
// archive origin); see cues.mjs.
let CUES = null;
+// The manifest's own media (`src` clips): set by buildVideo() beside CUES.
+let LOCAL = null;
const exists = (p) => access(p).then(() => true, () => false);
@@ -359,7 +364,8 @@ const HUMAN = {
fetch: (e) =>
e.cached
? ` source ${e.id}: ${e.source ?? "raw-cache"} ${e.reuse ?? "(exact window)"} covers ` +
- `${hms(e.from)}–${hms(e.to)} — no download` + (e.height ? ` (${e.height}p)` : "")
+ `${hms(e.from)}–${hms(e.to)} — no download` + (e.height ? ` (${e.height}p)` : "") +
+ (e.source === "audio" ? " — audio only, under a poster" : "")
: ` fetch ${e.id}: ${e.video} ${hms(e.from)}–${hms(e.to)} (network)`,
snap: (e) =>
` snap ${e.id}: ${e.start ? "start✓" : "start–"} ${e.end ? "end✓" : "end–"} ` +
@@ -438,10 +444,26 @@ export async function videoMeta(videoId, channelSlug, hints = {}) {
// this record is loaded for the title and the upload date anyway.
return {
title: d.title, uploadDate: d.uploadDate, webpageUrl: d.webpageUrl,
- duration: d.duration, channel: d.channel,
+ duration: d.duration, channel: d.channel, platform: d.platform ?? null,
};
}
+/**
+ * A clip's metadata, wherever it lives: a `src` clip's from its own cue file
+ * (local-media.mjs), every other clip's from its corpus record. Every reader
+ * of a clip's title, date and channel asks this, so a `src` clip is named the
+ * same in its header, its chapter and the deck.
+ */
+export async function clipMeta(entry, provenance = {}) {
+ if (hasLocalMedia(entry)) {
+ if (!LOCAL) throw new Error("clipMeta: no local media reader — it is set up by buildVideo()");
+ return LOCAL.meta(entry);
+ }
+ return videoMeta(entry.video, entry.channel ?? provenance?.channelSlug, {
+ siteChannel: entry.siteChannel, siteVideo: entry.siteVideo,
+ });
+}
+
// The CONTAINER's duration is max(video, audio), and the audio is longer: the
// AAC encoder pads the front with ~21 ms of decoder delay, and a video duration
// is rarely an exact multiple of the frame interval. Either way the excess is
@@ -469,6 +491,84 @@ export async function probeDuration(file, fps) {
return Number(stdout.trim());
}
+/**
+ * What a clip's source file holds: its duration, and whether it has a PICTURE
+ * -- a video stream that is not a cover image (an mp3's attached picture is
+ * one frame of album art, not footage).
+ *
+ * @returns {Promise<{ hasVideo: boolean, duration: number|null }>}
+ */
+export async function probeMedia(file) {
+ const { stdout } = await execFileP(FFPROBE, [
+ "-v", "error",
+ "-show_entries", "stream=codec_type:stream_disposition=attached_pic:format=duration",
+ "-of", "json", file,
+ ]);
+ const doc = JSON.parse(stdout);
+ const hasVideo = (doc.streams ?? []).some(
+ (st) => st.codec_type === "video" && !Number(st.disposition?.attached_pic),
+ );
+ const duration = Number(doc.format?.duration);
+ return { hasVideo, duration: Number.isFinite(duration) && duration > 0 ? duration : null };
+}
+
+// ---- the audio poster ------------------------------------------------------
+// A clip whose source has no picture -- a podcast episode, a channel that kept
+// only its sound (sources.mjs's audio tier), a `src` that is an mp3 -- plays
+// under a POSTER where the footage would be: the clip's own card (channel,
+// title, date: attributionParts, the header's resolver) drawn by renderCard at
+// exactly the picture box's size, with the sound's waveform moving along its
+// foot (`render.audioPoster.waveform`, default on).
+//
+// It stands in for `[0:v]` and nothing else: every filter after it -- the
+// letterbox, the header, the footer, the deck's framing -- runs on it as on
+// footage, and the encode is the same, so the segment's size, fps, pixel
+// format, SAR and audio layout are every other segment's and the concat
+// cannot tell it from one.
+
+/** Where the waveform sits in a WxH poster: along the foot, inside the card's margins. */
+export function posterWaveGeometry(W, H) {
+ const even = (n) => Math.max(2, 2 * Math.round(n / 2));
+ const x = Math.round(W * 0.09);
+ const width = even(W - 2 * x);
+ const height = even(H * 0.16);
+ return { x, y: H - height - Math.round(H * 0.07), width, height };
+}
+
+/**
+ * The filter parts that make the poster's picture, labelled `[pic]`: the
+ * still, then the waveform of input 0's sound over it.
+ *
+ * @param {{ posterIdx: number, box: { width: number, height: number }, render: any }} args
+ */
+export function audioPosterParts({ posterIdx, box, render }) {
+ if (render.audioPoster?.waveform === false) return [`[${posterIdx}:v]null[pic]`];
+ const g = posterWaveGeometry(box.width, box.height);
+ return [
+ `[0:a]aformat=channel_layouts=mono,` +
+ `showwaves=s=${g.width}x${g.height}:mode=cline:rate=${render.fps}:colors=${render.palette.accent}[wave]`,
+ `[${posterIdx}:v][wave]overlay=x=${g.x}:y=${g.y}[pic]`,
+ ];
+}
+
+/**
+ * Render the poster still and return its ffmpeg input: a `seconds`-long loop
+ * at the cut's fps, so the picture ends where the sound does.
+ */
+async function audioPosterInput({ entry, meta, provenance, render, outDir, box, seconds }) {
+ const parts = attributionParts(entry, meta ?? {}, provenance ?? {});
+ const card = {
+ id: `${entry.id}.poster`,
+ style: "chapter",
+ ...(parts.channel ? { kicker: parts.channel } : {}),
+ heading: parts.title || clipLabel(entry),
+ ...(parts.date ? { sub: parts.date } : {}),
+ };
+ // At the box's size, and with no rail: the card IS the picture box.
+ const png = await renderCard(card, { ...render, width: box.width, height: box.height, rail: undefined }, outDir, null);
+ return ["-loop", "1", "-framerate", String(render.fps), "-t", seconds.toFixed(3), "-i", png];
+}
+
// yt-dlp exits 101 on a clean early stop (break-on-existing / max-downloads).
// The repo treats that as success everywhere else; do the same here.
const ytdlpOk = (err) => err?.code === 101;
@@ -574,13 +674,14 @@ export function localSources({ rawDir, channelsDir, channelSlug = null, probe }
rawDir,
channelsDir,
slugOf,
- async resolve(entry, span, { exact = false } = {}) {
+ // `audio` admits the audio tier: sources.mjs `audioAllowed` decides it.
+ async resolve(entry, span, { exact = false, audio = false } = {}) {
const slug = slugOf(entry);
- const key = `${slug}/${entry.video}/${span.from}/${span.to}/${exact}`;
+ const key = `${slug}/${entry.video}/${span.from}/${span.to}/${exact}/${audio}`;
if (memo.has(key)) return memo.get(key);
const hit = await resolveLocalSource(
{ video: entry.video, slug, from: span.from, to: span.to },
- { rawDir, channelsDir, exact, probe },
+ { rawDir, channelsDir, exact, probe, audio },
);
// Hits only: a miss is about to be fetched into the raw cache, and the
// next clip asking for the same span must find that file.
@@ -590,19 +691,27 @@ export function localSources({ rawDir, channelsDir, channelSlug = null, probe }
};
}
+/** May this build reach the network at all? */
+const networkOn = (opts) => !opts.noNetwork && !opts.skipFetch;
+
/**
- * The clips of `entries` that no local source can serve, i.e. the ones that
- * would need a fetch: `--no-network` refuses the build on any of them before
- * a frame is rendered. `index` is the entry's position in the manifest's
- * timeline.
+ * Where each clip of `entries` would come from without the network: `missing`
+ * are the ones no local source serves -- `--no-network` refuses the build on
+ * any of them before a frame is rendered -- and `audio` the ones only their
+ * sound serves, which play under a poster. `index` is the entry's position in
+ * the manifest's timeline. A `src` clip is its own source and is in neither.
*/
-export async function clipsNeedingFetch(entries, timeline, render, opts, local) {
- const out = [];
+export async function planLocalSources(entries, timeline, render, opts, local) {
+ const missing = [];
+ const audio = [];
for (const entry of entries) {
- if (!isClipEntry(entry)) continue;
+ if (!isClipEntry(entry) || hasLocalMedia(entry)) continue;
const span = fetchSpan(entry, render, opts);
- if (await local.resolve(entry, span, { exact: !!opts.noReuse })) continue;
- out.push({
+ const hit = await local.resolve(entry, span, {
+ exact: !!opts.noReuse,
+ audio: audioAllowed({ entry, render, network: networkOn(opts) }),
+ });
+ const row = {
// By id: a variant's view may be a copy of the manifest's entry.
index: timeline.findIndex((e) => e === entry || (entry.id != null && e?.id === entry.id)),
id: entry.id,
@@ -610,9 +719,26 @@ export async function clipsNeedingFetch(entries, timeline, render, opts, local)
video: entry.video,
from: span.from,
to: span.to,
- });
+ };
+ if (!hit) missing.push(row);
+ else if (hit.kind === "audio") audio.push({ ...row, local: hit.name });
}
- return out;
+ return { missing, audio };
+}
+
+/** planLocalSources' `missing`: the clips that would need a fetch. */
+export async function clipsNeedingFetch(entries, timeline, render, opts, local) {
+ return (await planLocalSources(entries, timeline, render, opts, local)).missing;
+}
+
+/** One line per clip that plays from its sound alone, for the log. */
+export function audioOnlyMessage(audio) {
+ return (
+ `${audio.length} clip(s) play from audio only (a poster where the picture would be):\n` +
+ audio
+ .map((m) => ` timeline[${m.index}] ${m.id} ${m.slug ?? "(no channel)"}/${m.video} ${m.local}`)
+ .join("\n")
+ );
}
/** One line per clip the network would have to serve, for the refusal. */
@@ -636,6 +762,21 @@ async function fetchClip(entry, meta, render, local, opts) {
const { rawDir } = local;
const { from, to } = fetchSpan(entry, render, opts);
+ // A `src` clip IS its source: the whole file, [0, duration], beside the
+ // manifest. Nothing is looked up and nothing is fetched.
+ if (hasLocalMedia(entry)) {
+ const file = LOCAL.file(entry);
+ const duration = entry.srcDuration ?? (await probeMedia(file)).duration;
+ EMIT("fetch", {
+ id: entry.id, video: clipLabel(entry), from, to, cached: true, source: "src",
+ reuse: path.basename(file), local: file, window: [0, duration],
+ });
+ return {
+ path: file, fetchStart: 0, cached: true, source: "src",
+ scan: { from: Math.max(0, from), to: Number.isFinite(duration) ? Math.min(to, duration) : to },
+ };
+ }
+
// Shared across variants, and deliberately so: this is the only expensive
// thing in a build, and the two cuts overlap almost entirely.
const name = rawWindowName(entry.video, from, to);
@@ -646,7 +787,10 @@ async function fetchClip(entry, meta, render, local, opts) {
// named for exactly this span. fetchStart is the SOURCE's start, not the
// requested one -- every cut downstream is relative to it, so where the
// bytes came from is transparent.
- const hit = await local.resolve(entry, { from, to }, { exact: !!opts.noReuse });
+ // The audio tier only by audioAllowed's rule: before the network for a clip
+ // with no picture to fetch, otherwise only when there is no network to ask.
+ const audio = audioAllowed({ entry, meta, render, network: networkOn(opts) });
+ const hit = await local.resolve(entry, { from, to }, { exact: !!opts.noReuse, audio });
if (hit) {
EMIT("fetch", {
id: entry.id, video: entry.video, from, to, cached: true, source: hit.kind,
@@ -903,14 +1047,29 @@ async function buildClipSegment(entry, meta, render, dirs, opts, chrome, nodes,
// and the segment is the picture alone, framed into the box above the deck.
// The composition is overlaid on the whole concat later; nothing here knows
// about it. Same cut, same audio map, same encode as every other segment.
+ // No picture in the source: a poster stands in for `[0:v]` (see above).
+ const media = await probeMedia(raw);
+ const posterSeconds = Math.max(1 / render.fps, (media.duration != null ? Math.min(cutB, media.duration) : cutB) - cutA);
+ if (!media.hasVideo) EMIT("note", { message: `${entry.id}: no picture in ${path.basename(raw)} — a poster plays under its sound` });
+ const poster = (box, posterIdx) =>
+ media.hasVideo
+ ? null
+ : audioPosterInput({ entry, meta, provenance, render, outDir, box, seconds: posterSeconds })
+ .then((input) => ({ input, parts: audioPosterParts({ posterIdx, box, render }) }));
+
if (deckOn(render)) {
const fr = framing ?? segmentFraming(render);
+ const pst = await poster(fr.box, 1);
await execFileP(
FFMPEG,
[
"-nostdin", "-v", "error", "-y",
...cutArgs(raw, cutA, cutB),
- "-filter_complex", `[0:v]${deckFramingFilter(render, { feed: fr.layout === "feed" })}[v]`,
+ ...(pst ? pst.input : []),
+ "-filter_complex", [
+ ...(pst ? pst.parts : []),
+ `${pst ? "[pic]" : "[0:v]"}${deckFramingFilter(render, { feed: fr.layout === "feed" })}[v]`,
+ ].join(";"),
"-map", "[v]", "-map", "0:a",
...encodeArgs(render),
seg,
@@ -1014,16 +1173,24 @@ async function buildClipSegment(entry, meta, render, dirs, opts, chrome, nodes,
markIdx = nextIdx++;
inputs.push("-i", await markPng(render, brandMark.mark, path.join(outDir, "cards")));
}
+ // The poster, when the source has no picture: after everything else.
+ const pst = await poster({ width: VW, height: VH }, nextIdx);
+ if (pst) {
+ nextIdx += 1;
+ inputs.push(...pst.input);
+ }
+ const pic = pst ? "[pic]" : "[0:v]";
const parts = hasFooter
? [
- `[0:v]${base}[b]`,
+ ...(pst ? pst.parts : []),
+ `${pic}${base}[b]`,
`[b][${footerIdx}:v]overlay=0:${height - FH}[f]`,
`[${barIdx}:v]crop=w=${chrome.trackLen}:h=3:x='${chrome.trackLen}-(${fillExpr})':y=0[bar]`,
`[f][bar]overlay=x=${chrome.x0}:y=${trackAbsY - 1}[g]`,
`[g][${markerIdx}:v]overlay=x=${markX}:y=${trackAbsY - chrome.markerRadius}[q]`,
]
- : [`[0:v]${base}[q]`];
+ : [...(pst ? pst.parts : []), `${pic}${base}[q]`];
let q = "q";
if (brandMark) {
@@ -1075,6 +1242,8 @@ export async function readCutRecord(seg) {
// not scan) and it must keep its quiet zone (the white border is part of the
// symbol, not decoration).
async function qrForEntry(entry, provenance, render, outDir) {
+ // A `src` clip's file has no page on the archive: only its own citeUrl.
+ if (hasLocalMedia(entry) && !entry.citeUrl) return null;
const q = render.qr ?? {};
// A mirror's LOCAL slug is not the id the site serves, and a clip taken from a
// copy whose archived transcript is broken should point at the copy that reads —
@@ -3249,7 +3418,7 @@ export async function chapterTitle(entry, index, provenance, { deck = false } =
}
if (entry.type !== "clip") return entry.title ?? entry.heading ?? `Card ${index + 1}`;
try {
- const meta = await videoMeta(entry.video, entry.channel ?? provenance.channelSlug, { siteChannel: entry.siteChannel, siteVideo: entry.siteVideo });
+ const meta = await clipMeta(entry, provenance);
// The SAME overrides the header honours, resolved by the SAME functions. A
// chapter list that says 2019 for a clip whose burned-in line says 2016 is
// the mp4 disagreeing with itself, and only one of the two is on screen
@@ -3258,11 +3427,11 @@ export async function chapterTitle(entry, index, provenance, { deck = false } =
// lands in.
const who = channelName(entry, meta, provenance);
const date = entry.date ?? uploadDateToIso(meta.uploadDate);
- const title = String(entry.title ?? meta.title ?? entry.video);
+ const title = String(entry.title ?? meta.title ?? clipLabel(entry));
const tail = `${date} — ${title.length > 60 ? `${title.slice(0, 57)}…` : title}`.trim();
return who ? `${who} · ${tail}` : tail;
} catch {
- return `${index + 1}. ${entry.video}`;
+ return `${index + 1}. ${clipLabel(entry)}`;
}
}
@@ -3296,11 +3465,7 @@ export async function measureChromeSchedule({ manifest, entries, segments, D })
metas.push(null);
continue;
}
- metas.push(
- await videoMeta(e.video, e.channel ?? provenance.channelSlug, {
- siteChannel: e.siteChannel, siteVideo: e.siteVideo,
- }).catch(() => null),
- );
+ metas.push(await clipMeta(e, provenance).catch(() => null));
}
// The variant's posts, placed on the clips this cut plays with their real
// upload dates. A manifest without posts writes the schedule it always did.
@@ -3429,11 +3594,48 @@ async function assertConcatLength(file, expected, fps, what) {
* ffmpeg inside a request handler has no cancellation story, and a runaway
* grandchild would outlive the request that started it.
*/
+/**
+ * A cut's `src` clips (local-media.mjs), made ready before anything else runs:
+ * every path checked -- a refusal names each entry, before a fetch or a frame
+ * -- the reader set, and each clip's window filled in on this run's copy:
+ * `start`/`end` default to the file's two ends and must lie inside it.
+ */
+export async function prepareLocalMedia(timeline, manifestDir, render = {}) {
+ const allowAbsolute = render?.allowAbsoluteSrc === true;
+ const errors = await validateLocalMedia(timeline, manifestDir, { allowAbsolute, isClip: isClipEntry });
+ if (errors.length) throw new Error(`manifest: ${errors.join("; ")}`);
+ LOCAL = createLocalMedia({ baseDir: manifestDir, allowAbsolute });
+ for (const [i, e] of (timeline ?? []).entries()) {
+ if (!e || !isClipEntry(e) || !hasLocalMedia(e)) continue;
+ const who = `timeline[${i}] ${e.id}`;
+ const { duration } = await probeMedia(LOCAL.file(e)).catch(() => ({ duration: null }));
+ if (duration == null) {
+ errors.push(`${who}: ffprobe cannot read a duration from ${e.src}`);
+ continue;
+ }
+ e.srcDuration = duration;
+ e.start ??= 0;
+ e.end ??= duration;
+ if (!(Number.isFinite(e.start) && Number.isFinite(e.end) && e.start >= 0 && e.end > e.start)) {
+ errors.push(`${who}: start/end must be seconds with 0 ≤ start < end (got ${e.start}–${e.end})`);
+ } else if (e.end > duration + SRC_EPS) {
+ errors.push(`${who}: end ${e.end} is past the end of ${e.src} (${duration.toFixed(2)} s)`);
+ }
+ }
+ if (errors.length) throw new Error(`manifest: ${errors.join("; ")}`);
+}
+
export async function buildVideo({ manifestPath, opts = {}, out, only, fetchOnly } = {}) {
const variant = opts.variant ?? "sourced";
const whole = JSON.parse(await readFile(manifestPath, "utf8"));
const manifest = selectVariant(whole, variant);
const { render, provenance } = manifest;
+ // An `image` entry's `src` is relative to the MANIFEST, which is checked in
+ // beside the pictures it cites -- not to the cwd the build was started from.
+ // So is a clip's.
+ const manifestDir = path.dirname(path.resolve(manifestPath));
+ // A `src` clip's file and cues, checked and measured before anything else.
+ await prepareLocalMedia(manifest.timeline, manifestDir, render);
// A `render.chrome` that cannot be built is refused here, before a single
// fetch is spent. Absent, validateChrome has nothing to say.
if (render.chrome !== undefined && render.chrome !== null) assertChrome(render.chrome, render);
@@ -3450,9 +3652,6 @@ export async function buildVideo({ manifestPath, opts = {}, out, only, fetchOnly
const errors = validatePosts(whole.posts, whole.timeline ?? [], whole.render);
if (errors.length) throw new Error(`posts: ${errors.join("; ")}`);
}
- // An `image` entry's `src` is relative to the MANIFEST, which is checked in
- // beside the pictures it cites -- not to the cwd the build was started from.
- const manifestDir = path.dirname(path.resolve(manifestPath));
// Where each post's QR goes: its page on the archive (`posts.links`
// "archive", the default), which needs the archive channel that keeps it.
@@ -3517,8 +3716,9 @@ export async function buildVideo({ manifestPath, opts = {}, out, only, fetchOnly
// A still has nothing to fetch and is already on disk, so this is a no-op
// rather than an error: a bench that walks the timeline asking for each
// entry's window should not have to know which kinds have one.
- if (entry?.type === "image" || entry?.type === "teaser") {
- EMIT("note", { message: `${fetchOnly} is ${entry.type === "image" ? "an image" : "a teaser"} entry — nothing to fetch` });
+ if (entry?.type === "image" || entry?.type === "teaser" || (entry?.type === "clip" && hasLocalMedia(entry))) {
+ const what = entry.type === "image" ? "an image" : entry.type === "teaser" ? "a teaser" : "a clip with its own `src`";
+ EMIT("note", { message: `${fetchOnly} is ${what} entry — nothing to fetch` });
EMIT("done", { out: null, nothingToFetch: true });
return { out: null, failures: [] };
}
@@ -3569,8 +3769,11 @@ export async function buildVideo({ manifestPath, opts = {}, out, only, fetchOnly
// rebuild no segment fetch nothing, so they are not asked.)
if (opts.noNetwork && !opts.chaptersOnly && !opts.railOnly && !opts.chromeOnly && !opts.chromePreview) {
const want = manifest.timeline.filter((e) => !only || e.id === only);
- const missing = await clipsNeedingFetch(want, whole.timeline ?? [], render, opts, dirs.local);
+ const { missing, audio } = await planLocalSources(want, whole.timeline ?? [], render, opts, dirs.local);
if (missing.length) throw new Error(needsFetchMessage(missing));
+ // Satisfied, not missed: said once, up front, so a cut that will show a
+ // poster instead of a face is never a surprise at the end.
+ if (audio.length) EMIT("note", { message: audioOnlyMessage(audio) });
}
// Footer chrome is shared by every clip, so build it once up front. The deck
@@ -3784,9 +3987,9 @@ export async function buildVideo({ manifestPath, opts = {}, out, only, fetchOnly
: await buildLedgerSegment(entry, render, outDir, manifest.ledger, availability),
);
} else {
- const meta = await videoMeta(entry.video, entry.channel ?? provenance.channelSlug, { siteChannel: entry.siteChannel, siteVideo: entry.siteVideo });
+ const meta = await clipMeta(entry, provenance);
EMIT("clip", {
- id: entry.id, i, n: entries.length, video: entry.video,
+ id: entry.id, i, n: entries.length, video: clipLabel(entry),
section: entry.section, sectionEnter: !!entry.sectionEnter,
});
segments.push(
diff --git a/umtool/report-to-video/deck.mjs b/umtool/report-to-video/deck.mjs
@@ -546,6 +546,8 @@ export function deckText(entry, meta, provenance, deck, multiChannel) {
export function deckQrUrl(entry, provenance = {}, { links = DECK_DEFAULTS.qr.links, meta = null } = {}) {
if (entry.type === "clip") {
if (entry.citeUrl) return entry.citeUrl;
+ // A clip from a file beside the manifest (`src`) has no page anywhere.
+ if (entry.src != null) return null;
const original = links === "original" ? originalUrlAt(meta?.webpageUrl, entry.start) : null;
return original ??
`${provenance.siteOrigin}/?v=${encodeURIComponent(
diff --git a/umtool/report-to-video/local-media.mjs b/umtool/report-to-video/local-media.mjs
@@ -0,0 +1,203 @@
+// local-media.mjs — a clip whose media is a FILE BESIDE THE MANIFEST.
+//
+// Most clips name a recording in the corpus (`video`, `channel`), and the build
+// finds its cues, its metadata and its media through that. A clip may instead
+// carry its own:
+//
+// { "type": "clip", "id": "p01",
+// "src": "media/episode-12.mp3", // video or audio-only
+// "cues": "media/episode-12.json", // optional
+// "start": 61.2, "end": 74.9 } // optional: default the whole file
+//
+// Both paths are RELATIVE TO THE MANIFEST, like an image entry's `src`, and may
+// not climb out of the manifest's directory: a manifest is checked in beside
+// the media it cites, and a `../../` that happens to resolve on one machine is
+// a build that breaks on the next. An absolute path is refused too, unless the
+// manifest says `render.allowAbsoluteSrc: true` -- media too big to keep beside
+// the manifest (a podcast archive on another drive) is a decision to write
+// down, not an accident to discover.
+//
+// The window is the whole file, [0, its probed duration]; `start`/`end` cut
+// inside it and default to its two ends. No channel, no corpus lookup, no QR to
+// a site (the file has no page there) unless the entry carries `citeUrl`.
+//
+// `cues` is read in either of two shapes and normalised by ONE function, so the
+// sentence-widening and the cut-to-quote passes read a manifest-local cue file
+// exactly as they read the corpus's:
+//
+// - the corpus's `transcript.cues.json`: `{ title, channel, uploadDate,
+// duration, platform, cues: [{ start, end, text }] }`
+// - the transcript `scripts/parakeet-stitch.mjs` writes: `{ duration_seconds,
+// chunks, text, chunk_data: [{ start_time, end_time, text }] }`
+//
+// A LEAF, like sources.mjs: no imports from build-video.mjs, and no path
+// computed from the cwd -- the manifest's directory arrives as an argument.
+import { readFile, stat } from "node:fs/promises";
+import path from "node:path";
+
+/** Does this clip carry its own media (rather than a corpus `video`)? Ask it of clips only. */
+export const hasLocalMedia = (entry) => entry != null && entry.src != null;
+
+/** What a clip is called in a log line: its corpus video, else its file. */
+export const clipLabel = (entry) => entry?.video ?? entry?.src ?? entry?.id ?? "?";
+
+/**
+ * One cue document in either shape, as the corpus's: `{ cues: [{start, end,
+ * text}], duration?, title?, channel?, uploadDate?, platform?, webpageUrl? }`.
+ * A cue with no finite start or end is dropped, never guessed at; a document in
+ * neither shape throws.
+ *
+ * @param {any} doc parsed JSON
+ * @param {string} [label] what the error names
+ */
+export function normaliseCueDoc(doc, label = "cues") {
+ if (!doc || typeof doc !== "object") throw new Error(`${label}: not a cue document`);
+ let raw;
+ let duration;
+ if (Array.isArray(doc.cues)) {
+ raw = doc.cues.map((c) => ({ start: c?.start, end: c?.end, text: c?.text }));
+ duration = doc.duration;
+ } else if (Array.isArray(doc.chunk_data)) {
+ raw = doc.chunk_data.map((c) => ({ start: c?.start_time, end: c?.end_time, text: c?.text }));
+ duration = doc.duration_seconds;
+ } else {
+ throw new Error(
+ `${label}: neither a transcript.cues.json (\`cues\`) nor a parakeet transcript (\`chunk_data\`)`,
+ );
+ }
+ const cues = [];
+ for (const c of raw) {
+ const start = Number(c.start);
+ const end = Number(c.end);
+ if (!Number.isFinite(start) || !Number.isFinite(end)) continue;
+ cues.push({ start, end, text: String(c.text ?? "") });
+ }
+ cues.sort((a, b) => a.start - b.start);
+ const d = Number(duration);
+ const out = { cues };
+ if (Number.isFinite(d) && d > 0) out.duration = d;
+ for (const k of ["title", "channel", "uploadDate", "platform", "webpageUrl"]) {
+ if (typeof doc[k] === "string" && doc[k]) out[k] = doc[k];
+ }
+ return out;
+}
+
+/** Read and normalise one cue file. */
+export async function readCueFile(file, label = file) {
+ return normaliseCueDoc(JSON.parse(await readFile(file, "utf8")), label);
+}
+
+/**
+ * A manifest-relative path, resolved -- or why it is refused. Relative paths
+ * must stay inside `baseDir`; an absolute one needs `allowAbsolute`.
+ *
+ * @returns {{ file: string } | { error: string }}
+ */
+export function resolveManifestPath(baseDir, p, { allowAbsolute = false } = {}) {
+ if (typeof p !== "string" || !p.trim()) return { error: "must be a non-empty path" };
+ if (path.isAbsolute(p)) {
+ if (!allowAbsolute) {
+ return { error: `${p} is absolute; set render.allowAbsoluteSrc to allow it` };
+ }
+ return { file: path.normalize(p) };
+ }
+ const file = path.resolve(/* turbopackIgnore: true */ baseDir, p);
+ const rel = path.relative(/* turbopackIgnore: true */ baseDir, file);
+ if (rel === ".." || rel.startsWith(`..${path.sep}`) || path.isAbsolute(rel)) {
+ return { error: `${p} climbs out of the manifest's directory` };
+ }
+ return { file };
+}
+
+const isFile = (p) => stat(p).then((s) => s.isFile(), () => false);
+
+/**
+ * Every problem with the timeline's manifest-local media, as sentences naming
+ * the entry: a path that escapes or is absolute without leave, a file that is
+ * not there, a cue file in no known shape, `src` beside a `video`, `cues`
+ * without `src`. Empty when there is nothing wrong (or nothing local).
+ *
+ * @param {any[]} timeline
+ * @param {string} baseDir the manifest's directory
+ * @param {{ allowAbsolute?: boolean, isClip?: (e: any) => boolean }} [opts]
+ * `isClip` is the build's own predicate (its vocabulary is open: anything it
+ * has no other branch for is a clip); other entries' `src` is not this one.
+ */
+export async function validateLocalMedia(timeline, baseDir, { allowAbsolute = false, isClip = (e) => e?.type === "clip" } = {}) {
+ const errors = [];
+ for (const [i, e] of (timeline ?? []).entries()) {
+ if (!e || !isClip(e)) continue;
+ const who = `timeline[${i}] ${e.id ?? "(no id)"}`;
+ if (e.src == null) {
+ if (e.cues != null) errors.push(`${who}: \`cues\` belongs to a clip with \`src\``);
+ continue;
+ }
+ if (e.video != null) errors.push(`${who}: a clip takes \`src\` or \`video\`, not both`);
+ const src = resolveManifestPath(baseDir, e.src, { allowAbsolute });
+ if (src.error) errors.push(`${who}: src ${src.error}`);
+ else if (!(await isFile(src.file))) errors.push(`${who}: no media at ${src.file} (src: ${e.src})`);
+ if (e.cues == null) continue;
+ const cues = resolveManifestPath(baseDir, e.cues, { allowAbsolute });
+ if (cues.error) {
+ errors.push(`${who}: cues ${cues.error}`);
+ } else if (!(await isFile(cues.file))) {
+ errors.push(`${who}: no cue file at ${cues.file} (cues: ${e.cues})`);
+ } else {
+ try {
+ await readCueFile(cues.file, e.cues);
+ } catch (err) {
+ errors.push(`${who}: ${err?.message ?? err}`);
+ }
+ }
+ }
+ return errors;
+}
+
+/**
+ * The reader a build (or resolve-windows) holds for one manifest's local
+ * media: each entry's resolved file, its cue document (read once) and the
+ * metadata the header, the chapters and the deck read in place of a corpus
+ * record's. Assumes validateLocalMedia passed.
+ *
+ * @param {{ baseDir: string, allowAbsolute?: boolean }} config
+ */
+export function createLocalMedia({ baseDir, allowAbsolute = false }) {
+ const docs = new Map();
+ const resolveOrThrow = (entry, key) => {
+ const r = resolveManifestPath(baseDir, entry[key], { allowAbsolute });
+ if (r.error) throw new Error(`${entry.id}: ${key} ${r.error}`);
+ return r.file;
+ };
+ return {
+ baseDir,
+ /** The entry's media file, absolute. */
+ file: (entry) => resolveOrThrow(entry, "src"),
+ /** Its normalised cue document, or null when it names none. */
+ async doc(entry) {
+ if (entry.cues == null) return null;
+ const file = resolveOrThrow(entry, "cues");
+ if (!docs.has(file)) docs.set(file, readCueFile(file, entry.cues));
+ return docs.get(file);
+ },
+ /** Its cues alone ([] without a cue file). */
+ async cues(entry) {
+ return (await this.doc(entry))?.cues ?? [];
+ },
+ /**
+ * The metadata a corpus record would have given: the cue document's when it
+ * carries any, the file's name as the title otherwise. The entry's own
+ * `title`, `date` and `channelTitle` still win where the header reads them.
+ */
+ async meta(entry) {
+ const d = await this.doc(entry);
+ return {
+ title: d?.title ?? path.basename(entry.src, path.extname(entry.src)),
+ uploadDate: d?.uploadDate ?? null,
+ webpageUrl: null,
+ duration: d?.duration ?? null,
+ channel: d?.channel ?? null,
+ platform: d?.platform ?? null,
+ };
+ },
+ };
+}
diff --git a/umtool/report-to-video/render-cards.mjs b/umtool/report-to-video/render-cards.mjs
@@ -727,7 +727,11 @@ function rosterCellSvg(cell, x, y, g, pal, face) {
// will not scan) and it must keep its quiet zone (the white border is part of
// the symbol, not decoration).
export function qrUrlFor(entry, provenance) {
- if (entry.type === "clip") {
+ // A clip from a file beside the manifest (`src`) has no moment on the archive
+ // to link: its own citeUrl, else -- the rail draws a code on every tile --
+ // the sweep's, as a card's.
+ const momentless = entry.type === "clip" && entry.src != null && !entry.citeUrl;
+ if (entry.type === "clip" && !momentless) {
// A mirror's LOCAL slug is not the id the site serves, so an explicit
// per-clip citeUrl always wins over the derived one.
return (
@@ -779,7 +783,7 @@ async function qrTileStrip(entries, provenance, render, g, outDir) {
const images = [];
entries.forEach((e, i) => {
const y = i * TH;
- const isClip = e.type === "clip";
+ const isClip = e.type === "clip" && !(e.src != null && !e.citeUrl);
body.push(
`<rect x="0.5" y="${y + 0.5}" width="${g.TILEW - 1}" height="${TH - 1}" fill="${pal.bg}" ` +
`stroke="${pal.accent}" stroke-width="1"/>`,
diff --git a/umtool/report-to-video/resolve-windows.mjs b/umtool/report-to-video/resolve-windows.mjs
@@ -40,10 +40,16 @@
// once a corpus moves past its last publish — see cues.mjs.
//
// A clip entry may set `lockStart` / `lockEnd` to pin that edge exactly.
+//
+// A clip with its own media (`src`) reads its own `cues` file, relative to the
+// manifest, in either shape local-media.mjs normalises; one with no `cues`, or
+// with no `start`/`end` (it plays the whole file), is left as it is.
import { readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
import { createCueSource, siteOriginFromManifest } from "./cues.mjs";
+import { clipLabel, createLocalMedia, hasLocalMedia } from "./local-media.mjs";
const ENDS_SENTENCE = /[.!?]["'”’)\]]*\s*$/;
@@ -305,19 +311,36 @@ function retargetCiteUrl(entry, cite, channelSlug) {
* worth having; a flag that silently did both would make one of those two
* decisions on their behalf.
*/
-async function cutPass(manifest, { loadCues, slug, opts, write, force }) {
+/**
+ * Why a clip cannot be resolved against cues, or null when it can: a `src`
+ * clip with no `cues` file, or one with no window (it plays the whole file).
+ */
+export function unresolvable(e) {
+ if (!hasLocalMedia(e)) return null;
+ if (e.cues == null) return "no cues file";
+ if (!Number.isFinite(e.start) || !Number.isFinite(e.end)) return "no start/end (the whole file)";
+ return null;
+}
+
+async function cutPass(manifest, { cuesOf, slug, opts, write, force }) {
let changed = 0;
let unmatched = 0;
let cites = 0;
for (const e of manifest.timeline) {
if (e.type !== "clip") continue;
const id = String(e.id).padEnd(4);
+ const label = String(clipLabel(e)).padEnd(12);
+ const why = unresolvable(e);
+ if (why) {
+ console.log(`${id} ${label} ${why} — left as is`);
+ continue;
+ }
if (e.lockCut) {
- console.log(`${id} ${String(e.video).padEnd(12)} lockCut — left at ${e.cutStart}–${e.cutEnd}`);
+ console.log(`${id} ${label} lockCut — left at ${e.cutStart}–${e.cutEnd}`);
continue;
}
if (!force && (e.cutStart != null || e.cutEnd != null)) {
- console.log(`${id} ${String(e.video).padEnd(12)} already cut ${e.cutStart}–${e.cutEnd} (--force-cut to redo)`);
+ console.log(`${id} ${label} already cut ${e.cutStart}–${e.cutEnd} (--force-cut to redo)`);
// The cut stands, but a cite left behind by an EARLIER run of this pass
// still names a second the clip does not play. Aligning it needs no
// cues, no network and no re-derivation, so it is not worth a --force.
@@ -334,20 +357,19 @@ async function cutPass(manifest, { loadCues, slug, opts, write, force }) {
}
continue;
}
- const chan = e.channel ?? slug;
- const cues = await loadCues(e.video, chan, { siteChannel: e.siteChannel, siteVideo: e.siteVideo });
+ const cues = await cuesOf(e);
const r = cutToQuote(cues, e.quote, { start: e.start, end: e.end }, opts);
const extent = `${e.start.toFixed(1)}–${e.end.toFixed(1)}`;
if (!r.ok) {
unmatched += 1;
console.log(
- `${id} ${String(e.video).padEnd(12)} ${extent} UNMATCHED (${r.score.toFixed(2)}) — ${r.why}`,
+ `${id} ${label} ${extent} UNMATCHED (${r.score.toFixed(2)}) — ${r.why}`,
);
if (r.matched) console.log(` best partial: ${r.matched.slice(0, 120)}`);
continue;
}
console.log(
- `${id} ${String(e.video).padEnd(12)} ${extent} -> cut ${r.cutStart.toFixed(1)}–${r.cutEnd.toFixed(1)} ` +
+ `${id} ${label} ${extent} -> cut ${r.cutStart.toFixed(1)}–${r.cutEnd.toFixed(1)} ` +
`(${(r.cutEnd - r.cutStart).toFixed(1)}s, match ${r.score.toFixed(2)})`,
);
console.log(` ${r.matched.slice(0, 120)}`);
@@ -389,7 +411,7 @@ async function main() {
const opts = { maxLead: num("--max-lead", 8), maxTail: num("--max-tail", 12) };
const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
- const slug = manifest.provenance.channelSlug;
+ const slug = manifest.provenance?.channelSlug;
const cache = new Map();
// Cues come from a local corpus when there is one, and from the published
@@ -405,13 +427,23 @@ async function main() {
prefer: flag("--cue-source") ?? "auto",
log: (m) => console.error(` · ${m}`),
});
- const loadCues = (videoId, channelSlug, hints) =>
- cues.load(channelSlug, videoId, hints).then((r) => r.cues);
+ // A `src` clip's cues are its own file, beside the manifest; every other
+ // clip's are its corpus record's.
+ const local = createLocalMedia({
+ baseDir: path.dirname(path.resolve(manifestPath)),
+ allowAbsolute: manifest.render?.allowAbsoluteSrc === true,
+ });
+ const cuesOf = (e) =>
+ hasLocalMedia(e)
+ ? local.cues(e)
+ : cues
+ .load(e.channel ?? slug, e.video, { siteChannel: e.siteChannel, siteVideo: e.siteVideo })
+ .then((r) => r.cues);
// ---- the cut pass, which is a different question ----
if (argv.includes("--cut-to-quote")) {
const r = await cutPass(manifest, {
- loadCues,
+ cuesOf,
slug,
opts: { ...opts, minMatch: num("--min-match", 0.6) },
write: argv.includes("--write"),
@@ -441,18 +473,18 @@ async function main() {
// An author can trim a clip to land mid-cue on purpose — a cue often carries
// a whole paragraph, and cutting a quote short is an editorial decision.
// Widening would undo exactly that, so `lock` opts the clip out.
- if (e.lock) {
- console.log(`${e.id.padEnd(4)} ${e.video.padEnd(12)} locked, left at ${e.start.toFixed(1)}–${e.end.toFixed(1)}`);
+ const label = String(clipLabel(e)).padEnd(12);
+ const why = unresolvable(e);
+ if (why) {
+ console.log(`${e.id.padEnd(4)} ${label} ${why} — left as is`);
continue;
}
- const chan = e.channel ?? slug;
- const key = `${chan}/${e.video}`;
- if (!cache.has(key)) {
- cache.set(
- key,
- await loadCues(e.video, chan, { siteChannel: e.siteChannel, siteVideo: e.siteVideo }),
- );
+ if (e.lock) {
+ console.log(`${e.id.padEnd(4)} ${label} locked, left at ${e.start.toFixed(1)}–${e.end.toFixed(1)}`);
+ continue;
}
+ const key = hasLocalMedia(e) ? `src:${e.cues}` : `${e.channel ?? slug}/${e.video}`;
+ if (!cache.has(key)) cache.set(key, await cuesOf(e));
const cues = cache.get(key);
const before = { start: e.start, end: e.end };
@@ -474,7 +506,7 @@ async function main() {
Math.abs(w.start - before.start) > 0.05 || Math.abs(w.end - before.end) > 0.05;
if (moved) changed += 1;
console.log(
- `${e.id.padEnd(4)} ${e.video.padEnd(12)} ` +
+ `${e.id.padEnd(4)} ${label} ` +
`${before.start.toFixed(1)}–${before.end.toFixed(1)} -> ` +
`${w.start.toFixed(1)}–${w.end.toFixed(1)} (+${dLead}s lead, +${dTail}s tail, ${dur}s)`,
);
@@ -493,9 +525,11 @@ async function main() {
// The later clip's start is the deliberate one, so trim the earlier clip's tail.
const byVideo = new Map();
for (const e of manifest.timeline) {
- if (e.type !== "clip") continue;
- if (!byVideo.has(e.video)) byVideo.set(e.video, []);
- byVideo.get(e.video).push(e);
+ if (e.type !== "clip" || unresolvable(e)) continue;
+ // A `src` clip's recording is its file.
+ const rec = hasLocalMedia(e) ? `src:${e.src}` : e.video;
+ if (!byVideo.has(rec)) byVideo.set(rec, []);
+ byVideo.get(rec).push(e);
}
for (const [video, list] of byVideo) {
if (list.length < 2) continue;
diff --git a/umtool/report-to-video/sources.mjs b/umtool/report-to-video/sources.mjs
@@ -12,6 +12,18 @@
// pointer (`data/<id>/saved-video.json` -> `<dir>/<file>`),
// else a `data/<id>/source-media.<ext>` not yet moved there.
// It is the window [0, its probed duration].
+// 4. audio `data/<id>/audio.<ext>`: the recording's SOUND alone, the
+// window [0, its probed duration]. The build draws a poster
+// where the picture would be.
+//
+// THE AUDIO TIER IS ASKED ONLY WHEN THE CALLER SAYS SO (`audio: true`), and
+// `audioAllowed` below is the one rule for when: a clip with no picture to
+// fetch (`audioOnly: true`, `render.preferLocalAudio`, or a record whose
+// platform carries none), or a build that could not fetch one anyway
+// (`--no-network`, `--skip-fetch`, a record with no page to fetch from). Being
+// last, it never beats a picture already on disk; asked before the network, it
+// is what plays a podcast. Every transcribed video keeps an `audio.mp3`, so
+// without the rule a cut would quietly lose its pictures to it.
//
// A window qualifies only if it holds the REQUESTED span whole -- the clip plus
// the build's fetch pad -- to WIN_EPS. Within a tier the tightest wins (the
@@ -42,7 +54,29 @@ import { promisify } from "node:util";
const execFileP = promisify(execFile);
/** The source kinds, in the order they are consulted. */
-export const SOURCE_KINDS = ["raw-cache", "corpus-window", "saved-video"];
+export const SOURCE_KINDS = ["raw-cache", "corpus-window", "saved-video", "audio"];
+
+/**
+ * Platforms whose records have no picture: a feed of episodes. A clip from one
+ * reads its local audio BEFORE the network, since a fetch has no video to get.
+ */
+export const AUDIO_ONLY_PLATFORMS = new Set(["podcast", "feed", "rss"]);
+
+/**
+ * Should the audio tier be consulted for this clip? (See the note at the top.)
+ *
+ * @param {{ entry?: any, meta?: { platform?: string|null, webpageUrl?: string|null }|null,
+ * render?: any, network?: boolean }} args
+ * `network` is whether this build may fetch at all; `meta` is the record,
+ * when it has been read (without it, only the entry and render can say).
+ */
+export function audioAllowed({ entry = null, meta = null, render = null, network = true } = {}) {
+ if (entry?.audioOnly === true || render?.preferLocalAudio === true) return true;
+ if (AUDIO_ONLY_PLATFORMS.has(String(meta?.platform ?? "").toLowerCase())) return true;
+ if (!network) return true;
+ // A record with no page has nothing a fetch could get.
+ return meta != null && !meta.webpageUrl;
+}
// A window read back from a 2 dp name can sit a hair outside the request that
// produced it; the same tolerance resolve-windows.mjs and common's
@@ -196,6 +230,24 @@ export async function wholeContainersOf(videoDir) {
return out;
}
+// The sound files a video dir may hold: common/lib/mediaFiles.ts's anchored
+// `audio.<ext>` over AUDIO_EXTS, plus the `.webm`/`.mp4` an extract-to-mp3 that
+// failed leaves behind (sound only). Read in AUDIO_READ_PREFERENCE's order --
+// the file the transcribe path reads -- then by name.
+const AUDIO_FILE_RE = /^audio\.(?:mp3|m4a|aac|ogg|oga|opus|wav|flac|webm|mp4)$/i;
+export const AUDIO_PREFERENCE = ["audio.mp3", "audio.m4a", "audio.opus"];
+
+/** A video dir's audio files, best first. Not yet checked for existence. */
+export async function audioFilesOf(videoDir) {
+ const names = (await listRawNames(videoDir)).filter((n) => AUDIO_FILE_RE.test(n));
+ const rank = (n) => {
+ const i = AUDIO_PREFERENCE.indexOf(n);
+ return i < 0 ? AUDIO_PREFERENCE.length : i;
+ };
+ names.sort((a, b) => rank(a) - rank(b) || a.localeCompare(b));
+ return names.map((name) => ({ name, path: path.join(/* turbopackIgnore: true */ videoDir, name) }));
+}
+
/**
* Is there a readable file at `p`, through every link on the way? A dangling
* link, a missing file and an unanswering drive are all "no", never a throw.
@@ -263,10 +315,25 @@ async function savedVideoWindows({ video, slug, channelsDir, probe = ffprobeSour
return out;
}
+// The ONE best audio file, not every one: they are all the same recording, and
+// probing three formats of it would buy nothing.
+async function audioWindows({ video, slug, channelsDir, probe = ffprobeSource }) {
+ if (!channelsDir || !slug) return [];
+ for (const f of await audioFilesOf(videoDirOf(channelsDir, slug, video))) {
+ if (!(await present(f.path))) continue;
+ const duration = Number((await probe(f.path))?.duration);
+ if (!Number.isFinite(duration) || duration <= 0) continue;
+ return [{ name: f.name, path: f.path, from: 0, to: duration }];
+ }
+ return [];
+}
+
+// `audio: true` marks a tier consulted only when the caller allows it.
export const TIERS = [
{ kind: "raw-cache", windows: rawCacheWindows },
{ kind: "corpus-window", windows: corpusWindows },
{ kind: "saved-video", windows: savedVideoWindows },
+ { kind: "audio", windows: audioWindows, audio: true },
];
const asSource = (kind, w) => ({
@@ -289,24 +356,29 @@ const asSource = (kind, w) => ({
* kinds?: string[] }} [config]
* `exact` (`--no-reuse`) accepts only the raw-cache file named for exactly
* this span. `kinds` narrows the tiers consulted (default: all, in order).
+ * `audio` admits the audio tier (default false; see `audioAllowed`).
* @returns {Promise<null | { kind: string, path: string, name: string,
* windowStart: number, windowEnd: number, height?: number }>}
*/
export async function resolveLocalSource(want, config = {}) {
const { video, slug = null, from, to } = want;
- const { rawDir = null, channelsDir = null, exact = false, probe, kinds = null } = config;
+ const { rawDir = null, channelsDir = null, exact = false, probe, kinds = null, audio = false } = config;
if (!video || !Number.isFinite(from) || !Number.isFinite(to)) return null;
if (exact) {
- if (!rawDir) return null;
- const name = rawWindowName(video, from, to);
- const p = path.join(/* turbopackIgnore: true */ rawDir, name);
- return (await present(p)) ? asSource("raw-cache", { name, path: p, from, to }) : null;
+ if (rawDir) {
+ const name = rawWindowName(video, from, to);
+ const p = path.join(/* turbopackIgnore: true */ rawDir, name);
+ if (await present(p)) return asSource("raw-cache", { name, path: p, from, to });
+ }
+ // `--no-reuse` re-cuts no cached window, but the sound is not one.
+ if (!audio) return null;
}
const ctx = { video, slug, rawDir, channelsDir, probe };
for (const tier of TIERS) {
if (kinds && !kinds.includes(tier.kind)) continue;
+ if (tier.audio ? !audio : exact) continue;
const windows = (await tier.windows(ctx)).filter((w) => windowContains(w, from, to));
// The raw cache's EXACT name first, as it always was; then tightest.
const exactName = tier.kind === "raw-cache" ? rawWindowName(video, from, to) : null;
@@ -327,12 +399,15 @@ export async function resolveLocalSource(want, config = {}) {
*
* `probe` decides a whole container's span. The bench passes a cheap one (the
* cue doc's duration) and is asked only for a video that has a container.
+ *
+ * Not the audio tier: nearly every transcribed video keeps its sound, and a
+ * bench that called each of those "fetched" would never offer a fetch.
*/
export async function corpusWindowsOf({ video, slug, channelsDir, probe = ffprobeSource }) {
const ctx = { video, slug, channelsDir, probe };
const out = [];
for (const tier of TIERS) {
- if (tier.kind === "raw-cache") continue;
+ if (tier.kind === "raw-cache" || tier.audio) continue;
for (const w of await tier.windows(ctx)) {
if (await present(w.path)) out.push({ ...w, kind: tier.kind });
}
diff --git a/umtool/report-to-video/sources.test.mjs b/umtool/report-to-video/sources.test.mjs
@@ -11,6 +11,7 @@ import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
import {
SOURCE_KINDS,
TIERS,
+ audioAllowed,
channelsDirFor,
corpusWindowsOf,
rawWindowName,
@@ -19,12 +20,14 @@ import {
windowsFromBareNames,
} from "./sources.mjs";
import {
+ audioOnlyMessage,
buildVideo,
clipsNeedingFetch,
fetchSpan,
findContainingWindow,
localSources,
needsFetchMessage,
+ planLocalSources,
} from "./build-video.mjs";
const SLUG = "demo-channel";
@@ -62,7 +65,8 @@ async function pointTo(videoDir, storeDir, file, extra = {}) {
test("the tiers are listed in the order they are consulted", () => {
assert.deepEqual(TIERS.map((t) => t.kind), SOURCE_KINDS);
- assert.deepEqual(SOURCE_KINDS, ["raw-cache", "corpus-window", "saved-video"]);
+ assert.deepEqual(SOURCE_KINDS, ["raw-cache", "corpus-window", "saved-video", "audio"]);
+ assert.deepEqual(TIERS.filter((t) => t.audio).map((t) => t.kind), ["audio"]);
});
test("rawWindowName is the build's cache key, two decimals always", () => {
@@ -355,3 +359,155 @@ test("buildVideo --no-network refuses before rendering, naming every clip that w
await rm(t.root, { recursive: true, force: true });
}
});
+
+// ---- the audio tier ------------------------------------------------------------
+
+test("audio: asked only when allowed, the whole file, the preferred format", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.videoDir, "audio.m4a"));
+ await touch(path.join(t.videoDir, "audio.mp3"));
+ // Not sound files: a subtitle, a partial, the transcribe scratch.
+ await touch(path.join(t.videoDir, "audio.en-orig.vtt"));
+ await touch(path.join(t.videoDir, "audio.mp3.part"));
+ await touch(path.join(t.videoDir, "audio.tmp-123.mp3"));
+ const { probe, calls } = fakeProbe(900);
+ const want = { video: VIDEO, slug: SLUG, from: 100, to: 110 };
+ const cfg = { channelsDir: t.channelsDir, probe };
+ assert.equal(await resolveLocalSource(want, cfg), null, "not without audio: true");
+ assert.equal(calls.length, 0, "and nothing probed for it");
+ const hit = await resolveLocalSource(want, { ...cfg, audio: true });
+ assert.deepEqual(hit, {
+ kind: "audio", path: path.join(t.videoDir, "audio.mp3"), name: "audio.mp3", windowStart: 0, windowEnd: 900,
+ });
+ assert.deepEqual(calls, [path.join(t.videoDir, "audio.mp3")], "one file probed, not every format");
+ // Past the recording's end is not covered.
+ assert.equal(await resolveLocalSource({ ...want, from: 895, to: 905 }, { ...cfg, audio: true }), null);
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("audio: every picture on disk wins over the sound, even when the sound is allowed", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.videoDir, "audio.mp3"));
+ await touch(path.join(t.storeDir, "source-media.mp4"));
+ await pointTo(t.videoDir, t.storeDir, "source-media.mp4");
+ const cfg = { rawDir: t.rawDir, channelsDir: t.channelsDir, probe: fakeProbe(600).probe, audio: true };
+ const want = { video: VIDEO, slug: SLUG, from: 100, to: 110 };
+ assert.equal((await resolveLocalSource(want, cfg)).kind, "saved-video");
+ await touch(path.join(t.clipsDir, "90.00-120.00.mp4"));
+ assert.equal((await resolveLocalSource(want, cfg)).kind, "corpus-window");
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("audio: a dangling audio link is not there; --no-reuse still reaches the sound", async () => {
+ const t = await tree();
+ try {
+ await symlink(path.join("..", "..", "media", VIDEO, "audio.mp3"), path.join(t.videoDir, "audio.mp3"));
+ await touch(path.join(t.videoDir, "audio.opus"));
+ const { probe, calls } = fakeProbe(300);
+ const want = { video: VIDEO, slug: SLUG, from: 10, to: 20 };
+ const hit = await resolveLocalSource(want, { channelsDir: t.channelsDir, probe, audio: true });
+ assert.equal(hit.name, "audio.opus");
+ assert.deepEqual(calls, [path.join(t.videoDir, "audio.opus")]);
+ // exact: the raw file named for the span, else (allowed) the sound -- never a corpus window.
+ await touch(path.join(t.clipsDir, "0.00-60.00.mp4"));
+ const cfg = { rawDir: t.rawDir, channelsDir: t.channelsDir, probe, exact: true };
+ assert.equal(await resolveLocalSource(want, cfg), null);
+ assert.equal((await resolveLocalSource(want, { ...cfg, audio: true })).kind, "audio");
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("audio: the bench's corpus windows leave the sound out", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.videoDir, "audio.mp3"));
+ const got = await corpusWindowsOf({ video: VIDEO, slug: SLUG, channelsDir: t.channelsDir, probe: fakeProbe(90).probe });
+ assert.deepEqual(got, []);
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("audioAllowed: no picture to fetch, or no fetch to make", () => {
+ const web = { platform: "youtube", webpageUrl: "https://example.invalid/v" };
+ // A clip with a picture somewhere, and a network to get it: never the sound.
+ assert.equal(audioAllowed({ entry: {}, meta: web, render: {} }), false);
+ assert.equal(audioAllowed({ entry: {}, render: {} }), false, "no record read yet");
+ // No picture to fetch.
+ assert.equal(audioAllowed({ entry: { audioOnly: true }, meta: web }), true);
+ assert.equal(audioAllowed({ entry: {}, meta: web, render: { preferLocalAudio: true } }), true);
+ assert.equal(audioAllowed({ entry: {}, meta: { ...web, platform: "podcast" } }), true);
+ assert.equal(audioAllowed({ entry: {}, meta: { ...web, platform: "RSS" } }), true);
+ // No fetch to make: --no-network / --skip-fetch, or a record with no page.
+ assert.equal(audioAllowed({ entry: {}, meta: web, network: false }), true);
+ assert.equal(audioAllowed({ entry: {}, meta: { platform: "youtube", webpageUrl: null } }), true);
+});
+
+test("planLocalSources: an audio-only clip is satisfied, and named as such, never a miss", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.videoDir, "audio.mp3"));
+ const timeline = [
+ { id: "c1", type: "clip", video: VIDEO, start: 10, end: 20 },
+ { id: "c2", type: "clip", video: "def456", start: 10, end: 20 },
+ { id: "c3", type: "clip", video: VIDEO, start: 30, end: 40, audioOnly: true },
+ ];
+ const local = localSources({ rawDir: t.rawDir, channelsDir: t.channelsDir, channelSlug: SLUG, probe: fakeProbe(600).probe });
+ const render = { fetchPad: 3 };
+
+ // --no-network: the sound serves c1 and c3; c2 has nothing.
+ let plan = await planLocalSources(timeline, timeline, render, { noNetwork: true }, local);
+ assert.deepEqual(plan.missing.map((m) => m.id), ["c2"]);
+ assert.deepEqual(plan.audio.map((m) => [m.index, m.id, m.local]), [[0, "c1", "audio.mp3"], [2, "c3", "audio.mp3"]]);
+ assert.deepEqual(await clipsNeedingFetch(timeline, timeline, render, { noNetwork: true }, local), plan.missing);
+ const msg = audioOnlyMessage(plan.audio);
+ assert.match(msg, /2 clip\(s\) play from audio only/);
+ assert.match(msg, /timeline\[0\] c1 {2}demo-channel\/abc123 {2}audio\.mp3/);
+
+ // With the network: only the clip that asked for its sound gets it before a fetch.
+ plan = await planLocalSources(timeline, timeline, render, {}, local);
+ assert.deepEqual(plan.missing.map((m) => m.id), ["c1", "c2"]);
+ assert.deepEqual(plan.audio.map((m) => m.id), ["c3"]);
+ // ...and every clip of a cut that prefers local sound.
+ plan = await planLocalSources(timeline, timeline, { ...render, preferLocalAudio: true }, {}, local);
+ assert.deepEqual(plan.audio.map((m) => m.id), ["c1", "c3"]);
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("buildVideo --no-network: an audio-only clip is not in the refusal", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.videoDir, "audio.mp3"));
+ const manifestPath = path.join(t.root, "video.manifest.json");
+ await writeFile(manifestPath, JSON.stringify({
+ slug: "demo",
+ title: "Demo",
+ provenance: { channelSlug: SLUG, channelsDir: "channels", siteOrigin: "https://example.invalid" },
+ render: { width: 1280, height: 720, fps: 30, fetchPad: 3 },
+ timeline: [
+ { id: "c1", type: "clip", video: VIDEO, start: 10, end: 20 },
+ { id: "c2", type: "clip", video: "def456", start: 1, end: 4 },
+ ],
+ }));
+ await assert.rejects(
+ buildVideo({ manifestPath, opts: { noNetwork: true, probe: fakeProbe(600).probe } }),
+ (err) => {
+ assert.match(err.message, /1 clip\(s\) have no local source/);
+ assert.match(err.message, /c2/);
+ assert.doesNotMatch(err.message, /c1/);
+ return true;
+ },
+ );
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
