commit 0c3f1a34e16afc39ca20a8359c500f20e9d3ff80
parent f3546ca6814ee736b80cb59f52ef77f2a9964bb1
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sun, 4 Oct 2026 16:38:55 -0400
Merge render-local-sources (report-to-video resolves clip media from disk first: raw cache, corpus clip windows, saved-video container; --no-network lists every clip that would need a fetch and refuses)
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
10 files changed, 1100 insertions(+), 198 deletions(-)
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,7 @@
# Changelog
## [Unreleased]
+- **A report build cuts from media already on disk before it downloads anything, and `--no-network` makes sure it never does.** For each clip, `build-video.mjs` now looks, in order, in the project's own `out/clips-raw`, in the clip windows the editor fetched into the channel (`channels/<slug>/data/<id>/clips/`), and in a saved whole source video (through the saved-video store's pointer, or a `source-media` file still in the video's folder), and cuts from the first that holds the clip plus its fetch pad; only when none does is the window downloaded. A file that is a link to a drive that is not mounted counts as not there, and the next place is tried. The build prints one line per clip naming where its source came from (`raw-cache`, `corpus-window`, `saved-video`, or a network fetch). With `--no-network`, every clip's source is found before anything is rendered, and if any clip would need a download the build stops at once and lists each one (its position in the timeline, channel, video and the span it needs). umtool's clip bench reads the same three places, so a clip it shows as fetched is one the build cuts from without downloading.
- **umtool's report videos can show a highlighted sentence from a saved article.** `node umtool/report-to-video/shoot-page.mjs --page <saved page.html> --quote "<sentence>" --out <shot.png>` opens a web page saved to disk, finds the sentence in its text, highlights it and saves a PNG of the paragraph that holds it, ready to be a report manifest's `image` entry. `--batch <items.json> --out <dir>` does a list of `{ id, page, quote, context? }` at once and writes `<id>.png` for each plus a `results.json` recording each shot's crop, the matched text and the block it shot. The page is opened offline: nothing is fetched except files saved beside it, and its own scripts do not run unless `--js` is given. The sentence is found whether its quotes and apostrophes are curly or straight, across links and emphasis, and through non-breaking spaces, soft hyphens and line breaks in the page's source. A sentence that is not on the page is listed in `results.json` and on the terminal, and the run ends with an error rather than leaving it out. `--color` sets the highlight; `context` picks one occurrence of a sentence that appears more than once.
- **A clip or whole-recording fetch can name the tallest source video it wants.** The MCP's `fetch_clip` takes `maxHeight`, `fetch-via-editor.mjs` takes `--max-height`, and the editor's fetch endpoint takes `maxHeight`: a whole number of pixels from 144 to 2160; anything else is refused before anything is fetched. A window is fetched at or under that height (720 when none is given, as before). A whole recording asked for at 720 or less is saved as the **Video 720p** quality, and above 720 at the original quality; with no height it follows the channel's, else the global, source video quality, as before. umtool's whole-source fetch from the clip bench now asks at the report's `render.maxHeightSource`. A file already on disk is returned as it is and never fetched again for a different height; the answer now gives its height (a window's is read from the file, a whole recording's from what its persist recorded) and says when it is taller than the height asked for.
- **Persist a list of videos, across channels, to the saved-video store.** `pnpm ops persist-videos --json '{"items":[{"slug":"<channel>","id":"<video id>"}, …]}'` (or `--file list.json` for a long list) re-fetches the source container of each video that is not saved yet, the way **Persist source video** does on a video's page. `"format": "original"` or `"video_720"` picks the quality (default: each channel's own), and `"replace": "above-height"` also re-fetches a saved video whose recorded height is unknown or above that quality — the old file is removed only after the new one is saved, and the saved video keeps its retention class. Downloads run one at a time, one job per channel on that channel's download queue, with the usual gap between them (`"gapMs"` overrides it); `"minFreeMemMb"` holds each download until that much memory is free. A low disk or a rate limit stops the job, and running the same list again picks up where it left off: saved videos are skipped. `"dryRun": true` answers with what a run would do — saved, saved above the height, to fetch, no source URL, unknown — and starts nothing. Jobs show as **Persist videos**, can be drained, and can be retried from /jobs.
diff --git a/umtool/lib/projects/report.mjs b/umtool/lib/projects/report.mjs
@@ -8,6 +8,12 @@
import { readdir, readFile, stat } from "node:fs/promises";
import path from "node:path";
import { DEFAULT_VARIANT, cachedWindowsFor } from "umtool-report-to-video/build-video";
+import {
+ SHADOW_CHANNELS as SOURCES_SHADOW_CHANNELS,
+ channelsDirFor as sourcesChannelsDirFor,
+ corpusWindowsOf,
+ videoDirOf,
+} from "umtool-report-to-video/sources";
import { rawCacheOf } from "../report/raw-cache.mjs";
import { deliverablesState, finishCommand, outDirState, leftoversOf } from "../report/storage.mjs";
import { CHANNELS_DIR, inside } from "../paths.mjs";
@@ -44,7 +50,7 @@ export const MANIFEST_NAME = "video.manifest.json";
export const GLOBAL_CHANNELS_DIR = () => CHANNELS_DIR;
/** The conventional name make-shadow-channels.sh builds. */
-export const SHADOW_CHANNELS = ".shadow-channels";
+export const SHADOW_CHANNELS = SOURCES_SHADOW_CHANNELS;
// ---------------------------------------------------------------------------
// Which channels directory THIS project reads.
@@ -62,11 +68,11 @@ export const SHADOW_CHANNELS = ".shadow-channels";
// detected because it already exists and nothing had to change on disk for it
// to work. Neither is a guess: both are things the project itself wrote down.
// ---------------------------------------------------------------------------
+//
+// The rule itself is report-to-video/sources.mjs's, so the render looks for
+// local media in the tree this reads.
export function channelsDirFor(dir, manifest, { shadowExists = false } = {}) {
- const declared = manifest?.provenance?.channelsDir;
- if (declared) return path.resolve(dir, declared);
- if (shadowExists) return path.join(dir, SHADOW_CHANNELS);
- return GLOBAL_CHANNELS_DIR();
+ return sourcesChannelsDirFor(dir, manifest, { shadowExists, fallback: GLOBAL_CHANNELS_DIR() });
}
export const hasShadowChannels = (dir) =>
@@ -85,17 +91,6 @@ const DEAD_ORIGIN =
export const isDeadOrigin = (o) => !o || typeof o !== "string" || DEAD_ORIGIN.test(o);
const stat0 = (p) => stat(p).then((s) => s, () => null);
-const readJson0 = (p) =>
- readFile(p, "utf8").then(
- (t) => {
- try {
- return JSON.parse(t);
- } catch {
- return null;
- }
- },
- () => null,
- );
export const manifestPath = (dir) => path.join(dir, MANIFEST_NAME);
@@ -326,14 +321,27 @@ export async function readAvailability(dir) {
}
/**
- * The corpus directories that also hold windows of this manifest's videos.
+ * The windows OUTSIDE the project that also hold this manifest's videos, as
+ * `[{ video, windows }]` for rawCacheOf's `extraWindows`.
*
- * The editor fetches a clip window into channels/<slug>/data/<id>/clips/ --
- * managed, polite, provenanced, and REUSABLE, which is the whole point: a
- * window one report paid for is a window the next one does not. One entry per
- * distinct (channel, video) the timeline cites; rawCacheOf reads each once.
+ * Listed by the render's own module (report-to-video/sources.mjs), so the
+ * bench's "fetched" and the build's "local" are one set: the editor's window
+ * cache in channels/<slug>/data/<id>/clips/ -- managed, polite, provenanced,
+ * and REUSABLE, which is the whole point: a window one report paid for is a
+ * window the next one does not -- and a saved whole source, reached through
+ * the store's pointer (`full: true` puts the recording there, not in clips/)
+ * or still in the video dir. A dangling link -- a drive not mounted -- is no
+ * window, never an error. One entry per distinct (channel, video) the
+ * timeline cites.
+ *
+ * A WHOLE CONTAINER'S SPAN COMES FROM THE CUES, not from an ffprobe as the
+ * render's does: the cue doc is the archive's own record of how long the
+ * recording is, it is memoised against mtime (see below), and it is asked for
+ * only when a container is actually there -- which keeps the index load as
+ * cheap as it was. No duration is no window rather than a guess.
*/
export const clipWindowDirs = async (m, channelsDir) => {
+ const root = channelsDir ?? GLOBAL_CHANNELS_DIR();
const seen = new Set();
const out = [];
for (const e of clipsOf(m)) {
@@ -342,35 +350,13 @@ export const clipWindowDirs = async (m, channelsDir) => {
const key = `${chan}/${e.video}`;
if (seen.has(key)) continue;
seen.add(key);
- const videoDir = path.join(channelsDir ?? GLOBAL_CHANNELS_DIR(), chan, "data", e.video);
- out.push({ video: e.video, dir: path.join(videoDir, "clips") });
-
- // AND THE WHOLE SOURCE, when somebody asked the editor for one.
- //
- // `full: true` puts the entire recording in the saved-video STORE, not in
- // clips/ -- that is where big containers already live, with a retention
- // rule that leaves an explicitly-requested one alone. The store is on
- // another path (and possibly another drive), so the only way back to it is
- // the pointer the editor leaves in the video dir. `pointer.dir` is exactly
- // the shape rawCacheOf's SOURCE_MEDIA_RE branch handles.
- //
- // THE DURATION IS READ FROM THE CUES, and ONLY for a video that actually
- // has a pointer. rawCacheOf needs a finite duration to treat the container
- // as the window [0, duration]; the cue doc is the archive's own record of
- // how long the recording is. Reading it is the expensive thing this module
- // memoises against mtime (see below) -- so it is asked for the handful of
- // videos somebody fetched a full source for, and never for the rest, which
- // keeps the index load exactly as cheap as it was.
- const pointer = await readJson0(path.join(videoDir, "saved-video.json"));
- if (!pointer?.dir) continue;
- const cues = await readCues(path.join(videoDir, "transcript.cues.json"));
- const duration = Number(cues?.duration);
- // No duration is no entry rather than a guess: without one the cache
- // cannot say what the container holds, and claiming a span a file may not
- // cover is the one failure mode worse than a cache miss.
- if (Number.isFinite(duration) && duration > 0) {
- out.push({ video: e.video, dir: pointer.dir, duration });
- }
+ const cueFile = path.join(videoDirOf(root, chan, e.video), "transcript.cues.json");
+ const probe = async () => {
+ const duration = Number((await readCues(cueFile))?.duration);
+ return Number.isFinite(duration) && duration > 0 ? { duration } : null;
+ };
+ const windows = await corpusWindowsOf({ video: e.video, slug: chan, channelsDir: root, probe });
+ if (windows.length) out.push({ video: e.video, windows });
}
return out;
};
@@ -1156,7 +1142,7 @@ export async function readClipDetail(dir, { manifest = null } = {}) {
// ONE listing of out/clips-raw for the whole project. This used to be two
// readdirs PER CLIP -- the padded lookup and the full list -- so a forty-clip
// cut paid eighty directory reads to draw one page.
- const raw = await rawCacheOf(dir, { extraDirs: await clipWindowDirs(m, channelsDir) });
+ const raw = await rawCacheOf(dir, { extraWindows: await clipWindowDirs(m, channelsDir) });
const segDirs = segmentDirs(path.join(dir, "out"));
const segLists = await Promise.all(segDirs.map((d) => readdir(d).catch(() => [])));
const segWhich = segLists.findIndex((l) => l.length);
diff --git a/umtool/lib/report/raw-cache.mjs b/umtool/lib/report/raw-cache.mjs
@@ -13,27 +13,6 @@ import {
windowsFromNames,
} from "umtool-report-to-video/build-video";
-// A window file inside a CORPUS clips dir. Un-prefixed, because that directory
-// is already per-video: channels/<slug>/data/<id>/clips/<from>-<to>.mp4. The
-// project-local cache prefixes with the video id because one flat directory
-// holds every source. Same numbers, two namings, one predicate over both.
-const BARE_WINDOW_RE = /^(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)\.mp4$/;
-
-function windowsFromBareNames(names, dir) {
- const out = [];
- for (const name of names) {
- const m = BARE_WINDOW_RE.exec(name);
- if (!m) continue;
- const from = Number(m[1]);
- const to = Number(m[2]);
- if (!(to > from)) continue;
- out.push({ name, path: path.join(dir, name), from, to });
- }
- return out;
-}
-
-const SOURCE_MEDIA_RE = /^source-media\.[A-Za-z0-9]+$/;
-
/**
* The cache. The directory is keyed by VIDEO and a page asks about it per clip
* -- the bench page did two readdirs per clip, and a forty-clip cut paid eighty
@@ -48,30 +27,21 @@ const SOURCE_MEDIA_RE = /^source-media\.[A-Za-z0-9]+$/;
* ready to judge.
*
* @param {string} projectDir
- * @param {{ extraDirs?: { video: string, dir: string, duration?: number }[] }} [opts]
- * Directories OUTSIDE the project that also hold windows of a given video --
- * today, the corpus clips dir the editor fetches into. Each is read once,
- * here, so `windows()` stays synchronous and every existing caller is
- * unchanged. A `duration` makes a whole `source-media.*` container in that
- * directory count as the window [0, duration]: a full source contains every
- * window of its video, which is exactly what the predicate needs to know.
+ * @param {{ extraWindows?: { video: string, windows: { name: string, path: string, from: number, to: number }[] }[] }} [opts]
+ * Windows OUTSIDE the project that also hold a given video -- the corpus's
+ * clip windows and a saved whole source, as `clipWindowDirs` lists them
+ * through the render's own sources.mjs, so the bench and the build agree on
+ * what is local. A whole container is the window [0, duration]: a full
+ * source contains every window of its video, which is exactly what the
+ * predicate needs to know.
*/
-export async function rawCacheOf(projectDir, { extraDirs = [] } = {}) {
+export async function rawCacheOf(projectDir, { extraWindows = [] } = {}) {
const rawDir = path.join(projectDir, "out", "clips-raw");
const names = await listRawNames(rawDir);
const extra = new Map();
- for (const e of extraDirs) {
- if (!e?.video || !e?.dir) continue;
- const listed = await listRawNames(e.dir);
- const found = windowsFromBareNames(listed, e.dir);
- if (Number.isFinite(e.duration)) {
- const whole = listed.find((n) => SOURCE_MEDIA_RE.test(n));
- if (whole) {
- found.push({ name: whole, path: path.join(e.dir, whole), from: 0, to: Number(e.duration) });
- }
- }
- if (!found.length) continue;
- extra.set(e.video, [...(extra.get(e.video) ?? []), ...found]);
+ for (const e of extraWindows) {
+ if (!e?.video || !e?.windows?.length) continue;
+ extra.set(e.video, [...(extra.get(e.video) ?? []), ...e.windows]);
}
const byVideo = new Map();
const windows = (video) => {
diff --git a/umtool/lib/report/raw-cache.test.mjs b/umtool/lib/report/raw-cache.test.mjs
@@ -0,0 +1,101 @@
+// The clip bench's view of what is cached: out/clips-raw plus the corpus's
+// windows, listed by clipWindowDirs through the render's own sources.mjs --
+// so a window the bench calls fetched is one the build cuts from.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import test from "node:test";
+import path from "node:path";
+import os from "node:os";
+import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
+
+import { rawCacheOf } from "./raw-cache.mjs";
+import { clipWindowDirs } from "../projects/report.mjs";
+import { resolveLocalSource } from "umtool-report-to-video/sources";
+
+const SLUG = "demo-channel";
+
+async function fixture() {
+ const root = await mkdtemp(path.join(os.tmpdir(), "raw-cache-"));
+ const projectDir = path.join(root, "project");
+ const channelsDir = path.join(root, "channels");
+ const rawDir = path.join(projectDir, "out", "clips-raw");
+ await mkdir(rawDir, { recursive: true });
+ const video = async (id) => {
+ const dir = path.join(channelsDir, SLUG, "data", id);
+ await mkdir(path.join(dir, "clips"), { recursive: true });
+ return dir;
+ };
+ return { root, projectDir, channelsDir, rawDir, video };
+}
+
+const touch = (p) => writeFile(p, Buffer.alloc(16, 1));
+
+test("rawCacheOf with no corpus windows reads clips-raw alone", async () => {
+ const f = await fixture();
+ try {
+ await touch(path.join(f.rawDir, "abc123_0.00-30.00.mp4"));
+ const c = await rawCacheOf(f.projectDir);
+ assert.equal(c.containing("abc123", 5, 10).name, "abc123_0.00-30.00.mp4");
+ assert.equal(c.isFetched({ video: "abc123", start: 25, end: 31 }), false);
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+});
+
+test("clipWindowDirs: corpus windows and a saved source, and the render resolves the same files", async () => {
+ const f = await fixture();
+ try {
+ const a = await f.video("abc123");
+ await touch(path.join(a, "clips", "100.00-160.00.mp4"));
+ await touch(path.join(a, "clips", "100.00-160.00.json"));
+ // A window on a drive that is not there is no window.
+ await symlink(path.join(f.root, "gone.mp4"), path.join(a, "clips", "300.00-400.00.mp4"));
+
+ // A second video with its whole source in the store; its span is the cue
+ // doc's duration.
+ const b = await f.video("def456");
+ const store = path.join(f.root, "store", SLUG, "def456");
+ await mkdir(store, { recursive: true });
+ await touch(path.join(store, "source-media.mp4"));
+ await writeFile(path.join(b, "saved-video.json"), JSON.stringify({ dir: store, file: "source-media.mp4", bytes: 16 }));
+ await writeFile(path.join(b, "transcript.cues.json"), JSON.stringify({ duration: 1800, cues: [] }));
+
+ const m = {
+ provenance: { channelSlug: SLUG },
+ timeline: [
+ { id: "c1", type: "clip", video: "abc123", start: 110, end: 120 },
+ { id: "c2", type: "clip", video: "abc123", start: 320, end: 330 },
+ { id: "c3", type: "clip", video: "def456", start: 1000, end: 1010 },
+ { id: "c4", type: "clip", video: "nope99", start: 1, end: 2 },
+ ],
+ };
+ const extra = await clipWindowDirs(m, f.channelsDir);
+ assert.deepEqual(
+ extra.map((e) => [e.video, e.windows.map((w) => [w.kind, w.name, w.from, w.to])]),
+ [
+ ["abc123", [["corpus-window", "100.00-160.00.mp4", 100, 160]]],
+ ["def456", [["saved-video", "source-media.mp4", 0, 1800]]],
+ ],
+ );
+
+ const c = await rawCacheOf(f.projectDir, { extraWindows: extra });
+ assert.equal(c.isFetched(m.timeline[0]), true);
+ assert.equal(c.isFetched(m.timeline[1]), false, "the dangling window is not fetched");
+ assert.equal(c.isFetched(m.timeline[2]), true);
+ assert.equal(c.isFetched(m.timeline[3]), false);
+
+ // The render, asked the same question, finds the same files.
+ const probe = async () => ({ duration: 1800 });
+ for (const e of m.timeline.slice(0, 3)) {
+ const hit = await resolveLocalSource(
+ { video: e.video, slug: SLUG, from: e.start, to: e.end },
+ { channelsDir: f.channelsDir, probe },
+ );
+ const bench = c.containing(e.video, e.start, e.end);
+ assert.equal(hit?.path ?? null, bench?.path ?? null, e.id);
+ }
+ } finally {
+ await rm(f.root, { recursive: true, force: true });
+ }
+});
diff --git a/umtool/lib/report/serve.mjs b/umtool/lib/report/serve.mjs
@@ -69,7 +69,7 @@ export async function projectCache(project, manifest = null) {
const m = manifest ?? (await readManifest(project.dir));
const shadowExists = await hasShadowChannels(project.dir);
const channelsDir = channelsDirFor(project.dir, m, { shadowExists });
- return rawCacheOf(project.dir, { extraDirs: await clipWindowDirs(m, channelsDir) });
+ return rawCacheOf(project.dir, { extraWindows: await clipWindowDirs(m, channelsDir) });
}
/**
diff --git a/umtool/report-to-video/README.md b/umtool/report-to-video/README.md
@@ -108,12 +108,19 @@ Everything is cached by content, so iteration is cheap:
nothing at all. Only a window that escapes every cached file downloads again.
- **Preview one entry** — `--only <id>` builds a single segment and stops.
- **Work offline** — `--skip-fetch` fails loudly instead of downloading, so you
- can confirm you are working entirely from cache.
+ can confirm you are working entirely from cache. `--no-network` is the
+ stronger form: every clip's source is found on disk **before** anything is
+ rendered, and if even one clip would need a download the build refuses at
+ once, listing each such clip as `timeline[<i>] <id> <channel>/<video>
+ <from>–<to>` (the padded span it needs). It governs clip **media** only: cue
+ windows and metadata still come from wherever `--cue-source` says, so add
+ `--cue-source local` to keep those off the network too.
- **Fetch one clip, wide** — `--fetch-only <id> --pad 20` puts a generous window
in the cache without building anything. This is what the umtool clip bench runs,
and containing-window reuse is what makes that fetch double as the build's cache.
- **Force a refetch** — delete `out/clips-raw/`, or pass `--no-reuse` to require an
- exact-window file.
+ exact-window file (the corpus and saved-source tiers below are reuse too, and
+ are skipped).
- **Iterate on the rail** — `--rail-only` re-runs only the rail chain over a cached
`out/<slug>.prerail.mp4`; `--preview <start> <dur>` does the same over a window.
`--no-rail` builds the cut without one. See
@@ -135,6 +142,41 @@ cue and runs the search on to the *next* sentence. Left alone, every re-run grew
the same clip. Hence `EPS` in the end lookup and the 0.05 s deadband on applying a
change.
+### Where a clip's source comes from
+
+Before a clip is fetched, `sources.mjs` looks for its source on disk, in this
+order, and the first place holding the clip's **padded** span (its extent plus
+`fetchPad`, to the same 0.02 s tolerance) wins:
+
+1. **`raw-cache`** — this build's `out/clips-raw/<video>_<from>-<to>.mp4`: the
+ exact-window file first, else the tightest containing one, as above.
+2. **`corpus-window`** — the editor's window cache,
+ `channels/<slug>/data/<id>/clips/<from>-<to>.mp4` (`.mkv`/`.webm` from older
+ fetches; the `.json` sidecar and in-flight `.part.mp4` are ignored), tightest
+ first. This is where umtool's bench and the MCP's `fetch_clip` put windows.
+3. **`saved-video`** — the whole recording: the saved-video store's container
+ through `data/<id>/saved-video.json`, else a `data/<id>/source-media.<ext>`
+ not yet moved there. It counts as the window `[0, duration]`, the duration
+ (and picture height) read by one `ffprobe` — run only when tiers 1 and 2
+ missed.
+
+Only a miss in all three fetches. The channels tree is the one the clip bench
+reads for the project: `provenance.channelsDir`, else a `.shadow-channels/`
+beside the manifest, else `CHANNELS_DIR` / the checkout's `transcripts/channels`.
+A file reached through a link into `media/` (perhaps on another drive) is
+followed; a dangling one — a drive not mounted — is "not here", and the next
+candidate or tier answers. The build logs one line per clip naming the kind it
+used (`--progress ndjson`: the `fetch` event's `source`, which is `"network"`
+for a download, plus `local` and `window` for a hit).
+
+A corpus window or a whole source is wider than the fetch would have been, so
+silence detection measures just the padded span of it — the same seconds a
+fetch would have produced — rather than decoding a fifteen-minute window or a
+three-hour container. A raw-cache window is measured whole, as before, so no
+cached cut moves. umtool's bench lists tiers 2 and 3 through the same module
+(`clipWindowDirs`), with a container's span from its cue doc rather than an
+ffprobe, so what the bench calls fetched is what the build cuts from.
+
## Driven from umtool
The three CLIs are the source of truth and stay usable on their own; umtool drives
diff --git a/umtool/report-to-video/build-video.mjs b/umtool/report-to-video/build-video.mjs
@@ -26,6 +26,12 @@
// Fetched clips are cached by (video, start, end); re-running is cheap and only
// changed entries re-download. Delete out/clips-raw to force a refetch.
//
+// Before any fetch, a clip's source is looked for ON DISK (sources.mjs): this
+// build's out/clips-raw, then the corpus's channels/<slug>/data/<id>/clips/
+// windows the editor fetched, then a saved whole source. Only a miss in all
+// three goes to the network, and --no-network refuses the build up front if
+// any clip would.
+//
// In the app: not used. On the CLI:
// node umtool/report-to-video/build-video.mjs <manifest.json> [options]
//
@@ -33,6 +39,9 @@
// --out <dir> Output root (default: manifest dir + /out)
// --variant <name> Which cut to build (sourced | full; default sourced)
// --skip-fetch Fail instead of downloading anything not already cached
+// --no-network Find every clip's source on disk first (sources.mjs: the
+// raw cache, the corpus's clip windows, a saved source) and
+// refuse the build, listing each clip, if any needs a fetch
// --only <id> Build a single entry's segment and stop (for iterating)
// --no-xfade Hard cuts instead of crossfades (much faster; concat copy)
// --progress ndjson One JSON event per line instead of prose (for umtool)
@@ -76,7 +85,12 @@ import {
renderLedgerCard, ledgerRevealAt, ledgerSeconds,
cardWidth, contentWidth, reservedFooterHeight,
} from "./render-cards.mjs";
-import { createCueSource, siteOriginFromManifest } from "./cues.mjs";
+import { DEFAULT_CHANNELS_DIR, createCueSource, siteOriginFromManifest } from "./cues.mjs";
+// Where a clip's media is ALREADY on disk -- the build's raw cache, the
+// editor's corpus windows, the saved source -- asked before anything fetches.
+import {
+ SHADOW_CHANNELS, channelsDirFor, findContainingWindow, rawWindowName, resolveLocalSource,
+} from "./sources.mjs";
import { createPostChannelResolver, resolvePostLinks } from "./post-links.mjs";
import { ensureWriteDir } from "../lib/report/storage.mjs";
// The deck (`render.chrome`): its geometry, validation and schedule are pure
@@ -340,12 +354,13 @@ const HUMAN = {
start: (e) => `${e.title} — ${e.entries} entr(ies)`,
card: (e) => `card ${e.id}`,
clip: (e) => `clip ${e.id} (${e.video}) §${e.section}${e.sectionEnter ? " ⟶" : ""}`,
+ // One line per clip, naming where its source came from (`source`: a
+ // sources.mjs kind, or "network").
fetch: (e) =>
- e.reuse
- ? ` fetch ${e.id}: ${e.reuse} already covers ${hms(e.from)}–${hms(e.to)} — no download`
- : e.cached
- ? null
- : ` fetch ${e.id}: ${e.video} ${hms(e.from)}–${hms(e.to)}`,
+ e.cached
+ ? ` source ${e.id}: ${e.source ?? "raw-cache"} ${e.reuse ?? "(exact window)"} covers ` +
+ `${hms(e.from)}–${hms(e.to)} — no download` + (e.height ? ` (${e.height}p)` : "")
+ : ` fetch ${e.id}: ${e.video} ${hms(e.from)}–${hms(e.to)} (network)`,
snap: (e) =>
` snap ${e.id}: ${e.start ? "start✓" : "start–"} ${e.end ? "end✓" : "end–"} ` +
`(${Number(e.seconds).toFixed(1)}s)`,
@@ -470,66 +485,15 @@ const ytdlpOk = (err) => err?.code === 101;
// costs more than the 14s one that would also have done. The clip bench fetches
// deliberately wide, and this is what makes that generous fetch become the
// build's cache rather than a second one.
-const WINDOW_RE = /^(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)$/;
-
-// A window read back from a 2 dp manifest can sit a hair outside the file that
-// produced it; the same tolerance resolve-windows.mjs uses for the same reason.
-export const WIN_EPS = 0.02;
-
-/**
- * The question is asked FOUR times per page -- the build, the bench, the
- * project page's pill and the walk -- so it is one predicate in one place,
- * split into the I/O and the arithmetic. A caller with a directory listing
- * already in hand (the bench reads clips-raw ONCE per request and answers for
- * every clip) uses the pure halves; the two original functions are those halves
- * composed, and behave exactly as they did.
- */
-export async function listRawNames(rawDir) {
- try {
- return await readdir(rawDir);
- } catch {
- return [];
- }
-}
-
-/** The windows THIS video's files hold, parsed out of a directory listing. */
-export function windowsFromNames(names, rawDir, video) {
- const prefix = `${video}_`;
- const out = [];
- for (const name of names) {
- if (!name.startsWith(prefix) || !name.endsWith(".mp4")) continue;
- // The remainder must be exactly `a-b`, which is what stops a video id that
- // is a prefix of another (or one containing `_`) from claiming its files.
- const m = WINDOW_RE.exec(name.slice(prefix.length, -4));
- if (!m) continue;
- out.push({ name, path: path.join(rawDir, name), from: Number(m[1]), to: Number(m[2]) });
- }
- return out;
-}
-
-/** Does this cached file hold [from, to] whole, to the manifest's tolerance? */
-export function windowContains(w, from, to) {
- return !(w.from > from + WIN_EPS || w.to < to - WIN_EPS);
-}
-
-/** The tightest of `windows` containing [from, to], or null. */
-export function tightestContaining(windows, from, to) {
- let best = null;
- for (const w of windows) {
- if (!windowContains(w, from, to)) continue;
- if (!best || w.to - w.from < best.to - best.from) best = w;
- }
- return best;
-}
-
-export async function cachedWindowsFor(rawDir, video) {
- return windowsFromNames(await listRawNames(rawDir), rawDir, video);
-}
-
-/** The tightest cached file containing [from, to], or null. */
-export async function findContainingWindow(rawDir, video, from, to) {
- return tightestContaining(await cachedWindowsFor(rawDir, video), from, to);
-}
+// The cache's naming and the containment predicate live in sources.mjs now --
+// beside the corpus tiers the build also reads before it fetches -- and are
+// re-exported here under the names umtool has always imported them by. The
+// question is asked FOUR times per page (the build, the bench, the project
+// page's pill and the walk), so it is one predicate in one place.
+export {
+ WIN_EPS, cachedWindowsFor, findContainingWindow, listRawNames, rawWindowName,
+ tightestContaining, windowContains, windowsFromNames,
+} from "./sources.mjs";
// The yt-dlp argv for one clip window. The platform's fixed args come from
// common's table (`platformArgsForUrl`) -- the same copy every editor spawn
@@ -578,39 +542,130 @@ export function cutArgs(raw, a, b) {
return ["-ss", a.toFixed(3), "-to", b.toFixed(3), "-i", raw];
}
-async function fetchClip(entry, meta, render, rawDir, opts) {
- // Deliberately over-fetch: the snapping pass below needs room on both sides to
- // find a silence, and a clip that has no slack can only be cut where the cue
- // happened to break — which is what put words in half in the first place.
- // ONE EDGE AT A TIME. Extending a window is nearly always one-sided -- you
- // want the sentence that follows, not another twenty seconds of the lead-in
- // you already heard -- and a symmetric pad makes every "20 more" fetch twice
- // what was asked for. `--pad` stays the shorthand that sets both.
+/**
+ * The span a clip's source is needed over: its extent, widened by the fetch
+ * pad on each side.
+ *
+ * Deliberately over-fetched: the snapping pass needs room on both sides to
+ * find a silence, and a clip that has no slack can only be cut where the cue
+ * happened to break -- which is what put words in half in the first place.
+ * ONE EDGE AT A TIME. Extending a window is nearly always one-sided -- you
+ * want the sentence that follows, not another twenty seconds of the lead-in
+ * you already heard -- and a symmetric pad makes every "20 more" fetch twice
+ * what was asked for. `--pad` stays the shorthand that sets both.
+ */
+export function fetchSpan(entry, render, opts = {}) {
const pad = opts.pad ?? render.fetchPad ?? 3.0;
const padBefore = opts.padBefore ?? pad;
const padAfter = opts.padAfter ?? pad;
- const from = Math.max(0, entry.start - padBefore);
- const to = entry.end + padAfter;
+ return { from: Math.max(0, entry.start - padBefore), to: entry.end + padAfter };
+}
+
+/**
+ * Where this build looks for media already on disk: the raw cache, the
+ * channels tree and the clip's channel. One per build; `resolve` is memoised,
+ * so `--no-network`'s up-front pass and the clip's own fetch probe a saved
+ * container once between them.
+ */
+export function localSources({ rawDir, channelsDir, channelSlug = null, probe } = {}) {
+ const memo = new Map();
+ const slugOf = (entry) => entry.channel ?? channelSlug ?? null;
+ return {
+ rawDir,
+ channelsDir,
+ slugOf,
+ async resolve(entry, span, { exact = false } = {}) {
+ const slug = slugOf(entry);
+ const key = `${slug}/${entry.video}/${span.from}/${span.to}/${exact}`;
+ if (memo.has(key)) return memo.get(key);
+ const hit = await resolveLocalSource(
+ { video: entry.video, slug, from: span.from, to: span.to },
+ { rawDir, channelsDir, exact, probe },
+ );
+ // Hits only: a miss is about to be fetched into the raw cache, and the
+ // next clip asking for the same span must find that file.
+ if (hit) memo.set(key, hit);
+ return hit;
+ },
+ };
+}
+
+/**
+ * The clips of `entries` that no local source can serve, i.e. the ones that
+ * would need a fetch: `--no-network` refuses the build on any of them before
+ * a frame is rendered. `index` is the entry's position in the manifest's
+ * timeline.
+ */
+export async function clipsNeedingFetch(entries, timeline, render, opts, local) {
+ const out = [];
+ for (const entry of entries) {
+ if (!isClipEntry(entry)) continue;
+ const span = fetchSpan(entry, render, opts);
+ if (await local.resolve(entry, span, { exact: !!opts.noReuse })) continue;
+ out.push({
+ // By id: a variant's view may be a copy of the manifest's entry.
+ index: timeline.findIndex((e) => e === entry || (entry.id != null && e?.id === entry.id)),
+ id: entry.id,
+ slug: local.slugOf(entry),
+ video: entry.video,
+ from: span.from,
+ to: span.to,
+ });
+ }
+ return out;
+}
+
+/** One line per clip the network would have to serve, for the refusal. */
+export function needsFetchMessage(missing) {
+ return (
+ `--no-network: ${missing.length} clip(s) have no local source and would need a fetch:\n` +
+ missing
+ .map((m) =>
+ ` timeline[${m.index}] ${m.id} ${m.slug ?? "(no channel)"}/${m.video} ` +
+ `${m.from.toFixed(2)}–${m.to.toFixed(2)}`)
+ .join("\n")
+ );
+}
+
+// The timeline's vocabulary is OPEN; the build loop treats everything it has
+// no other branch for as a clip, and so does this.
+const NON_CLIP_TYPES = new Set(["card", "teaser", "image", "scroll", "chart", "ledger"]);
+const isClipEntry = (e) => !NON_CLIP_TYPES.has(e?.type);
+
+async function fetchClip(entry, meta, render, local, opts) {
+ const { rawDir } = local;
+ const { from, to } = fetchSpan(entry, render, opts);
// Shared across variants, and deliberately so: this is the only expensive
// thing in a build, and the two cuts overlap almost entirely.
- const name = `${entry.video}_${from.toFixed(2)}-${to.toFixed(2)}.mp4`;
+ const name = rawWindowName(entry.video, from, to);
const dest = path.join(rawDir, name);
- if (await exists(dest)) {
- EMIT("fetch", { id: entry.id, video: entry.video, from, to, cached: true });
- return { path: dest, fetchStart: from, cached: true };
- }
- if (!opts.noReuse) {
- const hit = await findContainingWindow(rawDir, entry.video, from, to);
- if (hit) {
- EMIT("fetch", {
- id: entry.id, video: entry.video, from, to, cached: true, reuse: hit.name,
- });
- // fetchStart is the CACHED file's start, not the requested one -- every cut
- // downstream is expressed relative to it, so reuse is transparent.
- return { path: hit.path, fetchStart: hit.from, cached: true };
- }
+
+ // ON DISK FIRST. The raw cache, then the editor's corpus windows, then a
+ // saved whole source (sources.mjs). `--no-reuse` takes only the raw file
+ // named for exactly this span. fetchStart is the SOURCE's start, not the
+ // requested one -- every cut downstream is relative to it, so where the
+ // bytes came from is transparent.
+ const hit = await local.resolve(entry, { from, to }, { exact: !!opts.noReuse });
+ if (hit) {
+ EMIT("fetch", {
+ id: entry.id, video: entry.video, from, to, cached: true, source: hit.kind,
+ ...(hit.kind !== "raw-cache" || hit.name !== name ? { reuse: hit.name } : {}),
+ local: hit.path,
+ window: [hit.windowStart, hit.windowEnd],
+ ...(hit.height ? { height: hit.height } : {}),
+ });
+ return {
+ path: hit.path,
+ fetchStart: hit.windowStart,
+ cached: true,
+ source: hit.kind,
+ // A whole container is hours long and silence detection decodes what it
+ // is given: give it the span a fetch would have produced, no more.
+ ...(hit.kind === "raw-cache" ? {} : { scan: { from: from - hit.windowStart, to: to - hit.windowStart } }),
+ };
}
+ if (opts.noNetwork) throw new Error(`--no-network set and no local source covers ${name}`);
if (opts.skipFetch) throw new Error(`--skip-fetch set and no cached window covers ${name}`);
const maxH = render.maxHeightSource;
@@ -633,7 +688,7 @@ async function fetchClip(entry, meta, render, rawDir, opts) {
}
};
- EMIT("fetch", { id: entry.id, video: entry.video, from, to, cached: false });
+ EMIT("fetch", { id: entry.id, video: entry.video, from, to, cached: false, source: "network" });
let err = await attempt([]);
// Rumble delivers HLS whose segments are named `.tar`, and ffmpeg 8's picky
@@ -662,13 +717,21 @@ async function fetchClip(entry, meta, render, rawDir, opts) {
if (!stray) throw new Error(`yt-dlp reported success but produced no file for ${entry.id}`);
await rename(path.join(dir, stray), dest);
}
- return { path: dest, fetchStart: from, cached: false };
+ return { path: dest, fetchStart: from, cached: false, source: "network" };
}
// Parse ffmpeg's silencedetect output into [{s, e}] intervals, in seconds
// relative to the start of the given file.
-async function detectSilence(file, render) {
+async function detectSilence(file, render, scan = null) {
const minDur = render.silenceMinDur ?? 0.09;
+ // A source wider than the fetch would have been (a corpus window, a whole
+ // saved container) is measured over just that span: seeked, audio only, and
+ // the silences shifted back onto the file's clock. A raw-cache window is
+ // measured whole, as it always was, so no cached cut moves.
+ const input = scan
+ ? ["-ss", scan.from.toFixed(3), "-to", scan.to.toFixed(3), "-i", file, "-vn"]
+ : ["-i", file];
+ const shift = scan ? scan.from : 0;
// The threshold has to be RELATIVE to the clip, not absolute. These are game
// streams: the gaps between words are full of game audio and music, so they
@@ -678,7 +741,7 @@ async function detectSilence(file, render) {
// and cut a few dB under its own mean instead.
const { stderr: volLog } = await execFileP(
FFMPEG,
- ["-nostdin", "-i", file, "-af", "volumedetect", "-f", "null", "-"],
+ ["-nostdin", ...input, "-af", "volumedetect", "-f", "null", "-"],
{ maxBuffer: 1 << 26 },
).catch((e) => ({ stderr: e.stderr ?? "" }));
const meanMatch = (volLog ?? "").match(/mean_volume:\s*(-?[\d.]+) dB/);
@@ -688,7 +751,7 @@ async function detectSilence(file, render) {
// ffmpeg exits 0 here, so stderr comes back on the resolved result.
const { stderr } = await execFileP(
FFMPEG,
- ["-nostdin", "-i", file, "-af", `silencedetect=noise=${noise.toFixed(1)}dB:d=${minDur}`, "-f", "null", "-"],
+ ["-nostdin", ...input, "-af", `silencedetect=noise=${noise.toFixed(1)}dB:d=${minDur}`, "-f", "null", "-"],
{ maxBuffer: 1 << 26 },
).catch((e) => ({ stderr: e.stderr ?? "" }));
const log = stderr ?? "";
@@ -697,10 +760,10 @@ async function detectSilence(file, render) {
let open = null;
for (const line of log.split("\n")) {
const s = line.match(/silence_start:\s*(-?[\d.]+)/);
- if (s) open = Number(s[1]);
+ if (s) open = Number(s[1]) + shift;
const e = line.match(/silence_end:\s*(-?[\d.]+)/);
if (e && open !== null) {
- out.push({ s: open, e: Number(e[1]) });
+ out.push({ s: open, e: Number(e[1]) + shift });
open = null;
}
}
@@ -755,7 +818,7 @@ const encodeArgsVideoOnly = (render) => [
async function buildClipSegment(entry, meta, render, dirs, opts, chrome, nodes, provenance, framing = null) {
const outDir = dirs.dir;
- const { path: raw, fetchStart } = await fetchClip(entry, meta, render, dirs.rawDir, opts);
+ const { path: raw, fetchStart, scan } = await fetchClip(entry, meta, render, dirs.local, opts);
const seg = path.join(outDir, "segments", `${entry.id}.mp4`);
const pal = render.palette;
const { width, height } = render;
@@ -799,7 +862,7 @@ async function buildClipSegment(entry, meta, render, dirs, opts, chrome, nodes,
const wantB = playTo - fetchStart;
const win = render.snapWindow ?? 1.6;
- const sil = await detectSilence(raw, render);
+ const sil = await detectSilence(raw, render, scan);
const a = snap(wantA, sil, "start", win);
const b = snap(wantB, sil, "end", win);
// Never let snapping invert or collapse the window.
@@ -3421,6 +3484,17 @@ export async function buildVideo({ manifestPath, opts = {}, out, only, fetchOnly
const outRoot = out ?? path.join(path.dirname(path.resolve(manifestPath)), "out");
const dirs = variantPaths(outRoot, manifest.slug, variant);
const outDir = dirs.dir;
+ // Where media may already be on disk. The channels tree is the one the clip
+ // bench reads for this project (sources.mjs channelsDirFor): the manifest's
+ // own `provenance.channelsDir`, a `.shadow-channels/` beside it, else the
+ // corpus the cue reader defaults to.
+ const channelsDir = opts.channelsDir ?? channelsDirFor(manifestDir, whole, {
+ shadowExists: await stat(path.join(manifestDir, SHADOW_CHANNELS)).then((st) => st.isDirectory(), () => false),
+ fallback: DEFAULT_CHANNELS_DIR,
+ });
+ dirs.local = localSources({
+ rawDir: dirs.rawDir, channelsDir, channelSlug: provenance?.channelSlug ?? null, probe: opts.probe,
+ });
// A project's out/ first, through ensureOutDir: with UMTOOL_MEDIA_DIR set it
// is a link to the media root, and the recursive mkdirs below would
@@ -3477,8 +3551,8 @@ export async function buildVideo({ manifestPath, opts = {}, out, only, fetchOnly
};
}
const meta = await videoMeta(entry.video, entry.channel ?? provenance.channelSlug, { siteChannel: entry.siteChannel, siteVideo: entry.siteVideo });
- const r = await fetchClip(entry, meta, render, dirs.rawDir, opts);
- EMIT("done", { out: r.path, fetchStart: r.fetchStart, cached: r.cached });
+ const r = await fetchClip(entry, meta, render, dirs.local, opts);
+ EMIT("done", { out: r.path, fetchStart: r.fetchStart, cached: r.cached, source: r.source });
return { out: r.path, failures: [] };
}
@@ -3489,6 +3563,16 @@ export async function buildVideo({ manifestPath, opts = {}, out, only, fetchOnly
return { out: r.out, failures: [] };
}
+ // `--no-network`: every clip's source is found on disk BEFORE anything is
+ // rendered, and a build that would have to fetch even one is refused here,
+ // naming each -- not twenty minutes in, at the first one. (The runs that
+ // rebuild no segment fetch nothing, so they are not asked.)
+ if (opts.noNetwork && !opts.chaptersOnly && !opts.railOnly && !opts.chromeOnly && !opts.chromePreview) {
+ const want = manifest.timeline.filter((e) => !only || e.id === only);
+ const missing = await clipsNeedingFetch(want, whole.timeline ?? [], render, opts, dirs.local);
+ if (missing.length) throw new Error(needsFetchMessage(missing));
+ }
+
// Footer chrome is shared by every clip, so build it once up front. The deck
// has none: it is the chrome. (`hyper` is the chart band's legacy switch,
// which assertChrome already refuses beside a deck; the `!deck` says so here
@@ -3941,7 +4025,7 @@ async function main() {
console.error(
"usage: build-video.mjs <manifest.json> [--out <dir>] [--variant sourced|full]\n" +
" [--only <id>] [--fetch-only <id>]\n" +
- " [--pad <s>] [--pad-before <s>] [--pad-after <s>] [--skip-fetch] [--no-xfade] [--no-chapters] [--chapters-only]\n" +
+ " [--pad <s>] [--pad-before <s>] [--pad-after <s>] [--skip-fetch] [--no-network] [--no-xfade] [--no-chapters] [--chapters-only]\n" +
" [--progress ndjson] [--continue-on-error] [--no-reuse]\n" +
" [--no-rail] [--rail-only] [--preview <start> <dur>]\n" +
" [--chrome-only] [--no-chrome] [--chrome-preview <at> <dur>] (render.chrome, the deck)\n" +
@@ -3962,6 +4046,7 @@ async function main() {
const opts = {
variant: flag("--variant") ?? "sourced",
skipFetch: argv.includes("--skip-fetch"),
+ noNetwork: argv.includes("--no-network"),
continueOnError: argv.includes("--continue-on-error"),
noXfade: argv.includes("--no-xfade"),
noChapters: argv.includes("--no-chapters"),
diff --git a/umtool/report-to-video/package.json b/umtool/report-to-video/package.json
@@ -32,6 +32,7 @@
"./render-cards": "./render-cards.mjs",
"./resolve-windows": "./resolve-windows.mjs",
"./shoot-page": "./shoot-page.mjs",
+ "./sources": "./sources.mjs",
"./verify-build": "./verify-build.mjs"
}
}
diff --git a/umtool/report-to-video/sources.mjs b/umtool/report-to-video/sources.mjs
@@ -0,0 +1,359 @@
+// sources.mjs — where a clip's media comes from when it is ALREADY ON DISK.
+//
+// A render needs a few seconds of a source around each clip. Those seconds are
+// usually somewhere local already, in one of three places, and fetching them
+// again is the one expensive thing a build does. So before anything asks the
+// network, the build asks this module, in this order:
+//
+// 1. raw-cache the build's own `out/clips-raw/<video>_<from>-<to>.mp4`
+// 2. corpus-window the editor's window cache,
+// `channels/<slug>/data/<id>/clips/<from>-<to>.<ext>`
+// 3. saved-video the whole source container: the saved-video store's
+// pointer (`data/<id>/saved-video.json` -> `<dir>/<file>`),
+// else a `data/<id>/source-media.<ext>` not yet moved there.
+// It is the window [0, its probed duration].
+//
+// A window qualifies only if it holds the REQUESTED span whole -- the clip plus
+// the build's fetch pad -- to WIN_EPS. Within a tier the tightest wins (the
+// build decodes what it is handed); across tiers the order above wins, so a
+// file this project already cut from keeps being the one it cuts from.
+//
+// The tiers are a list, not a chain of ifs: each is `{kind, windows(ctx)}`, and
+// the next kind of local source is one more entry. A tier is only listed when
+// every tier before it missed, so the saved-video probe (an ffprobe, perhaps on
+// a platter) is paid for only by a clip nothing nearer could serve.
+//
+// A file in `data/<id>/` may be a RELATIVE symlink into `media/`, which may be
+// an absolute symlink to another drive. Every candidate is `stat`ed through its
+// links before it is returned, and a dangling one -- an unmounted drive -- is
+// "not here", never an error: the build falls through to the next tier.
+//
+// A LEAF. Plain ESM with no imports from build-video.mjs or umtool's lib, so
+// the build, the clip bench (lib/projects/report.mjs) and lib/report/raw-cache
+// can all import it without a cycle, and so the bench and the render cannot
+// disagree about what is local. It never computes a path from the cwd: every
+// root arrives as an argument (see umtool/lib/paths.mjs on why that matters to
+// the Next build).
+import { execFile } from "node:child_process";
+import { readdir, readFile, stat } from "node:fs/promises";
+import path from "node:path";
+import { promisify } from "node:util";
+
+const execFileP = promisify(execFile);
+
+/** The source kinds, in the order they are consulted. */
+export const SOURCE_KINDS = ["raw-cache", "corpus-window", "saved-video"];
+
+// A window read back from a 2 dp name can sit a hair outside the request that
+// produced it; the same tolerance resolve-windows.mjs and common's
+// lib/clipWindow.ts use, for the same reason.
+export const WIN_EPS = 0.02;
+
+const WINDOW_RE = /^(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)$/;
+
+// ---- the build's raw cache (tier 1) ----------------------------------------
+// A raw clip's window is IN ITS NAME, which makes the file immutable and the
+// cache content-addressed. TWO DECIMALS, always: the name is a function of the
+// numbers and nothing else.
+
+/** The raw-cache file name for a window of `video`. The build's cache key. */
+export function rawWindowName(video, from, to) {
+ return `${video}_${from.toFixed(2)}-${to.toFixed(2)}.mp4`;
+}
+
+/** A directory listing, or [] for a directory that is not there. */
+export async function listRawNames(rawDir) {
+ try {
+ return await readdir(rawDir);
+ } catch {
+ return [];
+ }
+}
+
+/** The windows THIS video's files hold, parsed out of a clips-raw listing. */
+export function windowsFromNames(names, rawDir, video) {
+ const prefix = `${video}_`;
+ const out = [];
+ for (const name of names) {
+ if (!name.startsWith(prefix) || !name.endsWith(".mp4")) continue;
+ // The remainder must be exactly `a-b`, which is what stops a video id that
+ // is a prefix of another (or one containing `_`) from claiming its files.
+ const m = WINDOW_RE.exec(name.slice(prefix.length, -4));
+ if (!m) continue;
+ out.push({ name, path: path.join(/* turbopackIgnore: true */ rawDir, name), from: Number(m[1]), to: Number(m[2]) });
+ }
+ return out;
+}
+
+/** Does this window hold [from, to] whole, to the manifest's tolerance? */
+export function windowContains(w, from, to) {
+ return !(w.from > from + WIN_EPS || w.to < to - WIN_EPS);
+}
+
+/** The tightest of `windows` containing [from, to], or null. */
+export function tightestContaining(windows, from, to) {
+ let best = null;
+ for (const w of windows) {
+ if (!windowContains(w, from, to)) continue;
+ if (!best || w.to - w.from < best.to - best.from) best = w;
+ }
+ return best;
+}
+
+export async function cachedWindowsFor(rawDir, video) {
+ return windowsFromNames(await listRawNames(rawDir), rawDir, video);
+}
+
+/** The tightest cached file containing [from, to], or null. */
+export async function findContainingWindow(rawDir, video, from, to) {
+ return tightestContaining(await cachedWindowsFor(rawDir, video), from, to);
+}
+
+// ---- the corpus (tiers 2 and 3) --------------------------------------------
+
+/** A video's directory in a channels tree: `<channelsDir>/<slug>/data/<id>`. */
+export function videoDirOf(channelsDir, slug, video) {
+ return path.join(/* turbopackIgnore: true */ channelsDir, slug, "data", video);
+}
+
+/** Where the editor's fetch-window puts a video's windows (lib/clipWindow.ts). */
+export function corpusClipsDir(videoDir) {
+ return path.join(/* turbopackIgnore: true */ videoDir, "clips");
+}
+
+// The extensions a corpus window may wear -- common/lib/clipWindow.ts's
+// CLIP_WINDOW_EXTS. The editor writes `.mp4`; the other two are what an older
+// fetch left. Never `.json` (the sidecar) and never `.part.mp4` (in flight:
+// its stem is not `a-b`, so the pattern below refuses it).
+export const CORPUS_WINDOW_EXTS = [".mp4", ".mkv", ".webm"];
+
+/**
+ * The windows in a corpus clips dir: un-prefixed, because the directory is
+ * already per video -- `<from>-<to>.<ext>`. The same numbers as clips-raw's
+ * names, one predicate over both.
+ */
+export function windowsFromBareNames(names, dir) {
+ const out = [];
+ for (const name of names) {
+ const ext = CORPUS_WINDOW_EXTS.find((x) => name.endsWith(x));
+ if (!ext) continue;
+ const m = WINDOW_RE.exec(name.slice(0, -ext.length));
+ if (!m) continue;
+ const from = Number(m[1]);
+ const to = Number(m[2]);
+ if (!(to > from)) continue;
+ out.push({ name, path: path.join(/* turbopackIgnore: true */ dir, name), from, to });
+ }
+ return out;
+}
+
+export const SAVED_VIDEO_POINTER = "saved-video.json";
+
+// common/lib/mediaFiles.ts's anchored `source-media.<ext>`, over its
+// VIDEO_CONTAINER_EXTS: a picture is the point here, so an audio-only
+// container is not a source for this tier.
+const SOURCE_MEDIA_RE = /^source-media\.(?:mp4|webm|mkv|mov|m4v|ogv|avi)$/i;
+
+/**
+ * The saved-video store's pointer for a video, read the way
+ * common/lib/savedVideo.ts parses it (`dir` and `file`, both non-empty), or
+ * null. Re-implemented rather than imported: this runs under bare node.
+ * `height` is the format the persist recorded taking, when it recorded one.
+ */
+export async function readSavedVideoPointer(videoDir) {
+ let raw;
+ try {
+ raw = JSON.parse(await readFile(path.join(/* turbopackIgnore: true */ videoDir, SAVED_VIDEO_POINTER), "utf8"));
+ } catch {
+ return null;
+ }
+ if (!raw || typeof raw !== "object") return null;
+ if (typeof raw.dir !== "string" || raw.dir === "") return null;
+ if (typeof raw.file !== "string" || raw.file === "") return null;
+ const height = Number(raw.format?.height);
+ return {
+ dir: raw.dir,
+ file: raw.file,
+ path: path.join(/* turbopackIgnore: true */ raw.dir, raw.file),
+ ...(Number.isInteger(height) && height > 0 ? { height } : {}),
+ };
+}
+
+/**
+ * The whole-source containers a video has: the store's (through the pointer)
+ * first, then any `source-media.<ext>` still in the video dir. Not yet checked
+ * for existence -- that is `present`'s job, once, for every tier.
+ */
+export async function wholeContainersOf(videoDir) {
+ const out = [];
+ const pointer = await readSavedVideoPointer(videoDir);
+ if (pointer) out.push({ name: pointer.file, path: pointer.path, height: pointer.height });
+ const local = (await listRawNames(videoDir)).filter((n) => SOURCE_MEDIA_RE.test(n)).sort();
+ for (const name of local) {
+ const p = path.join(/* turbopackIgnore: true */ videoDir, name);
+ if (!out.some((c) => c.path === p)) out.push({ name, path: p });
+ }
+ return out;
+}
+
+/**
+ * Is there a readable file at `p`, through every link on the way? A dangling
+ * link, a missing file and an unanswering drive are all "no", never a throw.
+ */
+export async function present(p) {
+ try {
+ return (await stat(p)).isFile();
+ } catch {
+ return false;
+ }
+}
+
+/**
+ * The default probe: one ffprobe for the container's duration and its first
+ * video stream's height. Null when ffprobe cannot read it -- which makes the
+ * container "not a source", not a failed build.
+ */
+export async function ffprobeSource(file) {
+ try {
+ const { stdout } = await execFileP(process.env.FFPROBE_BIN ?? "ffprobe", [
+ "-v", "error", "-select_streams", "v:0",
+ "-show_entries", "stream=height:format=duration",
+ "-of", "json", file,
+ ]);
+ const doc = JSON.parse(stdout);
+ const duration = Number(doc?.format?.duration);
+ if (!Number.isFinite(duration) || duration <= 0) return null;
+ const height = Number(doc?.streams?.[0]?.height);
+ return { duration, ...(Number.isInteger(height) && height > 0 ? { height } : {}) };
+ } catch {
+ return null;
+ }
+}
+
+// ---- the tiers ---------------------------------------------------------------
+// Each lists a video's candidate windows `{name, path, from, to, height?}` for
+// one context `{video, slug, rawDir, channelsDir, probe}`. A tier that cannot
+// apply (no rawDir, no channelsDir or slug) lists nothing.
+
+async function rawCacheWindows({ video, rawDir }) {
+ if (!rawDir) return [];
+ return cachedWindowsFor(rawDir, video);
+}
+
+async function corpusWindows({ video, slug, channelsDir }) {
+ if (!channelsDir || !slug) return [];
+ const dir = corpusClipsDir(videoDirOf(channelsDir, slug, video));
+ return windowsFromBareNames(await listRawNames(dir), dir);
+}
+
+async function savedVideoWindows({ video, slug, channelsDir, probe = ffprobeSource }) {
+ if (!channelsDir || !slug) return [];
+ const out = [];
+ for (const c of await wholeContainersOf(videoDirOf(channelsDir, slug, video))) {
+ // Existence BEFORE the probe: an unmounted drive must not cost a timeout.
+ if (!(await present(c.path))) continue;
+ const info = await probe(c.path);
+ const duration = Number(info?.duration);
+ // No duration is no entry rather than a guess: claiming a span a file may
+ // not cover is the one failure worse than a miss.
+ if (!Number.isFinite(duration) || duration <= 0) continue;
+ const height = c.height ?? info?.height;
+ out.push({ name: c.name, path: c.path, from: 0, to: duration, ...(height ? { height } : {}) });
+ }
+ return out;
+}
+
+export const TIERS = [
+ { kind: "raw-cache", windows: rawCacheWindows },
+ { kind: "corpus-window", windows: corpusWindows },
+ { kind: "saved-video", windows: savedVideoWindows },
+];
+
+const asSource = (kind, w) => ({
+ kind,
+ path: w.path,
+ name: w.name,
+ windowStart: w.from,
+ windowEnd: w.to,
+ ...(w.height ? { height: w.height } : {}),
+});
+
+/**
+ * The local source for one span of one video, or null when nothing on disk
+ * holds it and only a fetch would.
+ *
+ * @param {{ video: string, slug?: string|null, from: number, to: number }} want
+ * `from`/`to` are the PADDED span -- what the build would otherwise fetch.
+ * @param {{ rawDir?: string|null, channelsDir?: string|null, exact?: boolean,
+ * probe?: (file: string) => Promise<{duration: number, height?: number}|null>,
+ * kinds?: string[] }} [config]
+ * `exact` (`--no-reuse`) accepts only the raw-cache file named for exactly
+ * this span. `kinds` narrows the tiers consulted (default: all, in order).
+ * @returns {Promise<null | { kind: string, path: string, name: string,
+ * windowStart: number, windowEnd: number, height?: number }>}
+ */
+export async function resolveLocalSource(want, config = {}) {
+ const { video, slug = null, from, to } = want;
+ const { rawDir = null, channelsDir = null, exact = false, probe, kinds = null } = config;
+ if (!video || !Number.isFinite(from) || !Number.isFinite(to)) return null;
+
+ if (exact) {
+ if (!rawDir) return null;
+ const name = rawWindowName(video, from, to);
+ const p = path.join(/* turbopackIgnore: true */ rawDir, name);
+ return (await present(p)) ? asSource("raw-cache", { name, path: p, from, to }) : null;
+ }
+
+ const ctx = { video, slug, rawDir, channelsDir, probe };
+ for (const tier of TIERS) {
+ if (kinds && !kinds.includes(tier.kind)) continue;
+ const windows = (await tier.windows(ctx)).filter((w) => windowContains(w, from, to));
+ // The raw cache's EXACT name first, as it always was; then tightest.
+ const exactName = tier.kind === "raw-cache" ? rawWindowName(video, from, to) : null;
+ windows.sort((a, b) =>
+ (b.name === exactName) - (a.name === exactName) || (a.to - a.from) - (b.to - b.from));
+ for (const w of windows) {
+ if (await present(w.path)) return asSource(tier.kind, w);
+ }
+ }
+ return null;
+}
+
+/**
+ * Every window of a video OUTSIDE the project -- the corpus tiers -- each
+ * tagged with its `kind` and checked to be there. What the clip bench folds
+ * into its view of the raw cache, so the bench's "fetched" and the render's
+ * "local" are the same set.
+ *
+ * `probe` decides a whole container's span. The bench passes a cheap one (the
+ * cue doc's duration) and is asked only for a video that has a container.
+ */
+export async function corpusWindowsOf({ video, slug, channelsDir, probe = ffprobeSource }) {
+ const ctx = { video, slug, channelsDir, probe };
+ const out = [];
+ for (const tier of TIERS) {
+ if (tier.kind === "raw-cache") continue;
+ for (const w of await tier.windows(ctx)) {
+ if (await present(w.path)) out.push({ ...w, kind: tier.kind });
+ }
+ }
+ return out;
+}
+
+// ---- which channels tree -------------------------------------------------------
+
+/** The conventional name make-shadow-channels.sh builds. */
+export const SHADOW_CHANNELS = ".shadow-channels";
+
+/**
+ * Which channels directory a project reads: `provenance.channelsDir` (resolved
+ * against the project dir), else a `.shadow-channels/` the project built, else
+ * `fallback` (the global corpus). The bench and the render both ask this, so
+ * they look for local media in the same tree.
+ */
+export function channelsDirFor(dir, manifest, { shadowExists = false, fallback = null } = {}) {
+ const declared = manifest?.provenance?.channelsDir;
+ if (declared) return path.resolve(/* turbopackIgnore: true */ dir, declared);
+ if (shadowExists) return path.join(/* turbopackIgnore: true */ dir, SHADOW_CHANNELS);
+ return fallback;
+}
diff --git a/umtool/report-to-video/sources.test.mjs b/umtool/report-to-video/sources.test.mjs
@@ -0,0 +1,357 @@
+// Tests for sources.mjs: where a clip's media is found on disk before the
+// build fetches it, and the build's use of it (fetchSpan, --no-network).
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import test from "node:test";
+import path from "node:path";
+import os from "node:os";
+import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
+
+import {
+ SOURCE_KINDS,
+ TIERS,
+ channelsDirFor,
+ corpusWindowsOf,
+ rawWindowName,
+ readSavedVideoPointer,
+ resolveLocalSource,
+ windowsFromBareNames,
+} from "./sources.mjs";
+import {
+ buildVideo,
+ clipsNeedingFetch,
+ fetchSpan,
+ findContainingWindow,
+ localSources,
+ needsFetchMessage,
+} from "./build-video.mjs";
+
+const SLUG = "demo-channel";
+const VIDEO = "abc123";
+
+async function tree() {
+ const root = await mkdtemp(path.join(os.tmpdir(), "rtv-sources-"));
+ const rawDir = path.join(root, "out", "clips-raw");
+ const channelsDir = path.join(root, "channels");
+ const videoDir = path.join(channelsDir, SLUG, "data", VIDEO);
+ const clipsDir = path.join(videoDir, "clips");
+ const storeDir = path.join(root, "store", SLUG, VIDEO);
+ for (const d of [rawDir, clipsDir, storeDir]) await mkdir(d, { recursive: true });
+ return { root, rawDir, channelsDir, videoDir, clipsDir, storeDir };
+}
+
+const touch = (p) => writeFile(p, Buffer.alloc(16, 1));
+
+// A probe that says how long a container is, and counts how often it is asked.
+function fakeProbe(duration = 600, extra = {}) {
+ const calls = [];
+ const probe = async (file) => {
+ calls.push(file);
+ return { duration, ...extra };
+ };
+ return { probe, calls };
+}
+
+async function pointTo(videoDir, storeDir, file, extra = {}) {
+ await writeFile(
+ path.join(videoDir, "saved-video.json"),
+ JSON.stringify({ storedAt: "2026-01-01T00:00:00Z", dir: storeDir, file, bytes: 16, ...extra }),
+ );
+}
+
+test("the tiers are listed in the order they are consulted", () => {
+ assert.deepEqual(TIERS.map((t) => t.kind), SOURCE_KINDS);
+ assert.deepEqual(SOURCE_KINDS, ["raw-cache", "corpus-window", "saved-video"]);
+});
+
+test("rawWindowName is the build's cache key, two decimals always", () => {
+ assert.equal(rawWindowName(VIDEO, 7, 21.5), "abc123_7.00-21.50.mp4");
+});
+
+test("raw-cache: the exact name first, else the tightest containing window", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.rawDir, rawWindowName(VIDEO, 10, 20)));
+ await touch(path.join(t.rawDir, rawWindowName(VIDEO, 0, 60)));
+ await touch(path.join(t.rawDir, rawWindowName(VIDEO, 9, 25)));
+ // A video id that is a prefix of this one must not claim its files.
+ await touch(path.join(t.rawDir, rawWindowName(`${VIDEO}x`, 9, 21)));
+ const exact = await resolveLocalSource({ video: VIDEO, slug: SLUG, from: 10, to: 20 }, { rawDir: t.rawDir });
+ assert.equal(exact.kind, "raw-cache");
+ assert.equal(exact.name, "abc123_10.00-20.00.mp4");
+ const tight = await resolveLocalSource({ video: VIDEO, slug: SLUG, from: 11, to: 22 }, { rawDir: t.rawDir });
+ assert.equal(tight.name, "abc123_9.00-25.00.mp4");
+ assert.equal(tight.windowStart, 9);
+ assert.equal(tight.windowEnd, 25);
+ // The re-export build-video has always offered answers the same.
+ assert.equal((await findContainingWindow(t.rawDir, VIDEO, 11, 22)).name, tight.name);
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("coverage is of the PADDED span, to WIN_EPS", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.clipsDir, "10.00-20.00.mp4"));
+ const ask = (from, to) =>
+ resolveLocalSource({ video: VIDEO, slug: SLUG, from, to }, { channelsDir: t.channelsDir });
+ // The clip alone fits; the clip with a 3 s pad either side does not.
+ assert.ok(await ask(13, 17));
+ assert.equal(await ask(10 - 3 + 0.5, 17 + 3), null, "the after-pad runs past 20");
+ assert.equal(await ask(7, 17), null, "the before-pad starts before 10");
+ // A 2 dp name read back a hair outside the request still counts.
+ assert.ok(await ask(9.99, 20.01));
+ assert.equal(await ask(9.9, 20), null);
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("corpus-window: bare names, the editor's three extensions, never a sidecar or a partial", () => {
+ const got = windowsFromBareNames(
+ ["1.00-2.00.mp4", "1.00-3.00.mkv", "1.00-4.00.webm", "1.00-5.00.json", "1.00-6.00.part.mp4", "6.00-5.00.mp4", "x.mp4"],
+ "/c",
+ );
+ assert.deepEqual(got.map((w) => w.name), ["1.00-2.00.mp4", "1.00-3.00.mkv", "1.00-4.00.webm"]);
+ assert.equal(got[0].path, path.join("/c", "1.00-2.00.mp4"));
+});
+
+test("precedence: raw-cache over corpus-window over saved-video, and no probe when a nearer tier hits", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.storeDir, "source-media.mp4"));
+ await pointTo(t.videoDir, t.storeDir, "source-media.mp4");
+ const { probe, calls } = fakeProbe(600);
+ const cfg = { rawDir: t.rawDir, channelsDir: t.channelsDir, probe };
+ const want = { video: VIDEO, slug: SLUG, from: 100, to: 110 };
+
+ let hit = await resolveLocalSource(want, cfg);
+ assert.equal(hit.kind, "saved-video");
+ assert.equal(calls.length, 1);
+
+ await touch(path.join(t.clipsDir, "90.00-120.00.mp4"));
+ calls.length = 0;
+ hit = await resolveLocalSource(want, cfg);
+ assert.equal(hit.kind, "corpus-window");
+ assert.equal(hit.windowStart, 90);
+ assert.equal(calls.length, 0, "the saved container is not probed when a window serves");
+
+ // Wider than the corpus window, and still preferred: it is this build's own.
+ await touch(path.join(t.rawDir, rawWindowName(VIDEO, 50, 200)));
+ hit = await resolveLocalSource(want, cfg);
+ assert.equal(hit.kind, "raw-cache");
+ assert.equal(hit.windowStart, 50);
+
+ // `kinds` narrows the tiers consulted.
+ hit = await resolveLocalSource(want, { ...cfg, kinds: ["corpus-window"] });
+ assert.equal(hit.kind, "corpus-window");
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("exact (--no-reuse): only the raw file named for exactly this span", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.rawDir, rawWindowName(VIDEO, 0, 60)));
+ await touch(path.join(t.clipsDir, "0.00-60.00.mp4"));
+ const want = { video: VIDEO, slug: SLUG, from: 10, to: 20 };
+ const cfg = { rawDir: t.rawDir, channelsDir: t.channelsDir, exact: true };
+ assert.equal(await resolveLocalSource(want, cfg), null);
+ await touch(path.join(t.rawDir, rawWindowName(VIDEO, 10, 20)));
+ const hit = await resolveLocalSource(want, cfg);
+ assert.equal(hit.kind, "raw-cache");
+ assert.equal(hit.name, "abc123_10.00-20.00.mp4");
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("saved-video: the pointer's container is the window [0, probed duration], height from the pointer", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.storeDir, "source-media.webm"));
+ await pointTo(t.videoDir, t.storeDir, "source-media.webm", { format: { preset: "video_720", height: 720 } });
+ const p = await readSavedVideoPointer(t.videoDir);
+ assert.equal(p.path, path.join(t.storeDir, "source-media.webm"));
+ assert.equal(p.height, 720);
+
+ const { probe } = fakeProbe(300.5, { height: 1080 });
+ const cfg = { channelsDir: t.channelsDir, probe };
+ const hit = await resolveLocalSource({ video: VIDEO, slug: SLUG, from: 250, to: 300 }, cfg);
+ assert.deepEqual(hit, {
+ kind: "saved-video",
+ path: path.join(t.storeDir, "source-media.webm"),
+ name: "source-media.webm",
+ windowStart: 0,
+ windowEnd: 300.5,
+ height: 720,
+ });
+ // Past the container's end is not covered.
+ assert.equal(await resolveLocalSource({ video: VIDEO, slug: SLUG, from: 290, to: 310 }, cfg), null);
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("saved-video: a source-media container still in the video dir, through a relative link into media/", async () => {
+ const t = await tree();
+ try {
+ const mediaDir = path.join(t.channelsDir, SLUG, "media", VIDEO);
+ await mkdir(mediaDir, { recursive: true });
+ await touch(path.join(mediaDir, "source-media.mp4"));
+ await symlink(path.join("..", "..", "media", VIDEO, "source-media.mp4"), path.join(t.videoDir, "source-media.mp4"));
+ // Not a source: an audio-only container, and yt-dlp's scratch.
+ await touch(path.join(t.videoDir, "source-media.m4a"));
+ await touch(path.join(t.videoDir, "source-media.temp.mp4"));
+ const { probe, calls } = fakeProbe(120, { height: 480 });
+ const hit = await resolveLocalSource({ video: VIDEO, slug: SLUG, from: 5, to: 15 }, { channelsDir: t.channelsDir, probe });
+ assert.equal(hit.kind, "saved-video");
+ assert.equal(hit.name, "source-media.mp4");
+ assert.equal(hit.height, 480, "no pointer format, so the probe's height");
+ assert.deepEqual(calls, [path.join(t.videoDir, "source-media.mp4")]);
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("a dangling link is not found -- never an error -- and the next candidate or tier answers", async () => {
+ const t = await tree();
+ try {
+ // The tightest corpus window is a link to a drive that is not there.
+ await symlink(path.join(t.root, "unmounted", "w.mp4"), path.join(t.clipsDir, "10.00-20.00.mp4"));
+ await touch(path.join(t.clipsDir, "0.00-40.00.mp4"));
+ let hit = await resolveLocalSource({ video: VIDEO, slug: SLUG, from: 11, to: 19 }, { channelsDir: t.channelsDir });
+ assert.equal(hit.name, "0.00-40.00.mp4");
+
+ // A pointer into an unmounted store and a dangling in-dir link: no source,
+ // and the probe is never asked about either.
+ await pointTo(t.videoDir, path.join(t.root, "unmounted-store"), "source-media.mp4");
+ await symlink(path.join("..", "..", "media", VIDEO, "source-media.mkv"), path.join(t.videoDir, "source-media.mkv"));
+ const { probe, calls } = fakeProbe(9999);
+ hit = await resolveLocalSource({ video: VIDEO, slug: SLUG, from: 100, to: 110 }, { channelsDir: t.channelsDir, probe });
+ assert.equal(hit, null);
+ assert.equal(calls.length, 0);
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("a container the probe cannot measure is no source rather than a guess", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.storeDir, "source-media.mp4"));
+ await pointTo(t.videoDir, t.storeDir, "source-media.mp4");
+ const hit = await resolveLocalSource(
+ { video: VIDEO, slug: SLUG, from: 0, to: 1 },
+ { channelsDir: t.channelsDir, probe: async () => null },
+ );
+ assert.equal(hit, null);
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("no channel slug or no channels tree: only the raw cache can answer", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.clipsDir, "0.00-40.00.mp4"));
+ assert.equal(await resolveLocalSource({ video: VIDEO, slug: null, from: 1, to: 2 }, { channelsDir: t.channelsDir }), null);
+ assert.equal(await resolveLocalSource({ video: VIDEO, slug: SLUG, from: 1, to: 2 }, { rawDir: t.rawDir }), null);
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("corpusWindowsOf: the corpus tiers' windows, tagged, present ones only", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.rawDir, rawWindowName(VIDEO, 0, 10)));
+ await touch(path.join(t.clipsDir, "5.00-15.00.mp4"));
+ await symlink(path.join(t.root, "gone.mp4"), path.join(t.clipsDir, "1.00-2.00.mp4"));
+ await touch(path.join(t.storeDir, "source-media.mp4"));
+ await pointTo(t.videoDir, t.storeDir, "source-media.mp4");
+ const got = await corpusWindowsOf({ video: VIDEO, slug: SLUG, channelsDir: t.channelsDir, probe: fakeProbe(90).probe });
+ assert.deepEqual(
+ got.map((w) => [w.kind, w.name, w.from, w.to]),
+ [
+ ["corpus-window", "5.00-15.00.mp4", 5, 15],
+ ["saved-video", "source-media.mp4", 0, 90],
+ ],
+ );
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("channelsDirFor: declared, then the shadow tree, then the fallback", () => {
+ assert.equal(channelsDirFor("/p", { provenance: { channelsDir: "../c" } }, { fallback: "/g" }), path.resolve("/p", "../c"));
+ assert.equal(channelsDirFor("/p", {}, { shadowExists: true, fallback: "/g" }), "/p/.shadow-channels");
+ assert.equal(channelsDirFor("/p", {}, { fallback: "/g" }), "/g");
+});
+
+// ---- the build's side --------------------------------------------------------
+
+test("fetchSpan: the extent widened by the pad, one edge at a time, clamped at 0", () => {
+ const e = { start: 10, end: 20 };
+ assert.deepEqual(fetchSpan(e, {}), { from: 7, to: 23 });
+ assert.deepEqual(fetchSpan(e, { fetchPad: 1 }), { from: 9, to: 21 });
+ assert.deepEqual(fetchSpan(e, { fetchPad: 1 }, { pad: 5, padAfter: 2 }), { from: 5, to: 22 });
+ assert.deepEqual(fetchSpan({ start: 1, end: 2 }, {}), { from: 0, to: 5 });
+});
+
+test("clipsNeedingFetch: the clips no tier serves, with their timeline index", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.clipsDir, "0.00-40.00.mp4"));
+ const timeline = [
+ { id: "t0", type: "card" },
+ { id: "c1", type: "clip", video: VIDEO, start: 10, end: 20 },
+ { id: "c2", type: "clip", video: "def456", start: 10, end: 20 },
+ { id: "c3", type: "clip", video: VIDEO, start: 38, end: 45 },
+ ];
+ const local = localSources({ rawDir: t.rawDir, channelsDir: t.channelsDir, channelSlug: SLUG });
+ const missing = await clipsNeedingFetch(timeline, timeline, { fetchPad: 3 }, {}, local);
+ assert.deepEqual(missing.map((m) => [m.index, m.id, m.slug, m.video, m.from, m.to]), [
+ [2, "c2", SLUG, "def456", 7, 23],
+ [3, "c3", SLUG, VIDEO, 35, 48],
+ ]);
+ const msg = needsFetchMessage(missing);
+ assert.match(msg, /--no-network: 2 clip\(s\)/);
+ assert.match(msg, /timeline\[2\] c2 {2}demo-channel\/def456 {2}7\.00–23\.00/);
+ assert.match(msg, /timeline\[3\] c3 {2}demo-channel\/abc123 {2}35\.00–48\.00/);
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
+
+test("buildVideo --no-network refuses before rendering, naming every clip that would need a fetch", async () => {
+ const t = await tree();
+ try {
+ await touch(path.join(t.clipsDir, "0.00-40.00.mp4"));
+ const manifestPath = path.join(t.root, "video.manifest.json");
+ await writeFile(manifestPath, JSON.stringify({
+ slug: "demo",
+ title: "Demo",
+ provenance: { channelSlug: SLUG, channelsDir: "channels", siteOrigin: "https://example.invalid" },
+ render: { width: 1280, height: 720, fps: 30, fetchPad: 3 },
+ timeline: [
+ { id: "c1", type: "clip", video: VIDEO, start: 10, end: 20 },
+ { id: "c2", type: "clip", video: "def456", start: 1, end: 4 },
+ ],
+ }));
+ await assert.rejects(
+ buildVideo({ manifestPath, opts: { noNetwork: true } }),
+ (err) => {
+ assert.match(err.message, /1 clip\(s\) have no local source/);
+ assert.match(err.message, /timeline\[1\] c2 {2}demo-channel\/def456 {2}0\.00–7\.00/);
+ assert.doesNotMatch(err.message, /c1/);
+ return true;
+ },
+ );
+ } finally {
+ await rm(t.root, { recursive: true, force: true });
+ }
+});
