commit b80edbeddd56875575592576441537a98f5cda9c
parent 20edeac3f8defb9b7e1401560aa697f9ef2df602
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 5 Oct 2026 02:50:02 -0400
common: the corpus half of the local-source lookup (lib/evidenceClip.mjs); report-to-video's sources.mjs and cutArgs take it from there
The corpus-window, saved-video and audio tiers, the window predicates, the
probe and the input cut arguments move into one plain-ESM module in common,
which umtool re-exports under the same names; the build's raw cache stays
umtool's own tier in front of them.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
4 files changed, 453 insertions(+), 251 deletions(-)
diff --git a/common/lib/evidenceClip.mjs b/common/lib/evidenceClip.mjs
@@ -0,0 +1,400 @@
+// evidenceClip.mjs — where a cited span's media already is in the corpus, and
+// the cut that takes the span out of it.
+//
+// Two consumers ask the same question — "which file on this disk holds these
+// seconds of this record?" — and must get the same answer:
+// - umtool's report-to-video build (report-to-video/sources.mjs), which asks
+// its own raw cache first and then these tiers;
+// - the report-site prepare step (lib/evidenceClip-server.ts), which cuts a
+// self-hosted evidence clip of every cited span.
+// So the corpus half of the lookup lives HERE, once, and sources.mjs imports
+// and re-exports it. Plain ESM with no app imports: umtool's scripts run under
+// bare node, which cannot load a `.ts` file.
+//
+// THE CORPUS TIERS, in the order they are consulted:
+//
+// 1. corpus-window the editor's window cache,
+// `channels/<slug>/data/<id>/clips/<from>-<to>.<ext>`
+// (lib/clipWindow.ts)
+// 2. saved-video the whole source container: the saved-video store's
+// pointer (`data/<id>/saved-video.json` -> `<dir>/<file>`),
+// else a `data/<id>/source-media.<ext>` not yet moved there.
+// It is the window [0, its probed duration].
+// 3. audio `data/<id>/audio.<ext>`: the recording's SOUND alone, the
+// window [0, its probed duration]. Consulted only when the
+// caller allows it (`audio: true` on the tier): being last,
+// it never beats a picture already on disk.
+//
+// A window qualifies only if it holds the REQUESTED span whole — the span plus
+// its pad — to WIN_EPS. Within a tier the tightest wins; across tiers the order
+// above wins.
+//
+// A file in `data/<id>/` may be a RELATIVE symlink into `media/`, which may be
+// an absolute symlink to another drive. Every candidate is `stat`ed through its
+// links before it is returned, and a dangling one — an unmounted drive — is
+// "not here", never an error: the lookup falls through to the next tier.
+//
+// It never computes a path from the cwd: every root arrives as an argument
+// (umtool's Next build bundles this file, and a cwd join makes its tracer walk
+// the corpus).
+import { execFile } from "node:child_process";
+import { readdir, readFile, stat } from "node:fs/promises";
+import path from "node:path";
+import { promisify } from "node:util";
+
+const execFileP = promisify(execFile);
+
+/**
+ * Platforms whose records have no picture: a feed of episodes. A span of one
+ * is served by its sound.
+ */
+export const AUDIO_ONLY_PLATFORMS = new Set(["podcast", "feed", "rss"]);
+
+// A window read back from a 2 dp name can sit a hair outside the request that
+// produced it; the same tolerance lib/clipWindow.ts and resolve-windows.mjs
+// use, for the same reason.
+export const WIN_EPS = 0.02;
+
+export const WINDOW_RE = /^(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)$/;
+
+/**
+ * A directory listing, or [] for a directory that is not there.
+ * @param {string} dir
+ * @returns {Promise<string[]>}
+ */
+export async function listNames(dir) {
+ try {
+ return await readdir(dir);
+ } catch {
+ return [];
+ }
+}
+
+/**
+ * @typedef {{ name: string, path: string, from: number, to: number, height?: number }} SourceWindow
+ * @typedef {{ kind: string, path: string, name: string, windowStart: number, windowEnd: number, height?: number }} LocalSource
+ * @typedef {(file: string) => Promise<{ duration: number, height?: number } | null>} SourceProbe
+ * @typedef {{ video: string, slug?: string | null, channelsDir?: string | null, rawDir?: string | null, probe?: SourceProbe }} TierContext
+ * @typedef {{ kind: string, windows: (ctx: TierContext) => Promise<SourceWindow[]>, audio?: boolean }} SourceTier
+ */
+
+/**
+ * Does this window hold [from, to] whole, to the tolerance?
+ * @param {{ from: number, to: number }} w
+ * @param {number} from
+ * @param {number} to
+ */
+export function windowContains(w, from, to) {
+ return !(w.from > from + WIN_EPS || w.to < to - WIN_EPS);
+}
+
+/**
+ * The tightest of `windows` containing [from, to], or null.
+ * @template {{ from: number, to: number }} W
+ * @param {W[]} windows
+ * @param {number} from
+ * @param {number} to
+ * @returns {W | null}
+ */
+export function tightestContaining(windows, from, to) {
+ /** @type {W | null} */
+ let best = null;
+ for (const w of windows) {
+ if (!windowContains(w, from, to)) continue;
+ if (!best || w.to - w.from < best.to - best.from) best = w;
+ }
+ return best;
+}
+
+/**
+ * A video's directory in a channels tree: `<channelsDir>/<slug>/data/<id>`.
+ * @param {string} channelsDir
+ * @param {string} slug
+ * @param {string} video
+ */
+export function videoDirOf(channelsDir, slug, video) {
+ return path.join(/* turbopackIgnore: true */ channelsDir, slug, "data", video);
+}
+
+/**
+ * Where the editor's fetch-window puts a video's windows (lib/clipWindow.ts).
+ * @param {string} videoDir
+ */
+export function corpusClipsDir(videoDir) {
+ return path.join(/* turbopackIgnore: true */ videoDir, "clips");
+}
+
+// The extensions a corpus window may wear -- lib/clipWindow.ts's
+// CLIP_WINDOW_EXTS. The editor writes `.mp4`; the other two are what an older
+// fetch left. Never `.json` (the sidecar) and never `.part.mp4` (in flight:
+// its stem is not `a-b`, so the pattern refuses it).
+export const CORPUS_WINDOW_EXTS = [".mp4", ".mkv", ".webm"];
+
+/**
+ * The windows in a corpus clips dir: un-prefixed, because the directory is
+ * already per video -- `<from>-<to>.<ext>`.
+ * @param {string[]} names
+ * @param {string} dir
+ * @returns {SourceWindow[]}
+ */
+export function windowsFromBareNames(names, dir) {
+ /** @type {SourceWindow[]} */
+ const out = [];
+ for (const name of names) {
+ const ext = CORPUS_WINDOW_EXTS.find((x) => name.endsWith(x));
+ if (!ext) continue;
+ const m = WINDOW_RE.exec(name.slice(0, -ext.length));
+ if (!m) continue;
+ const from = Number(m[1]);
+ const to = Number(m[2]);
+ if (!(to > from)) continue;
+ out.push({ name, path: path.join(/* turbopackIgnore: true */ dir, name), from, to });
+ }
+ return out;
+}
+
+export const SAVED_VIDEO_POINTER = "saved-video.json";
+
+// lib/mediaFiles.ts's anchored `source-media.<ext>`, over its
+// VIDEO_CONTAINER_EXTS: a picture is the point here, so an audio-only
+// container is not a source for this tier.
+const SOURCE_MEDIA_RE = /^source-media\.(?:mp4|webm|mkv|mov|m4v|ogv|avi)$/i;
+
+/**
+ * The saved-video store's pointer for a video, read the way lib/savedVideo.ts
+ * parses it (`dir` and `file`, both non-empty), or null. Re-implemented rather
+ * than imported: this runs under bare node. `height` is the format the persist
+ * recorded taking, when it recorded one.
+ * @param {string} videoDir
+ * @returns {Promise<{ dir: string, file: string, path: string, height?: number } | null>}
+ */
+export async function readSavedVideoPointer(videoDir) {
+ let raw;
+ try {
+ raw = JSON.parse(await readFile(path.join(/* turbopackIgnore: true */ videoDir, SAVED_VIDEO_POINTER), "utf8"));
+ } catch {
+ return null;
+ }
+ if (!raw || typeof raw !== "object") return null;
+ if (typeof raw.dir !== "string" || raw.dir === "") return null;
+ if (typeof raw.file !== "string" || raw.file === "") return null;
+ const height = Number(raw.format?.height);
+ return {
+ dir: raw.dir,
+ file: raw.file,
+ path: path.join(/* turbopackIgnore: true */ raw.dir, raw.file),
+ ...(Number.isInteger(height) && height > 0 ? { height } : {}),
+ };
+}
+
+/**
+ * The whole-source containers a video has: the store's (through the pointer)
+ * first, then any `source-media.<ext>` still in the video dir. Not yet checked
+ * for existence -- that is `present`'s job, once, for every tier.
+ * @param {string} videoDir
+ * @returns {Promise<{ name: string, path: string, height?: number }[]>}
+ */
+export async function wholeContainersOf(videoDir) {
+ /** @type {{ name: string, path: string, height?: number }[]} */
+ const out = [];
+ const pointer = await readSavedVideoPointer(videoDir);
+ if (pointer) out.push({ name: pointer.file, path: pointer.path, height: pointer.height });
+ const local = (await listNames(videoDir)).filter((n) => SOURCE_MEDIA_RE.test(n)).sort();
+ for (const name of local) {
+ const p = path.join(/* turbopackIgnore: true */ videoDir, name);
+ if (!out.some((c) => c.path === p)) out.push({ name, path: p });
+ }
+ return out;
+}
+
+// The sound files a video dir may hold: lib/mediaFiles.ts's anchored
+// `audio.<ext>` over AUDIO_EXTS, plus the `.webm`/`.mp4` an extract-to-mp3 that
+// failed leaves behind (sound only). Read in AUDIO_PREFERENCE's order -- the
+// file the transcribe path reads -- then by name.
+const AUDIO_FILE_RE = /^audio\.(?:mp3|m4a|aac|ogg|oga|opus|wav|flac|webm|mp4)$/i;
+export const AUDIO_PREFERENCE = ["audio.mp3", "audio.m4a", "audio.opus"];
+
+/**
+ * A video dir's audio files, best first. Not yet checked for existence.
+ * @param {string} videoDir
+ * @returns {Promise<{ name: string, path: string }[]>}
+ */
+export async function audioFilesOf(videoDir) {
+ const names = (await listNames(videoDir)).filter((n) => AUDIO_FILE_RE.test(n));
+ const rank = (/** @type {string} */ n) => {
+ const i = AUDIO_PREFERENCE.indexOf(n);
+ return i < 0 ? AUDIO_PREFERENCE.length : i;
+ };
+ names.sort((a, b) => rank(a) - rank(b) || a.localeCompare(b));
+ return names.map((name) => ({ name, path: path.join(/* turbopackIgnore: true */ videoDir, name) }));
+}
+
+/**
+ * Is there a readable file at `p`, through every link on the way? A dangling
+ * link, a missing file and an unanswering drive are all "no", never a throw.
+ * @param {string} p
+ */
+export async function present(p) {
+ try {
+ return (await stat(p)).isFile();
+ } catch {
+ return false;
+ }
+}
+
+/**
+ * The default probe: one ffprobe for the container's duration and its first
+ * video stream's height. Null when ffprobe cannot read it -- which makes the
+ * container "not a source", not a failure.
+ * @param {string} file
+ * @param {string} [bin] the ffprobe binary (default: FFPROBE_BIN, else `ffprobe`)
+ * @returns {Promise<{ duration: number, height?: number } | null>}
+ */
+export async function ffprobeSource(file, bin = process.env.FFPROBE_BIN ?? "ffprobe") {
+ try {
+ const { stdout } = await execFileP(bin, [
+ "-v", "error", "-select_streams", "v:0",
+ "-show_entries", "stream=height:format=duration",
+ "-of", "json", file,
+ ]);
+ const doc = JSON.parse(stdout);
+ const duration = Number(doc?.format?.duration);
+ if (!Number.isFinite(duration) || duration <= 0) return null;
+ const height = Number(doc?.streams?.[0]?.height);
+ return { duration, ...(Number.isInteger(height) && height > 0 ? { height } : {}) };
+ } catch {
+ return null;
+ }
+}
+
+// ---- the tiers ---------------------------------------------------------------
+// Each lists a video's candidate windows `{name, path, from, to, height?}` for
+// one context `{video, slug, channelsDir, probe}`. A tier that cannot apply (no
+// channelsDir or slug) lists nothing.
+
+/** @param {TierContext} ctx */
+async function corpusWindows({ video, slug, channelsDir }) {
+ if (!channelsDir || !slug) return [];
+ const dir = corpusClipsDir(videoDirOf(channelsDir, slug, video));
+ return windowsFromBareNames(await listNames(dir), dir);
+}
+
+/** @param {TierContext} ctx */
+async function savedVideoWindows({ video, slug, channelsDir, probe = ffprobeSource }) {
+ if (!channelsDir || !slug) return [];
+ /** @type {SourceWindow[]} */
+ const out = [];
+ for (const c of await wholeContainersOf(videoDirOf(channelsDir, slug, video))) {
+ // Existence BEFORE the probe: an unmounted drive must not cost a timeout.
+ if (!(await present(c.path))) continue;
+ const info = await probe(c.path);
+ const duration = Number(info?.duration);
+ // No duration is no entry rather than a guess: claiming a span a file may
+ // not cover is the one failure worse than a miss.
+ if (!Number.isFinite(duration) || duration <= 0) continue;
+ const height = c.height ?? info?.height;
+ out.push({ name: c.name, path: c.path, from: 0, to: duration, ...(height ? { height } : {}) });
+ }
+ return out;
+}
+
+// The ONE best audio file, not every one: they are all the same recording, and
+// probing three formats of it would buy nothing.
+/** @param {TierContext} ctx */
+async function audioWindows({ video, slug, channelsDir, probe = ffprobeSource }) {
+ if (!channelsDir || !slug) return [];
+ for (const f of await audioFilesOf(videoDirOf(channelsDir, slug, video))) {
+ if (!(await present(f.path))) continue;
+ const duration = Number((await probe(f.path))?.duration);
+ if (!Number.isFinite(duration) || duration <= 0) continue;
+ return [{ name: f.name, path: f.path, from: 0, to: duration }];
+ }
+ return [];
+}
+
+/**
+ * The corpus tiers, in order. `audio: true` marks a tier consulted only when
+ * the caller allows it.
+ * @type {SourceTier[]}
+ */
+export const CORPUS_TIERS = [
+ { kind: "corpus-window", windows: corpusWindows },
+ { kind: "saved-video", windows: savedVideoWindows },
+ { kind: "audio", windows: audioWindows, audio: true },
+];
+
+/**
+ * @param {string} kind
+ * @param {SourceWindow} w
+ * @returns {LocalSource}
+ */
+export const asSource = (kind, w) => ({
+ kind,
+ path: w.path,
+ name: w.name,
+ windowStart: w.from,
+ windowEnd: w.to,
+ ...(w.height ? { height: w.height } : {}),
+});
+
+/**
+ * The first source among `tiers` holding [from, to] whole and present on disk.
+ * A tier is only listed when every tier before it missed, so the saved-video
+ * probe (an ffprobe, perhaps on a platter) is paid for only by a span nothing
+ * nearer could serve.
+ *
+ * @param {SourceTier[]} tiers
+ * @param {TierContext} ctx
+ * @param {number} from
+ * @param {number} to
+ * @param {{ allow?: (tier: SourceTier) => boolean, preferName?: (tier: SourceTier) => string | null }} [opts]
+ * `allow` skips a tier (default: every tier but an audio one);
+ * `preferName` names, per tier, a file that wins over the tightest.
+ * @returns {Promise<LocalSource | null>}
+ */
+export async function resolveFromTiers(tiers, ctx, from, to, opts = {}) {
+ const { allow = (tier) => !tier.audio, preferName = () => null } = opts;
+ for (const tier of tiers) {
+ if (!allow(tier)) continue;
+ const windows = (await tier.windows(ctx)).filter((w) => windowContains(w, from, to));
+ const exactName = preferName(tier);
+ windows.sort((a, b) =>
+ Number(b.name === exactName) - Number(a.name === exactName) || (a.to - a.from) - (b.to - b.from));
+ for (const w of windows) {
+ if (await present(w.path)) return asSource(tier.kind, w);
+ }
+ }
+ return null;
+}
+
+/**
+ * The corpus source for one span of one record, or null when nothing on disk
+ * holds it.
+ *
+ * @param {{ video: string, slug: string, from: number, to: number }} want
+ * `from`/`to` are the PADDED span.
+ * @param {{ channelsDir: string, probe?: SourceProbe, audio?: boolean }} config
+ * `audio` admits the audio tier (default false).
+ * @returns {Promise<LocalSource | null>}
+ */
+export async function resolveCorpusSource(want, config) {
+ const { video, slug, from, to } = want;
+ const { channelsDir, probe, audio = false } = config;
+ if (!video || !slug || !Number.isFinite(from) || !Number.isFinite(to)) return null;
+ return resolveFromTiers(CORPUS_TIERS, { video, slug, channelsDir, probe }, from, to, {
+ allow: (tier) => (tier.audio ? audio : true),
+ });
+}
+
+/**
+ * The input half of a cut of [a, b] seconds of `raw`: seek on the input, so a
+ * re-encode starts on exactly the asked-for frame. report-to-video's segment
+ * pass and the evidence cutter both use it, so the seconds a video renders and
+ * the seconds a moment page plays are the same arithmetic.
+ * @param {string} raw
+ * @param {number} a
+ * @param {number} b
+ */
+export function cutArgs(raw, a, b) {
+ return ["-ss", a.toFixed(3), "-to", b.toFixed(3), "-i", raw];
+}
diff --git a/common/package.json b/common/package.json
@@ -32,6 +32,7 @@
"./components/virtualizer": "./components/virtualizer.ts",
"./components/*": "./components/*.tsx",
"./lib/detectPlatform.mjs": "./lib/detectPlatform.mjs",
+ "./lib/evidenceClip.mjs": "./lib/evidenceClip.mjs",
"./lib/ports.mjs": "./lib/ports.mjs",
"./lib/report/verdicts.mjs": "./lib/report/verdicts.mjs",
"./lib/toolProbe.mjs": "./lib/toolProbe.mjs",
diff --git a/umtool/report-to-video/build-video.mjs b/umtool/report-to-video/build-video.mjs
@@ -92,6 +92,7 @@ import {
cardWidth, contentWidth, reservedFooterHeight,
} from "./render-cards.mjs";
import { DEFAULT_CHANNELS_DIR, createCueSource, siteOriginFromManifest } from "./cues.mjs";
+import { cutArgs } from "yt-dlp-transcript-common/lib/evidenceClip.mjs";
// Where a clip's media is ALREADY on disk -- the build's raw cache, the
// editor's corpus windows, the saved source -- asked before anything fetches.
import {
@@ -648,9 +649,9 @@ export function clipFetchArgs({ url, from, to, fmt, dest, extra = [] }) {
* @param {number} a seconds INTO that file where the clip starts
* @param {number} b seconds into it where the clip ends
*/
-export function cutArgs(raw, a, b) {
- return ["-ss", a.toFixed(3), "-to", b.toFixed(3), "-i", raw];
-}
+// Common's (lib/evidenceClip.mjs) since a report site cuts its evidence clips
+// with the same arguments: re-exported, never re-spelled.
+export { cutArgs };
/**
* The span a clip's source is needed over: its extent, widened by the fetch
diff --git a/umtool/report-to-video/sources.mjs b/umtool/report-to-video/sources.mjs
@@ -46,27 +46,52 @@
// links before it is returned, and a dangling one -- an unmounted drive -- is
// "not here", never an error: the build falls through to the next tier.
//
+// THE CORPUS HALF IS COMMON'S (common/lib/evidenceClip.mjs): tiers 2-4, the
+// window predicates and the probe live there, once, because the report-site
+// prepare step (common/lib/evidenceClip-server.ts) asks the same question of
+// the same disk and must get the same answer. This module adds the build's own
+// raw cache in front of them and re-exports the rest, so every importer here
+// keeps its names.
+//
// A LEAF. Plain ESM with no imports from build-video.mjs or umtool's lib, so
// the build, the clip bench (lib/projects/report.mjs) and lib/report/raw-cache
// can all import it without a cycle, and so the bench and the render cannot
// disagree about what is local. It never computes a path from the cwd: every
// root arrives as an argument (see umtool/lib/paths.mjs on why that matters to
// the Next build).
-import { execFile } from "node:child_process";
-import { readdir, readFile, stat } from "node:fs/promises";
import path from "node:path";
-import { promisify } from "node:util";
-
-const execFileP = promisify(execFile);
+import {
+ AUDIO_ONLY_PLATFORMS,
+ CORPUS_TIERS,
+ WINDOW_RE,
+ listNames,
+ present,
+ resolveFromTiers,
+ windowContains,
+ tightestContaining,
+ ffprobeSource,
+} from "yt-dlp-transcript-common/lib/evidenceClip.mjs";
+
+export {
+ AUDIO_ONLY_PLATFORMS,
+ AUDIO_PREFERENCE,
+ CORPUS_WINDOW_EXTS,
+ SAVED_VIDEO_POINTER,
+ WIN_EPS,
+ audioFilesOf,
+ corpusClipsDir,
+ ffprobeSource,
+ present,
+ readSavedVideoPointer,
+ tightestContaining,
+ videoDirOf,
+ wholeContainersOf,
+ windowContains,
+ windowsFromBareNames,
+} from "yt-dlp-transcript-common/lib/evidenceClip.mjs";
/** The source kinds, in the order they are consulted. */
-export const SOURCE_KINDS = ["raw-cache", "corpus-window", "saved-video", "audio"];
-
-/**
- * Platforms whose records have no picture: a feed of episodes. A clip from one
- * reads its local audio BEFORE the network, since a fetch has no video to get.
- */
-export const AUDIO_ONLY_PLATFORMS = new Set(["podcast", "feed", "rss"]);
+export const SOURCE_KINDS = ["raw-cache", ...CORPUS_TIERS.map((t) => t.kind)];
/**
* May the audio tier serve this clip, and as what? (See the note at the top.)
@@ -91,13 +116,6 @@ export function audioUse({ entry = null, meta = null, render = null, network = t
/** audioUse's yes or no: should the audio tier be consulted at all? */
export const audioAllowed = (args = {}) => audioUse(args) !== null;
-// A window read back from a 2 dp name can sit a hair outside the request that
-// produced it; the same tolerance resolve-windows.mjs and common's
-// lib/clipWindow.ts use, for the same reason.
-export const WIN_EPS = 0.02;
-
-const WINDOW_RE = /^(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)$/;
-
// ---- the build's raw cache (tier 1) ----------------------------------------
// A raw clip's window is IN ITS NAME, which makes the file immutable and the
// cache content-addressed. TWO DECIMALS, always: the name is a function of the
@@ -109,13 +127,7 @@ export function rawWindowName(video, from, to) {
}
/** A directory listing, or [] for a directory that is not there. */
-export async function listRawNames(rawDir) {
- try {
- return await readdir(rawDir);
- } catch {
- return [];
- }
-}
+export const listRawNames = listNames;
/** The windows THIS video's files hold, parsed out of a clips-raw listing. */
export function windowsFromNames(names, rawDir, video) {
@@ -132,21 +144,6 @@ export function windowsFromNames(names, rawDir, video) {
return out;
}
-/** Does this window hold [from, to] whole, to the manifest's tolerance? */
-export function windowContains(w, from, to) {
- return !(w.from > from + WIN_EPS || w.to < to - WIN_EPS);
-}
-
-/** The tightest of `windows` containing [from, to], or null. */
-export function tightestContaining(windows, from, to) {
- let best = null;
- for (const w of windows) {
- if (!windowContains(w, from, to)) continue;
- if (!best || w.to - w.from < best.to - best.from) best = w;
- }
- return best;
-}
-
export async function cachedWindowsFor(rawDir, video) {
return windowsFromNames(await listRawNames(rawDir), rawDir, video);
}
@@ -156,145 +153,6 @@ export async function findContainingWindow(rawDir, video, from, to) {
return tightestContaining(await cachedWindowsFor(rawDir, video), from, to);
}
-// ---- the corpus (tiers 2 and 3) --------------------------------------------
-
-/** A video's directory in a channels tree: `<channelsDir>/<slug>/data/<id>`. */
-export function videoDirOf(channelsDir, slug, video) {
- return path.join(/* turbopackIgnore: true */ channelsDir, slug, "data", video);
-}
-
-/** Where the editor's fetch-window puts a video's windows (lib/clipWindow.ts). */
-export function corpusClipsDir(videoDir) {
- return path.join(/* turbopackIgnore: true */ videoDir, "clips");
-}
-
-// The extensions a corpus window may wear -- common/lib/clipWindow.ts's
-// CLIP_WINDOW_EXTS. The editor writes `.mp4`; the other two are what an older
-// fetch left. Never `.json` (the sidecar) and never `.part.mp4` (in flight:
-// its stem is not `a-b`, so the pattern below refuses it).
-export const CORPUS_WINDOW_EXTS = [".mp4", ".mkv", ".webm"];
-
-/**
- * The windows in a corpus clips dir: un-prefixed, because the directory is
- * already per video -- `<from>-<to>.<ext>`. The same numbers as clips-raw's
- * names, one predicate over both.
- */
-export function windowsFromBareNames(names, dir) {
- const out = [];
- for (const name of names) {
- const ext = CORPUS_WINDOW_EXTS.find((x) => name.endsWith(x));
- if (!ext) continue;
- const m = WINDOW_RE.exec(name.slice(0, -ext.length));
- if (!m) continue;
- const from = Number(m[1]);
- const to = Number(m[2]);
- if (!(to > from)) continue;
- out.push({ name, path: path.join(/* turbopackIgnore: true */ dir, name), from, to });
- }
- return out;
-}
-
-export const SAVED_VIDEO_POINTER = "saved-video.json";
-
-// common/lib/mediaFiles.ts's anchored `source-media.<ext>`, over its
-// VIDEO_CONTAINER_EXTS: a picture is the point here, so an audio-only
-// container is not a source for this tier.
-const SOURCE_MEDIA_RE = /^source-media\.(?:mp4|webm|mkv|mov|m4v|ogv|avi)$/i;
-
-/**
- * The saved-video store's pointer for a video, read the way
- * common/lib/savedVideo.ts parses it (`dir` and `file`, both non-empty), or
- * null. Re-implemented rather than imported: this runs under bare node.
- * `height` is the format the persist recorded taking, when it recorded one.
- */
-export async function readSavedVideoPointer(videoDir) {
- let raw;
- try {
- raw = JSON.parse(await readFile(path.join(/* turbopackIgnore: true */ videoDir, SAVED_VIDEO_POINTER), "utf8"));
- } catch {
- return null;
- }
- if (!raw || typeof raw !== "object") return null;
- if (typeof raw.dir !== "string" || raw.dir === "") return null;
- if (typeof raw.file !== "string" || raw.file === "") return null;
- const height = Number(raw.format?.height);
- return {
- dir: raw.dir,
- file: raw.file,
- path: path.join(/* turbopackIgnore: true */ raw.dir, raw.file),
- ...(Number.isInteger(height) && height > 0 ? { height } : {}),
- };
-}
-
-/**
- * The whole-source containers a video has: the store's (through the pointer)
- * first, then any `source-media.<ext>` still in the video dir. Not yet checked
- * for existence -- that is `present`'s job, once, for every tier.
- */
-export async function wholeContainersOf(videoDir) {
- const out = [];
- const pointer = await readSavedVideoPointer(videoDir);
- if (pointer) out.push({ name: pointer.file, path: pointer.path, height: pointer.height });
- const local = (await listRawNames(videoDir)).filter((n) => SOURCE_MEDIA_RE.test(n)).sort();
- for (const name of local) {
- const p = path.join(/* turbopackIgnore: true */ videoDir, name);
- if (!out.some((c) => c.path === p)) out.push({ name, path: p });
- }
- return out;
-}
-
-// The sound files a video dir may hold: common/lib/mediaFiles.ts's anchored
-// `audio.<ext>` over AUDIO_EXTS, plus the `.webm`/`.mp4` an extract-to-mp3 that
-// failed leaves behind (sound only). Read in AUDIO_READ_PREFERENCE's order --
-// the file the transcribe path reads -- then by name.
-const AUDIO_FILE_RE = /^audio\.(?:mp3|m4a|aac|ogg|oga|opus|wav|flac|webm|mp4)$/i;
-export const AUDIO_PREFERENCE = ["audio.mp3", "audio.m4a", "audio.opus"];
-
-/** A video dir's audio files, best first. Not yet checked for existence. */
-export async function audioFilesOf(videoDir) {
- const names = (await listRawNames(videoDir)).filter((n) => AUDIO_FILE_RE.test(n));
- const rank = (n) => {
- const i = AUDIO_PREFERENCE.indexOf(n);
- return i < 0 ? AUDIO_PREFERENCE.length : i;
- };
- names.sort((a, b) => rank(a) - rank(b) || a.localeCompare(b));
- return names.map((name) => ({ name, path: path.join(/* turbopackIgnore: true */ videoDir, name) }));
-}
-
-/**
- * Is there a readable file at `p`, through every link on the way? A dangling
- * link, a missing file and an unanswering drive are all "no", never a throw.
- */
-export async function present(p) {
- try {
- return (await stat(p)).isFile();
- } catch {
- return false;
- }
-}
-
-/**
- * The default probe: one ffprobe for the container's duration and its first
- * video stream's height. Null when ffprobe cannot read it -- which makes the
- * container "not a source", not a failed build.
- */
-export async function ffprobeSource(file) {
- try {
- const { stdout } = await execFileP(process.env.FFPROBE_BIN ?? "ffprobe", [
- "-v", "error", "-select_streams", "v:0",
- "-show_entries", "stream=height:format=duration",
- "-of", "json", file,
- ]);
- const doc = JSON.parse(stdout);
- const duration = Number(doc?.format?.duration);
- if (!Number.isFinite(duration) || duration <= 0) return null;
- const height = Number(doc?.streams?.[0]?.height);
- return { duration, ...(Number.isInteger(height) && height > 0 ? { height } : {}) };
- } catch {
- return null;
- }
-}
-
// ---- the tiers ---------------------------------------------------------------
// Each lists a video's candidate windows `{name, path, from, to, height?}` for
// one context `{video, slug, rawDir, channelsDir, probe}`. A tier that cannot
@@ -305,58 +163,8 @@ async function rawCacheWindows({ video, rawDir }) {
return cachedWindowsFor(rawDir, video);
}
-async function corpusWindows({ video, slug, channelsDir }) {
- if (!channelsDir || !slug) return [];
- const dir = corpusClipsDir(videoDirOf(channelsDir, slug, video));
- return windowsFromBareNames(await listRawNames(dir), dir);
-}
-
-async function savedVideoWindows({ video, slug, channelsDir, probe = ffprobeSource }) {
- if (!channelsDir || !slug) return [];
- const out = [];
- for (const c of await wholeContainersOf(videoDirOf(channelsDir, slug, video))) {
- // Existence BEFORE the probe: an unmounted drive must not cost a timeout.
- if (!(await present(c.path))) continue;
- const info = await probe(c.path);
- const duration = Number(info?.duration);
- // No duration is no entry rather than a guess: claiming a span a file may
- // not cover is the one failure worse than a miss.
- if (!Number.isFinite(duration) || duration <= 0) continue;
- const height = c.height ?? info?.height;
- out.push({ name: c.name, path: c.path, from: 0, to: duration, ...(height ? { height } : {}) });
- }
- return out;
-}
-
-// The ONE best audio file, not every one: they are all the same recording, and
-// probing three formats of it would buy nothing.
-async function audioWindows({ video, slug, channelsDir, probe = ffprobeSource }) {
- if (!channelsDir || !slug) return [];
- for (const f of await audioFilesOf(videoDirOf(channelsDir, slug, video))) {
- if (!(await present(f.path))) continue;
- const duration = Number((await probe(f.path))?.duration);
- if (!Number.isFinite(duration) || duration <= 0) continue;
- return [{ name: f.name, path: f.path, from: 0, to: duration }];
- }
- return [];
-}
-
// `audio: true` marks a tier consulted only when the caller allows it.
-export const TIERS = [
- { kind: "raw-cache", windows: rawCacheWindows },
- { kind: "corpus-window", windows: corpusWindows },
- { kind: "saved-video", windows: savedVideoWindows },
- { kind: "audio", windows: audioWindows, audio: true },
-];
-
-const asSource = (kind, w) => ({
- kind,
- path: w.path,
- name: w.name,
- windowStart: w.from,
- windowEnd: w.to,
- ...(w.height ? { height: w.height } : {}),
-});
+export const TIERS = [{ kind: "raw-cache", windows: rawCacheWindows }, ...CORPUS_TIERS];
/**
* The local source for one span of one video, or null when nothing on disk
@@ -382,26 +190,18 @@ export async function resolveLocalSource(want, config = {}) {
if (rawDir) {
const name = rawWindowName(video, from, to);
const p = path.join(/* turbopackIgnore: true */ rawDir, name);
- if (await present(p)) return asSource("raw-cache", { name, path: p, from, to });
+ if (await present(p)) return { kind: "raw-cache", path: p, name, windowStart: from, windowEnd: to };
}
// `--no-reuse` re-cuts no cached window, but the sound is not one.
if (!audio) return null;
}
- const ctx = { video, slug, rawDir, channelsDir, probe };
- for (const tier of TIERS) {
- if (kinds && !kinds.includes(tier.kind)) continue;
- if (tier.audio ? !audio : exact) continue;
- const windows = (await tier.windows(ctx)).filter((w) => windowContains(w, from, to));
- // The raw cache's EXACT name first, as it always was; then tightest.
- const exactName = tier.kind === "raw-cache" ? rawWindowName(video, from, to) : null;
- windows.sort((a, b) =>
- (b.name === exactName) - (a.name === exactName) || (a.to - a.from) - (b.to - b.from));
- for (const w of windows) {
- if (await present(w.path)) return asSource(tier.kind, w);
- }
- }
- return null;
+ // The raw cache's EXACT name first, as it always was; then tightest.
+ const exactName = rawWindowName(video, from, to);
+ return resolveFromTiers(TIERS, { video, slug, rawDir, channelsDir, probe }, from, to, {
+ allow: (tier) => (!kinds || kinds.includes(tier.kind)) && (tier.audio ? audio : !exact),
+ preferName: (tier) => (tier.kind === "raw-cache" ? exactName : null),
+ });
}
/**
@@ -419,8 +219,8 @@ export async function resolveLocalSource(want, config = {}) {
export async function corpusWindowsOf({ video, slug, channelsDir, probe = ffprobeSource }) {
const ctx = { video, slug, channelsDir, probe };
const out = [];
- for (const tier of TIERS) {
- if (tier.kind === "raw-cache" || tier.audio) continue;
+ for (const tier of CORPUS_TIERS) {
+ if (tier.audio) continue;
for (const w of await tier.windows(ctx)) {
if (await present(w.path)) out.push({ ...w, kind: tier.kind });
}
