// A CLIP WINDOW: a few seconds of a video's source media, fetched on purpose // and kept beside the video it came from. // // umtool's clip bench used to run yt-dlp itself into its own project-local // `out/clips-raw`. Every tool that wanted the same seconds paid for them again, // and none of those fetches went through the editor's cookie policy, its // per-platform sleeps or its 429 cooldown. A window now lands in the corpus — // // channels//data//clips/-.mp4 // channels//data//clips/-.json (who asked, and why) // // — where any tool can reuse it and the video page can show it. // // THE LOAD-BEARING RULE: a window fetch never writes metadata.info.json. // buildIndex.ts keys a video's presence in the LMDB index on that file's mtime // (controller/buildIndex.ts, the `metaMs` stat that `continue`s when it // throws), so writing one would put an undownloaded video into the index on the // strength of thirty seconds of audio. That is why the fetch assembles its own // argv rather than going through outputArgsForUrl / downloadOneManaged. // // Everything a `clips/` subdirectory has to survive is a consequence of two // facts, both re-verified when this landed: // * the video-dir predicates in lib/mediaFiles.ts are ANCHORED to one segment // after a fixed base name (`audio.`, `source-media.`), so a // directory named `clips` matches none of them, and readVideoFiles // (lib/videoStatus.ts) filters a plain readdir with exactly those; // * controller/scanCorruptMedia.ts filters the same listing by EXTENSION, and // `clips` has none. // The saved-video walks (pruneSavedVideos, savedVideoInventory) are // pointer-driven and iterate the VIDEO dirs under data/, one level above this. // // This module is pure (string + number math) so a client component can import // it; the filesystem half lives in clipWindow-server.ts. import path from "node:path"; // The subdirectory inside a video dir that holds fetched windows. export const CLIPS_DIR_NAME = "clips"; // A window read back from a 2 dp name can sit a hair outside the request that // produced it. The same tolerance umtool's build uses for the same reason // (report-to-video/build-video.mjs WIN_EPS) — the two have to agree or a file // one of them fetched is invisible to the other. export const WIN_EPS = 0.02; // The widest window one request may ask for. Fifteen minutes is far past any // citation and well short of "you meant to download the video" — which is what // the full-source path is for, and which goes to the saved-video store instead. // // HERE rather than beside the action that enforces it: `videoActions.ts` is a // "use server" module, and every export of one must be an async function — a // constant there is a build error, not a lint. export const MAX_CLIP_WINDOW_SECONDS = 900; // The source heights a fetch may cap itself at (`maxHeight` on the fetch-window // request, fetch_clip and fetch-via-editor). 144 is the lowest rung a source // offers; 2160 is 4K. A whole number in between, or the request is refused — // it lands in a yt-dlp `-f` selector, so it is checked, not coerced. export const MIN_FETCH_MAX_HEIGHT = 144; export const MAX_FETCH_MAX_HEIGHT = 2160; export function isFetchMaxHeight(v: unknown): v is number { return ( typeof v === "number" && Number.isInteger(v) && v >= MIN_FETCH_MAX_HEIGHT && v <= MAX_FETCH_MAX_HEIGHT ); } export function clipsDirFor(videoDir: string): string { return path.join(videoDir, CLIPS_DIR_NAME); } // TWO DECIMALS, ALWAYS. The name IS the window: a reader parses it back rather // than opening a sidecar, so the formatting has to be a function of the numbers // and nothing else. export function clipWindowName(from: number, to: number): string { return `${from.toFixed(2)}-${to.toFixed(2)}`; } export function clipWindowFile(from: number, to: number): string { return `${clipWindowName(from, to)}.mp4`; } export function clipWindowSidecar(from: number, to: number): string { return `${clipWindowName(from, to)}.json`; } const WINDOW_RE = /^(\d+(?:\.\d+)?)-(\d+(?:\.\d+)?)$/; // Parse `-.mp4` (or the bare `-`) back into its numbers, or // null for anything else in the directory — a sidecar, a `.part`, a stray. // THE EXTENSIONS A WINDOW MAY WEAR. `clipWindowFile` writes `.mp4` and only // `.mp4` (the format is pinned to H.264 so both sides address the same file — // see the header), so the other two are not speculation about a future format: // they are what a window fetched by an older build, or by a hand-run yt-dlp // that ignored the pin, is called. A parser that could not read those names // made those files invisible to `listClipWindows` AND to `evictClipWindows`, // which is the worse half — bytes nothing counts and nothing can remove. // // NOT `.json`: a sidecar is removed with the window it describes and must // never parse as one itself. export const CLIP_WINDOW_EXTS = [".mp4", ".mkv", ".webm"] as const; // ONE PREDICATE, so the lister and the evictor cannot disagree about what a // window is. They did: `listClipWindows` pre-filtered on `.mp4` while the // parser had been widened, so a `.mkv` window was invisible to the video page // and to `findContainingClipWindow` (the fetch dedup — it would have paid for // those seconds again) while `evictClipWindows` could see and delete it. A file // one half of the code owns and the other cannot see is the worst of both. export function hasClipWindowExt(name: string): boolean { return CLIP_WINDOW_EXTS.some((ext) => name.endsWith(ext)); } function stripClipExt(name: string): string { for (const ext of CLIP_WINDOW_EXTS) { if (name.endsWith(ext)) return name.slice(0, -ext.length); } return name; } export function parseClipWindowName( name: string, ): { from: number; to: number } | null { const stem = stripClipExt(name); const m = WINDOW_RE.exec(stem); if (!m) return null; const from = Number(m[1]); const to = Number(m[2]); if (!Number.isFinite(from) || !Number.isFinite(to) || to <= from) return null; return { from, to }; } // Who asked for this window, and why. Written beside the file so the video page // can say "downloaded by umtool for /, because " // without a job log — a job log is pruned, and the window is not. export type ClipProvenance = { // The tool that asked. "umtool" today; the field exists so a second one is // distinguishable from it without a schema change. requestedBy: string; // The report manifest and the clip within it this window was fetched for. manifest?: string; clipId?: string; // The operator-facing sentence: why this clip needs these seconds. reason?: string; requestedAt?: string; // The pad, in seconds, the requester added around the clip's own window. pad?: number; bytes?: number; fetchedAt?: string; // The argv the fetch actually ran, minus the binary. For the case where a // window looks wrong and the question is which format selector produced it. ytdlp?: { args: string[] }; // WHERE THE BYTES CAME FROM, when it was not the network: "saved-video" for // a window cut from the video's saved container (release 21 D2, // lib/savedVideoWindow-server.ts). Absent on every fetched window. source?: ClipWindowSource; }; export const CLIP_WINDOW_SOURCES = ["saved-video"] as const; export type ClipWindowSource = (typeof CLIP_WINDOW_SOURCES)[number]; const str = (v: unknown): string | undefined => typeof v === "string" && v !== "" ? v : undefined; const num = (v: unknown): number | undefined => typeof v === "number" && Number.isFinite(v) ? v : undefined; // TOLERANT, like every other sidecar parse in this repo: a hand-edited or // half-written file degrades to "no provenance", never to a crashed page. export function parseClipProvenance(raw: unknown): ClipProvenance | null { if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null; const r = raw as Record; const requestedBy = str(r.requestedBy); if (!requestedBy) return null; const out: ClipProvenance = { requestedBy }; const manifest = str(r.manifest); if (manifest) out.manifest = manifest; const clipId = str(r.clipId); if (clipId) out.clipId = clipId; const reason = str(r.reason); if (reason) out.reason = reason; const requestedAt = str(r.requestedAt); if (requestedAt) out.requestedAt = requestedAt; const pad = num(r.pad); if (pad !== undefined) out.pad = pad; const bytes = num(r.bytes); if (bytes !== undefined) out.bytes = bytes; const fetchedAt = str(r.fetchedAt); if (fetchedAt) out.fetchedAt = fetchedAt; const y = r.ytdlp; if (y && typeof y === "object" && Array.isArray((y as { args?: unknown }).args)) { const args = (y as { args: unknown[] }).args.filter( (a): a is string => typeof a === "string", ); out.ytdlp = { args }; } if ((CLIP_WINDOW_SOURCES as readonly unknown[]).includes(r.source)) { out.source = r.source as ClipWindowSource; } return out; } export type ClipWindowSpan = { from: number; to: number }; // Does this window hold [from, to] whole, to the naming tolerance? export function clipWindowContains( w: ClipWindowSpan, from: number, to: number, ): boolean { return !(w.from > from + WIN_EPS || w.to < to - WIN_EPS); } // The TIGHTEST window containing [from, to], or null. // // Tightest rather than widest for the same reason the build picks it: a // consumer decodes the whole file, so a 40-second container costs more than the // 14-second one that would also have done. export function tightestClipWindow( windows: readonly T[], from: number, to: number, ): T | null { let best: T | null = null; for (const w of windows) { if (!clipWindowContains(w, from, to)) continue; if (!best || w.to - w.from < best.to - best.from) best = w; } return best; }