commit ce1b8f401cb24c10f5156aaeae0691836f945489
parent 45a37f16b987efe990b0ab460a84ff0ab1039931
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 21 Sep 2026 01:36:21 -0400
cut: a confirmed clip is already on this disk, so stop fetching it again
The written report ships clips/<id>.mp4 and the first 69 of them were fetched
one per clip with yt-dlp --download-sections. Every one of those seconds was
already paid for: nobody can judge a clip until its window is cached, so a
CONFIRMED clip is by definition one whose frames are on this disk.
So cutClipFromCache() asks the project cache for the tightest window holding
the clip and cuts it out. A clip with no containing window is not an error and
is not fetched here -- it is the bench's own "not fetched" state, which has a
managed download behind it (the editor's, with the cookie policy and the
per-platform sleeps), and this module must never grow a second one.
The ffmpeg cut is build-video's, not a second spelling of it: cutArgs() is
lifted out of buildClipSegment and exported, so the render's segment pass and
this share the one piece of arithmetic that turns a clip's absolute seconds
into an offset into a cached file. FFMPEG_BIN/FFPROBE_BIN go with it, so a
fixture's stub binaries are wired in one place.
Stream copy first, then MEASURE it. A copy can only begin on a keyframe: a
window fetched with --force-keyframes-at-cuts has one exactly where the clip
starts and the copy is exact and free, while a window fetched any other way --
or a clip whose edges moved on the bench after the fetch -- begins seconds
early, which is the wrong sentence rather than a rounding error. Probing the
result answers that in one number, where probing the window's keyframes would
only have predicted it.
The two share-batch encode profiles come off ~/reports/elfpire-eva's
reencode.py, ported into encode.mjs so there is ONE place holding them rather
than a python file beside one project's deliverables.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
4 files changed, 317 insertions(+), 2 deletions(-)
diff --git a/umtool/bin/cut-from-cache.mjs b/umtool/bin/cut-from-cache.mjs
@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+// Cut ONE clip out of the window already on disk.
+//
+// node bin/cut-from-cache.mjs --project <id> --clip <id> [--reencode|--no-reencode]
+//
+// A CLI so the Deliver panel's "cut the confirmed clips" can be ONE job with
+// one step per clip: lib/jobs.ts then owns the sequencing, the `k of n` the UI
+// reads off stepIndex, the timeout, and a Stop that kills the process group
+// rather than a promise nobody can interrupt. A loop inside a request handler
+// would have had to reinvent all four.
+//
+// Never fetches. A clip with no containing window exits 3 and says so, which
+// is a fact about the cache and not a failure of the cut.
+import process from "node:process";
+import { resolveProject } from "../lib/projects/core.mjs";
+import { cutClipFromCache } from "../lib/report/cut.mjs";
+
+const argv = process.argv.slice(2);
+const val = (flag) => {
+ const i = argv.indexOf(flag);
+ return i < 0 ? null : argv[i + 1];
+};
+
+const projectArg = val("--project");
+const clipId = val("--clip");
+if (!projectArg || !clipId) {
+ console.error("usage: cut-from-cache.mjs --project <id> --clip <id> [--reencode|--no-reencode]");
+ process.exit(2);
+}
+
+const r = await resolveProject(projectArg);
+if (!r.project) {
+ console.error(`no project matches "${projectArg}"`);
+ process.exit(2);
+}
+
+const reencode = argv.includes("--reencode")
+ ? "always"
+ : argv.includes("--no-reencode")
+ ? "never"
+ : "auto";
+
+const res = await cutClipFromCache(r.project, clipId, { reencode });
+if (!res.ok) {
+ console.error(`${clipId}: ${res.error}`);
+ // 3 is "nothing on disk holds this clip", which the panel already lists
+ // under its own heading with a fetch link. Every other failure is a 1.
+ process.exit(res.reason === "not-fetched" ? 3 : 1);
+}
+console.log(
+ `CUT-OK ${res.id} ${res.seconds?.toFixed(2)}s (wanted ${res.want.toFixed(2)}s) ` +
+ `${res.mode} from ${res.window.name}`,
+);
diff --git a/umtool/lib/report/cut.mjs b/umtool/lib/report/cut.mjs
@@ -0,0 +1,152 @@
+// Cutting ONE clip out of the window already on disk.
+//
+// The deliverable a written report ships is `clips/<id>.mp4` -- the players in
+// report.html read exactly that -- and the first batch of them was fetched one
+// per clip with `yt-dlp --download-sections`. That is a second download of
+// seconds already paid for: the bench fetches a generous window per clip into
+// `out/clips-raw` (or, now, into the corpus) before anybody can judge it, and
+// every confirmed clip therefore already has its own frames on this disk.
+//
+// So this cuts from the cache and never touches the network. A clip with no
+// containing window is NOT cut here and is not an error either -- it is a clip
+// nobody has fetched yet, which is the bench's own "not fetched" state and has
+// its own button.
+//
+// The ffmpeg cut itself is build-video's: `cutArgs()` is the render's segment
+// pass, exported rather than copied, so the seconds this writes and the seconds
+// the video renders are the same arithmetic.
+import { execFile } from "node:child_process";
+import { mkdir, rename, rm } from "node:fs/promises";
+import path from "node:path";
+import { promisify } from "node:util";
+import { FFMPEG_BIN, cutArgs } from "umtool-report-to-video/build-video";
+import { clipsOf, readManifest } from "../projects/report.mjs";
+import { ACCURATE_CUT_ARGS, probeSeconds } from "./encode.mjs";
+import { projectCache } from "./serve.mjs";
+
+const execFileP = promisify(execFile);
+
+/** Where the report's own players look. Not configurable: build.py hardcodes it. */
+export const CLIPS_DIR = "clips";
+
+/**
+ * How far out a stream-copied cut may land before it is re-encoded.
+ *
+ * A copy can only start on a keyframe. A window fetched with
+ * `--force-keyframes-at-cuts` has one exactly where the clip begins, so the
+ * copy is exact and free; a window fetched any other way -- or a clip whose
+ * edges were MOVED on the bench after the fetch -- has one wherever the encoder
+ * put it, and the copy silently begins seconds early. That is not a rounding
+ * error to tolerate: it is the wrong sentence.
+ */
+export const CUT_TOLERANCE = 0.05;
+
+/** ffmpeg gets a generous cap; a cut of a cached window is seconds of work. */
+const CUT_TIMEOUT_MS = 5 * 60_000;
+
+/**
+ * Cut `clipId` of `project` out of the cached window that contains it.
+ *
+ * @param {{ id: string, dir: string }} project
+ * @param {string} clipId
+ * @param {{ reencode?: "auto" | "always" | "never", manifest?: object | null,
+ * cache?: Awaited<ReturnType<typeof projectCache>> | null }} [opts]
+ * `reencode` is the operator's override of the keyframe test above:
+ * "always" for a window whose copy is known to be wrong, "never" for a
+ * re-cut that must not lose a generation.
+ * @returns {Promise<{ ok: boolean, id: string, reason?: string, error?: string,
+ * path?: string, rel?: string, seconds?: number, want?: number,
+ * mode?: "copy" | "reencode", window?: { name: string, from: number, to: number } }>}
+ */
+export async function cutClipFromCache(
+ project,
+ clipId,
+ { reencode = "auto", manifest = null, cache = null } = {},
+) {
+ const m = manifest ?? (await readManifest(project.dir));
+ if (!m) return { ok: false, id: clipId, reason: "no-manifest", error: "no manifest" };
+ const clip = clipsOf(m).find((e) => e.id === clipId);
+ if (!clip) return { ok: false, id: clipId, reason: "no-clip", error: "no such clip" };
+
+ const start = Number(clip.start);
+ const end = Number(clip.end);
+ if (!Number.isFinite(start) || !Number.isFinite(end) || !(end > start)) {
+ return { ok: false, id: clipId, reason: "no-window", error: "this clip has no window" };
+ }
+
+ // THE EXTENT, not the cut-to-quote. `cutStart`/`cutEnd` is what the VIDEO
+ // plays; a written report's player is the reviewed extent, which is what
+ // clips.json carries and what the caption under it describes.
+ const c = cache ?? (await projectCache(project, m));
+ const win = c.containing(clip.video, start, end);
+ if (!win) {
+ // Not an error. Nobody has fetched this one yet, and the bench has a
+ // button for exactly that.
+ return {
+ ok: false,
+ id: clipId,
+ reason: "not-fetched",
+ error: "no cached window holds this clip end to end",
+ };
+ }
+
+ const want = end - start;
+ const a = Math.max(0, start - win.from);
+ const b = a + want;
+ const dir = path.join(project.dir, CLIPS_DIR);
+ await mkdir(dir, { recursive: true });
+ const out = path.join(dir, `${clipId}.mp4`);
+ const tmp = path.join(dir, `.${clipId}.cutting.mp4`);
+
+ const run = async (args) => {
+ await rm(tmp, { force: true });
+ await execFileP(
+ FFMPEG_BIN,
+ ["-nostdin", "-v", "error", "-y", ...cutArgs(win.path, a, b), ...args, tmp],
+ { maxBuffer: 1 << 24, timeout: CUT_TIMEOUT_MS },
+ );
+ return probeSeconds(tmp);
+ };
+
+ let mode = reencode === "always" ? "reencode" : "copy";
+ let got = null;
+ try {
+ if (mode === "copy") {
+ // `-avoid_negative_ts make_zero` so the copied packets' timestamps start
+ // at zero: without it a copy that began on an earlier keyframe carries
+ // the window's own clock into the file, and every player disagrees about
+ // how long it is.
+ got = await run(["-c", "copy", "-avoid_negative_ts", "make_zero", "-movflags", "+faststart"]);
+ const off = got == null ? Infinity : Math.abs(got - want);
+ if (off > CUT_TOLERANCE && reencode !== "never") {
+ // The window's keyframes are not where this clip's edges are, so the
+ // copy is the wrong seconds. Pay for one generation and get the cut
+ // that was asked for.
+ mode = "reencode";
+ got = await run(ACCURATE_CUT_ARGS);
+ }
+ } else {
+ got = await run(ACCURATE_CUT_ARGS);
+ }
+ } catch (e) {
+ await rm(tmp, { force: true });
+ return {
+ ok: false,
+ id: clipId,
+ reason: "ffmpeg",
+ error: e instanceof Error ? e.message : String(e),
+ };
+ }
+
+ await rename(tmp, out);
+ return {
+ ok: true,
+ id: clipId,
+ path: out,
+ rel: path.posix.join(CLIPS_DIR, `${clipId}.mp4`),
+ seconds: got == null ? null : Number(got.toFixed(3)),
+ want: Number(want.toFixed(3)),
+ mode,
+ window: { name: win.name, from: win.from, to: win.to },
+ };
+}
diff --git a/umtool/lib/report/encode.mjs b/umtool/lib/report/encode.mjs
@@ -0,0 +1,83 @@
+// The encode profiles, in ONE place.
+//
+// These came off `share-report-clips/reencode.py` in ~/reports/elfpire-eva,
+// which is where the first share batch was actually encoded and therefore the
+// only honest source for "what the last batch looked like". Ported rather than
+// spawned: a python file living beside one project's deliverables is not a
+// profile the next project can reach, and two copies of an x264 command line
+// is exactly the drift that makes batch 2 not match batch 1.
+//
+// `std` and `small` are that script's two profiles, argument for argument.
+// `accurate` is neither: it is the re-encode a CUT falls back to when the cut
+// cannot be stream-copied, and it deliberately keeps the source's geometry --
+// an `orig/` file is "the window as fetched", and rescaling it here would make
+// the folder's own description untrue.
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { FFPROBE_BIN } from "umtool-report-to-video/build-video";
+
+const execFileP = promisify(execFile);
+
+/** Scale-and-pad to an exact frame, the shape both share profiles use. */
+const box = (w, h) =>
+ `scale=w=${w}:h=${h}:force_original_aspect_ratio=decrease,` +
+ `pad=${w}:${h}:(ow-iw)/2:(oh-ih)/2,format=yuv420p`;
+
+export const SHARE_PROFILES = {
+ // Exactly 1280x720, faststart, a level every phone and every forum player
+ // will take. This is the one people download.
+ std: {
+ label: "1280x720 · crf 23",
+ args: [
+ "-vf", box(1280, 720),
+ "-c:v", "libx264", "-profile:v", "high", "-level", "4.0",
+ "-preset", "medium", "-crf", "23", "-r", "30", "-g", "60",
+ "-c:a", "aac", "-b:a", "128k", "-ar", "44100", "-ac", "2",
+ "-movflags", "+faststart",
+ ],
+ },
+ // 640x360 under a 300k ceiling: the copy that survives an upload limit.
+ small: {
+ label: "640x360 · crf 28 · 300k cap",
+ args: [
+ "-vf", box(640, 360),
+ "-c:v", "libx264", "-profile:v", "main", "-level", "3.1",
+ "-preset", "medium", "-crf", "28", "-maxrate", "300k", "-bufsize", "600k",
+ "-r", "30", "-g", "60",
+ "-c:a", "aac", "-b:a", "64k", "-ar", "44100", "-ac", "2",
+ "-movflags", "+faststart",
+ ],
+ },
+};
+
+/** The encode names a batch produces, beside the untouched `orig`. */
+export const SHARE_VARIANTS = ["orig", ...Object.keys(SHARE_PROFILES)];
+
+/**
+ * The re-encode a cut falls back to when the window's keyframes are in the
+ * wrong places. No scaling and no frame-rate change: the point of this file is
+ * that its first and last frames are the seconds that were asked for.
+ */
+export const ACCURATE_CUT_ARGS = [
+ "-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-pix_fmt", "yuv420p",
+ "-c:a", "aac", "-b:a", "160k",
+ "-movflags", "+faststart",
+];
+
+/**
+ * A file's duration in seconds, from the container.
+ *
+ * build-video's probeDuration() counts VIDEO FRAMES, because a rendered
+ * segment's length has to agree with the timeline it is concatenated into.
+ * Nothing here is concatenated: the question is "did this cut come out the
+ * length it was asked for", and the container's own answer is the one a
+ * downloader will see.
+ */
+export async function probeSeconds(file) {
+ const { stdout } = await execFileP(FFPROBE_BIN, [
+ "-v", "error", "-show_entries", "format=duration",
+ "-of", "default=nw=1:nk=1", file,
+ ]);
+ const n = Number(String(stdout).trim());
+ return Number.isFinite(n) ? n : null;
+}
diff --git a/umtool/report-to-video/build-video.mjs b/umtool/report-to-video/build-video.mjs
@@ -77,6 +77,12 @@ const FFMPEG = process.env.FFMPEG_BIN ?? "ffmpeg";
const FFPROBE = process.env.FFPROBE_BIN ?? "ffprobe";
const QRENCODE = process.env.QRENCODE_BIN ?? "qrencode";
+// The binaries, under the names the rest of umtool calls them by. A second
+// module reading FFMPEG_BIN for itself would be a second place a fixture's
+// stub has to be wired in, and the one that forgot would shell out to the real
+// ffmpeg in the middle of a test.
+export { FFMPEG as FFMPEG_BIN, FFPROBE as FFPROBE_BIN };
+
// Cue windows and per-video metadata come from a local corpus when there is one
// and from the published archive otherwise, so this runs in a clone with no
// `transcripts/` directory. Built once main() has the manifest (it carries the
@@ -252,7 +258,7 @@ async function videoMeta(videoId, channelSlug, hints = {}) {
// The video stream's frame COUNT is the number the timeline actually runs on,
// so derive the duration from it. nb_frames is absent on some demuxers; fall
// back to the container rather than failing a build over a probe.
-async function probeDuration(file, fps) {
+export async function probeDuration(file, fps) {
if (fps) {
const { stdout } = await execFileP(FFPROBE, [
"-v", "error", "-select_streams", "v:0", "-show_entries", "stream=nb_frames",
@@ -345,6 +351,27 @@ export async function findContainingWindow(rawDir, video, from, to) {
return tightestContaining(await cachedWindowsFor(rawDir, video), from, to);
}
+/**
+ * WHERE A CLIP IS CUT OUT OF A CACHED WINDOW, as ffmpeg input arguments.
+ *
+ * `-ss`/`-to` BEFORE `-i`, and both matter. Before the input, ffmpeg seeks the
+ * demuxer rather than decoding and discarding, which is the difference between
+ * seconds and minutes on a 45-minute window; `-to` before the input is then
+ * measured on the same clock as the `-ss`, i.e. in the SOURCE FILE's seconds,
+ * which is what an offset into a cached window is.
+ *
+ * Exported because two things cut a clip out of a window now -- the render's
+ * segment pass and the bench's "cut confirmed clips from cache" -- and a second
+ * spelling of this is a second set of seconds that can drift from the first.
+ *
+ * @param {string} raw the cached window file
+ * @param {number} a seconds INTO that file where the clip starts
+ * @param {number} b seconds into it where the clip ends
+ */
+export function cutArgs(raw, a, b) {
+ return ["-ss", a.toFixed(3), "-to", b.toFixed(3), "-i", raw];
+}
+
async function fetchClip(entry, meta, render, rawDir, opts) {
// Deliberately over-fetch: the snapping pass below needs room on both sides to
// find a silence, and a clip that has no slack can only be cut where the cue
@@ -687,7 +714,7 @@ async function buildClipSegment(entry, meta, render, dirs, opts, chrome, nodes,
// eof_action=repeat holds the strip's last frame, which is the parked bar.
const barT = Math.max(0.2, cutB - cutA - 0.25);
- const inputs = ["-ss", cutA.toFixed(3), "-to", cutB.toFixed(3), "-i", raw];
+ const inputs = cutArgs(raw, cutA, cutB);
let nextIdx = 1;
let footerIdx, markerIdx, barIdx, qrIdx;
if (hasFooter) {
