commit ecbf82580039695405f6ef5de8627e090a5fd98e
parent b4265b0b618b17f52fdd12d01e84a5e6ff62c264
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 18 Aug 2026 22:31:21 -0400
umtool: the build driver — four steps, per-clip progress, and no orphans
Building a report video from the project page. The client sends a project id and
a PRESET NAME; it never sends a path, an argv or an env map, and the output path
is computed server-side -- the same contract /api/browse/build already keeps.
The chain is four steps, and step 1 earns its place:
1. check every source is still fetchable yt-dlp --simulate, no bytes
2. resolve windows (DRY — nothing written)
3. build --progress ndjson --continue-on-error
4. verify the file that came out new: verify-build.mjs
Availability is a STEP, not a preamble somebody remembers. It is the one fact
about a manifest that goes stale in both directions, it costs seconds, and
without it a dead source is discovered twenty minutes and a dozen paid-for
fetches into the build. Resolve runs DRY: a widener silently rewriting windows
somebody just set in the bench is exactly the surprise `lock` exists to prevent,
so applying is a separate explicit action.
verify-build.mjs is new because a build can exit 0 and still be wrong: a concat
that produced nothing, a chapter pass that dropped markers, a timeline that lost
a clip because --continue-on-error let it. Each looks like success at the
terminal and like a finished video in a directory listing. It measures the
deliverable against the manifest -- duration, chapters == entries, a length floor
-- and it passes on the real quartering-gout cut (705.1s, 14 chapters for 14
entries).
Per-clip progress, because "step 3 of 4, running" is not progress when step 3 is
twenty minutes on nineteen clips. The build step renders a grid, one box per
entry, lit by the pipeline's own NDJSON events.
An existing deliverable is never destroyed to make a new one. build-video always
passes -y, so an output NEWER than its manifest is refused (409, needsReplace);
with replace=1 it is stamped aside as out/<slug>.<YYYYMMDD-HHMM>.mp4, the shape
promote already uses for a demoted cut.
The whole thing is testable OFFLINE. The fixture writes stub YTDLP_BIN and
QRENCODE_BIN as node scripts (bash needed awk for fractional window arithmetic
and three layers of quoting inside a generated file, which got mangled once).
The stub reports `gone1` removed the way a deleted upload is, so
`source-unavailable` has a true answer: gone-fixture fails at step 1 having
encoded nothing, and writes availability.json so the inbox can report it without
running yt-dlp itself. A full 4-clip build — cache reuse, three stub fetches, QR
overlay, concat, chapters, verify — runs in 3 seconds with no network.
Cancel kills the process GROUP. build-video shells out, so the thing burning CPU
is a grandchild that child.kill() leaves running -- the same failure the diarize
backfill had. The spec asserts no `build-video.mjs` survives, and polls for it
rather than asserting once: the kill is SIGTERM then SIGKILL five seconds later,
so "gone" is a state it reaches, and racing that was green one run and red the
next.
Three fixture projects now, not one. report-fixture is read-only for the index
and decision specs, bench-fixture is written by the clip bench, build-fixture is
built. Sharing made each suite's result depend on which file playwright ran
first. The index spec's kind count is now asserted as "the chip's number equals
the number of cards" -- the actual invariant -- so adding a fixture no longer
edits an unrelated spec.
e2e: 134 passed.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Diffstat:
10 files changed, 941 insertions(+), 13 deletions(-)
diff --git a/scripts/report-to-video/package.json b/scripts/report-to-video/package.json
@@ -7,13 +7,15 @@
"bin": {
"report-build-video": "./build-video.mjs",
"report-resolve-windows": "./resolve-windows.mjs",
- "report-check-availability": "./check-availability.mjs"
+ "report-check-availability": "./check-availability.mjs",
+ "report-verify-build": "./verify-build.mjs"
},
"exports": {
"./resolve-windows": "./resolve-windows.mjs",
"./build-video": "./build-video.mjs",
"./render-cards": "./render-cards.mjs",
"./check-availability": "./check-availability.mjs",
- "./package.json": "./package.json"
+ "./package.json": "./package.json",
+ "./verify-build": "./verify-build.mjs"
}
}
diff --git a/scripts/report-to-video/verify-build.mjs b/scripts/report-to-video/verify-build.mjs
@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+// verify-build.mjs — is the file that came out the file that was asked for?
+//
+// A build can exit 0 and still be wrong in ways nothing else notices: a concat
+// that produced a zero-length file, a chapter pass that silently dropped
+// markers, a timeline that lost a clip because --continue-on-error let it. Each
+// of those looks like success at the terminal and like a finished video in a
+// directory listing.
+//
+// So the last step of a build measures the deliverable and compares it to the
+// manifest. Cheap (one ffprobe) and the only thing that closes the loop.
+//
+// node scripts/report-to-video/verify-build.mjs <manifest.json> [--out <dir>] [--json]
+
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { readFile, stat } from "node:fs/promises";
+import path from "node:path";
+
+const execFileP = promisify(execFile);
+const FFPROBE = process.env.FFPROBE_BIN ?? "ffprobe";
+
+export async function verifyBuild(manifestPath, { outDir } = {}) {
+ const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
+ const dir = outDir ?? path.join(path.dirname(path.resolve(manifestPath)), "out");
+ const file = path.join(dir, `${manifest.slug}.mp4`);
+ const problems = [];
+
+ const st = await stat(file).catch(() => null);
+ if (!st) return { ok: false, file, problems: [`${file} does not exist`] };
+ if (st.size < 1024) problems.push(`${file} is ${st.size} bytes`);
+
+ const { stdout } = await execFileP(FFPROBE, [
+ "-v", "error",
+ "-show_entries", "format=duration,size",
+ "-show_chapters",
+ "-of", "json",
+ file,
+ ], { maxBuffer: 1 << 24 });
+ const probe = JSON.parse(stdout);
+ const duration = Number(probe.format?.duration ?? 0);
+ const chapters = (probe.chapters ?? []).length;
+ const entries = (manifest.timeline ?? []).length;
+
+ if (!(duration > 0)) problems.push("duration is not greater than zero");
+
+ // Every timeline entry becomes a chapter, so a mismatch means the timeline and
+ // the file disagree about what is in it -- which is exactly the failure
+ // --continue-on-error is allowed to cause and must never cause silently.
+ if (chapters > 0 && chapters !== entries) {
+ problems.push(`${chapters} chapter(s) for ${entries} timeline entr(ies) — the cut is missing something`);
+ }
+
+ // A rough floor: the sum of the windows, less the crossfades. Well under the
+ // real duration because snapping moves the cuts, but a file that came out at
+ // half the expected length did not build what was asked for.
+ const wanted = (manifest.timeline ?? []).reduce(
+ (n, e) => n + (e.type === "card" ? (e.seconds ?? 0) : Math.max(0, (e.end ?? 0) - (e.start ?? 0))),
+ 0,
+ );
+ if (wanted > 0 && duration < wanted * 0.5) {
+ problems.push(`${duration.toFixed(1)}s out of a timeline that asks for about ${wanted.toFixed(0)}s`);
+ }
+
+ return { ok: problems.length === 0, file, duration, chapters, entries, size: st.size, problems };
+}
+
+async function main() {
+ const argv = process.argv.slice(2);
+ const manifestPath = argv.find((a) => !a.startsWith("--"));
+ if (!manifestPath) {
+ console.error("usage: verify-build.mjs <manifest.json> [--out <dir>] [--json]");
+ process.exit(2);
+ }
+ const i = argv.indexOf("--out");
+ const res = await verifyBuild(manifestPath, { outDir: i >= 0 ? argv[i + 1] : undefined });
+
+ if (argv.includes("--json")) {
+ console.log(JSON.stringify(res, null, 2));
+ } else {
+ console.log(
+ `${res.file}\n ${res.duration?.toFixed(1) ?? "?"}s · ${res.chapters ?? 0} chapter(s) for ` +
+ `${res.entries ?? 0} entr(ies) · ${((res.size ?? 0) / 1e6).toFixed(1)} MB`,
+ );
+ for (const p of res.problems) console.log(` ** ${p}`);
+ if (res.ok) console.log(" ok");
+ }
+ process.exit(res.ok ? 0 : 1);
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+ main().catch((err) => {
+ console.error(err.message ?? err);
+ process.exit(1);
+ });
+}
diff --git a/umtool/app/api/report/build/route.ts b/umtool/app/api/report/build/route.ts
@@ -0,0 +1,141 @@
+import { rename, stat } from "node:fs/promises";
+import path from "node:path";
+import { cancelJob, getJob, jobView, recentJobs, runningJob, startJob } from "@/lib/jobs";
+import { PRESETS, buildSteps } from "@/lib/report/driver.mjs";
+import { clipsOf, readManifest } from "@/lib/projects/report.mjs";
+import { projectRef } from "@/lib/projects";
+
+export const dynamic = "force-dynamic";
+
+// Building a report video.
+//
+// The client sends a PROJECT ID and a PRESET NAME. It never sends a path, an
+// argv or an env map -- the same contract /api/browse/build keeps, and the
+// reason the output path is computed here rather than accepted.
+//
+// POLLING, NOT STREAMING, like every other job in this app. The progress that
+// matters is per-clip, and that arrives as NDJSON events the driver collects.
+
+/** `20260817-1408`, the stamp shape promote already uses for a demoted cut. */
+function stamp(d = new Date()): string {
+ const p = (n: number) => String(n).padStart(2, "0");
+ return `${d.getFullYear()}${p(d.getMonth() + 1)}${p(d.getDate())}-${p(d.getHours())}${p(d.getMinutes())}`;
+}
+
+export async function GET(request: Request) {
+ const url = new URL(request.url);
+ const id = url.searchParams.get("job");
+ const headers = { "cache-control": "no-store" };
+ if (!id) {
+ const running = runningJob();
+ return Response.json(
+ {
+ running: running ? jobView(running) : null,
+ jobs: recentJobs(5).map((j) => jobView(j, j.log.length, j.events.length)),
+ presets: Object.entries(PRESETS).map(([k, v]) => ({ id: k, label: v.label })),
+ },
+ { headers },
+ );
+ }
+ const job = getJob(id);
+ if (!job) return Response.json({ error: "no such job" }, { status: 404 });
+ return Response.json(
+ jobView(job, Number(url.searchParams.get("since") ?? 0), Number(url.searchParams.get("sinceEvent") ?? 0)),
+ { headers },
+ );
+}
+
+export async function POST(request: Request) {
+ const url = new URL(request.url);
+ const dry = url.searchParams.get("dry") === "1";
+ const replace = url.searchParams.get("replace") === "1";
+ const cancel = url.searchParams.get("cancel");
+
+ if (cancel) {
+ // Safe at any point, and worth saying why: every artefact is
+ // content-addressed -- a fetched window by its window, a segment by its clip
+ // id -- so re-running skips whatever finished. A cancelled build is paused.
+ return Response.json({ cancelled: cancelJob(cancel) });
+ }
+
+ const body = (await request.json().catch(() => ({}))) as Record<string, unknown>;
+ const projectId = String(body.project ?? "");
+ const preset = String(body.preset ?? "fast");
+ const only = body.only ? String(body.only) : null;
+ const skipFetch = !!body.skipFetch;
+
+ if (!(preset in PRESETS)) {
+ return Response.json(
+ { error: `preset must be one of ${Object.keys(PRESETS).join(", ")}` },
+ { status: 400 },
+ );
+ }
+
+ const project = await projectRef(projectId);
+ if (!project) return Response.json({ error: "no such project" }, { status: 404 });
+ const manifest = await readManifest(project.dir);
+ if (!manifest) return Response.json({ error: "no manifest" }, { status: 400 });
+ if (only && !clipsOf(manifest).some((e: { id: string }) => e.id === only)) {
+ return Response.json({ error: `no clip with id ${only}` }, { status: 400 });
+ }
+
+ const clipCount = clipsOf(manifest).length;
+ const steps = buildSteps(project, { preset, only, skipFetch, clipCount });
+ const view = {
+ project: project.id,
+ preset,
+ only,
+ steps: steps.map((s: { label: string; argv: string[]; cwd: string; timeoutMs?: number }) => ({
+ label: s.label,
+ argv: s.argv,
+ cwd: s.cwd,
+ timeoutMs: s.timeoutMs ?? null,
+ })),
+ };
+
+ // The exact argv before anything runs. Same contract as /api/mix/render?dry=1.
+ if (dry) return Response.json({ dry: true, ...view }, { headers: { "cache-control": "no-store" } });
+
+ // ---- the overwrite guard --------------------------------------------------
+ //
+ // build-video.mjs always passes -y. A deliverable that cost an hour of network
+ // fetches must not be destroyed to make a new one, so an existing output that
+ // is NEWER than the manifest is refused; on ?replace=1 it is stamped aside
+ // rather than overwritten.
+ const finalPath = path.join(project.dir, "out", `${manifest.slug}.mp4`);
+ if (!only) {
+ const [fin, man] = await Promise.all([
+ stat(finalPath).catch(() => null),
+ stat(path.join(project.dir, "video.manifest.json")).catch(() => null),
+ ]);
+ if (fin && man && fin.mtimeMs >= man.mtimeMs) {
+ if (!replace) {
+ return Response.json(
+ {
+ error:
+ `${path.basename(finalPath)} is newer than the manifest — building would overwrite ` +
+ "a deliverable nothing has asked to change. Pass replace=1 to stamp it aside.",
+ needsReplace: true,
+ },
+ { status: 409 },
+ );
+ }
+ await rename(finalPath, finalPath.replace(/\.mp4$/, `.${stamp()}.mp4`));
+ }
+ }
+
+ const running = runningJob();
+ if (running) {
+ return Response.json(
+ { error: `a job is already running (${running.kind})`, running: jobView(running) },
+ { status: 409 },
+ );
+ }
+
+ try {
+ const job = startJob(`build ${project.id} (${preset})`, steps);
+ return Response.json({ ok: true, ...view, job: jobView(job) }, { headers: { "cache-control": "no-store" } });
+ } catch (e) {
+ return Response.json({ error: e instanceof Error ? e.message : String(e) }, { status: 409 });
+ }
+}
diff --git a/umtool/components/projects/ReportBuildChain.tsx b/umtool/components/projects/ReportBuildChain.tsx
@@ -0,0 +1,308 @@
+"use client";
+
+import { useCallback, useEffect, useRef, useState } from "react";
+import { badgeVariants } from "@/components/ui/badge";
+import { buttonVariants } from "@/components/ui/button";
+
+// ---------------------------------------------------------------------------
+// Running a build, and watching it per clip.
+//
+// BuildChain's step boxes are the right model for the chain -- preflight,
+// resolve, build, verify -- but the build step alone is twenty minutes of work
+// on nineteen clips, and "step 3 of 4, running" is not progress. So the build
+// step also renders a grid: one box per timeline entry, lit by the NDJSON events
+// the pipeline emits (fetch / snap / segment / entry-failed).
+//
+// Polling, not streaming, like every other job here.
+// ---------------------------------------------------------------------------
+
+type StepView = { label: string; argv: string[]; cwd: string; timeoutMs: number | null };
+type Ev = Record<string, unknown> & { ev: string; id?: string };
+type JobView = {
+ id: string;
+ state: "running" | "done" | "failed";
+ stepIndex: number;
+ steps: StepView[];
+ error: string | null;
+ log: string[];
+ next: number;
+ events: Ev[];
+ nextEvent: number;
+};
+
+type Preset = { id: string; label: string };
+
+const CLIP_STATE = {
+ pending: "border-[var(--color-line)] text-[var(--color-dim)]",
+ fetching: "border-[var(--color-meter)] text-[var(--color-meter)]",
+ cutting: "border-[var(--color-sel)] text-[var(--color-sel)]",
+ done: "border-[var(--color-good)] text-[var(--color-good)]",
+ failed: "border-[var(--color-bad)] text-[var(--color-bad)]",
+} as const;
+type ClipState = keyof typeof CLIP_STATE;
+
+export default function ReportBuildChain({
+ project,
+ entries,
+}: {
+ project: string;
+ entries: { id: string; kind: string }[];
+}) {
+ const [presets, setPresets] = useState<Preset[]>([]);
+ const [preset, setPreset] = useState("fast");
+ const [only, setOnly] = useState("");
+ const [dry, setDry] = useState<StepView[] | null>(null);
+ const [job, setJob] = useState<JobView | null>(null);
+ const [events, setEvents] = useState<Ev[]>([]);
+ const [error, setError] = useState<string | null>(null);
+ const [needsReplace, setNeedsReplace] = useState(false);
+ const [busy, setBusy] = useState(false);
+ const since = useRef(0);
+ const sinceEvent = useRef(0);
+
+ useEffect(() => {
+ void fetch("/api/report/build", { cache: "no-store" })
+ .then((r) => r.json())
+ .then((j) => {
+ setPresets(j.presets ?? []);
+ // Adopt a build already running, so a reload does not lose it.
+ if (j.running) {
+ setJob(j.running as JobView);
+ setEvents((j.running as JobView).events ?? []);
+ }
+ })
+ .catch(() => {});
+ }, []);
+
+ useEffect(() => {
+ if (!job || job.state !== "running") return;
+ const t = setInterval(async () => {
+ const r = await fetch(
+ `/api/report/build?job=${job.id}&since=${since.current}&sinceEvent=${sinceEvent.current}`,
+ { cache: "no-store" },
+ );
+ if (!r.ok) return;
+ const j = (await r.json()) as JobView;
+ since.current = j.next;
+ sinceEvent.current = j.nextEvent;
+ setEvents((prev) => [...prev, ...(j.events ?? [])]);
+ setJob((prev) => (prev ? { ...j, log: [...prev.log, ...j.log] } : j));
+ }, 800);
+ return () => clearInterval(t);
+ }, [job]);
+
+ const post = useCallback(
+ async (qs: string, extra: Record<string, unknown> = {}) => {
+ setBusy(true);
+ setError(null);
+ const r = await fetch(`/api/report/build${qs}`, {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ project, preset, only: only || null, ...extra }),
+ });
+ const j = (await r.json()) as Record<string, unknown>;
+ setBusy(false);
+ if (!r.ok) {
+ setError(String(j.error ?? r.status));
+ setNeedsReplace(!!j.needsReplace);
+ return null;
+ }
+ setNeedsReplace(false);
+ return j;
+ },
+ [project, preset, only],
+ );
+
+ const clipStates = new Map<string, ClipState>();
+ for (const e of events) {
+ const id = e.id as string | undefined;
+ if (!id) continue;
+ if (e.ev === "fetch") clipStates.set(id, e.cached ? "cutting" : "fetching");
+ else if (e.ev === "clip" || e.ev === "card") clipStates.set(id, "cutting");
+ else if (e.ev === "snap") clipStates.set(id, "cutting");
+ else if (e.ev === "segment") clipStates.set(id, "done");
+ else if (e.ev === "entry-failed") clipStates.set(id, "failed");
+ }
+ const failed = events.filter((e) => e.ev === "entry-failed");
+
+ return (
+ <section className="space-y-2 rounded border border-[var(--color-line)] bg-[var(--color-panel)] p-3">
+ <div className="flex flex-wrap items-center gap-2">
+ <span className="micro">build</span>
+ <select
+ value={preset}
+ onChange={(e) => setPreset(e.target.value)}
+ data-preset=""
+ className="rounded border border-[var(--color-line)] bg-[var(--color-panel-2)] px-2 py-1 text-[12px]"
+ >
+ {presets.map((p) => (
+ <option key={p.id} value={p.id}>
+ {p.label}
+ </option>
+ ))}
+ </select>
+ {preset === "preview" && (
+ <select
+ value={only}
+ onChange={(e) => setOnly(e.target.value)}
+ data-only=""
+ className="rounded border border-[var(--color-line)] bg-[var(--color-panel-2)] px-2 py-1 font-mono text-[12px]"
+ >
+ <option value="">— which clip —</option>
+ {entries.filter((e) => e.kind === "clip").map((e) => (
+ <option key={e.id} value={e.id}>
+ {e.id}
+ </option>
+ ))}
+ </select>
+ )}
+
+ <button
+ type="button"
+ data-action="dry"
+ className={buttonVariants({ size: "sm" })}
+ disabled={busy}
+ onClick={async () => {
+ const j = await post("?dry=1");
+ if (j) setDry(j.steps as StepView[]);
+ }}
+ >
+ show the command
+ </button>
+ <button
+ type="button"
+ data-action="run"
+ className={buttonVariants({ variant: "primary", size: "sm" })}
+ disabled={busy || job?.state === "running"}
+ onClick={async () => {
+ const j = await post("");
+ if (j?.job) {
+ since.current = 0;
+ sinceEvent.current = 0;
+ setEvents([]);
+ setJob(j.job as JobView);
+ }
+ }}
+ >
+ run
+ </button>
+ {job?.state === "running" && (
+ <button
+ type="button"
+ data-action="cancel"
+ className={buttonVariants({ variant: "destructive", size: "sm" })}
+ onClick={() => void fetch(`/api/report/build?cancel=${job.id}`, { method: "POST" })}
+ >
+ cancel
+ </button>
+ )}
+ </div>
+
+ {error && (
+ <p data-build-error="" className="text-[12px] text-[var(--color-bad)]">
+ {error}
+ {needsReplace && (
+ <>
+ {" "}
+ <button
+ type="button"
+ data-action="replace"
+ className={buttonVariants({ variant: "destructive", size: "sm" })}
+ onClick={async () => {
+ const j = await post("?replace=1");
+ if (j?.job) {
+ since.current = 0;
+ sinceEvent.current = 0;
+ setEvents([]);
+ setJob(j.job as JobView);
+ }
+ }}
+ >
+ stamp it aside and build
+ </button>
+ </>
+ )}
+ </p>
+ )}
+
+ {dry && !job && (
+ <pre
+ data-dry=""
+ className="max-h-56 overflow-auto rounded border border-[var(--color-line)] bg-[var(--color-ink)] p-2 font-mono text-[11px] text-[var(--color-dim)]"
+ >
+ {dry.map((s) => `# ${s.label}\n${s.argv.join(" ")}\n`).join("\n")}
+ </pre>
+ )}
+
+ {job && (
+ <div className="space-y-2">
+ <ol className="flex flex-wrap gap-1.5">
+ {job.steps.map((s, i) => (
+ <li
+ key={s.label}
+ data-step={i}
+ data-step-state={
+ job.state !== "running" && i <= job.stepIndex
+ ? job.state
+ : i < job.stepIndex
+ ? "done"
+ : i === job.stepIndex
+ ? "running"
+ : "pending"
+ }
+ className={badgeVariants({
+ variant:
+ i < job.stepIndex || (job.state === "done" && i <= job.stepIndex)
+ ? "meter"
+ : i === job.stepIndex && job.state === "running"
+ ? "on"
+ : job.state === "failed" && i === job.stepIndex
+ ? "blocking"
+ : "neutral",
+ })}
+ >
+ {s.label}
+ </li>
+ ))}
+ </ol>
+
+ {/* One box per entry, lit by the pipeline's own events. "Step 3 of 4,
+ running" is not progress when step 3 is twenty minutes long. */}
+ <div className="flex flex-wrap gap-1">
+ {entries.map((e) => {
+ const st = clipStates.get(e.id) ?? "pending";
+ return (
+ <span
+ key={e.id}
+ data-clip-box={e.id}
+ data-clip-state={st}
+ className={`rounded border px-1.5 py-0.5 font-mono text-[10px] ${CLIP_STATE[st]}`}
+ >
+ {e.id}
+ </span>
+ );
+ })}
+ </div>
+
+ {failed.length > 0 && (
+ <p data-build-failed="" className="text-[12px] text-[var(--color-bad)]">
+ {failed.length} entr{failed.length === 1 ? "y" : "ies"} failed (
+ {failed.map((f) => String(f.id)).join(", ")}). Everything buildable was built and the
+ segments are kept — but the timeline was NOT concatenated, because a finished file
+ quietly missing a citation looks complete.
+ </p>
+ )}
+
+ <pre className="max-h-56 overflow-auto rounded border border-[var(--color-line)] bg-[var(--color-ink)] p-2 font-mono text-[11px] text-[var(--color-dim)]">
+ {job.log.slice(-120).join("\n")}
+ </pre>
+
+ <div className="micro" data-job-state={job.state}>
+ {job.state}
+ {job.error ? ` — ${job.error}` : ""}
+ </div>
+ </div>
+ )}
+ </section>
+ );
+}
diff --git a/umtool/components/projects/ReportProject.tsx b/umtool/components/projects/ReportProject.tsx
@@ -2,6 +2,7 @@ import Link from "next/link";
import BrowseHeader from "@/components/BrowseHeader";
import { fmtAgo, fmtBytes } from "@/lib/format";
import { readClipDetail } from "@/lib/projects/report.mjs";
+import ReportBuildChain from "./ReportBuildChain";
import { decisionsForProject } from "@/lib/projects";
import { badgeVariants, type BadgeVariants } from "@/components/ui/badge";
import type { Severity } from "@/lib/decisions";
@@ -141,6 +142,12 @@ export default async function ReportProject({
</div>
</section>
+ {/* --- building it ---------------------------------------------- */}
+ <ReportBuildChain
+ project={project.id}
+ entries={entries.map((e) => ({ id: e.id, kind: e.kind }))}
+ />
+
{/* --- the timeline --------------------------------------------- */}
<section>
<h2 className="micro mb-1.5">
diff --git a/umtool/e2e/build.spec.ts b/umtool/e2e/build.spec.ts
@@ -0,0 +1,188 @@
+import { test, expect } from "@playwright/test";
+import { execFileSync } from "node:child_process";
+import { existsSync, readdirSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+// ---------------------------------------------------------------------------
+// The build driver.
+//
+// Every one of these runs OFFLINE. The fixture writes stub YTDLP_BIN and
+// QRENCODE_BIN and playwright.config points the server at them, so the chain --
+// preflight, resolve, build, verify -- exercises its real code with no network
+// and a deterministic answer. The stub reports `gone1` removed the way a deleted
+// upload is, which is what gives the preflight's blocking path a true answer
+// rather than a plausible one.
+//
+// build-fixture is its own project. The bench specs write windows and the index
+// specs assert them; a build stamps deliverables aside and gets refused when one
+// is newer than its manifest. Sharing would make each suite's result depend on
+// the other's order.
+// ---------------------------------------------------------------------------
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const UMTOOL = path.join(HERE, "..");
+const FIXTURE = path.join(UMTOOL, ".e2e-song");
+const PROJECT = "reports/build-fixture";
+const OUT = path.join(FIXTURE, "reports", "build-fixture", "out");
+const FINAL = path.join(OUT, "build-fixture.mp4");
+
+type Job = { id: string; state: string; error: string | null };
+
+async function waitFor(
+ request: { get: (u: string) => Promise<{ json: () => Promise<unknown> }> },
+ id: string,
+ ms = 120_000,
+) {
+ const until = Date.now() + ms;
+ while (Date.now() < until) {
+ const j = (await (await request.get(`/api/report/build?job=${id}`)).json()) as Job & {
+ events: { ev: string; id?: string }[];
+ };
+ if (j.state !== "running") return j;
+ await new Promise((r) => setTimeout(r, 400));
+ }
+ throw new Error("the job never finished");
+}
+
+test("a dry run prints the exact chain, and runs nothing", async ({ request }) => {
+ const r = await request.post("/api/report/build?dry=1", {
+ data: { project: PROJECT, preset: "fast" },
+ });
+ const j = (await r.json()) as { dry: boolean; steps: { label: string; argv: string[]; timeoutMs: number }[] };
+ expect(j.dry).toBe(true);
+
+ // Preflight is a STEP, not a preamble: it is the one fact that goes stale in
+ // both directions, and without it a dead source is found twenty minutes and a
+ // dozen paid-for fetches in.
+ expect(j.steps[0].argv.join(" ")).toContain("check-availability.mjs");
+ expect(j.steps[1].argv.join(" ")).toContain("resolve-windows.mjs");
+ // Dry: the widener is never handed --write from here. Applying is explicit.
+ expect(j.steps[1].argv).not.toContain("--write");
+ expect(j.steps[2].argv.join(" ")).toContain("build-video.mjs");
+ expect(j.steps[2].argv).toContain("--continue-on-error");
+ expect(j.steps.at(-1)!.argv.join(" ")).toContain("verify-build.mjs");
+
+ // The build asks for more than the 15-minute default, which exists to catch
+ // the accidental hour-long job and would SIGKILL a real 19-clip run.
+ expect(j.steps[2].timeoutMs).toBeGreaterThanOrEqual(15 * 60_000);
+});
+
+test("a build runs end to end, offline, and the file is verified", async ({ request }) => {
+ const start = await request.post("/api/report/build", {
+ data: { project: PROJECT, preset: "fast" },
+ });
+ expect(start.ok()).toBeTruthy();
+ const { job } = (await start.json()) as { job: Job };
+
+ const done = await waitFor(request, job.id);
+ expect(done.state, done.error ?? "").toBe("done");
+ expect(existsSync(FINAL)).toBe(true);
+
+ // Per-clip progress, from the pipeline's own NDJSON. "Step 3 of 4, running"
+ // is not progress when step 3 is the twenty-minute one.
+ const events = (done as unknown as { events: { ev: string; id?: string }[] }).events ?? [];
+ expect(events.some((e) => e.ev === "segment" && e.id === "c01")).toBe(true);
+ expect(events.some((e) => e.ev === "segment" && e.id === "c02")).toBe(true);
+ expect(events.some((e) => e.ev === "done")).toBe(true);
+
+ // c01's window is already cached, so the build must REUSE it rather than ask
+ // the stub for it -- the same containment rule the bench's wide fetch relies on.
+ const fetches = events.filter((e) => e.ev === "fetch");
+ expect(fetches.find((e) => e.id === "c01")).toMatchObject({ cached: true });
+});
+
+test("an existing deliverable is not destroyed to make a new one", async ({ request }) => {
+ // The previous test built it, so out/ is now newer than the manifest.
+ expect(existsSync(FINAL)).toBe(true);
+
+ const refused = await request.post("/api/report/build", {
+ data: { project: PROJECT, preset: "fast" },
+ });
+ expect(refused.status()).toBe(409);
+ const j = (await refused.json()) as { needsReplace: boolean; error: string };
+ expect(j.needsReplace).toBe(true);
+ expect(j.error).toContain("newer than the manifest");
+
+ // With replace=1 the old file is STAMPED ASIDE, never overwritten -- it cost
+ // an hour of network fetches in the real case.
+ const ok = await request.post("/api/report/build?replace=1", {
+ data: { project: PROJECT, preset: "fast" },
+ });
+ expect(ok.ok()).toBeTruthy();
+ const stamped = readdirSync(OUT).filter((n) => /^build-fixture\.\d{8}-\d{4}\.mp4$/.test(n));
+ expect(stamped.length).toBeGreaterThan(0);
+
+ const { job } = (await ok.json()) as { job: Job };
+ const done = await waitFor(request, job.id);
+ expect(done.state, done.error ?? "").toBe("done");
+});
+
+test("a source that is gone blocks the build at step 1", async ({ request }) => {
+ const start = await request.post("/api/report/build", {
+ data: { project: "reports/gone-fixture", preset: "fast" },
+ });
+ expect(start.ok()).toBeTruthy();
+ const { job } = (await start.json()) as { job: Job };
+
+ const done = await waitFor(request, job.id);
+ // Failed at the preflight, having encoded nothing.
+ expect(done.state).toBe("failed");
+ expect(existsSync(path.join(FIXTURE, "reports", "gone-fixture", "out", "gone-fixture.mp4"))).toBe(
+ false,
+ );
+
+ // And it wrote down what it found, so the inbox can report it without running
+ // yt-dlp itself.
+ const avail = path.join(FIXTURE, "reports", "gone-fixture", "out", "availability.json");
+ expect(existsSync(avail)).toBe(true);
+});
+
+test("a second build is refused while one is running, and cancel leaves no orphan", async ({
+ request,
+}) => {
+ const first = await request.post("/api/report/build?replace=1", {
+ data: { project: PROJECT, preset: "final" },
+ });
+ expect(first.ok()).toBeTruthy();
+ const { job } = (await first.json()) as { job: Job };
+
+ // One job, process-wide. Two builds would interleave in one out/segments, and
+ // two in different projects would still fight over yt-dlp and the CPU.
+ const second = await request.post("/api/report/build", {
+ data: { project: "reports/report-fixture", preset: "fast" },
+ });
+ expect(second.status()).toBe(409);
+
+ await request.post(`/api/report/build?cancel=${job.id}`);
+ const done = await waitFor(request, job.id, 60_000);
+ expect(done.state).toBe("failed");
+
+ // The GRANDCHILDREN are the point. build-video shells out, so child.kill()
+ // reaps the node process and leaves yt-dlp and ffmpeg running -- the same
+ // failure the diarize backfill had. Killing the process GROUP is what stops it.
+ //
+ // pgrep is run WITHOUT a shell on purpose. Going through `bash -lc` puts the
+ // pattern into bash's own command line, so pgrep matches the shell that is
+ // asking -- a self-match that reads exactly like the orphan it is looking for.
+ //
+ // Polled rather than asserted once: the kill is SIGTERM to the group and
+ // SIGKILL five seconds later, so "gone" is a state it reaches rather than one
+ // it is in the instant the job reports failed. Racing that produced a red on
+ // one run and a green on the next.
+ const gone = async () => {
+ for (let i = 0; i < 30; i += 1) {
+ let out = "";
+ try {
+ out = execFileSync("pgrep", ["-af", "build-video.mjs"]).toString().trim();
+ } catch {
+ return ""; // pgrep exits 1 when nothing matches, which is the good case
+ }
+ if (!out) return "";
+ await new Promise((r) => setTimeout(r, 400));
+ }
+ return execFileSync("pgrep", ["-af", "build-video.mjs"]).toString().trim();
+ };
+ const strays = await gone();
+ expect(strays, `left running: ${strays}`).toBe("");
+});
diff --git a/umtool/e2e/fixtures/make-fixture.mjs b/umtool/e2e/fixtures/make-fixture.mjs
@@ -624,6 +624,13 @@ const CUES = {
[15, 18, "trailing off and then"],
[18, 21, "it lands at last."],
],
+ // Cited by gone-fixture. Its cue file exists (so the manifest is readable) but
+ // the stub yt-dlp reports it removed, which is what gives `source-unavailable`
+ // a true answer rather than a plausible one.
+ gone1: [
+ [0, 3, "This upload has since been deleted."],
+ [3, 6, "But its transcript is still in the archive."],
+ ],
vid2: [
[0, 3, "no punctuation anywhere in this upload"],
[3, 6, "the asr never emitted a full stop"],
@@ -651,6 +658,21 @@ for (const [vid, rows] of Object.entries(CUES)) {
);
}
+// A font the header's drawtext can actually load, or no header.
+//
+// build-video.mjs draws the citation line with `fontfile='<render.fontRegular>'`
+// and an empty one is a filtergraph error, not a missing label. Rather than
+// assume a font, look for one and honestly set headerHeight: 0 when there is
+// none -- which is itself a documented manifest configuration ("a cut whose
+// sources are listed elsewhere does not need its own attribution burnt in").
+const FONT_CANDIDATES = [
+ "/usr/share/fonts/TTF/FiraSans-Regular.ttf",
+ "/usr/share/fonts/TTF/DejaVuSans.ttf",
+ "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
+ "/usr/share/fonts/liberation/LiberationSans-Regular.ttf",
+];
+const FONT = FONT_CANDIDATES.find((f) => existsSync(f)) ?? null;
+
const manifest = (slug, title, provenance, timeline) => ({
schemaVersion: 1,
slug,
@@ -658,7 +680,25 @@ const manifest = (slug, title, provenance, timeline) => ({
subtitle: "a fixture",
generatedOn: "2026-01-01",
provenance: { channelSlug: "testchan", ...provenance },
- render: { width: 1920, height: 1080, fps: 30, audioRate: 48000, audioChannels: 2, fetchPad: 3 },
+ render: {
+ width: 640,
+ height: 360,
+ fps: 15,
+ audioRate: 48000,
+ audioChannels: 2,
+ maxHeightSource: 360,
+ fetchPad: 3,
+ snapWindow: 1.6,
+ transition: 0.2,
+ crf: 30,
+ preset: "ultrafast",
+ // No timelineNodes, so no footer -- which is what keeps ImageMagick out of
+ // the fixture build entirely.
+ footerHeight: 0,
+ headerHeight: FONT ? 24 : 0,
+ ...(FONT ? { fontRegular: FONT, fontBold: FONT } : {}),
+ palette: { bg: "#12100c", fg: "#f6f1e6", muted: "#a2957f", accent: "#c8752a", amber: "#ffc860" },
+ },
timelineNodes: [],
timeline,
});
@@ -736,6 +776,112 @@ const BENCH = writeProject(
]),
);
+// -- STUB BINARIES, so a build is offline and deterministic --------------------
+//
+// The pipeline shells out to yt-dlp for the availability preflight and for every
+// fetch. Neither belongs in a test: the first needs the network and the second
+// needs somebody else's server to still be serving. YTDLP_BIN and QRENCODE_BIN
+// already exist as overrides for exactly this, so the fixture provides both.
+//
+// They are NODE scripts, not shell. The yt-dlp stub has to parse
+// `--download-sections *FROM-TO` and do fractional arithmetic on it, and doing
+// that in bash means awk, which means three layers of quoting inside a
+// generated file. It got mangled once; this cannot.
+//
+// The stub gives `source-unavailable` a TRUE answer: any URL naming `gone1`
+// fails the way a removed upload does, so a manifest citing it is genuinely
+// blocked rather than assumed to be.
+const BIN = path.join(dest, "bin");
+mkdirSync(BIN, { recursive: true });
+
+writeFileSync(
+ path.join(BIN, "yt-dlp"),
+ `#!/usr/bin/env node
+// Fixture stub for yt-dlp. Deterministic, offline.
+import { spawnSync } from "node:child_process";
+import { mkdirSync } from "node:fs";
+import path from "node:path";
+
+const argv = process.argv.slice(2);
+const all = argv.join(" ");
+
+if (all.includes("gone1")) {
+ process.stderr.write("ERROR: [youtube] gone1: Video unavailable. This video has been removed by the uploader\\n");
+ process.exit(1);
+}
+if (argv.includes("--simulate")) process.exit(0);
+
+const out = argv[argv.indexOf("-o") + 1];
+if (!out || argv.indexOf("-o") < 0) {
+ process.stderr.write("stub: no -o\\n");
+ process.exit(2);
+}
+const sec = argv[argv.indexOf("--download-sections") + 1] ?? "*0-5";
+const [from, to] = sec.replace(/^\\*/, "").split("-").map(Number);
+const dur = Math.max(1, (to || 5) - (from || 0));
+
+mkdirSync(path.dirname(out), { recursive: true });
+const r = spawnSync(
+ "ffmpeg",
+ ["-nostdin", "-v", "error", "-y",
+ "-f", "lavfi", "-i", \`color=c=darkgreen:size=320x180:rate=15:duration=\${dur}\`,
+ "-f", "lavfi", "-i", \`sine=frequency=440:duration=\${dur}\`,
+ "-t", String(dur),
+ "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-ar", "48000", "-ac", "2", out],
+ { stdio: "inherit" },
+);
+process.exit(r.status ?? 1);
+`,
+ { mode: 0o755 },
+);
+
+writeFileSync(
+ path.join(BIN, "qrencode"),
+ `#!/usr/bin/env node
+// Fixture stub for qrencode: a real code is not needed to prove one was overlaid.
+import { spawnSync } from "node:child_process";
+import { mkdirSync } from "node:fs";
+import path from "node:path";
+
+const argv = process.argv.slice(2);
+const i = argv.indexOf("-o");
+if (i < 0) process.exit(2);
+const out = argv[i + 1];
+mkdirSync(path.dirname(out), { recursive: true });
+const r = spawnSync(
+ "ffmpeg",
+ ["-nostdin", "-v", "error", "-y", "-f", "lavfi", "-i", "color=c=white:size=64x64", "-frames:v", "1", out],
+ { stdio: "inherit" },
+);
+process.exit(r.status ?? 1);
+`,
+ { mode: 0o755 },
+);
+
+// A THIRD copy, for the build specs.
+//
+// Same reason bench-fixture exists: a build writes out/, stamps deliverables
+// aside and is refused when one is newer than its manifest. Sharing a project
+// with the bench specs would make each suite's result depend on the other's
+// order, which has already cost one confusing red.
+const BUILD = writeProject(
+ "build-fixture",
+ manifest("build-fixture", "The Build Fixture", { siteOrigin: "https://archive.example" }, [
+ { type: "clip", id: "c01", video: "vid1", start: 3.0, end: 6.0, cite: 3, section: 0, lock: true, quote: "and because" },
+ { type: "clip", id: "c02", video: "vid1", start: 9.0, end: 12.0, cite: 9, section: 0, lock: true, quote: "another whole sentence" },
+ ]),
+);
+
+// A cut whose source is GONE. The preflight must block it, and must block it
+// before anything encodes -- which is the whole reason it is step 1 rather than
+// a preamble somebody remembers to run.
+writeProject(
+ "gone-fixture",
+ manifest("gone-fixture", "A Source That Is Gone", { siteOrigin: "https://archive.example" }, [
+ { type: "clip", id: "c01", video: "gone1", start: 0, end: 3, cite: 0, section: 0, lock: true, quote: "deleted" },
+ ]),
+);
+
mkdirSync(path.join(reports, "bike-fixture"), { recursive: true });
writeFileSync(
path.join(reports, "bike-fixture", "sweep-report.md"),
@@ -767,11 +913,13 @@ ff([
"-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-ar", "48000", "-ac", "2",
path.join(REPORT, "out", "clips-raw", "vid1_0.00-9.00.mp4"),
]);
-mkdirSync(path.join(BENCH, "out", "clips-raw"), { recursive: true });
-copyFileSync(
- path.join(REPORT, "out", "clips-raw", "vid1_0.00-9.00.mp4"),
- path.join(BENCH, "out", "clips-raw", "vid1_0.00-9.00.mp4"),
-);
+for (const dir of [BENCH, BUILD]) {
+ mkdirSync(path.join(dir, "out", "clips-raw"), { recursive: true });
+ copyFileSync(
+ path.join(REPORT, "out", "clips-raw", "vid1_0.00-9.00.mp4"),
+ path.join(dir, "out", "clips-raw", "vid1_0.00-9.00.mp4"),
+ );
+}
console.log(`fixture at ${dest}`);
if (planned) console.log(` planned clip (used in a build): ${planned}`);
@@ -787,6 +935,7 @@ console.log(` flagged source: ${flagged ? flagged.video : "none — no asr/"}`)
console.log(` SONG_CODE_DIR=${path.join(dest, "code")}`);
console.log(` SONG_DIR=${path.join(dest, "data")}`);
console.log(` SONG_REPORTS_DIR=${reports}`);
+console.log(` YTDLP_BIN=${path.join(BIN, "yt-dlp")} QRENCODE_BIN=${path.join(BIN, "qrencode")}`);
console.log(` CHANNELS_DIR=${CHANNELS} (testchan/vid1 punctuated, vid2 not)`);
console.log(` projects: report-fixture (4 clips, 1 mid-sentence), no-origin-fixture,`);
console.log(` localhost-fixture, bike-fixture (sweep), find/ (shadowed),`);
diff --git a/umtool/e2e/projects.spec.ts b/umtool/e2e/projects.spec.ts
@@ -30,7 +30,15 @@ const UMTOOL = path.join(HERE, "..");
test("the index lists every kind, and says which state each project is in", async ({ page }) => {
await page.goto("/browse");
- await expect(page.locator("[data-kind='report-video']")).toHaveCount(6);
+ // The chip's number must EQUAL the number of cards, because the counts come
+ // from the unfiltered set. Asserting that relationship rather than a magic
+ // number is what stops every new fixture project from editing this spec.
+ const cards = await page.locator("[data-kind='report-video']").count();
+ expect(cards).toBeGreaterThan(1);
+ await expect(page.getByRole("link", { name: /^report video \d+$/ })).toHaveText(
+ `report video ${cards}`,
+ );
+
// NOT an exact count: browse.spec.ts creates a song through /api/browse/init,
// so the number here depends on what else has run. What matters is that the
// kind is present and that the fixture's own songs are in it.
@@ -62,7 +70,7 @@ test("a kind chip filters, and the counts do not move when it does", async ({ pa
await chip.click();
await expect(page).toHaveURL(/kind=report-video/);
await expect(page.locator("[data-kind='song']")).toHaveCount(0);
- await expect(page.locator("[data-kind='report-video']")).toHaveCount(6);
+ await expect(page.locator("[data-kind='report-video']").first()).toBeVisible();
// Counts come from the UNFILTERED set on purpose: a chip whose number changes
// when you click a different chip moves under the cursor.
diff --git a/umtool/lib/report/driver.mjs b/umtool/lib/report/driver.mjs
@@ -45,7 +45,12 @@ export const buildTimeoutMs = (clipCount, xfade) =>
* bench is exactly the surprise `lock` exists to prevent. Applying is a second,
* explicit action.
*/
-export function buildSteps(project, { preset = "fast", only = null, skipFetch = false, env = {} } = {}) {
+/**
+ * @param {{ id: string, dir: string }} project
+ * @param {{ preset?: string, only?: string | null, skipFetch?: boolean, env?: Record<string,string>, clipCount?: number }} [opts]
+ * @returns {import("../trim").Step[]}
+ */
+export function buildSteps(project, { preset = "fast", only = null, skipFetch = false, env = {}, clipCount = 20 } = {}) {
const p = PRESETS[preset] ?? PRESETS.fast;
const manifest = path.join(project.dir, "video.manifest.json");
const outDir = path.join(project.dir, "out");
@@ -86,13 +91,34 @@ export function buildSteps(project, { preset = "fast", only = null, skipFetch =
label: p.label,
argv: buildArgv,
ndjson: true,
- timeoutMs: buildTimeoutMs(project.clipCount ?? 20, p.xfade),
+ timeoutMs: buildTimeoutMs(clipCount, p.xfade),
});
+ // A build can exit 0 and still be wrong: a concat that produced nothing, a
+ // chapter pass that dropped markers, a timeline that lost a clip because
+ // --continue-on-error let it. Each looks like success at the terminal.
+ //
+ // Skipped for a one-clip preview, which deliberately does not produce a
+ // deliverable to measure.
+ if (!p.only || !only) {
+ steps.push({
+ ...base,
+ label: "verify the file that came out",
+ argv: ["node", script("verify-build.mjs"), manifest, "--out", outDir],
+ timeoutMs: 5 * 60_000,
+ });
+ }
+
return steps;
}
-/** Fetch ONE clip's window, wide. What the bench's "fetch more" runs. */
+/**
+ * Fetch ONE clip's window, wide. What the bench's "fetch more" runs.
+ * @param {{ dir: string }} project
+ * @param {string} clipId
+ * @param {number} pad
+ * @returns {import("../trim").Step[]}
+ */
export function fetchSteps(project, clipId, pad) {
return [
{
diff --git a/umtool/playwright.config.ts b/umtool/playwright.config.ts
@@ -38,6 +38,9 @@ export default defineConfig({
// var. CHANNELS_DIR has to be said explicitly: it is where a report
// video's cue files live, and its default is the real 3 GB corpus.
`CHANNELS_DIR=${FIXTURE}/channels ` +
+ // Stub binaries, so a build spec is offline and deterministic. The
+ // pipeline already reads both as overrides; the fixture writes them.
+ `YTDLP_BIN=${FIXTURE}/bin/yt-dlp QRENCODE_BIN=${FIXTURE}/bin/qrencode ` +
`NEXT_DIST_DIR=.next-e2e pnpm exec next dev --port ${PORT}`,
port: PORT,
reuseExistingServer: false,
