commit dcb54a161f8e17bacb5a2276d6948f3b93b4544b
parent a6fa44f737ee52e55687e02814668b46b38823f4
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 18 Aug 2026 13:16:29 -0400
debox-bg.mjs: cut a game's own text-box freezes out of a background bed
On videos/yoshi/notes.json the user asked for the green "there are very
dangerous donut lifts" popup to stop lingering: "The whole background holds
still while it's up, so can we trim so there's only about 1 second of the held
frame for each text box?"
The bed is a Yoshi's Island longplay and the game pauses itself behind its own
text boxes -- five freezes between 2.75s and 13.95s, the longest 3.82s. So the
tool keeps keepSeconds of each held stretch and drops the rest.
WHY IT IS INVISIBLE. Every frame inside a freezedetect run is the same picture,
so removing frames from the MIDDLE of a run and resuming at the run's end
splices a frozen frame onto the frame that broke the freeze -- the cut the
source already had, one frozen frame earlier. Cutting anywhere else is a jump.
That is checked rather than argued: the frames either side of every proposed
cut are decoded and compared before anything is encoded, and a cut whose span
is not flat aborts the run.
Frames, not timestamps, per pk3.sh's rule. And snapped rather than floored:
freeze boundaries are frame PTS, but they arrive as decimal seconds and
8.683333 * 60 is 520.99998 in binary floating point -- a floor there keeps 59
frames of a freeze where 60 were asked for, which is exactly how the first run
came out one frame short.
On yoshi-bg.mp4 at keepSeconds=1.0: keep frames 0-309, 397-456, 490-580,
750-809, 837-8396 = 8081 frames = 134.683s, -5.267s. Re-running freezedetect on
the result, the longest held stretch anywhere is 1.067s (1s plus four frames --
the re-encode smooths near-identical frames into the noise floor) against 3.82s
before.
yoshi-rebuild.sh's BG_VIDEO becomes ${BG_VIDEO:-...} like TP/PLANF/OUT/TL
already are, so a treated bed renders against the same arrangement without
editing the recipe. slice-out.mjs is the wrong tool for any of this: it cuts a
finished video, picture AND sound, and here the sound is the song.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Diffstat:
2 files changed, 163 insertions(+), 1 deletion(-)
diff --git a/umtool/song/debox-bg.mjs b/umtool/song/debox-bg.mjs
@@ -0,0 +1,159 @@
+#!/usr/bin/env node
+// Cut the held frames out of a background bed.
+//
+// node debox-bg.mjs <in.mp4> <out.mp4> [keepSeconds=1.0] [noise=0.003] [minFreeze=0.5]
+//
+// A gameplay bed recorded off a real playthrough pauses whenever the game puts a
+// text box on screen -- the game stops the world to let you read. Yoshi's Island
+// does it twice in the first fourteen seconds and holds the picture for nearly
+// four seconds at a stretch. Looped behind a song that keeps cutting back to it,
+// that reads as a broken video rather than a game waiting for a button.
+//
+// So: keep <keepSeconds> of each held stretch -- a beat to read the box -- and
+// drop the rest.
+//
+// WHY THIS IS INVISIBLE, and why it is only invisible done exactly this way.
+// Every frame inside a freezedetect run is the same picture. Removing frames
+// from the MIDDLE of a run and resuming at the run's end therefore splices a
+// frozen frame onto the very frame that broke the freeze -- which is the cut the
+// source already had, one frozen frame earlier. Cut anywhere else and it is a
+// jump. That property is checked, not asserted: before rendering, the frame
+// either side of every proposed cut is decoded and compared, and a cut whose
+// span is not actually flat aborts the run.
+//
+// FRAMES, NOT TIMESTAMPS. The bed is 60fps and pk3.sh's rule holds here -- every
+// piece gets quantised to a frame. A cut expressed in seconds lands half a frame
+// off and then the "identical" frames either side are two different pictures.
+// So the whole plan is computed in frame indices and handed to ffmpeg as
+// `select='between(n,...)'`, which counts frames and cannot round.
+//
+// Nothing is written over: the input bed is untouched and the result is a new
+// file. slice-out.mjs is the wrong tool for this job -- it cuts a finished video,
+// picture AND sound together, and here the sound is the song.
+import { execFileSync, spawnSync } from "node:child_process";
+import { existsSync } from "node:fs";
+
+const [IN, OUT] = process.argv.slice(2);
+const KEEP = Number(process.argv[4] ?? 1.0);
+const NOISE = Number(process.argv[5] ?? 0.003);
+const MIN_FREEZE = Number(process.argv[6] ?? 0.5);
+if (!IN || !OUT) {
+ console.error("usage: debox-bg.mjs <in.mp4> <out.mp4> [keepSeconds] [noise] [minFreeze]");
+ process.exit(1);
+}
+if (!existsSync(IN)) { console.error(`no such file: ${IN}`); process.exit(1); }
+
+const probe = (entries, stream) => execFileSync("ffprobe", ["-v", "error",
+ ...(stream ? ["-select_streams", stream] : []), "-show_entries", entries,
+ "-of", "default=nw=1:nk=1", IN], { encoding: "utf8" }).trim().split("\n");
+
+const [num, den] = probe("stream=r_frame_rate", "v:0")[0].split("/").map(Number);
+const FPS = num / (den || 1);
+const DUR = Number(probe("format=duration")[0]);
+const NFRAMES = Math.round(DUR * FPS);
+// SNAPPED to the nearest frame, not floored. Every timestamp freezedetect prints
+// is a frame's own PTS, so it is already on a frame boundary -- but it arrives as
+// decimal seconds, and 8.683333 * 60 is 520.99998 in binary floating point. A
+// floor turns that into frame 520 and quietly keeps 59 frames of a freeze where
+// 60 were asked for. Rounding is not a fudge here; it is reading a frame-quantised
+// number as the frame it names. Anything genuinely between frames is far enough
+// from an integer to be caught rather than snapped.
+const frameOf = (t) => {
+ const x = t * FPS;
+ const r = Math.round(x);
+ if (Math.abs(x - r) > 0.01) throw new Error(`timestamp ${t}s is not on a frame boundary at ${FPS}fps`);
+ return r;
+};
+
+console.log(`${IN}`);
+console.log(` ${FPS}fps, ${DUR.toFixed(3)}s, ${NFRAMES} frames`);
+console.log(` freezedetect n=${NOISE}:d=${MIN_FREEZE}, keeping ${KEEP}s of each held stretch\n`);
+
+// ---- what is held ----------------------------------------------------------
+// spawnSync, because freezedetect reports on STDERR and execFileSync hands back
+// stdout -- which is empty here and reads as "no freezes" rather than as a
+// mistake. The whole tool would then decline to cut anything and say so calmly.
+const run = spawnSync("ffmpeg", ["-nostdin", "-v", "info", "-i", IN,
+ "-vf", `freezedetect=n=${NOISE}:d=${MIN_FREEZE}`, "-map", "0:v", "-f", "null", "-"],
+ { encoding: "utf8", maxBuffer: 1 << 26 });
+if (run.status !== 0) { console.error(run.stderr ?? "ffmpeg failed"); process.exit(1); }
+const log = run.stderr ?? "";
+
+const freezes = [];
+let pending = null;
+for (const line of log.split("\n")) {
+ const s = line.match(/freeze_start:\s*([\d.]+)/);
+ const e = line.match(/freeze_end:\s*([\d.]+)/);
+ if (s) pending = { start: Number(s[1]) };
+ if (e && pending) { pending.end = Number(e[1]); freezes.push(pending); pending = null; }
+}
+// A freeze still open at EOF has no end line; it runs to the last frame.
+if (pending) { pending.end = DUR; freezes.push(pending); }
+if (!freezes.length) { console.error("no freezes detected — nothing to do"); process.exit(1); }
+
+// ---- the plan, in frames ---------------------------------------------------
+// A run shorter than the beat we mean to keep is left ALONE. Trimming it would
+// be trading a held frame nobody notices for a splice, which is a bad trade.
+const cuts = [];
+for (const f of freezes) {
+ const from = frameOf(f.start) + Math.round(KEEP * FPS); // first frame dropped
+ const to = frameOf(f.end); // first frame kept again
+ if (to - from < 1) continue;
+ cuts.push({ ...f, from, to });
+}
+
+const keeps = [];
+let at = 0;
+for (const c of cuts) { if (c.from > at) keeps.push([at, c.from - 1]); at = c.to; }
+if (at <= NFRAMES - 1) keeps.push([at, NFRAMES - 1]);
+
+const kept = keeps.reduce((t, [a, b]) => t + (b - a + 1), 0);
+console.log(" freeze (bed t) length action");
+for (const f of freezes) {
+ const c = cuts.find((x) => x.start === f.start);
+ const act = c
+ ? `cut ${(c.from / FPS).toFixed(3)}→${(c.to / FPS).toFixed(3)} (−${((c.to - c.from) / FPS).toFixed(3)}s)`
+ : "kept whole — already under the keep";
+ console.log(` ${f.start.toFixed(3)} → ${f.end.toFixed(3)} ${(f.end - f.start).toFixed(2)}s ${act}`);
+}
+console.log(`\n keep frames: ${keeps.map(([a, b]) => `${a}-${b}`).join(", ")}`);
+console.log(` ${kept} frames = ${(kept / FPS).toFixed(3)}s (−${((NFRAMES - kept) / FPS).toFixed(3)}s)\n`);
+
+// ---- prove every cut lands inside a flat stretch ---------------------------
+// The claim is that the last frame kept and the last frame dropped are the same
+// picture, because both sit inside one freeze. If that is false the splice is a
+// visible jump, so it is measured before anything is encoded.
+const grayAt = (frame) => execFileSync("ffmpeg", ["-nostdin", "-v", "error",
+ "-ss", String(frame / FPS), "-i", IN, "-frames:v", "1",
+ "-vf", "scale=160:90,format=gray", "-f", "rawvideo", "-"], { maxBuffer: 1 << 20 });
+const meanAbs = (a, b) => {
+ if (a.length !== b.length || !a.length) return Infinity;
+ let t = 0;
+ for (let i = 0; i < a.length; i += 1) t += Math.abs(a[i] - b[i]);
+ return t / a.length;
+};
+
+let bad = 0;
+for (const c of cuts) {
+ const d = meanAbs(grayAt(c.from - 1), grayAt(c.to - 1));
+ const ok = d < 1.0;
+ if (!ok) bad += 1;
+ console.log(` ${ok ? "ok " : "REFUSE"} cut at frame ${c.from}: frames ${c.from - 1} and ${c.to - 1} differ by ${d.toFixed(3)}`);
+}
+if (bad) {
+ console.error(`\n${bad} cut(s) would splice two DIFFERENT pictures. Refusing.`);
+ console.error("Raise minFreeze or lower noise so the runs found are really flat.");
+ process.exit(1);
+}
+
+// ---- rebuild ---------------------------------------------------------------
+const select = keeps.map(([a, b]) => `between(n\\,${a}\\,${b})`).join("+");
+execFileSync("ffmpeg", ["-nostdin", "-v", "error", "-y", "-i", IN,
+ "-vf", `select='${select}',setpts=N/FRAME_RATE/TB`, "-fps_mode", "cfr", "-r", String(FPS),
+ "-an", "-c:v", "libx264", "-preset", "medium", "-crf", "18", "-pix_fmt", "yuv420p", OUT],
+ { stdio: "inherit" });
+
+const outDur = Number(execFileSync("ffprobe", ["-v", "error", "-show_entries", "format=duration",
+ "-of", "csv=p=0", OUT], { encoding: "utf8" }).trim());
+console.log(`\n${OUT}`);
+console.log(` ${outDur.toFixed(3)}s (expected ${(kept / FPS).toFixed(3)}s)`);
diff --git a/umtool/song/yoshi-rebuild.sh b/umtool/song/yoshi-rebuild.sh
@@ -50,7 +50,10 @@ for(const v of p.voices){const t={};
# TAIL_FADE is NOT set: this cut ends on the reprise, not on a loop point, so it
# has an ending of its own. VIDEO_TAIL_FADE follows TAIL_FADE, so the picture is
# left alone too.
-PLAN=$PLANF BG_VIDEO=$PWD/intro/yoshi-bg.mp4 BG_FIT=pad LAYOUT=bg \
+# BG_VIDEO overridable like TP/PLANF/OUT/TL above, so a treated bed -- say one
+# debox-bg.mjs has taken the game's own text-box freezes out of -- can be
+# rendered against this exact arrangement without editing the recipe.
+PLAN=$PLANF BG_VIDEO=${BG_VIDEO:-$PWD/intro/yoshi-bg.mp4} BG_FIT=pad LAYOUT=bg \
node render-poly.mjs "$T/yoshi-${PLANF%.json}-body.mp4" | tail -3
LEAD=$(node -e "
