#!/usr/bin/env node // Cut the held frames out of a background bed. // // node debox-bg.mjs [keepSeconds=1.0] [noise=0.003] [minFreeze=0.5] // // A gameplay bed recorded off a real playthrough pauses whenever the game puts a // text box on screen -- the game stops the world to let you read. Yoshi's Island // does it twice in the first fourteen seconds and holds the picture for nearly // four seconds at a stretch. Looped behind a song that keeps cutting back to it, // that reads as a broken video rather than a game waiting for a button. // // So: keep of each held stretch -- a beat to read the box -- and // drop the rest. // // WHY THIS IS INVISIBLE, and why it is only invisible done exactly this way. // Every frame inside a freezedetect run is the same picture. Removing frames // from the MIDDLE of a run and resuming at the run's end therefore splices a // frozen frame onto the very frame that broke the freeze -- which is the cut the // source already had, one frozen frame earlier. Cut anywhere else and it is a // jump. That property is checked, not asserted: before rendering, the frame // either side of every proposed cut is decoded and compared, and a cut whose // span is not actually flat aborts the run. // // FRAMES, NOT TIMESTAMPS. The bed is 60fps and pk3.sh's rule holds here -- every // piece gets quantised to a frame. A cut expressed in seconds lands half a frame // off and then the "identical" frames either side are two different pictures. // So the whole plan is computed in frame indices and handed to ffmpeg as // `select='between(n,...)'`, which counts frames and cannot round. // // Nothing is written over: the input bed is untouched and the result is a new // file. slice-out.mjs is the wrong tool for this job -- it cuts a finished video, // picture AND sound together, and here the sound is the song. import { execFileSync, spawnSync } from "node:child_process"; import { existsSync } from "node:fs"; const [IN, OUT] = process.argv.slice(2); const KEEP = Number(process.argv[4] ?? 1.0); const NOISE = Number(process.argv[5] ?? 0.003); const MIN_FREEZE = Number(process.argv[6] ?? 0.5); if (!IN || !OUT) { console.error("usage: debox-bg.mjs [keepSeconds] [noise] [minFreeze]"); process.exit(1); } if (!existsSync(IN)) { console.error(`no such file: ${IN}`); process.exit(1); } const probe = (entries, stream) => execFileSync("ffprobe", ["-v", "error", ...(stream ? ["-select_streams", stream] : []), "-show_entries", entries, "-of", "default=nw=1:nk=1", IN], { encoding: "utf8" }).trim().split("\n"); const [num, den] = probe("stream=r_frame_rate", "v:0")[0].split("/").map(Number); const FPS = num / (den || 1); const DUR = Number(probe("format=duration")[0]); const NFRAMES = Math.round(DUR * FPS); // SNAPPED to the nearest frame, not floored. Every timestamp freezedetect prints // is a frame's own PTS, so it is already on a frame boundary -- but it arrives as // decimal seconds, and 8.683333 * 60 is 520.99998 in binary floating point. A // floor turns that into frame 520 and quietly keeps 59 frames of a freeze where // 60 were asked for. Rounding is not a fudge here; it is reading a frame-quantised // number as the frame it names. Anything genuinely between frames is far enough // from an integer to be caught rather than snapped. const frameOf = (t) => { const x = t * FPS; const r = Math.round(x); if (Math.abs(x - r) > 0.01) throw new Error(`timestamp ${t}s is not on a frame boundary at ${FPS}fps`); return r; }; console.log(`${IN}`); console.log(` ${FPS}fps, ${DUR.toFixed(3)}s, ${NFRAMES} frames`); console.log(` freezedetect n=${NOISE}:d=${MIN_FREEZE}, keeping ${KEEP}s of each held stretch\n`); // ---- what is held ---------------------------------------------------------- // spawnSync, because freezedetect reports on STDERR and execFileSync hands back // stdout -- which is empty here and reads as "no freezes" rather than as a // mistake. The whole tool would then decline to cut anything and say so calmly. const run = spawnSync("ffmpeg", ["-nostdin", "-v", "info", "-i", IN, "-vf", `freezedetect=n=${NOISE}:d=${MIN_FREEZE}`, "-map", "0:v", "-f", "null", "-"], { encoding: "utf8", maxBuffer: 1 << 26 }); if (run.status !== 0) { console.error(run.stderr ?? "ffmpeg failed"); process.exit(1); } const log = run.stderr ?? ""; const freezes = []; let pending = null; for (const line of log.split("\n")) { const s = line.match(/freeze_start:\s*([\d.]+)/); const e = line.match(/freeze_end:\s*([\d.]+)/); if (s) pending = { start: Number(s[1]) }; if (e && pending) { pending.end = Number(e[1]); freezes.push(pending); pending = null; } } // A freeze still open at EOF has no end line; it runs to the last frame. if (pending) { pending.end = DUR; freezes.push(pending); } if (!freezes.length) { console.error("no freezes detected — nothing to do"); process.exit(1); } // ---- the plan, in frames --------------------------------------------------- // A run shorter than the beat we mean to keep is left ALONE. Trimming it would // be trading a held frame nobody notices for a splice, which is a bad trade. const cuts = []; for (const f of freezes) { const from = frameOf(f.start) + Math.round(KEEP * FPS); // first frame dropped const to = frameOf(f.end); // first frame kept again if (to - from < 1) continue; cuts.push({ ...f, from, to }); } const keeps = []; let at = 0; for (const c of cuts) { if (c.from > at) keeps.push([at, c.from - 1]); at = c.to; } if (at <= NFRAMES - 1) keeps.push([at, NFRAMES - 1]); const kept = keeps.reduce((t, [a, b]) => t + (b - a + 1), 0); console.log(" freeze (bed t) length action"); for (const f of freezes) { const c = cuts.find((x) => x.start === f.start); const act = c ? `cut ${(c.from / FPS).toFixed(3)}→${(c.to / FPS).toFixed(3)} (−${((c.to - c.from) / FPS).toFixed(3)}s)` : "kept whole — already under the keep"; console.log(` ${f.start.toFixed(3)} → ${f.end.toFixed(3)} ${(f.end - f.start).toFixed(2)}s ${act}`); } console.log(`\n keep frames: ${keeps.map(([a, b]) => `${a}-${b}`).join(", ")}`); console.log(` ${kept} frames = ${(kept / FPS).toFixed(3)}s (−${((NFRAMES - kept) / FPS).toFixed(3)}s)\n`); // ---- prove every cut lands inside a flat stretch --------------------------- // The claim is that the last frame kept and the last frame dropped are the same // picture, because both sit inside one freeze. If that is false the splice is a // visible jump, so it is measured before anything is encoded. const grayAt = (frame) => execFileSync("ffmpeg", ["-nostdin", "-v", "error", "-ss", String(frame / FPS), "-i", IN, "-frames:v", "1", "-vf", "scale=160:90,format=gray", "-f", "rawvideo", "-"], { maxBuffer: 1 << 20 }); const meanAbs = (a, b) => { if (a.length !== b.length || !a.length) return Infinity; let t = 0; for (let i = 0; i < a.length; i += 1) t += Math.abs(a[i] - b[i]); return t / a.length; }; let bad = 0; for (const c of cuts) { const d = meanAbs(grayAt(c.from - 1), grayAt(c.to - 1)); const ok = d < 1.0; if (!ok) bad += 1; console.log(` ${ok ? "ok " : "REFUSE"} cut at frame ${c.from}: frames ${c.from - 1} and ${c.to - 1} differ by ${d.toFixed(3)}`); } if (bad) { console.error(`\n${bad} cut(s) would splice two DIFFERENT pictures. Refusing.`); console.error("Raise minFreeze or lower noise so the runs found are really flat."); process.exit(1); } // ---- rebuild --------------------------------------------------------------- const select = keeps.map(([a, b]) => `between(n\\,${a}\\,${b})`).join("+"); execFileSync("ffmpeg", ["-nostdin", "-v", "error", "-y", "-i", IN, "-vf", `select='${select}',setpts=N/FRAME_RATE/TB`, "-fps_mode", "cfr", "-r", String(FPS), "-an", "-c:v", "libx264", "-preset", "medium", "-crf", "18", "-pix_fmt", "yuv420p", OUT], { stdio: "inherit" }); const outDur = Number(execFileSync("ffprobe", ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", OUT], { encoding: "utf8" }).trim()); console.log(`\n${OUT}`); console.log(` ${outDur.toFixed(3)}s (expected ${(kept / FPS).toFixed(3)}s)`);