// The crossfade concat keeps every clip's sound on its picture. // // An encoded segment's audio is routinely a few to ~20 ms off its video, and // `acrossfade` joins by the SOUND's length while `xfade` offsets come from the // PICTURE's. Unpinned, the difference accumulates clip by clip. These segments // make it large on purpose -- each one's audio is 30 ms short -- and put a tone // exactly 1.0 s into each: in the joined cut, the third tone must still start // 1.0 s after the third segment does. // // Run with: pnpm test:scripts import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; import { xfadeGraph } from "./build-video.mjs"; const have = spawnSync("ffmpeg", ["-version"]).status === 0; const RENDER = { width: 320, height: 180, fps: 30, audioRate: 48000, audioChannels: 2 }; function segment(dir, name, seconds) { const out = path.join(dir, `${name}.mp4`); const a = (seconds - 0.03).toFixed(3); const r = spawnSync("ffmpeg", [ "-v", "error", "-y", "-f", "lavfi", "-i", `color=c=gray:s=320x180:r=30:d=${seconds}`, // Silence, then a tone from exactly 1.0 s; the stream is 30 ms short. "-f", "lavfi", "-i", `sine=f=1000:r=48000:d=${a}`, "-filter_complex", `[1:a]volume=enable='lt(t,1)':volume=0,aformat=channel_layouts=stereo[a]`, "-map", "0:v", "-map", "[a]", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-shortest", out, ]); assert.equal(r.status, 0, String(r.stderr)); return out; } /** The time (s) the tone's third onset starts in a file: the first loud window after `after`. */ function onsetAfter(file, after) { const r = spawnSync("ffmpeg", ["-v", "error", "-i", file, "-ac", "1", "-ar", "8000", "-f", "s16le", "-"], { maxBuffer: 1 << 26 }); const pcm = new Int16Array(r.stdout.buffer, r.stdout.byteOffset, r.stdout.length / 2); const w = 40; // 5 ms windows for (let i = Math.floor(after * 8000); i + w < pcm.length; i += w) { let sum = 0; for (let k = i; k < i + w; k += 1) sum += pcm[k] * pcm[k]; if (Math.sqrt(sum / w) > 1000) return i / 8000; } return null; } test("the crossfade concat keeps every clip's sound on its picture", { skip: !have && "no ffmpeg" }, () => { const dir = mkdtempSync(path.join(tmpdir(), "av-sync-")); try { const durs = [4, 4, 4]; const segs = durs.map((d, i) => segment(dir, `s${i}`, d)); const D = 0.5; const run = (graph, out) => { const r = spawnSync("ffmpeg", [ "-v", "error", "-y", ...segs.flatMap((s) => ["-i", s]), "-filter_complex", graph.parts.join(";"), "-map", graph.vlab, "-map", graph.alab, "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", out, ]); assert.equal(r.status, 0, String(r.stderr)); return out; }; // The third segment starts at 2 × (4 − 0.5) = 7.0 s; its tone at 8.0 s. const pinned = onsetAfter(run(xfadeGraph(durs, D, null, RENDER), path.join(dir, "pinned.mp4")), 7.6); assert.ok(Math.abs(pinned - 8.0) <= 0.01, `pinned onset ${pinned}, expected 8.0`); // The unpinned graph (what the concat wrote before) lands each later clip // 30 ms earlier per clip before it: the drift this guards against. const unpinned = { parts: [ `[0:v][1:v]xfade=transition=fade:duration=${D}:offset=3.500[v1]`, `[0:a][1:a]acrossfade=d=${D}:c1=tri:c2=tri[a1]`, `[v1][2:v]xfade=transition=fade:duration=${D}:offset=7.000[v2]`, `[a1][2:a]acrossfade=d=${D}:c1=tri:c2=tri[a2]`, ], vlab: "[v2]", alab: "[a2]", }; const drift = onsetAfter(run(unpinned, path.join(dir, "unpinned.mp4")), 7.6); assert.ok(8.0 - drift >= 0.045, `unpinned onset ${drift} should be ~60 ms early`); } finally { rmSync(dir, { recursive: true, force: true }); } });