import { spawn } from "node:child_process"; import { mkdir } from "node:fs/promises"; import path from "node:path"; import { SONG_REPORTS, WRITE_ROOTS, labelFor, resolveInRoots } from "./paths"; import { readJson, writeJsonAtomic, withStateLock } from "./state"; import { stateFile } from "./paths"; import { probeMedia } from "./media"; // --------------------------------------------------------------------------- // The mix bench: one body render, one background, and the four numbers that // decide how they meet. // // This is the mux step of the per-tune build scripts, made interactive. Those // scripts hard-code it: // // [1:a]volume='if(lt(t,H),1.0,DUCK)':eval=frame ... amix ... alimiter // // and every one of H, DUCK, the head trim and the end has been arrived at by // rendering, listening, and editing a shell variable. Two of those round trips // shipped defects that a picture would have prevented: a Pokemon handover set // 1.14s late off a pitch contour, so the game's alarm played AND then ours did; // and a Yoshi cut whose end point was chosen by a TIME_LIMIT constant rather // than by hearing where the tune should stop. // // So the numbers live here, the preview in the browser applies the SAME ramp // this file renders, and the render is the same ffmpeg call the scripts make. // What you hear is what gets written. // --------------------------------------------------------------------------- export type MixSpec = { /** The render carrying the picture and the song. Required. */ body: string; /** The track carrying the ambient/game audio, or null for body-only. */ bg: string | null; /** Seconds, in body clock: bg plays at bgGain before this, duck after. */ handover: number; /** Seconds of linear ramp at the handover. 0 is a hard cut, which clicks. */ fade: number; bgGain: number; duck: number; /** Trim off the head of the output. */ start: number; /** Where the output ends. 0 means "the body's own end". */ end: number; /** Output path; relative names land in the reports directory. */ out: string; }; export const DEFAULT_SPEC: MixSpec = { body: "", bg: null, handover: 0, fade: 0.12, bgGain: 1, duck: 0, start: 0, end: 0, out: "", }; const num = (v: unknown, fallback: number) => { const n = Number(v); return Number.isFinite(n) ? n : fallback; }; export function normaliseSpec(raw: Partial | undefined): MixSpec { const s = raw ?? {}; return { body: String(s.body ?? ""), bg: s.bg ? String(s.bg) : null, handover: Math.max(0, num(s.handover, 0)), fade: Math.min(5, Math.max(0, num(s.fade, DEFAULT_SPEC.fade))), bgGain: Math.min(4, Math.max(0, num(s.bgGain, 1))), duck: Math.min(4, Math.max(0, num(s.duck, 0))), start: Math.max(0, num(s.start, 0)), end: Math.max(0, num(s.end, 0)), out: String(s.out ?? ""), }; } /** * The background's gain as an ffmpeg `volume` expression, in SOURCE time. * * Applied before any trim, exactly as the build scripts do, so `handover` means * the same thing whether or not a head trim is in play. A `fade` of 0 is the * scripts' original hard switch; anything above 0 ramps, because a hard cut * from 1.0 to 0.0 mid-waveform is an audible click and every one of these * handovers lands on a moment somebody is listening closely to. */ export function bgVolumeExpr(spec: Pick): string { const { handover: h, bgGain: a, duck: b } = spec; const f = spec.fade; const A = a.toFixed(4); const B = b.toFixed(4); const H = h.toFixed(4); if (f <= 0.0005) return `if(lt(t,${H}),${A},${B})`; const F = f.toFixed(4); return `if(lt(t,${H}),${A},if(lt(t,${H}+${F}),${A}+(${B}-${A})*(t-${H})/${F},${B}))`; } export type ResolvedMix = { spec: MixSpec; bodyPath: string; bgPath: string | null; outPath: string; duration: number; /** A head trim cannot be a stream copy: an output seek lands on a keyframe. */ reencode: boolean; args: string[]; }; /** Resolve a spec to real paths and the exact ffmpeg argv that will run. */ export async function resolveMix(raw: Partial): Promise { const spec = normaliseSpec(raw); const bodyPath = resolveInRoots(spec.body); if (!bodyPath) throw new Error(`body is not inside a known root: ${spec.body || "(unset)"}`); const bgPath = spec.bg ? resolveInRoots(spec.bg) : null; if (spec.bg && !bgPath) throw new Error(`background is not inside a known root: ${spec.bg}`); const body = await probeMedia(bodyPath); if (!body.hasAudio) throw new Error(`${body.label} has no audio track`); const end = spec.end > 0 ? Math.min(spec.end, body.duration) : body.duration; if (end - spec.start < 0.05) throw new Error(`nothing left between start ${spec.start}s and end ${end}s`); const duration = +(end - spec.start).toFixed(3); const outName = spec.out.trim(); if (!outName) throw new Error("an output name is required"); if (outName.includes("..")) throw new Error("output name may not contain .."); const outAbs = path.isAbsolute(outName) ? outName : path.join(SONG_REPORTS, outName); // The WRITE roots, deliberately narrower than the read roots. Report videos // became readable so their clips could be mixed; their out/ directories hold // deliverables that cost an hour of network fetches each, and one typo here // would overwrite one. Refuse rather than clamp: a refused path must never // silently become a different path. const outPath = resolveInRoots(outAbs, "write"); if (!outPath) throw new Error(`output would land outside ${WRITE_ROOTS.join(", ")}`); if (outPath === bodyPath || (bgPath && outPath === bgPath)) { throw new Error("refusing to write the output over one of its own inputs"); } if (!/\.(mp4|mkv|mov)$/i.test(outPath)) throw new Error("output must be .mp4, .mkv or .mov"); const reencode = spec.start > 0.001; const args = ["-nostdin", "-v", "error", "-y", "-i", bodyPath]; if (bgPath) args.push("-i", bgPath); if (bgPath) { // normalize=0 keeps the song at unity when the background drops away -- // amix's default would otherwise turn every duck into a volume swell on the // remaining input, which reads as the mix breathing. args.push( "-filter_complex", `[1:a]volume='${bgVolumeExpr(spec)}':eval=frame[game];` + `[0:a]anull[song];` + `[song][game]amix=inputs=2:duration=first:normalize=0,alimiter=limit=0.98:level=disabled[a]`, "-map", "0:v", "-map", "[a]", ); } else { args.push("-map", "0:v", "-map", "0:a"); } if (spec.start > 0.001) args.push("-ss", spec.start.toFixed(3)); args.push("-t", duration.toFixed(3)); if (reencode) { args.push("-c:v", "libx264", "-preset", "medium", "-crf", "21", "-pix_fmt", "yuv420p", "-video_track_timescale", "30000"); } else { args.push("-c:v", "copy"); } args.push("-c:a", "aac", "-b:a", "192k", "-ar", "48000", "-ac", "2", "-movflags", "+faststart", outPath); return { spec, bodyPath, bgPath, outPath, duration, reencode, args }; } // --------------------------------------------------------------------------- // Render jobs. // // A stream-copy remux is a couple of seconds; a head trim re-encodes and can be // minutes. Holding a fetch open for either is the difference between a UI that // looks busy and one that looks hung, so the POST starts a job and returns, and // the client polls. In-memory on purpose: a render that does not survive a // server restart is a render worth running again. // --------------------------------------------------------------------------- export type RenderJob = { id: string; state: "running" | "done" | "failed"; startedAt: number; endedAt: number | null; out: string; label: string; cmd: string; error: string | null; duration: number; }; // On globalThis for the reason lib/jobs.ts gives: one registry per process, // not one per route entry, so the dashboard sees what /api/mix/render started. type RenderRegistry = { jobs: Map; seq: number }; const g = globalThis as typeof globalThis & { __umtoolRenders?: RenderRegistry }; const reg: RenderRegistry = (g.__umtoolRenders ??= { jobs: new Map(), seq: 0 }); const jobs = reg.jobs; export function getJob(id: string): RenderJob | null { return jobs.get(id) ?? null; } export function recentJobs(limit = 8): RenderJob[] { return [...jobs.values()].sort((a, b) => b.startedAt - a.startedAt).slice(0, limit); } export async function startRender(raw: Partial): Promise { const r = await resolveMix(raw); await mkdir(path.dirname(r.outPath), { recursive: true }); reg.seq += 1; const id = `mix${reg.seq}`; const job: RenderJob = { id, state: "running", startedAt: Date.now(), endedAt: null, out: r.outPath, label: labelFor(r.outPath), cmd: `ffmpeg ${r.args.join(" ")}`, error: null, duration: r.duration, }; jobs.set(id, job); const ff = spawn("ffmpeg", r.args); let err = ""; ff.stderr.on("data", (b: Buffer) => { err += b.toString(); if (err.length > 8000) err = err.slice(-8000); }); ff.on("error", (e) => { job.state = "failed"; job.error = String(e); job.endedAt = Date.now(); }); ff.on("close", (code) => { job.endedAt = Date.now(); if (code === 0) { job.state = "done"; } else { job.state = "failed"; job.error = err.trim() || `ffmpeg exited ${code}`; } }); return job; } // --------------------------------------------------------------------------- // Saved settings, so reopening a pair does not mean re-deriving four numbers. // --------------------------------------------------------------------------- const SESSIONS = () => stateFile("mix-sessions.json"); export type MixSessions = { last: string | null; byPair: Record }; export const pairKey = (body: string, bg: string | null) => `${body}||${bg ?? ""}`; export async function readMixSessions(): Promise { return readJson(SESSIONS(), { last: null, byPair: {} }); } export async function saveMixSession(raw: Partial): Promise { const spec = normaliseSpec(raw); if (!spec.body) throw new Error("a body is required to save a mix session"); return withStateLock(async () => { const cur = await readJson(SESSIONS(), { last: null, byPair: {} }); const key = pairKey(spec.body, spec.bg); const next: MixSessions = { last: key, byPair: { ...cur.byPair, [key]: spec } }; await writeJsonAtomic(SESSIONS(), next); return next; }); }