#!/usr/bin/env node // Draw the run voice's CONTOUR with its own clips: each note's portrait appears at a // position set by its pitch -- low notes low and to the left, high notes up and to the // right -- so a rising run steps up-and-right and a falling one steps back down. // // This exists because the climb is hard to HEAR: one unique um per note means a 1-2 // semitone step is swamped by the timbral jump between clips. Pitch, loudness, // brightness and pan are all being used to carry it; position is the one cue that // does not compete for the ear at all. // // Each portrait is held longer than its 90ms note (TRAIL), so consecutive ones overlap // and the line is DRAWN rather than strobing one element around the screen. // // node climb-panel.mjs // TRAIL=0.35 how long a portrait stays after its note // TILE_W=200 portrait width; height follows 16:9 // MARGIN=10 how close the line may get to the frame edge // EXTENT=0.72 how far out the figure reaches, so nothing flattens against an edge // MIN_RUN=4 shortest directional run worth drawing import { readFileSync, mkdirSync, existsSync } from "node:fs"; import { execFileSync } from "node:child_process"; import path from "node:path"; import { SONG_DATA } from "./paths.mjs"; const [PLANF, VOICE, IN, OUT] = process.argv.slice(2); const A = Number(process.env.A ?? 0), B = Number(process.env.B ?? 1e9); const OFFSET = Number(process.env.OFFSET ?? 0); const TRAIL = Number(process.env.TRAIL ?? 0.35); const TILE_W = Number(process.env.TILE_W ?? 200); const TILE_H = Number(process.env.TILE_H ?? Math.round(TILE_W * 9 / 16)); // the frame; the line clamps to its edges rather than to an inner box const W = Number(process.env.W ?? 1280), H = Number(process.env.H ?? 720); const TMP = path.join(SONG_DATA, "climbfx"); if (!existsSync(TMP)) mkdirSync(TMP, { recursive: true }); const plan = JSON.parse(readFileSync(PLANF, "utf8")); const v = plan.voices.find((x) => x.name === VOICE) ?? plan.voices[0]; // The strokes are for the stretches where the MELODY is out. // // They exist to fill the melody's rests -- 20.3% of the song, six ~2.1s stretches // where only the quiet accompaniment sounds. While the melody is singing, the run is // an inner voice again and goes back to its ordinary fixed panel; a line sweeping the // frame under a melody that is already carrying the tune competes with it. // // GATE_PLAN names a plan and GATE_VOICE a voice inside it; a run note is only drawn // where that voice is SILENT. const GATE_PLAN = process.env.GATE_PLAN ?? ""; const GATE_VOICE = process.env.GATE_VOICE ?? "melody"; let gateBusy = () => false; if (GATE_PLAN) { const gp = JSON.parse(readFileSync(GATE_PLAN, "utf8")); const gv = gp.voices.find((x) => x.name === GATE_VOICE); if (!gv) throw new Error(`no ${GATE_VOICE} in ${GATE_PLAN}`); const spans = gv.plan .map((n) => [n.slotStart, n.slotStart + Math.min(n.slotDur, n.soundedDur ?? n.slotDur)]) .sort((a, b) => a[0] - b[0]); gateBusy = (t) => spans.some(([a, b]) => t >= a - 0.02 && t < b + 0.02); } const notes = [...v.plan].filter((n) => n.slotStart >= A && n.slotStart <= B) .sort((a, b) => a.slotStart - b.slotStart) .filter((n) => !gateBusy(n.slotStart)); const mids = notes.map((n) => n.targetMidi); const lo = Math.min(...mids), hi = Math.max(...mids); // Only draw where the line is actually GOING somewhere. // // The run is not a continuous climb -- it rises, falls, and sits still. Drawing every // note puts portraits on screen the whole time, which is the same visual density as // the normal panels and says nothing about direction. Keeping only monotonic runs of // MIN_RUN or more notes means the figure appears when there is a climb (or a descent) // to show and is absent otherwise, so its arrival is itself the signal. const MIN_RUN = Number(process.env.MIN_RUN ?? 4); // Once the gate is on, "the next note" is not necessarily the next note in TIME: the // melody's rests are seconds apart, so the last note of one rest and the first note of // the next sit side by side in this array. Chaining them would draw one stroke across // a silence that is not there. A run therefore also breaks on a time discontinuity. const MAX_GAP = Number(process.env.MAX_GAP ?? 0.5); const adjacent = (a, b) => b.slotStart - (a.slotStart + a.slotDur) <= MAX_GAP; const dir = (a, b) => (adjacent(a, b) ? Math.sign(b.targetMidi - a.targetMidi) : 0); const runs = []; for (let i = 0; i < notes.length - 1; i += 1) { const d = dir(notes[i], notes[i + 1]); if (d === 0) continue; let j = i + 1; while (j < notes.length - 1 && dir(notes[j], notes[j + 1]) === d) j += 1; if (j - i + 1 >= MIN_RUN) runs.push({ a: i, b: j, d, notes: notes.slice(i, j + 1) }); i = j - 1; } const keep = new Set(); for (const r of runs) for (let k = r.a; k <= r.b; k += 1) keep.add(k); const shown = notes.filter((_, i) => keep.has(i)); console.log(`${notes.length} notes, targetMidi ${lo}-${hi}, trail ${TRAIL}s`); console.log(`${runs.length} directional runs of >=${MIN_RUN} notes -> drawing ${shown.length} of ${notes.length}` + ` (${runs.filter((r) => r.d > 0).length} rising, ${runs.filter((r) => r.d < 0).length} falling)`); notes.length = 0; notes.push(...shown); for (const [i, n] of notes.entries()) { n._f = path.join(TMP, `c${String(i).padStart(3, "0")}_${TILE_W}.jpg`); if (!existsSync(n._f)) { const at = (n.srcStart + Math.min(0.08, (n.srcDur ?? 0.3) / 2)).toFixed(3); execFileSync("ffmpeg", ["-nostdin", "-v", "error", "-y", "-ss", at, "-i", path.join(SONG_DATA, "media", `${n.video}.mp4`), "-frames:v", "1", "-vf", `scale=${TILE_W - 2}:${TILE_H - 2}:force_original_aspect_ratio=decrease,pad=${TILE_W}:${TILE_H}:(ow-iw)/2:(oh-ih)/2`, n._f]); } } console.log(`extracted ${notes.length} portraits at ${TILE_W}x${TILE_H}`); // TWO STROKES OF AN X, and only one of them exists at a time. // // Chaining runs corner to corner cannot avoid flat legs: any path that visits four // corners in sequence has to travel along an EDGE for half of them (bottom-left to // top-left is straight up the side), which is the "too flat on the sides" fault. A // figure eight drawn as one continuous path always has that problem. // // So do not chain them. Each run is a COMPLETE diagonal stroke, starting fresh at its // own corner: a rise climbs bottom-left to top-right, a fall drops top-left to // bottom-right. The two strokes form an X, and because the runs are separated in time // they are never both on screen -- the X is traced across the passage rather than // drawn at once. Every leg is a full diagonal, so nothing can flatten. const M = Number(process.env.MARGIN ?? 10); const EXTENT = Number(process.env.EXTENT ?? 0.72); // BOX=W:H:X:Y draws the figure inside a SUB-RECTANGLE of the frame rather than across // the whole of it. The vertical cut needs this: there the strokes belong over the // gameplay screen (1080x972 at 0,541), not sprawled across the voice boxes and the // bands, which are doing their own jobs. const BOX = (process.env.BOX ?? "").split(":").filter(Boolean).map(Number); const [BW, BH, BX, BY] = BOX.length === 4 ? BOX : [W, H, 0, 0]; const cx = BW / 2, cy = BH / 2; const xR = cx + (BW - M - TILE_W / 2 - cx) * EXTENT, xL = cx - (cx - M - TILE_W / 2) * EXTENT; const yT = cy - (cy - M - TILE_H / 2) * EXTENT, yB = cy + (BH - M - TILE_H / 2 - cy) * EXTENT; if (BOX.length === 4) console.log(`drawing inside a ${BW}x${BH} box at (${BX},${BY})`); const idxOf = new Map(notes.map((n, i) => [n, i])); const walk = new Array(notes.length); for (const r of runs) { // the stroke goes the way the music goes const from = r.d > 0 ? { x: xL, y: yB } : { x: xL, y: yT }; const to = r.d > 0 ? { x: xR, y: yT } : { x: xR, y: yB }; r.notes.forEach((n, j) => { const f = r.notes.length > 1 ? j / (r.notes.length - 1) : 0.5; walk[idxOf.get(n)] = { x: from.x + (to.x - from.x) * f, y: from.y + (to.y - from.y) * f }; }); } for (let i = 0; i < walk.length; i += 1) if (!walk[i]) walk[i] = { x: cx, y: cy }; console.log(`${runs.length} strokes of an X (extent ${EXTENT}, margin ${M}px); rises climb left-to-right, falls drop left-to-right`); const parts = []; let last = "0:v"; notes.forEach((n, i) => { const x = Math.round(BX + walk[i].x - TILE_W / 2); const y = Math.round(BY + walk[i].y - TILE_H / 2); const a = (n.slotStart - OFFSET).toFixed(3); const b = (n.slotStart + n.slotDur + TRAIL - OFFSET).toFixed(3); parts.push(`[${i + 1}:v]scale=${TILE_W}:${TILE_H},pad=${TILE_W + 4}:${TILE_H + 4}:2:2:white@0.9[p${i}]`); parts.push(`[${last}][p${i}]overlay=x=${x - 2}:y=${y - 2}:enable='between(t,${a},${b})'[o${i}]`); last = `o${i}`; }); const args = ["-nostdin", "-v", "warning", "-y", "-i", IN]; for (const n of notes) args.push("-loop", "1", "-i", n._f); const DUR = execFileSync("ffprobe", ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", IN]).toString().trim(); args.push("-filter_complex", `${parts.join(";")};[${last}]null[v]`, "-map", "[v]", "-map", "0:a?", "-t", DUR, "-c:v", "libx264", "-preset", "medium", "-crf", "18", "-pix_fmt", "yuv420p", "-c:a", "copy", "-movflags", "+faststart", OUT); console.log(`overlaying ${notes.length} portraits along the contour...`); execFileSync("ffmpeg", args, { stdio: ["ignore", "inherit", "inherit"] }); console.log(`wrote ${OUT}`);