commit 7085e0db117c4329ddae47acc3bd804eef870489
parent 5f42c57ecb7910e5b4eb427371bd86edab88955f
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 1 Oct 2026 15:00:24 -0400
Merge deck/bench-b1 (slice B1) — the clip bench plays every bounded range through Web Audio from a server-side ffmpeg decode (stops on the scheduled end: 0 of 20 trials off, was ~230 ms over), the mute mark (m / pick on waveform / nudge / clear) saved through the window route, updateClip muteFrom; e2e 97/97; reviewed
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
10 files changed, 1574 insertions(+), 52 deletions(-)
diff --git a/umtool/app/api/report/audio/route.ts b/umtool/app/api/report/audio/route.ts
@@ -0,0 +1,114 @@
+import { spawn } from "node:child_process";
+import { stat } from "node:fs/promises";
+import { absOf, pickWindow, resolveClip, windowsFor } from "@/lib/report/serve.mjs";
+import { MAX_DECODE_SPAN, wavHeader } from "@/lib/report/playback.mjs";
+
+export const dynamic = "force-dynamic";
+
+// A cached source window's SOUND, decoded, for the bench to play exactly.
+//
+// The bench plays every bounded range -- a selection, an edge, a line -- from
+// a decoded buffer with Web Audio, because an <video> element stopped from
+// `timeupdate` overran the end by up to a quarter of a second. This is that
+// buffer: the same file /api/report/raw serves (picked by the same membership
+// rule), as 16-bit PCM WAV, which every browser decodes without a codec.
+//
+// DECODED BY FFMPEG, NOT BY THE BROWSER. The build cuts with ffmpeg, so a cut
+// set by ear has to be set against ffmpeg's timeline: a browser's own AAC
+// decode may or may not honour the file's edit list, and the encoder's
+// priming samples are tens of milliseconds -- the size of the error this
+// exists to remove.
+//
+// `from`/`to` are ABSOLUTE source seconds, clamped to the file. A span longer
+// than MAX_DECODE_SPAN is refused rather than served: a whole recording in the
+// saved-video store is hours, and the bench asks for a window around the
+// selection instead (decodeSpan).
+
+/** Matches the build's `render.audioRate` default; the browser resamples anyway. */
+const RATE = 48000;
+const CHANNELS = 2;
+
+export async function GET(request: Request) {
+ const url = new URL(request.url);
+ const r = await resolveClip(url.searchParams.get("project") ?? "", url.searchParams.get("clip") ?? "");
+ if ("error" in r) return Response.json({ error: r.error }, { status: r.status });
+
+ const windows = await windowsFor(r.project, r.clip);
+ const win = pickWindow(windows, url.searchParams.get("file"));
+ if (!win) return Response.json({ error: "no cached window for this clip" }, { status: 404 });
+ const abs = absOf(win);
+ if (!abs) return Response.json({ error: "outside the roots" }, { status: 400 });
+ const st = await stat(abs).catch(() => null);
+ if (!st) return Response.json({ error: "gone" }, { status: 404 });
+
+ const num = (k: string, dflt: number) => {
+ const v = url.searchParams.get(k);
+ const n = v == null || v === "" ? dflt : Number(v);
+ return Number.isFinite(n) ? n : Number.NaN;
+ };
+ const from = Math.max(win.from, num("from", win.from));
+ const to = Math.min(win.to, num("to", win.to));
+ if (!Number.isFinite(from) || !Number.isFinite(to) || to <= from) {
+ return Response.json({ error: "bad span" }, { status: 400 });
+ }
+ if (to - from > MAX_DECODE_SPAN + 0.5) {
+ return Response.json(
+ { error: `asked for ${Math.round(to - from)} s; the bench decodes at most ${MAX_DECODE_SPAN} s at once` },
+ { status: 400 },
+ );
+ }
+
+ // Input seeking with a decode is sample-accurate in ffmpeg: it seeks to the
+ // keyframe before and discards up to the requested time.
+ const chunks: Buffer[] = [];
+ let err = "";
+ const code = await new Promise<number>((resolve) => {
+ const ff = spawn("ffmpeg", [
+ "-nostdin", "-v", "error",
+ "-ss", (from - win.from).toFixed(6),
+ "-i", abs,
+ "-t", (to - from).toFixed(6),
+ "-map", "a:0",
+ "-ac", String(CHANNELS), "-ar", String(RATE),
+ "-f", "s16le", "-acodec", "pcm_s16le", "-",
+ ]);
+ ff.stdout.on("data", (b: Buffer) => chunks.push(b));
+ ff.stderr.on("data", (b: Buffer) => {
+ err += b.toString();
+ });
+ ff.on("error", (e) => {
+ err += String(e);
+ resolve(-1);
+ });
+ ff.on("close", (c) => resolve(c ?? -1));
+ });
+ const pcm = Buffer.concat(chunks);
+ const frameBytes = CHANNELS * 2;
+ const dataBytes = pcm.length - (pcm.length % frameBytes);
+ if (code !== 0 || dataBytes === 0) {
+ // Said in the bench, which then plays through the element: the message is
+ // the reason the playback is approximate.
+ const why = /matches no streams|does not contain any stream/i.test(err)
+ ? "this file has no audio track"
+ : err.trim().split("\n").pop() || `ffmpeg exited ${code}`;
+ return Response.json({ error: why }, { status: 422 });
+ }
+
+ const body = new Uint8Array(44 + dataBytes);
+ body.set(wavHeader({ channels: CHANNELS, sampleRate: RATE, dataBytes }), 0);
+ body.set(pcm.subarray(0, dataBytes), 44);
+ return new Response(body, {
+ headers: {
+ "content-type": "audio/wav",
+ // The window is in the file's name, so the same request is the same
+ // sound: it may be cached for as long as the raw file is.
+ "cache-control": "private, max-age=3600, immutable",
+ // What was actually decoded, in absolute source seconds: the client
+ // places the buffer on the source clock from these, not from what it
+ // asked for.
+ "x-audio-from": String(from),
+ "x-audio-to": String(from + dataBytes / frameBytes / RATE),
+ "x-window": win.name,
+ },
+ });
+}
diff --git a/umtool/app/api/report/clip/route.ts b/umtool/app/api/report/clip/route.ts
@@ -59,6 +59,7 @@ export async function GET(request: Request) {
cutStart: clip.cutStart ?? null,
cutEnd: clip.cutEnd ?? null,
lockCut: !!clip.lockCut,
+ muteFrom: clip.muteFrom ?? null,
},
view,
windows: windows.map((w: { name: string; from: number; to: number }) => ({
diff --git a/umtool/app/api/report/window/route.ts b/umtool/app/api/report/window/route.ts
@@ -47,6 +47,10 @@ export async function PUT(request: Request) {
"cutStart",
"cutEnd",
"lockCut",
+ // Where the sound fades out for the rest of the clip while the picture
+ // plays on, in source seconds; empty or null clears it. Checked by the
+ // writer against the clip's window, like the cut.
+ "muteFrom",
// Whether the walk has looked at this clip: "confirmed", or empty to clear
// it. A non-empty `correction` is the other answer and needs no value.
"verdict",
diff --git a/umtool/components/projects/ClipBench.tsx b/umtool/components/projects/ClipBench.tsx
@@ -11,6 +11,16 @@ import { buttonVariants } from "@/components/ui/button";
// would only find out twenty minutes into a build.
import { attributionLine } from "umtool-report-to-video/attribution";
import {
+ MUTE_FADE,
+ START_LEAD,
+ bufferSchedule,
+ covers,
+ decodeSpan,
+ elementShouldStop,
+ muteRamp,
+ playheadAt,
+} from "@/lib/report/playback.mjs";
+import {
DeckFrame,
NeutralFrame,
composePreview,
@@ -100,6 +110,12 @@ type Clip = {
cutEnd: number | null;
/** The cut is deliberate; `resolve-windows --cut-to-quote` leaves it alone. */
lockCut: boolean;
+ /**
+ * The MUTE MARK, in source seconds: from here to the end of the clip the
+ * sound fades out and the picture plays on. Set by ear, at the last silence
+ * before a finale's ending sound. Absent means the clip plays with its sound.
+ */
+ muteFrom: number | null;
/** "confirmed" / "incorrect", or null for "nobody has looked at this yet". */
verdict: "confirmed" | "incorrect" | null;
/** What the on-screen panel says over this clip. Absent means the auto text. */
@@ -180,6 +196,7 @@ const fromEntry = (prev: Clip, e: Record<string, unknown>): Clip => ({
cutStart: e.cutStart == null ? null : Number(e.cutStart),
cutEnd: e.cutEnd == null ? null : Number(e.cutEnd),
lockCut: !!e.lockCut,
+ muteFrom: e.muteFrom == null ? null : Number(e.muteFrom),
verdict: e.verdict === "confirmed" || e.verdict === "incorrect" ? e.verdict : null,
onscreen: (e.onscreen as Onscreen | undefined) ?? null,
});
@@ -303,6 +320,62 @@ const clock = (t: number) => {
return `${Math.floor(m / 60) > 0 ? `${Math.floor(m / 60)}:${String(m % 60).padStart(2, "0")}` : m}:${String(s % 60).padStart(2, "0")}`;
};
+/** What `save window` (and a confirmation that moved something) writes. */
+type WindowPatch = {
+ start: number;
+ end: number;
+ cutStart?: string;
+ cutEnd?: string;
+ /** A number to set the mark, "" to clear it; absent leaves it alone. */
+ muteFrom?: number | string;
+};
+
+/** A decoded span of the cached window, placed on the source clock. */
+type Decoded = { name: string; span: { from: number; to: number }; buf: AudioBuffer };
+
+/**
+ * What is playing, and what the last playback measured.
+ *
+ * On the page as data attributes (`data-play-*`), because "did it stop where
+ * the selection ends" is the claim this bench now makes, and a spec has to be
+ * able to check it against the audio clock rather than against a feeling.
+ */
+type PlayInfo = {
+ engine: "webaudio" | "element";
+ state: "playing" | "stopped";
+ from: number;
+ /** The scheduled end, in source seconds. */
+ to: number;
+ rate: number;
+ /** Context clock: when the source starts and when it is told to stop. */
+ ctxStart: number | null;
+ ctxStop: number | null;
+ /**
+ * Where the playback had reached, in source seconds, when the page heard it
+ * end: the element's own position at its pause, or the context clock at
+ * `ended` -- which reaches the page a task later than the sound stopped, so
+ * for Web Audio it is an upper bound and `ctxStop` is the stop itself.
+ */
+ endedAt: number | null;
+};
+
+/** The playback in flight. One at a time: a new one stops the last. */
+type Session = {
+ gen: number;
+ engine: "webaudio" | "element";
+ node: AudioBufferSourceNode | null;
+ gain: GainNode | null;
+ raf: number;
+ timers: number[];
+ from: number;
+ to: number;
+ rate: number;
+ t0: number;
+ ctxStop: number;
+ /** The element's own mute, put back when the picture stops following. */
+ mutedBefore: boolean;
+};
+
export default function ClipBench({ data }: { data: ClipBenchData }) {
const [clip, setClip] = useState<Clip>(data.clip);
const [windows, setWindows] = useState<Win[]>(data.windows);
@@ -310,6 +383,13 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
const [proposed, setProposed] = useState(data.proposed);
const [view, setView] = useState(data.view);
const [sel, setSel] = useState({ from: data.clip.start, to: data.clip.end });
+ // The mute mark as drafted. Like the edges it is unsaved until `save window`
+ // or `y`, and it is what the bench's own playback mutes at -- so a mark is
+ // heard before it is written.
+ const [mute, setMute] = useState<number | null>(data.clip.muteFrom);
+ // Armed: the next click on the waveform places the mark instead of moving
+ // an edge.
+ const [mutePick, setMutePick] = useState(false);
const [peaks, setPeaks] = useState<Peaks | null>(null);
// The cues AROUND the cached window: what is coming, read before paying for
// the media. One request, widened only when somebody asks.
@@ -319,6 +399,10 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
const [cutScore, setCutScore] = useState<number | null>(null);
const [peekPad, setPeekPad] = useState(PEEK_STEP);
const [playhead, setPlayhead] = useState<number | null>(null);
+ const playheadRef = useRef<number | null>(null);
+ useEffect(() => {
+ playheadRef.current = playhead;
+ }, [playhead]);
const [note, setNote] = useState<string | null>(null);
const [busy, setBusy] = useState<string | null>(null);
const [dirty, setDirty] = useState(false);
@@ -356,7 +440,6 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
const token = useRef(data.token);
// And they must not interleave: same read-modify-write, one clip.
const saving = useRef<Promise<boolean>>(Promise.resolve(true));
- const stopAt = useRef<number | null>(null);
// `x` answers "no" by putting the cursor in the note, which is the answer.
const correctionBox = useRef<HTMLTextAreaElement | null>(null);
const segVideo = useRef<HTMLVideoElement | null>(null);
@@ -532,20 +615,366 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
}, [playback, segment, cached, renderedOpen, deckPreview]);
// ---- audition -----------------------------------------------------------
- const play = useCallback(
- (from: number, to: number) => {
+ //
+ // EVERY BOUNDED RANGE PLAYS FROM DECODED AUDIO. A selection, an edge, a line
+ // of the transcript, the auto-audition: each is an AudioBufferSourceNode
+ // started at an exact offset and stopped at an exact time on the audio
+ // clock, so what you hear ends where the selection ends, to the sample. The
+ // <video> used to play these itself and was stopped from `timeupdate`, which
+ // fires every 15-250 ms -- the stop overran by up to a quarter of a second,
+ // and by a different amount every time, which is exactly what makes a cut
+ // between two words impossible to judge by ear. (The old HTML chooser had
+ // this right; it is the same technique.)
+ //
+ // The picture FOLLOWS, muted, and the playhead is read off the audio clock.
+ // When the decode fails the element plays instead, stopped per animation
+ // frame by remaining time, and the bench says so.
+
+ /** The decoded audio for the cached window: one decode per window. */
+ const decoded = useRef<Decoded | null>(null);
+ const decoding = useRef<{ key: string; p: Promise<Decoded> } | null>(null);
+ const actx = useRef<AudioContext | null>(null);
+ const session = useRef<Session | null>(null);
+ const playGen = useRef(0);
+ // Pauses the bench asked for itself and has not yet heard the event of. A
+ // `pause` event is a task, so it arrives AFTER the next playback has started:
+ // without the count, stopping one playback to start the next would read as
+ // somebody pausing the picture, and stop the new one.
+ const selfPauses = useRef(0);
+ const pauseVideo = useCallback((el: HTMLVideoElement) => {
+ if (el.paused) return;
+ selfPauses.current += 1;
+ el.pause();
+ }, []);
+ // Read by play() at the moment it schedules, so a mark moved since the last
+ // render is the one you hear -- without re-creating play() on every nudge.
+ const muteRef = useRef<number | null>(data.clip.muteFrom);
+ const [audio, setAudio] = useState<{ state: "idle" | "loading" | "ready" | "failed"; why?: string }>({
+ state: "idle",
+ });
+ const [playInfo, setPlayInfo] = useState<PlayInfo | null>(null);
+
+ /**
+ * The cached window's sound over (at least) `want`, decoded once.
+ *
+ * Through /api/report/audio -- ffmpeg's decode of the same file the player
+ * loads, as PCM -- and decoded here on an OfflineAudioContext, so no audio
+ * output is opened before somebody asks to hear something. A buffer belongs
+ * to no context and plays in the real one.
+ */
+ const ensureDecoded = useCallback(
+ async (want: { from: number; to: number }): Promise<Decoded> => {
+ if (!cached) throw new Error("nothing is cached for this clip");
+ const have = decoded.current;
+ if (have && have.name === cached.name && covers(have.span, want.from, want.to)) return have;
+ const span = decodeSpan({ from: cached.from, to: cached.to }, want);
+ const key = `${cached.name}|${span.from}|${span.to}`;
+ if (decoding.current?.key === key) return decoding.current.p;
+ const p = (async () => {
+ setAudio({ state: "loading" });
+ const r = await fetch(
+ `/api/report/audio?project=${encodeURIComponent(data.project)}&clip=${encodeURIComponent(clip.id)}` +
+ `&file=${encodeURIComponent(cached.name)}&from=${span.from}&to=${span.to}`,
+ );
+ if (!r.ok) {
+ const j = (await r.json().catch(() => null)) as { error?: string } | null;
+ throw new Error(j?.error ?? `the audio route answered ${r.status}`);
+ }
+ const from = Number(r.headers.get("x-audio-from") ?? span.from);
+ const bytes = await r.arrayBuffer();
+ const Offline =
+ typeof window !== "undefined" ? (window.OfflineAudioContext ?? null) : null;
+ if (!Offline) throw new Error("this browser has no Web Audio");
+ const buf = await new Offline(2, 1, 48000).decodeAudioData(bytes);
+ const d: Decoded = { name: cached.name, span: { from, to: from + buf.duration }, buf };
+ decoded.current = d;
+ setAudio({ state: "ready" });
+ return d;
+ })();
+ decoding.current = { key, p };
+ p.catch((e: unknown) => {
+ if (decoding.current?.p === p) decoding.current = null;
+ setAudio({ state: "failed", why: e instanceof Error ? e.message : String(e) });
+ });
+ return p;
+ },
+ [cached, data.project, clip.id],
+ );
+
+ // A different window -- a wider one after "fetch more", or the first after a
+ // fetch -- is different audio: drop the old decode and start the new one
+ // now, so the first play after it does not wait.
+ const cachedName = cached?.name ?? null;
+ useEffect(() => {
+ decoded.current = null;
+ decoding.current = null;
+ setAudio({ state: "idle" });
+ if (!cachedName) return;
+ ensureDecoded({ from: clip.start, to: clip.end }).catch(() => {
+ /* said in the bench; play() falls back to the element */
+ });
+ // ensureDecoded changes identity with `cached`, which is this effect's
+ // whole subject; keying on the NAME is what stops a refresh() that returns
+ // the same window from decoding it again.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [cachedName]);
+
+ /** Stop whatever is playing, Web Audio or element, and let go of it. */
+ const stopPlayback = useCallback(() => {
+ const s = session.current;
+ if (!s) return;
+ session.current = null;
+ cancelAnimationFrame(s.raf);
+ for (const t of s.timers) clearTimeout(t);
+ if (s.node) {
+ s.node.onended = null;
+ try {
+ s.node.stop();
+ } catch {
+ /* never started, or already stopped */
+ }
+ s.node.disconnect();
+ s.gain?.disconnect();
+ }
+ const el = video.current;
+ if (el) {
+ pauseVideo(el);
+ el.muted = s.mutedBefore;
+ }
+ setPlayInfo((pi) => (pi && pi.state === "playing" ? { ...pi, state: "stopped" } : pi));
+ }, [pauseVideo]);
+
+ /**
+ * The fallback: the element, stopped per animation frame by REMAINING TIME.
+ *
+ * `timeupdate` was the old stop and is the overrun this replaces; a frame
+ * tick that stops once the end is under half a frame away is the best the
+ * element can do, and the bench says that this is what is playing.
+ */
+ const playElement = useCallback(
+ (from: number, to: number, gen: number) => {
const el = video.current;
if (!el || !cached) return;
+ const rate = playback.rate;
el.currentTime = Math.max(0, from - fetchStart);
- el.playbackRate = playback.rate;
- stopAt.current = to;
+ el.playbackRate = rate;
+ const s: Session = {
+ gen,
+ engine: "element",
+ node: null,
+ gain: null,
+ raf: 0,
+ timers: [],
+ from,
+ to,
+ rate,
+ t0: 0,
+ ctxStop: 0,
+ mutedBefore: el.muted,
+ };
+ session.current = s;
+ setPlayInfo({ engine: "element", state: "playing", from, to, rate, ctxStart: null, ctxStop: null, endedAt: null });
+ const stopNow = () => {
+ if (session.current !== s) return;
+ pauseVideo(el);
+ session.current = null;
+ cancelAnimationFrame(s.raf);
+ for (const t of s.timers) clearTimeout(t);
+ el.muted = s.mutedBefore;
+ setPlayInfo((pi) => (pi ? { ...pi, state: "stopped", endedAt: el.currentTime + fetchStart } : pi));
+ };
+ const tick = () => {
+ if (session.current !== s) return;
+ const t = el.currentTime + fetchStart;
+ setPlayhead(t);
+ // The mark, as near as a frame tick gets it.
+ const m = muteRef.current;
+ el.muted = s.mutedBefore || (m != null && t >= m);
+ if (!el.paused && elementShouldStop(t, to, rate)) {
+ stopNow();
+ return;
+ }
+ // Inside the last few frames, a timer for the REMAINING time: it lands
+ // between frames, where the next tick would land up to a frame late.
+ const remaining = (to - t) / rate;
+ if (!el.paused && remaining < 0.1 && !s.timers.length) {
+ s.timers.push(window.setTimeout(stopNow, remaining * 1000));
+ }
+ s.raf = requestAnimationFrame(tick);
+ };
+ s.raf = requestAnimationFrame(tick);
// A rejected play() is normal, not a bug: Chrome refuses unmuted audio on
// a document nobody has interacted with (a typed URL, a fresh tab), and
// an unhandled rejection in that case would be noise. The seek has
// already happened either way.
void el.play().catch(() => {});
},
- [cached, fetchStart, playback.rate],
+ [cached, fetchStart, playback.rate, pauseVideo],
+ );
+
+ const play = useCallback(
+ async (from: number, to: number) => {
+ const gen = (playGen.current += 1);
+ stopPlayback();
+ if (!cached || !(to > from)) return;
+ const rate = playback.rate;
+ // The context is opened HERE, inside the click or the key that asked:
+ // that gesture is what lets it start.
+ let ac = actx.current;
+ if (!ac && typeof window !== "undefined" && window.AudioContext) {
+ ac = new window.AudioContext();
+ actx.current = ac;
+ }
+ const resumed = ac && ac.state !== "running" ? ac.resume().catch(() => {}) : null;
+ let d: Decoded;
+ try {
+ if (!ac) throw new Error("this browser has no Web Audio");
+ d = await ensureDecoded({ from, to });
+ } catch {
+ if (gen === playGen.current) playElement(from, to, gen);
+ return;
+ }
+ if (gen !== playGen.current) return;
+ // Longer than one decode: the element, rather than a second decode in
+ // the middle of a click.
+ if (!covers(d.span, from, to)) {
+ playElement(from, to, gen);
+ return;
+ }
+ if (resumed) await Promise.race([resumed, new Promise((res) => setTimeout(res, 300))]);
+ // No gesture has ever reached this document, so the context may not
+ // start: the same refusal a muted-autoplay policy gives the element, and
+ // just as silent.
+ if (gen !== playGen.current || ac!.state !== "running") return;
+ const ctx = ac!;
+ const sch = bufferSchedule(d.span, d.buf.duration, from, to, rate);
+ if (!sch) return;
+
+ const node = ctx.createBufferSource();
+ node.buffer = d.buf;
+ node.playbackRate.value = rate;
+ const gain = ctx.createGain();
+ node.connect(gain);
+ gain.connect(ctx.destination);
+ // Ahead of now, so start and stop stay on the clock they were computed
+ // on: a start in the past begins late at the SAME offset.
+ const t0 = ctx.currentTime + START_LEAD;
+ const ctxStop = t0 + sch.wall;
+ const m = muteRamp(muteRef.current, from, from + sch.duration, rate);
+ if (m) {
+ if (m.at <= 0) gain.gain.setValueAtTime(0, t0);
+ else {
+ gain.gain.setValueAtTime(1, t0 + m.at);
+ gain.gain.linearRampToValueAtTime(0, t0 + m.at + m.fade);
+ }
+ }
+ // The stop is a TIME on the context's clock, not start()'s duration
+ // argument: that one is buffer content, and a clock time means the same
+ // thing at every speed.
+ node.start(t0, sch.offset);
+ node.stop(ctxStop);
+
+ const el = video.current;
+ const s: Session = {
+ gen,
+ engine: "webaudio",
+ node,
+ gain,
+ raf: 0,
+ timers: [],
+ from,
+ to: from + sch.duration,
+ rate,
+ t0,
+ ctxStop,
+ mutedBefore: el?.muted ?? false,
+ };
+ session.current = s;
+ setPlayInfo({
+ engine: "webaudio",
+ state: "playing",
+ from,
+ to: s.to,
+ rate,
+ ctxStart: t0,
+ ctxStop,
+ endedAt: null,
+ });
+
+ // The picture follows, muted. It is not what is being judged; a few
+ // milliseconds of drift between the two is not worth a sync loop.
+ if (el) {
+ el.muted = true;
+ el.playbackRate = rate;
+ el.currentTime = Math.max(0, from - fetchStart);
+ s.timers.push(
+ window.setTimeout(() => {
+ if (session.current === s) void el.play().catch(() => {});
+ }, START_LEAD * 1000),
+ );
+ }
+
+ node.onended = () => {
+ if (session.current !== s) return;
+ // How late the context clock reads at `ended`, mapped back onto the
+ // source: what the playback measured, for the bench's own record.
+ const late = ctx.currentTime - ctxStop;
+ session.current = null;
+ cancelAnimationFrame(s.raf);
+ for (const t of s.timers) clearTimeout(t);
+ node.disconnect();
+ gain.disconnect();
+ if (el) {
+ pauseVideo(el);
+ el.muted = s.mutedBefore;
+ }
+ setPlayhead(s.to);
+ setPlayInfo((pi) => (pi ? { ...pi, state: "stopped", endedAt: s.to + Math.max(0, late) * rate } : pi));
+ };
+
+ // The playhead at ~30 Hz, not every frame: each update re-renders the
+ // bench, and the line moving smoothly is not worth that at 60.
+ //
+ // And the picture is pulled back to the sound when it drifts by more
+ // than a quarter second -- which it does after a seek into a file that
+ // is only partly loaded, where the element starts late by however long
+ // the bytes took.
+ let painted = -1;
+ let checked = t0;
+ const tick = () => {
+ if (session.current !== s) return;
+ const now = ctx.currentTime;
+ const pos = playheadAt(from, t0, now, rate, sch.duration);
+ if (now - painted >= 1 / 30) {
+ painted = now;
+ setPlayhead(pos);
+ }
+ if (el && now - checked >= 0.5 && now > t0) {
+ checked = now;
+ const want = pos - fetchStart;
+ if (!el.seeking && Math.abs(el.currentTime - want) > 0.25) el.currentTime = want;
+ }
+ s.raf = requestAnimationFrame(tick);
+ };
+ s.raf = requestAnimationFrame(tick);
+ },
+ [cached, fetchStart, playback.rate, ensureDecoded, playElement, stopPlayback, pauseVideo],
+ );
+
+ // A speed change mid-playback would leave the picture and the sound at two
+ // rates: stop, and the next play is at the new one.
+ useEffect(() => {
+ stopPlayback();
+ }, [playback.rate, stopPlayback]);
+
+ // Let go of the audio output with the bench.
+ useEffect(
+ () => () => {
+ stopPlayback();
+ void actx.current?.close().catch(() => {});
+ actx.current = null;
+ },
+ [stopPlayback],
);
// ---- auto-audition -------------------------------------------------------
@@ -579,20 +1008,40 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
};
}, [playback.auto, cached, clip.id, clip.start, clip.end, play]);
+ // The element's own controls: unbounded "play from here", which stays on the
+ // element. Its clock moves the playhead only while nothing bounded is
+ // playing -- during a Web Audio playback the muted picture would otherwise
+ // fight the audio clock for it. Pausing the picture by hand stops the sound
+ // it is following.
useEffect(() => {
const el = video.current;
if (!el) return;
+ // While it plays by itself. A paused element's `timeupdate` is the one its
+ // own pause fires -- after a Web Audio playback, that is the muted picture
+ // stopping wherever it had got to, which is not where the sound stopped.
const tick = () => {
- const t = el.currentTime + fetchStart;
- setPlayhead(t);
- if (stopAt.current != null && t >= stopAt.current) {
- el.pause();
- stopAt.current = null;
+ if (!session.current && !el.paused) setPlayhead(el.currentTime + fetchStart);
+ };
+ // A scrub on the paused element's own bar.
+ const seeked = () => {
+ if (!session.current) setPlayhead(el.currentTime + fetchStart);
+ };
+ const paused = () => {
+ if (selfPauses.current > 0) {
+ selfPauses.current -= 1;
+ return;
}
+ if (session.current) stopPlayback();
};
el.addEventListener("timeupdate", tick);
- return () => el.removeEventListener("timeupdate", tick);
- }, [fetchStart]);
+ el.addEventListener("seeked", seeked);
+ el.addEventListener("pause", paused);
+ return () => {
+ el.removeEventListener("timeupdate", tick);
+ el.removeEventListener("seeked", seeked);
+ el.removeEventListener("pause", paused);
+ };
+ }, [fetchStart, stopPlayback, cachedName]);
// ---- the selection ------------------------------------------------------
//
@@ -655,6 +1104,12 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
setSel({ from: next.start, to: next.end });
setDirty(false);
}
+ // The mark goes back to what was STORED -- rounded, or cleared because
+ // the window no longer held it.
+ if (patch.muteFrom !== undefined) {
+ setMute(next.muteFrom);
+ muteRef.current = next.muteFrom;
+ }
// Re-sync only the fields this save carried, and from what the writer
// actually stored -- which is trimmed, rounded, or gone.
setDraft((d) => {
@@ -745,13 +1200,13 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
// longer contains, so the rule lives here rather than in each of them.
const windowMoved =
Math.abs(round2(sel.from) - clip.start) > 0.02 || Math.abs(round2(sel.to) - clip.end) > 0.02;
+ // The mark moved, set or cleared since the last save.
+ const muteMoved =
+ (mute == null) !== (clip.muteFrom == null) ||
+ (mute != null && clip.muteFrom != null && Math.abs(round2(mute) - clip.muteFrom) > 0.005);
+ const unsaved = dirty || muteMoved;
- const windowPatch = useCallback((): {
- start: number;
- end: number;
- cutStart?: string;
- cutEnd?: string;
- } => {
+ const windowPatch = useCallback((): WindowPatch => {
const start = round2(sel.from);
const end = round2(sel.to);
const cutOutside =
@@ -761,8 +1216,16 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
// The extent is the judgement being made right now; the cut was derived
// from a wider one and is no longer inside it. Clearing it in the SAME
// patch is what keeps the writer's rule and the screen agreeing.
- return cutOutside ? { start, end, cutStart: "", cutEnd: "" } : { start, end };
- }, [sel.from, sel.to, clip.cutStart, clip.cutEnd]);
+ const out: WindowPatch = cutOutside
+ ? { start, end, cutStart: "", cutEnd: "" }
+ : { start, end };
+ // The mute mark rides the same patch, by the same rule: a mark the new
+ // extent does not hold is cleared rather than refused.
+ const markOutside = mute != null && (mute < start - 0.02 || mute > end + 0.02);
+ if (markOutside) out.muteFrom = "";
+ else if (muteMoved) out.muteFrom = mute == null ? "" : round2(mute);
+ return out;
+ }, [sel.from, sel.to, clip.cutStart, clip.cutEnd, mute, muteMoved]);
// ---- the walk's verdict ---------------------------------------------------
//
@@ -776,7 +1239,7 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
// is a judgement about THAT window, and the advance would otherwise walk
// away from it -- so the edges go in the SAME patch as the verdict rather
// than needing `save window` pressed first. One write, one token.
- const win = windowMoved ? windowPatch() : null;
+ const win = windowMoved || muteMoved ? windowPatch() : null;
// A note survives a confirmation. It stops being a complaint and becomes
// what it now says it is: why this clip is here in the shape it is in.
const ok = await save({ verdict: "confirmed", ...(win ?? {}) });
@@ -788,7 +1251,7 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
: "saved — the window you moved was saved with it",
);
if (ok && data.next) router.push(`/browse/${data.project}/clip/${data.next}`);
- }, [save, router, data.project, data.next, windowMoved, windowPatch]);
+ }, [save, router, data.project, data.next, windowMoved, muteMoved, windowPatch]);
const rejectClip = useCallback(() => {
setNeedNote(true);
@@ -807,6 +1270,76 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
el.setSelectionRange(el.value.length, el.value.length);
}, [clip, save]);
+ // ---- the mute mark ---------------------------------------------------------
+ //
+ // A finale's last clip plays its picture to the end, but its ending sound is
+ // not wanted: `muteFrom` fades the sound out from a source second to the end
+ // of the clip. It is set BY EAR, at the last silence before that sound, so it
+ // is placed the ways an edge is -- at the playhead, by a click on the
+ // waveform, by a nudge -- and every playback here mutes at it, so a mark is
+ // heard before it is saved.
+
+ /** Re-aim the playback in flight at a new mark, from now. */
+ const retargetMute = useCallback((t: number | null) => {
+ const s = session.current;
+ const ctx = actx.current;
+ if (!s || s.engine !== "webaudio" || !s.gain || !ctx) return;
+ const now = ctx.currentTime;
+ const g = s.gain.gain;
+ g.cancelScheduledValues(now);
+ const pos = playheadAt(s.from, s.t0, now, s.rate, s.to - s.from);
+ const m = t == null ? null : muteRamp(t, pos, s.to, s.rate);
+ if (!m) {
+ g.setValueAtTime(1, now);
+ return;
+ }
+ if (m.at <= 0) {
+ // Already past the mark: fade out from here.
+ g.setValueAtTime(g.value, now);
+ g.linearRampToValueAtTime(0, now + MUTE_FADE / s.rate);
+ return;
+ }
+ g.setValueAtTime(1, now);
+ g.setValueAtTime(1, now + m.at);
+ g.linearRampToValueAtTime(0, now + m.at + m.fade);
+ }, []);
+
+ /**
+ * Put the mark at `t`, inside the selection. `audition` plays across it --
+ * three seconds of sound and two of what should now be silence -- the way a
+ * moved edge plays the edge it moved.
+ */
+ const placeMute = useCallback(
+ (t: number, audition: boolean) => {
+ const at = round2(Math.min(sel.to, Math.max(sel.from, t)));
+ setMute(at);
+ muteRef.current = at;
+ setMutePick(false);
+ if (audition) void play(Math.max(sel.from, at - 3), Math.min(sel.to, at + 2));
+ else retargetMute(at);
+ },
+ [sel.from, sel.to, play, retargetMute],
+ );
+
+ const clearMute = useCallback(() => {
+ setMute(null);
+ muteRef.current = null;
+ setMutePick(false);
+ retargetMute(null);
+ }, [retargetMute]);
+
+ /** `m`: where you are listening. Anywhere outside the selection is refused. */
+ const muteAtPlayhead = useCallback(() => {
+ // Through a ref: the playhead moves every frame while something plays,
+ // and a callback keyed on it would re-bind the keyboard every frame.
+ const at = playheadRef.current;
+ if (at == null || at < sel.from - 0.02 || at > sel.to + 0.02) {
+ setNote("the playhead is not inside the selection — play to the spot, or pick it on the waveform");
+ return;
+ }
+ placeMute(at, false);
+ }, [sel.from, sel.to, placeMute]);
+
// ---- keyboard -----------------------------------------------------------
useEffect(() => {
const nudge = (which: "from" | "to", by: number) =>
@@ -841,12 +1374,14 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
case ">": nudge("to", step); break;
case " ":
e.preventDefault();
- play(sel.from, sel.to);
+ void play(sel.from, sel.to);
break;
case "r":
case "R":
setSel({ from: clip.start, to: clip.end });
setDirty(false);
+ setMute(clip.muteFrom);
+ muteRef.current = clip.muteFrom;
break;
// Walking the cut. Reviewing a whole video is nineteen clips in a row,
// and going back to the project page between each one is nineteen round
@@ -882,6 +1417,28 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
case "A":
choosePlayback((pb) => ({ ...pb, auto: !pb.auto }));
break;
+ // The mute mark: `m` at the playhead, `M` clears it, and `;` `'` move
+ // it like an edge (shift for 0.5 s), each playing across it.
+ case "m":
+ muteAtPlayhead();
+ break;
+ case "M":
+ clearMute();
+ break;
+ case ";":
+ case ":":
+ case "'":
+ case '"':
+ if (mute == null) {
+ setNote("no mute mark yet — m sets one at the playhead");
+ break;
+ }
+ placeMute(mute + (e.key === ";" || e.key === ":" ? -step : step), true);
+ break;
+ case "Escape":
+ if (!mutePick) return;
+ setMutePick(false);
+ break;
default:
return;
}
@@ -893,6 +1450,7 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
sel,
clip.start,
clip.end,
+ clip.muteFrom,
onSel,
play,
router,
@@ -902,6 +1460,11 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
confirmClip,
rejectClip,
choosePlayback,
+ mute,
+ mutePick,
+ muteAtPlayhead,
+ clearMute,
+ placeMute,
]);
const refresh = useCallback(async (): Promise<ClipBenchData | null> => {
@@ -1129,10 +1692,14 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
const saveWindow = useCallback(() => {
const patch = windowPatch();
void save(patch).then((ok) => {
- if (ok && patch.cutStart !== undefined)
- setNote("saved — the cut no longer fitted this window and was cleared");
+ if (!ok) return;
+ const cleared = [
+ patch.cutStart !== undefined ? "the cut" : null,
+ patch.muteFrom === "" && mute != null ? "the mute mark" : null,
+ ].filter(Boolean);
+ if (cleared.length) setNote(`saved — ${cleared.join(" and ")} no longer fitted this window and ${cleared.length > 1 ? "were" : "was"} cleared`);
});
- }, [windowPatch, save]);
+ }, [windowPatch, save, mute]);
// ---- the warnings --------------------------------------------------------
const endCue = cues.find((c) => sel.to >= c.start - 0.02 && sel.to <= c.end + 0.02) ?? null;
@@ -1373,29 +1940,38 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
unsaved — was {hms(clip.start)} – {hms(clip.end)}
</span>
)}
+ {muteMoved && (
+ <span data-mute-unsaved="" className="num text-[var(--color-dirty)]">
+ mute mark unsaved — was {clip.muteFrom == null ? "none" : hms(clip.muteFrom)}
+ </span>
+ )}
<span className="flex flex-wrap items-center gap-1.5">
<button
type="button"
+ data-save-window=""
className={buttonVariants({ variant: "primary", size: "sm" })}
- disabled={!dirty || !!busy}
+ disabled={!unsaved || !!busy}
onClick={saveWindow}
>
save window
</button>
<button
type="button"
+ data-play-selection=""
className={buttonVariants({ size: "sm" })}
- onClick={() => play(sel.from, sel.to)}
+ onClick={() => void play(sel.from, sel.to)}
>
play selection
</button>
<button
type="button"
className={buttonVariants({ size: "sm" })}
- disabled={!dirty}
+ disabled={!unsaved}
onClick={() => {
setSel({ from: clip.start, to: clip.end });
setDirty(false);
+ setMute(clip.muteFrom);
+ muteRef.current = clip.muteFrom;
}}
>
reset
@@ -1446,7 +2022,41 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
>
auto-audition {playback.auto ? "on" : "off"}
</button>
+ {/* What is playing, and what it measured. The attributes are the
+ bench's record of its last playback, on the audio clock --
+ the claim is "stops where the selection ends", and this is
+ where it can be checked. */}
+ <span
+ data-playback=""
+ data-audio-state={audio.state}
+ data-play-engine={playInfo?.engine ?? ""}
+ data-play-state={playInfo?.state ?? ""}
+ data-play-from={playInfo?.from ?? ""}
+ data-play-to={playInfo?.to ?? ""}
+ data-play-rate={playInfo?.rate ?? ""}
+ data-play-ctx-start={playInfo?.ctxStart ?? ""}
+ data-play-ctx-stop={playInfo?.ctxStop ?? ""}
+ data-play-ended-at={playInfo?.endedAt ?? ""}
+ className="micro"
+ title={
+ audio.state === "failed"
+ ? undefined
+ : "every bounded playback is the decoded audio, started and stopped on the audio clock: it ends where the selection ends, to the sample"
+ }
+ >
+ {audio.state === "ready"
+ ? "exact playback"
+ : audio.state === "loading"
+ ? "decoding the audio…"
+ : null}
+ </span>
</span>
+ {audio.state === "failed" && (
+ <span data-playback-fallback="" className="text-[11px] text-[var(--color-dirty)]">
+ exact playback unavailable — {audio.why}. The video plays instead, stopped by the
+ frame: up to half a frame either side of the end.
+ </span>
+ )}
{/* ---- EXTENT above, CUT here ----
The window row says how much of the recording is worth having.
This says what will actually play, and offers to derive it
@@ -1493,10 +2103,60 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
</button>
)}
</span>
+ {/* ---- the MUTE MARK ----
+ Beside the cut because it is the same kind of decision -- what
+ the finished clip does with its seconds -- and drafted like the
+ edges: heard at once, written by `save window` or `y`. */}
+ <span
+ data-mute={mute ?? ""}
+ className="flex flex-wrap items-center gap-1.5"
+ title="From the mark to the end of the clip the sound fades out and the picture plays on. Set it at the last silence before the ending sound."
+ >
+ {mute != null ? (
+ <span data-mute-from={mute} className="num text-[var(--color-text)]">
+ mute from {hms(mute)}{" "}
+ <span className="text-[var(--color-dim)]">
+ ({Math.max(0, sel.to - mute).toFixed(2)}s silent to the end)
+ </span>
+ </span>
+ ) : (
+ <span className="text-[var(--color-dim)]">sound to the end</span>
+ )}
+ <button
+ type="button"
+ data-mute-set=""
+ className={buttonVariants({ size: "sm" })}
+ title="put the mute mark at the playhead"
+ onClick={muteAtPlayhead}
+ >
+ mute from here
+ </button>
+ <button
+ type="button"
+ data-mute-pick={mutePick ? "armed" : ""}
+ aria-pressed={mutePick}
+ className={buttonVariants({ variant: mutePick ? "primary" : "outline", size: "sm" })}
+ title="the next click on the waveform places the mute mark (Esc cancels)"
+ onClick={() => setMutePick((v) => !v)}
+ >
+ {mutePick ? "click the waveform…" : "pick on waveform"}
+ </button>
+ {mute != null && (
+ <button
+ type="button"
+ data-mute-clear=""
+ className={buttonVariants({ size: "sm" })}
+ onClick={clearMute}
+ >
+ clear mute
+ </button>
+ )}
+ </span>
<span className="micro">
<kbd>[</kbd> <kbd>]</kbd> start · <kbd>,</kbd> <kbd>.</kbd> end — each plays the edge
it moved · <kbd>space</kbd> the whole selection · <kbd>R</kbd> reset · <kbd>-</kbd>{" "}
- <kbd>=</kbd> speed · <kbd>a</kbd> auto — shift for 0.5s
+ <kbd>=</kbd> speed · <kbd>a</kbd> auto · <kbd>m</kbd> mute from the playhead,{" "}
+ <kbd>;</kbd> <kbd>'</kbd> move it, <kbd>M</kbd> clear — shift for 0.5s
</span>
{busy && <span className="text-[var(--color-meter)]">{busy}</span>}
{note && (
@@ -1507,19 +2167,58 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
</div>
{/* ---- the instrument ---- */}
- <Waveform
- view={view}
- sel={sel}
- cand={{ from: clip.start, to: clip.end }}
- words={cues.map((c) => ({ start: c.start, end: c.end, w: c.text.slice(0, 24) }))}
- peaks={peaks}
- playhead={playhead}
- height={104}
- onSel={onSel}
- onReachEdge={() => {
- /* widening is a FETCH here, not a redraw -- see the button below */
- }}
- />
+ <div className="relative">
+ <Waveform
+ view={view}
+ sel={sel}
+ cand={{ from: clip.start, to: clip.end }}
+ words={cues.map((c) => ({ start: c.start, end: c.end, w: c.text.slice(0, 24) }))}
+ peaks={peaks}
+ playhead={playhead}
+ height={104}
+ onSel={onSel}
+ onReachEdge={() => {
+ /* widening is a FETCH here, not a redraw -- see the button below */
+ }}
+ />
+ {/* The mute mark over the waveform: a line where the sound stops
+ and a hatch over what is silent to the end of the selection.
+ In the interaction colour, like the handles -- it is an edit,
+ not a reading -- and never in the way of a drag. */}
+ {mute != null && mute >= view.from && mute <= view.to && (
+ <>
+ <div
+ data-mute-span=""
+ className="pointer-events-none absolute top-0 h-full [background:repeating-linear-gradient(135deg,color-mix(in_srgb,var(--color-dim)_22%,transparent)_0_3px,transparent_3px_7px)]"
+ style={{
+ left: pct(mute),
+ width: `calc(${pct(Math.max(mute, sel.to))} - ${pct(mute)})`,
+ }}
+ />
+ <div
+ data-mute-marker={mute}
+ className="pointer-events-none absolute top-0 h-full border-l-2 border-dashed border-[var(--color-sel)]"
+ style={{ left: pct(mute) }}
+ >
+ <span className="absolute left-1 top-0.5 rounded bg-[var(--color-panel-2)] px-1 font-mono text-[9px] leading-tight text-[var(--color-sel)]">
+ mute
+ </span>
+ </div>
+ </>
+ )}
+ {/* Armed: the next click places the mark rather than an edge. */}
+ {mutePick && (
+ <div
+ data-mute-pick-layer=""
+ className="absolute inset-0 cursor-crosshair rounded-md outline outline-1 outline-[var(--color-sel)]"
+ onPointerDown={(e) => {
+ const r = e.currentTarget.getBoundingClientRect();
+ const frac = Math.min(1, Math.max(0, (e.clientX - r.left) / Math.max(1, r.width)));
+ placeMute(view.from + frac * span, true);
+ }}
+ />
+ )}
+ </div>
{/* Offered at the edge of the cache, as always -- and also whenever
the rail has something to read past it. Having just read the next
@@ -1767,7 +2466,7 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
if (!atMaxPad) void fetchMore(padsForCue(c));
return;
}
- play(c.start, Math.min(c.end + 4, view.to));
+ void play(c.start, Math.min(c.end + 4, view.to));
}}
className={`flex w-full gap-2 px-2 py-1 text-left hover:bg-[color-mix(in_srgb,var(--color-sel)_10%,transparent)] ${
inSel ? "bg-[color-mix(in_srgb,var(--color-sel)_14%,transparent)]" : ""
diff --git a/umtool/components/projects/ClipBenchPage.tsx b/umtool/components/projects/ClipBenchPage.tsx
@@ -88,6 +88,11 @@ export default async function ClipBenchPage({
cutStart: entry.cutStart ?? null,
cutEnd: entry.cutEnd ?? null,
lockCut: !!entry.lockCut,
+ // The mute mark, from the manifest's own entry: readClipDetail's rows
+ // carry the fields it reads, and this is not one of them.
+ muteFrom:
+ ((manifest.timeline ?? []) as { id: string; muteFrom?: number }[]).find((e) => e.id === clipId)
+ ?.muteFrom ?? null,
verdict:
entry.verdict === "confirmed" || entry.verdict === "incorrect" ? entry.verdict : null,
// From the manifest's own entry: what the on-screen panel says over this
diff --git a/umtool/e2e/clip-bench.spec.ts b/umtool/e2e/clip-bench.spec.ts
@@ -57,6 +57,7 @@ const readClipIn = (project: string, id: string) => {
verdict?: string;
cutStart?: number;
cutEnd?: number;
+ muteFrom?: number;
}[];
};
return m.timeline.find((e) => e.id === id)!;
@@ -804,13 +805,266 @@ test("moving the end auditions the END", async ({ page }) => {
await page.locator("body").press(".");
// Clamped to the cached file, which ends at 9.00 -- a drag never downloads.
const to = Math.min(before.end + 0.05, 9);
- const t = await page
- .getByTestId("clip-video")
- .evaluate((el: HTMLVideoElement) => el.currentTime);
- // vid1_0.00-9.00 starts at 0.00, so file time IS source time here. The four
- // seconds ENDING on the new edge, not the four after the start.
- expect(t).toBeGreaterThan(to - 4 - 0.4);
- expect(t).toBeLessThan(to - 4 + 2);
+ // The four seconds ENDING on the new edge, not the four after the start --
+ // read from the bench's own record of what it scheduled, on the audio clock.
+ const playback = page.locator("[data-playback]");
+ await expect(playback).toHaveAttribute("data-play-engine", "webaudio", { timeout: 15_000 });
+ expect(Number(await playback.getAttribute("data-play-to"))).toBeCloseTo(to, 2);
+ expect(Number(await playback.getAttribute("data-play-from"))).toBeCloseTo(to - 4, 2);
+ // And the picture follows it there, muted: vid1_0.00-9.00 starts at 0.00,
+ // so file time IS source time.
+ await expect
+ .poll(() => page.getByTestId("clip-video").evaluate((el: HTMLVideoElement) => el.currentTime))
+ .toBeGreaterThan(to - 4 - 0.4);
+});
+
+// ---------------------------------------------------------------------------
+// Exact playback.
+//
+// Every bounded range plays from the DECODED audio with Web Audio, started and
+// stopped on the audio clock. The element it replaces was stopped from
+// `timeupdate`, which overran the end by up to a quarter of a second -- the
+// difference between a cut between two words and one into the next.
+//
+// The proof is the signal, not the bench's word for it: TAP (below) is an
+// AudioWorklet put between the bench's audio and the speakers that records the
+// first and last NON-ZERO frame it is handed, on the context's own frame
+// clock. The bench publishes when it scheduled the stop (`data-play-ctx-stop`);
+// the two must agree to within one render quantum (128 frames).
+//
+// vid1 is a 440 Hz tone with silences at 2.9-3.1 and 5.9-6.1, so a selection
+// ending at 5.00 ends INSIDE the tone: the last sound is the stop, not a
+// silence that happened to come first.
+// ---------------------------------------------------------------------------
+
+const TAP = `(() => {
+ const code = \`class Tap extends AudioWorkletProcessor {
+ constructor() { super(); this.first = -1; this.last = -1;
+ this.port.onmessage = (e) => {
+ if (e.data === "reset") { this.first = -1; this.last = -1; }
+ if (e.data === "read") this.port.postMessage({ first: this.first, last: this.last, sr: sampleRate });
+ };
+ }
+ process(inputs) {
+ const ch = inputs[0] && inputs[0][0];
+ if (ch) for (let i = 0; i < ch.length; i += 1) if (ch[i] !== 0) {
+ const f = currentFrame + i; if (this.first < 0) this.first = f; this.last = f;
+ }
+ return true;
+ }
+ }
+ registerProcessor("tap", Tap);\`;
+ const url = URL.createObjectURL(new Blob([code], { type: "application/javascript" }));
+ const Orig = window.AudioContext;
+ if (!Orig) return;
+ const conn = AudioNode.prototype.connect;
+ window.__tapRead = () => new Promise((res) => {
+ const n = window.__tapNode;
+ if (!n) return res(null);
+ n.port.onmessage = (e) => res(e.data);
+ n.port.postMessage("read");
+ });
+ window.__tapReset = () => window.__tapNode && window.__tapNode.port.postMessage("reset");
+ window.AudioContext = class extends Orig {
+ constructor(...a) {
+ super(...a);
+ const ctx = this;
+ ctx.audioWorklet.addModule(url).then(() => {
+ const n = new AudioWorkletNode(ctx, "tap", { outputChannelCount: [1] });
+ conn.call(n, ctx.destination);
+ ctx.__tap = n;
+ window.__tapNode = n;
+ for (const src of ctx.__pending || []) conn.call(src, n);
+ ctx.__pending = [];
+ });
+ }
+ };
+ AudioNode.prototype.connect = function (dest, ...rest) {
+ const r = conn.call(this, dest, ...rest);
+ const tap = this.context && this.context.__tap;
+ if (dest === this.context.destination && this !== tap) {
+ if (tap) conn.call(this, tap);
+ else (this.context.__pending = this.context.__pending || []).push(this);
+ }
+ return r;
+ };
+})();`;
+
+type Tap = { first: number; last: number; sr: number } | null;
+const tapRead = (page: import("@playwright/test").Page) =>
+ page.evaluate(() => (window as unknown as { __tapRead: () => Promise<Tap> }).__tapRead());
+const tapReset = (page: import("@playwright/test").Page) =>
+ page.evaluate(() => (window as unknown as { __tapReset: () => void }).__tapReset());
+
+/** The bench's record of its last playback, as numbers. */
+const lastPlay = async (page: import("@playwright/test").Page) => {
+ const p = page.locator("[data-playback]");
+ const n = async (k: string) => Number(await p.getAttribute(`data-play-${k}`));
+ return {
+ engine: await p.getAttribute("data-play-engine"),
+ from: await n("from"),
+ to: await n("to"),
+ rate: await n("rate"),
+ ctxStart: await n("ctx-start"),
+ ctxStop: await n("ctx-stop"),
+ };
+};
+
+/** Start a playback with `go` and wait until the bench says it has stopped. */
+const playThrough = async (page: import("@playwright/test").Page, go: () => Promise<void>) => {
+ const p = page.locator("[data-playback]");
+ await tapReset(page);
+ await go();
+ await expect(p).toHaveAttribute("data-play-state", "playing", { timeout: 10_000 });
+ await expect(p).toHaveAttribute("data-play-state", "stopped", { timeout: 20_000 });
+ // `ended` reaches the page a task after the audio thread stopped; the tap's
+ // reply is one more message behind it.
+ await page.waitForTimeout(100);
+ return { play: await lastPlay(page), tap: await tapRead(page) };
+};
+
+/** Put c01 back to its fixture window with no mute mark. */
+const resetC01 = async (request: import("@playwright/test").APIRequestContext) => {
+ const { token: t } = await token(request, "c01");
+ const r = await request.put("/api/report/window", {
+ data: { project: PROJECT, clip: "c01", start: 3, end: 6, muteFrom: "", token: t },
+ });
+ expect(r.ok()).toBeTruthy();
+};
+
+test("a play-selection stops within one audio render quantum of its end, at any speed", async ({
+ page,
+ request,
+}) => {
+ await resetC01(request);
+ await page.addInitScript(TAP);
+ await page.goto(bench("c01"));
+ await expect(page.locator("[data-playback]")).toHaveAttribute("data-audio-state", "ready", {
+ timeout: 15_000,
+ });
+ await keyboardLive(page);
+ // 6.00 -> 5.00 (shift-, is half a second): the end now lies in the tone.
+ // Each press auditions the end it moved, which also opens the audio output
+ // and puts the tap in the path.
+ await page.locator("body").press("Shift+Comma");
+ await page.locator("body").press("Shift+Comma");
+ await expect
+ .poll(async () => Number(await page.locator("[data-playback]").getAttribute("data-play-to")), {
+ timeout: 10_000,
+ })
+ .toBeCloseTo(5, 3);
+ await expect(page.locator("[data-playback]")).toHaveAttribute("data-play-state", "stopped", {
+ timeout: 10_000,
+ });
+
+ for (const rate of ["1", "2"]) {
+ await page.locator("[data-playback-rate]").selectOption(rate);
+ const { play, tap } = await playThrough(page, () => page.locator("[data-play-selection]").click());
+ expect(play.engine).toBe("webaudio");
+ expect(play.from).toBeCloseTo(3, 3);
+ expect(play.to).toBeCloseTo(5, 3);
+ expect(play.rate).toBe(Number(rate));
+ expect(tap, "the tap saw the bench's audio").not.toBeNull();
+ const { first, last, sr } = tap!;
+ expect(last).toBeGreaterThan(first);
+ const quantum = 128;
+ // The STOP: the last sound is the scheduled stop, to within a quantum.
+ const stopFrame = Math.round(play.ctxStop * sr);
+ expect(Math.abs(last + 1 - stopFrame), `stopped ${last + 1 - stopFrame} frames from the end at ${rate}x`).toBeLessThanOrEqual(quantum);
+ // And nothing sounded before the scheduled start. (Not "the first sound
+ // IS the start": 3.00 is inside the silence at 2.9-3.1, so the first
+ // non-zero frame is the tone coming back.)
+ expect(first).toBeGreaterThanOrEqual(Math.round(play.ctxStart * sr) - quantum);
+ }
+});
+
+test("the mute mark: picked on the waveform, heard at once, saved, shown, cleared", async ({
+ page,
+ request,
+}) => {
+ await resetC01(request);
+ await page.addInitScript(TAP);
+ await page.goto(bench("c01"));
+ const playback = page.locator("[data-playback]");
+ await expect(playback).toHaveAttribute("data-audio-state", "ready", { timeout: 15_000 });
+ await expect(page.locator("[data-mute]")).toHaveAttribute("data-mute", "");
+ await expect(page.locator("[data-mute-marker]")).toHaveCount(0);
+
+ // Armed, the next click on the waveform places the mark: at 5.00 of the
+ // cached 0.00-9.00, the tone between the two silences.
+ await page.locator("[data-mute-pick]").click();
+ await expect(page.locator("[data-mute-pick=armed]")).toBeVisible();
+ const layer = page.locator("[data-mute-pick-layer]");
+ const box = (await layer.boundingBox())!;
+ const { play, tap } = await playThrough(page, () =>
+ page.mouse.click(box.x + (box.width * 5) / 9, box.y + box.height / 2),
+ );
+ const marker = page.locator("[data-mute-marker]");
+ await expect(marker).toBeVisible();
+ const mark = Number(await marker.getAttribute("data-mute-marker"));
+ expect(mark).toBeGreaterThan(4.9);
+ expect(mark).toBeLessThan(5.1);
+ await expect(page.locator("[data-mute-pick-layer]")).toHaveCount(0);
+ await expect(page.locator("[data-mute-unsaved]")).toContainText("was none");
+
+ // The pick plays ACROSS the mark -- up to three seconds before it (here
+ // from the selection's start), to the end of the selection -- and the sound
+ // goes at the mark: the last non-zero frame is where the fade (0.05 s) runs
+ // out, not the 6.00 the tone runs on to.
+ expect(play.engine).toBe("webaudio");
+ expect(play.from).toBeCloseTo(Math.max(3, mark - 3), 2);
+ expect(play.to).toBeCloseTo(6, 2);
+ const { last, sr } = tap!;
+ const silentAt = play.ctxStart + (mark - play.from) + 0.05;
+ expect(Math.abs((last + 1) / sr - silentAt), "silent at the mark, not at the end").toBeLessThan(0.01);
+
+ // Saved by `save window`, like the edges; the manifest has it.
+ await page.locator("[data-save-window]").click();
+ await expect(page.locator("[data-bench-note]")).toContainText("saved");
+ expect(readClip("c01").muteFrom).toBeCloseTo(mark, 2);
+ await expect(page.locator("[data-mute-unsaved]")).toHaveCount(0);
+
+ // A reload shows the saved mark.
+ await page.reload();
+ await expect(page.locator("[data-mute-marker]")).toHaveAttribute("data-mute-marker", String(readClip("c01").muteFrom));
+ await expect(page.locator("[data-mute-from]")).toBeVisible();
+
+ // Moved like an edge, and cleared.
+ await keyboardLive(page);
+ await page.locator("body").press("Shift+Semicolon");
+ await expect(page.locator("[data-mute-marker]")).toHaveAttribute(
+ "data-mute-marker",
+ String(Number((readClip("c01").muteFrom! - 0.5).toFixed(2))),
+ );
+ await page.locator("[data-mute-clear]").click();
+ await expect(page.locator("[data-mute-marker]")).toHaveCount(0);
+ await expect(page.locator("[data-mute-unsaved]")).toBeVisible();
+ await page.locator("[data-save-window]").click();
+ await expect(page.locator("[data-bench-note]")).toContainText("saved");
+ expect(readClip("c01").muteFrom).toBeUndefined();
+
+ // The writer's rule, through the route: inside the clip, or refused.
+ const { token: t } = await token(request, "c01");
+ const out = await request.put("/api/report/window", {
+ data: { project: PROJECT, clip: "c01", muteFrom: 8, token: t },
+ });
+ expect(out.status()).toBe(400);
+ expect(((await out.json()) as { error: string }).error).toMatch(/must lie inside the clip 3–6/);
+});
+
+test("a decode that fails falls back to the element, and the bench says so", async ({ page }) => {
+ await page.route("**/api/report/audio**", (r) =>
+ r.fulfill({ status: 422, json: { error: "this file has no audio track" } }),
+ );
+ await page.goto(bench("c01"));
+ const fallback = page.locator("[data-playback-fallback]");
+ await expect(fallback).toBeVisible({ timeout: 15_000 });
+ await expect(fallback).toContainText("this file has no audio track");
+ await page.locator("[data-play-selection]").click();
+ await expect(page.locator("[data-playback]")).toHaveAttribute("data-play-engine", "element");
+ await expect
+ .poll(() => page.getByTestId("clip-video").evaluate((el: HTMLVideoElement) => !el.paused))
+ .toBe(true);
});
test("the playback speed is this browser's, and it survives a reload", async ({ page }) => {
@@ -841,6 +1095,11 @@ test("auto-audition plays the clip you walk onto", async ({ page, request }) =>
await page.locator("[data-clip-nav=next]").click();
await expect(page.locator("[data-bench=c04]")).toBeVisible();
+ // The whole clip, from the decoded audio -- and the picture with it.
+ const playback = page.locator("[data-playback]");
+ await expect(playback).toHaveAttribute("data-play-engine", "webaudio", { timeout: 15_000 });
+ expect(Number(await playback.getAttribute("data-play-from"))).toBeCloseTo(15, 2);
+ expect(Number(await playback.getAttribute("data-play-to"))).toBeCloseTo(18, 2);
await expect
.poll(
() => page.getByTestId("clip-video").evaluate((el: HTMLVideoElement) => !el.paused),
diff --git a/umtool/lib/report/manifest.mjs b/umtool/lib/report/manifest.mjs
@@ -31,6 +31,7 @@ import {
} from "umtool-report-to-video/ledger-totals";
import { isCalendarDate } from "umtool-report-to-video/attribution";
import { normalizeOnscreen, validateChrome, validatePosts } from "umtool-report-to-video/deck";
+import { parseMuteFrom } from "./playback.mjs";
// Its own write queue, not lib/state.ts's.
//
@@ -281,6 +282,29 @@ export async function updateClip(dir, clipId, patch, { token = null } = {}) {
}
}
+ // ---- the mute mark ------------------------------------------------------
+ //
+ // `muteFrom`: from this source second to the end of the clip the sound
+ // fades out and the picture plays on -- set by ear, in the bench, at the
+ // last silence before a finale's ending sound. Inside the EXTENT, like the
+ // cut, and checked against the entry AFTER the patch: a window save that
+ // leaves the mark outside is refused rather than keeping a mark that no
+ // longer says anything about the clip. Empty or null deletes it.
+ if (patch.muteFrom !== undefined) {
+ const v = parseMuteFrom(patch.muteFrom, entry.start, entry.end);
+ if (v == null) delete entry.muteFrom;
+ else entry.muteFrom = v;
+ } else if ((patch.start !== undefined || patch.end !== undefined) && entry.muteFrom != null) {
+ try {
+ parseMuteFrom(entry.muteFrom, entry.start, entry.end);
+ } catch {
+ throw new Error(
+ `the mute mark ${entry.muteFrom} must lie inside the window ` +
+ `${entry.start}–${entry.end} — widen the window, or clear the mark`,
+ );
+ }
+ }
+
// ---- the walk's verdict -------------------------------------------------
//
// Whether somebody has LOOKED at this clip and said the description is what
diff --git a/umtool/lib/report/manifest.test.mjs b/umtool/lib/report/manifest.test.mjs
@@ -181,6 +181,42 @@ test("updateClip: onscreen is normalised, and empty or null deletes the key", as
}
});
+test("updateClip: muteFrom is a number inside the clip, rounded; empty or null deletes the key", async () => {
+ const dir = await project();
+ try {
+ const res = await updateClip(dir, "c01", { muteFrom: 18.456 });
+ assert.equal(res.entry.muteFrom, 18.46);
+ assert.equal(entry(await read(dir), "c01").muteFrom, 18.46);
+ // A string from a form is a number too.
+ await updateClip(dir, "c01", { muteFrom: "17.5" });
+ assert.equal(entry(await read(dir), "c01").muteFrom, 17.5);
+
+ const before = await readRaw(dir);
+ // Outside [start, end], and not a number: refused, nothing written.
+ await assert.rejects(updateClip(dir, "c01", { muteFrom: 25 }), /must lie inside the clip 10–20/);
+ await assert.rejects(updateClip(dir, "c01", { muteFrom: 5 }), /must lie inside/);
+ await assert.rejects(updateClip(dir, "c01", { muteFrom: "later" }), /must be a number/);
+ // A window that would leave the mark outside it is refused too...
+ await assert.rejects(updateClip(dir, "c01", { end: 17 }), /mute mark 17.5 must lie inside the window 10–17/);
+ assert.equal(await readRaw(dir), before);
+ // ...unless the same patch moves or clears it.
+ await updateClip(dir, "c01", { end: 17, muteFrom: 16 });
+ assert.equal(entry(await read(dir), "c01").muteFrom, 16);
+ // Checked against the window AFTER the patch: a wider window and a mark in
+ // the new seconds land together.
+ await updateClip(dir, "c01", { end: 22, muteFrom: 21 });
+ assert.equal(entry(await read(dir), "c01").muteFrom, 21);
+
+ await updateClip(dir, "c01", { muteFrom: "" });
+ assert.equal("muteFrom" in entry(await read(dir), "c01"), false);
+ await updateClip(dir, "c01", { muteFrom: 12 });
+ await updateClip(dir, "c01", { muteFrom: null });
+ assert.equal("muteFrom" in entry(await read(dir), "c01"), false);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+});
+
const DECK = { engine: "hyperframes", layout: "deck", deck: { height: 180, title: { size: 60 } } };
test("updateChrome: round trip stores the block as given; null removes it", async () => {
diff --git a/umtool/lib/report/playback.mjs b/umtool/lib/report/playback.mjs
@@ -0,0 +1,228 @@
+// The clip bench's bounded playback, as arithmetic.
+//
+// The bench used to play a range on the <video> element and stop it from
+// `timeupdate`, which fires every 15–250 ms: what you heard ran past the end
+// by up to a quarter of a second, and by a different amount every time. That
+// is the difference between a cut that lands between two words and one that
+// swallows the next syllable, and it could not be judged by ear.
+//
+// So a bounded range plays from the DECODED audio with Web Audio, the old
+// um-triage chooser's technique: an AudioBufferSourceNode started at an exact
+// buffer offset and stopped at an exact context time, sample-accurately. The
+// picture follows along, muted. These are the numbers that decide where in the
+// buffer to start, when to stop, where the playhead is, and when the finale's
+// mute mark goes quiet. Pure: no DOM, no Web Audio, so they are unit-tested and
+// the component only applies them.
+//
+// EVERYTHING IS IN ABSOLUTE SOURCE SECONDS except where a name says `wall`
+// (seconds of the listener's time, which is source time divided by the rate)
+// or `ctx` (the AudioContext's clock).
+
+/** Decoding is per window, but never more than this many seconds at once. */
+export const MAX_DECODE_SPAN = 120;
+
+/**
+ * Around the selection, when a window is too long to decode whole (a whole
+ * recording fetched into the saved-video store): enough to drag an edge and
+ * hear it without a second decode.
+ */
+export const DECODE_MARGIN = 30;
+
+/**
+ * How far ahead of "now" a playback is scheduled, in wall seconds.
+ *
+ * A start scheduled in the past is started late at the SAME offset, so every
+ * time computed from it would be early by however late it was. Scheduling a
+ * little ahead keeps `start` and `stop` on the clock they were computed on.
+ */
+export const START_LEAD = 0.03;
+
+/**
+ * The mute mark's fade, in source seconds: short enough to read as "the sound
+ * stops here", long enough not to click.
+ */
+export const MUTE_FADE = 0.05;
+
+/** One Web Audio render quantum, in frames. */
+export const RENDER_QUANTUM = 128;
+
+const round3 = (n) => Math.round(n * 1000) / 1000;
+
+/**
+ * Which span of a cached window to decode.
+ *
+ * The whole window when it is short enough -- one decode then serves every
+ * drag. A longer one decodes the selection plus a margin each side, clamped to
+ * the window, and capped: a decode is held in memory as 32-bit floats, and two
+ * minutes of stereo is already ~46 MB.
+ *
+ * @param {{from: number, to: number}} win the cached file's span
+ * @param {{from: number, to: number}} want the range about to be played
+ * @param {{max?: number, margin?: number}} [opts]
+ * @returns {{from: number, to: number}}
+ */
+export function decodeSpan(win, want, { max = MAX_DECODE_SPAN, margin = DECODE_MARGIN } = {}) {
+ if (win.to - win.from <= max) return { from: win.from, to: win.to };
+ let from = Math.max(win.from, want.from - margin);
+ let to = Math.min(win.to, Math.max(want.to, want.from) + margin);
+ if (to - from > max) {
+ // A selection wider than the cap: from its start, as much as fits.
+ from = Math.max(win.from, Math.min(want.from, win.to - max));
+ to = Math.min(win.to, from + max);
+ }
+ return { from: round3(from), to: round3(to) };
+}
+
+/**
+ * Does a decoded span hold this whole range? A hair of tolerance, because the
+ * decoded span's end is a frame count divided by a rate.
+ *
+ * @param {{from: number, to: number} | null} span
+ * @param {number} from
+ * @param {number} to
+ */
+export function covers(span, from, to) {
+ if (!span) return false;
+ return from >= span.from - 0.005 && to <= span.to + 0.005;
+}
+
+/**
+ * Where in the buffer to start, how much source to play, and how long that
+ * takes at this rate.
+ *
+ * `duration` is SOURCE seconds and `wall` is how long it lasts at `rate`. The
+ * stop is scheduled at `start + wall` on the context's clock rather than
+ * passed to `start()` as a duration: the spec reads that argument as buffer
+ * content, but a stop time on the context clock means one thing at any rate.
+ *
+ * @param {{from: number, to: number}} span the decoded buffer's absolute span
+ * @param {number} bufferSeconds the buffer's own duration
+ * @param {number} from
+ * @param {number} to
+ * @param {number} [rate]
+ * @returns {{offset: number, duration: number, wall: number} | null} null when
+ * nothing of the range is in the buffer
+ */
+export function bufferSchedule(span, bufferSeconds, from, to, rate = 1) {
+ const r = rate > 0 ? rate : 1;
+ const offset = Math.max(0, Math.min(bufferSeconds, from - span.from));
+ const duration = Math.min(bufferSeconds - offset, to - Math.max(from, span.from));
+ if (!(duration > 0)) return null;
+ return { offset, duration, wall: duration / r };
+}
+
+/**
+ * The playhead, in source seconds, from the audio clock.
+ *
+ * @param {number} from where the playback started, in source seconds
+ * @param {number} t0 the context time it started at
+ * @param {number} now the context time now
+ * @param {number} rate
+ * @param {number} duration source seconds the playback lasts
+ */
+export function playheadAt(from, t0, now, rate, duration) {
+ const r = rate > 0 ? rate : 1;
+ const played = Math.max(0, Math.min(duration, (now - t0) * r));
+ return from + played;
+}
+
+/**
+ * When the mute mark silences a playback of [from, to], as offsets in WALL
+ * seconds from its start.
+ *
+ * null the mark is not in this range (or there is none)
+ * { at: 0, fade: 0 } the range starts at or after the mark: silent
+ * from the first sample, the picture still plays
+ * { at, fade } full level until `at`, then a linear ramp to
+ * nothing over `fade`
+ *
+ * @param {number | null | undefined} muteFrom
+ * @param {number} from
+ * @param {number} to
+ * @param {number} [rate]
+ * @param {number} [fade] source seconds
+ * @returns {{at: number, fade: number} | null}
+ */
+export function muteRamp(muteFrom, from, to, rate = 1, fade = MUTE_FADE) {
+ if (muteFrom == null || !Number.isFinite(muteFrom)) return null;
+ const r = rate > 0 ? rate : 1;
+ if (muteFrom >= to) return null;
+ if (muteFrom <= from) return { at: 0, fade: 0 };
+ return { at: (muteFrom - from) / r, fade: fade / r };
+}
+
+/**
+ * The element fallback's stop test, run once per animation frame.
+ *
+ * Stopping at the first frame PAST the end overruns by up to a frame (and the
+ * element's own clock reports late on top of that). Stopping as soon as the
+ * end is less than half a frame away splits the error both ways instead: never
+ * more than half a frame early or late, at any rate.
+ *
+ * @param {number} t the element's position, in source seconds
+ * @param {number} stopAt
+ * @param {number} [rate]
+ * @param {number} [frame] wall seconds per animation frame
+ */
+export function elementShouldStop(t, stopAt, rate = 1, frame = 1 / 60) {
+ const r = rate > 0 ? rate : 1;
+ return t + (frame * r) / 2 >= stopAt;
+}
+
+/**
+ * A mute mark as the WRITER reads it: a number of source seconds inside the
+ * clip's extent, rounded like an edge -- or `null` to delete the key.
+ *
+ * `null` and `""` mean "no mark". Anything else must be a finite number in
+ * [start, end] (a hair of tolerance, the same 0.02 the cut's check allows).
+ *
+ * @param {unknown} raw
+ * @param {number} start
+ * @param {number} end
+ * @returns {number | null}
+ */
+export function parseMuteFrom(raw, start, end) {
+ if (raw === null || raw === "") return null;
+ const v = typeof raw === "number" ? raw : typeof raw === "string" ? Number(raw.trim()) : Number.NaN;
+ if (!Number.isFinite(v)) {
+ throw new Error(`muteFrom must be a number of source seconds, or empty to clear it (got \`${String(raw)}\`)`);
+ }
+ if (v < start - 0.02 || v > end + 0.02) {
+ throw new Error(
+ `muteFrom ${v} must lie inside the clip ${start}–${end} — the mark mutes from there to the clip's end`,
+ );
+ }
+ return Math.round(Math.min(end, Math.max(start, v)) * 100) / 100;
+}
+
+/**
+ * A RIFF/WAVE header for interleaved 16-bit PCM.
+ *
+ * Written by hand because ffmpeg writing WAV to a pipe cannot seek back to fill
+ * in the sizes, and a header that says "unknown length" is one more thing a
+ * decoder may or may not forgive.
+ *
+ * @param {{channels: number, sampleRate: number, dataBytes: number}} f
+ * @returns {Uint8Array} 44 bytes
+ */
+export function wavHeader({ channels, sampleRate, dataBytes }) {
+ const b = new Uint8Array(44);
+ const v = new DataView(b.buffer);
+ const tag = (at, s) => {
+ for (let i = 0; i < 4; i += 1) b[at + i] = s.charCodeAt(i);
+ };
+ tag(0, "RIFF");
+ v.setUint32(4, 36 + dataBytes, true);
+ tag(8, "WAVE");
+ tag(12, "fmt ");
+ v.setUint32(16, 16, true);
+ v.setUint16(20, 1, true); // PCM
+ v.setUint16(22, channels, true);
+ v.setUint32(24, sampleRate, true);
+ v.setUint32(28, sampleRate * channels * 2, true);
+ v.setUint16(32, channels * 2, true);
+ v.setUint16(34, 16, true);
+ tag(36, "data");
+ v.setUint32(40, dataBytes, true);
+ return b;
+}
diff --git a/umtool/lib/report/playback.test.mjs b/umtool/lib/report/playback.test.mjs
@@ -0,0 +1,152 @@
+// The clip bench's bounded playback, as arithmetic.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ MAX_DECODE_SPAN,
+ MUTE_FADE,
+ bufferSchedule,
+ covers,
+ decodeSpan,
+ elementShouldStop,
+ muteRamp,
+ parseMuteFrom,
+ playheadAt,
+ wavHeader,
+} from "./playback.mjs";
+
+const near = (a, b, eps = 1e-9) => assert.ok(Math.abs(a - b) <= eps, `${a} ≉ ${b}`);
+
+test("decodeSpan: a short window decodes whole, whatever is being played", () => {
+ assert.deepEqual(decodeSpan({ from: 24019.6, to: 24032.6 }, { from: 24022.6, to: 24029.6 }), {
+ from: 24019.6,
+ to: 24032.6,
+ });
+ // Exactly at the cap is still whole.
+ assert.deepEqual(decodeSpan({ from: 0, to: MAX_DECODE_SPAN }, { from: 5, to: 6 }), {
+ from: 0,
+ to: MAX_DECODE_SPAN,
+ });
+});
+
+test("decodeSpan: a long window decodes the selection plus a margin, clamped and capped", () => {
+ const whole = { from: 0, to: 7200 };
+ assert.deepEqual(decodeSpan(whole, { from: 3600, to: 3610 }), { from: 3570, to: 3640 });
+ // Clamped at the recording's start.
+ assert.deepEqual(decodeSpan(whole, { from: 10, to: 20 }), { from: 0, to: 50 });
+ // Clamped at its end.
+ assert.deepEqual(decodeSpan(whole, { from: 7190, to: 7200 }), { from: 7160, to: 7200 });
+ // A selection wider than the cap: from its start, as much as fits.
+ const wide = decodeSpan(whole, { from: 1000, to: 1300 });
+ assert.deepEqual(wide, { from: 1000, to: 1000 + MAX_DECODE_SPAN });
+ // ...and never past the window's end.
+ assert.deepEqual(decodeSpan(whole, { from: 7150, to: 7400 }), { from: 7120, to: 7200 });
+ assert.deepEqual(decodeSpan(whole, { from: 7000, to: 7400 }), { from: 7000, to: 7000 + MAX_DECODE_SPAN });
+});
+
+test("covers: the decoded span holds the whole range, with a hair of tolerance", () => {
+ const span = { from: 10, to: 20 };
+ assert.equal(covers(span, 10, 20), true);
+ assert.equal(covers(span, 12, 19.999), true);
+ assert.equal(covers(span, 9.9, 15), false);
+ assert.equal(covers(span, 15, 20.1), false);
+ assert.equal(covers(null, 1, 2), false);
+});
+
+test("bufferSchedule: offset into the buffer, source seconds to play, and how long that takes", () => {
+ const span = { from: 24019.6, to: 24032.6 };
+ const s = bufferSchedule(span, 13, 24022.6, 24029.6, 1);
+ near(s.offset, 3, 1e-6);
+ near(s.duration, 7, 1e-6);
+ near(s.wall, 7, 1e-6);
+ // Speed changes how long it takes, not what is played.
+ const fast = bufferSchedule(span, 13, 24022.6, 24029.6, 1.5);
+ near(fast.offset, 3, 1e-6);
+ near(fast.duration, 7, 1e-6);
+ near(fast.wall, 7 / 1.5, 1e-6);
+ const slow = bufferSchedule(span, 13, 24022.6, 24029.6, 0.75);
+ near(slow.wall, 7 / 0.75, 1e-6);
+ // A range running past the buffer stops at the buffer.
+ near(bufferSchedule(span, 13, 24030, 24040, 1).duration, 2.6, 1e-6);
+ // Nothing of the range in the buffer.
+ assert.equal(bufferSchedule(span, 13, 24040, 24050, 1), null);
+ // A nonsense rate is read as 1, not as a division by zero.
+ near(bufferSchedule(span, 13, 24022.6, 24029.6, 0).wall, 7, 1e-6);
+});
+
+test("playheadAt: from the audio clock, in source seconds, held at the end", () => {
+ near(playheadAt(100, 5, 5, 1, 4), 100);
+ near(playheadAt(100, 5, 6, 1, 4), 101);
+ near(playheadAt(100, 5, 6, 2, 4), 102);
+ // Before the scheduled start (the lead): at the start, not before it.
+ near(playheadAt(100, 5, 4.98, 1, 4), 100);
+ // Past the end: held at the end.
+ near(playheadAt(100, 5, 60, 1, 4), 104);
+});
+
+test("muteRamp: when the mark silences a playback, in wall seconds from its start", () => {
+ assert.equal(muteRamp(null, 10, 20), null);
+ assert.equal(muteRamp(undefined, 10, 20), null);
+ // At or past the end of the range: nothing to mute in it.
+ assert.equal(muteRamp(20, 10, 20), null);
+ assert.equal(muteRamp(25, 10, 20), null);
+ // At or before the start: silent throughout, the picture still plays.
+ assert.deepEqual(muteRamp(10, 10, 20), { at: 0, fade: 0 });
+ assert.deepEqual(muteRamp(5, 10, 20), { at: 0, fade: 0 });
+ // Inside: full level until the mark, then the fade.
+ const m = muteRamp(16, 10, 20, 1);
+ near(m.at, 6);
+ near(m.fade, MUTE_FADE);
+ // At 2x the mark arrives in half the time, and so does the fade.
+ const f = muteRamp(16, 10, 20, 2);
+ near(f.at, 3);
+ near(f.fade, MUTE_FADE / 2);
+});
+
+test("elementShouldStop: stops when the end is under half a frame away, at any rate", () => {
+ const frame = 1 / 60;
+ assert.equal(elementShouldStop(9.9, 10, 1, frame), false);
+ assert.equal(elementShouldStop(10 - frame / 2 - 0.001, 10, 1, frame), false);
+ assert.equal(elementShouldStop(10 - frame / 2 + 0.001, 10, 1, frame), true);
+ assert.equal(elementShouldStop(10.2, 10, 1, frame), true);
+ // At 2x a frame covers twice the source, so the stop comes a frame earlier.
+ assert.equal(elementShouldStop(10 - frame + 0.001, 10, 2, frame), true);
+});
+
+test("parseMuteFrom: a number inside the clip, rounded like an edge, or null to delete", () => {
+ assert.equal(parseMuteFrom(null, 10, 20), null);
+ assert.equal(parseMuteFrom("", 10, 20), null);
+ assert.equal(parseMuteFrom(15.456, 10, 20), 15.46);
+ assert.equal(parseMuteFrom(" 15.5 ", 10, 20), 15.5);
+ assert.equal(parseMuteFrom(10, 10, 20), 10);
+ assert.equal(parseMuteFrom(20, 10, 20), 20);
+ // The 0.02 tolerance an edge gets, clamped back inside.
+ assert.equal(parseMuteFrom(20.01, 10, 20), 20);
+ assert.equal(parseMuteFrom(9.99, 10, 20), 10);
+ assert.throws(() => parseMuteFrom(20.5, 10, 20), /must lie inside the clip 10–20/);
+ assert.throws(() => parseMuteFrom(9, 10, 20), /must lie inside/);
+ assert.throws(() => parseMuteFrom("soon", 10, 20), /must be a number/);
+ assert.throws(() => parseMuteFrom(true, 10, 20), /must be a number/);
+ assert.throws(() => parseMuteFrom(Number.NaN, 10, 20), /must be a number/);
+});
+
+test("wavHeader: 44 bytes of RIFF/WAVE for interleaved 16-bit PCM", () => {
+ const h = wavHeader({ channels: 2, sampleRate: 48000, dataBytes: 192000 });
+ assert.equal(h.length, 44);
+ const v = new DataView(h.buffer);
+ const tag = (at) => String.fromCharCode(...h.slice(at, at + 4));
+ assert.equal(tag(0), "RIFF");
+ assert.equal(v.getUint32(4, true), 36 + 192000);
+ assert.equal(tag(8), "WAVE");
+ assert.equal(tag(12), "fmt ");
+ assert.equal(v.getUint16(20, true), 1);
+ assert.equal(v.getUint16(22, true), 2);
+ assert.equal(v.getUint32(24, true), 48000);
+ assert.equal(v.getUint32(28, true), 48000 * 4);
+ assert.equal(v.getUint16(32, true), 4);
+ assert.equal(v.getUint16(34, true), 16);
+ assert.equal(tag(36), "data");
+ assert.equal(v.getUint32(40, true), 192000);
+});
