commit 48d297306c446967c86d590790df0ddbc58c1c3f
parent 654a78c5cabb44f53af6dcb96819daa676c126f3
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 1 Oct 2026 14:31:59 -0400
umtool: the clip bench plays every bounded range from decoded audio, and sets a clip's mute mark
Play selection, the edge auditions, the auto-audition and a transcript line
play from the window's decoded audio with Web Audio: an AudioBufferSourceNode
started at an exact buffer offset and stopped at an exact time on the
context clock, so the sound ends where the range ends. The stop is a context
time rather than start()'s duration argument, so it is exact at every speed.
The picture follows muted and the playhead reads the audio clock. The decode
is per window (the whole window up to 120 s, else the selection plus 30 s
each side) and is dropped when the window changes.
When the decode fails the element plays instead, stopped per animation frame
by remaining time (half a frame either side) with a timer in the last frames,
and the bench says that playback is approximate and why. The element's own
controls (unbounded) stay on the element.
The mute mark (`muteFrom`) is placed at the playhead (m), by a click on the
waveform (pick on waveform), or moved like an edge (; and ', shift for 0.5 s),
and cleared (M, clear mute). It is drafted like the edges and written by
`save window` or a confirmation; a mark the new window does not hold is
cleared in the same patch. Every playback mutes at it with a 0.05 s fade, and
a mark moved during a playback re-aims it.
The bench records each playback as data-play-* attributes. The specs check
the stop against an AudioWorklet tap of the bench's output: within one render
quantum at 1x and 2x.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
3 files changed, 979 insertions(+), 52 deletions(-)
diff --git a/umtool/components/projects/ClipBench.tsx b/umtool/components/projects/ClipBench.tsx
@@ -11,6 +11,16 @@ import { buttonVariants } from "@/components/ui/button";
// would only find out twenty minutes into a build.
import { attributionLine } from "umtool-report-to-video/attribution";
import {
+ MUTE_FADE,
+ START_LEAD,
+ bufferSchedule,
+ covers,
+ decodeSpan,
+ elementShouldStop,
+ muteRamp,
+ playheadAt,
+} from "@/lib/report/playback.mjs";
+import {
DeckFrame,
NeutralFrame,
composePreview,
@@ -100,6 +110,12 @@ type Clip = {
cutEnd: number | null;
/** The cut is deliberate; `resolve-windows --cut-to-quote` leaves it alone. */
lockCut: boolean;
+ /**
+ * The MUTE MARK, in source seconds: from here to the end of the clip the
+ * sound fades out and the picture plays on. Set by ear, at the last silence
+ * before a finale's ending sound. Absent means the clip plays with its sound.
+ */
+ muteFrom: number | null;
/** "confirmed" / "incorrect", or null for "nobody has looked at this yet". */
verdict: "confirmed" | "incorrect" | null;
/** What the on-screen panel says over this clip. Absent means the auto text. */
@@ -180,6 +196,7 @@ const fromEntry = (prev: Clip, e: Record<string, unknown>): Clip => ({
cutStart: e.cutStart == null ? null : Number(e.cutStart),
cutEnd: e.cutEnd == null ? null : Number(e.cutEnd),
lockCut: !!e.lockCut,
+ muteFrom: e.muteFrom == null ? null : Number(e.muteFrom),
verdict: e.verdict === "confirmed" || e.verdict === "incorrect" ? e.verdict : null,
onscreen: (e.onscreen as Onscreen | undefined) ?? null,
});
@@ -303,6 +320,62 @@ const clock = (t: number) => {
return `${Math.floor(m / 60) > 0 ? `${Math.floor(m / 60)}:${String(m % 60).padStart(2, "0")}` : m}:${String(s % 60).padStart(2, "0")}`;
};
+/** What `save window` (and a confirmation that moved something) writes. */
+type WindowPatch = {
+ start: number;
+ end: number;
+ cutStart?: string;
+ cutEnd?: string;
+ /** A number to set the mark, "" to clear it; absent leaves it alone. */
+ muteFrom?: number | string;
+};
+
+/** A decoded span of the cached window, placed on the source clock. */
+type Decoded = { name: string; span: { from: number; to: number }; buf: AudioBuffer };
+
+/**
+ * What is playing, and what the last playback measured.
+ *
+ * On the page as data attributes (`data-play-*`), because "did it stop where
+ * the selection ends" is the claim this bench now makes, and a spec has to be
+ * able to check it against the audio clock rather than against a feeling.
+ */
+type PlayInfo = {
+ engine: "webaudio" | "element";
+ state: "playing" | "stopped";
+ from: number;
+ /** The scheduled end, in source seconds. */
+ to: number;
+ rate: number;
+ /** Context clock: when the source starts and when it is told to stop. */
+ ctxStart: number | null;
+ ctxStop: number | null;
+ /**
+ * Where the playback had reached, in source seconds, when the page heard it
+ * end: the element's own position at its pause, or the context clock at
+ * `ended` -- which reaches the page a task later than the sound stopped, so
+ * for Web Audio it is an upper bound and `ctxStop` is the stop itself.
+ */
+ endedAt: number | null;
+};
+
+/** The playback in flight. One at a time: a new one stops the last. */
+type Session = {
+ gen: number;
+ engine: "webaudio" | "element";
+ node: AudioBufferSourceNode | null;
+ gain: GainNode | null;
+ raf: number;
+ timers: number[];
+ from: number;
+ to: number;
+ rate: number;
+ t0: number;
+ ctxStop: number;
+ /** The element's own mute, put back when the picture stops following. */
+ mutedBefore: boolean;
+};
+
export default function ClipBench({ data }: { data: ClipBenchData }) {
const [clip, setClip] = useState<Clip>(data.clip);
const [windows, setWindows] = useState<Win[]>(data.windows);
@@ -310,6 +383,13 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
const [proposed, setProposed] = useState(data.proposed);
const [view, setView] = useState(data.view);
const [sel, setSel] = useState({ from: data.clip.start, to: data.clip.end });
+ // The mute mark as drafted. Like the edges it is unsaved until `save window`
+ // or `y`, and it is what the bench's own playback mutes at -- so a mark is
+ // heard before it is written.
+ const [mute, setMute] = useState<number | null>(data.clip.muteFrom);
+ // Armed: the next click on the waveform places the mark instead of moving
+ // an edge.
+ const [mutePick, setMutePick] = useState(false);
const [peaks, setPeaks] = useState<Peaks | null>(null);
// The cues AROUND the cached window: what is coming, read before paying for
// the media. One request, widened only when somebody asks.
@@ -356,7 +436,6 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
const token = useRef(data.token);
// And they must not interleave: same read-modify-write, one clip.
const saving = useRef<Promise<boolean>>(Promise.resolve(true));
- const stopAt = useRef<number | null>(null);
// `x` answers "no" by putting the cursor in the note, which is the answer.
const correctionBox = useRef<HTMLTextAreaElement | null>(null);
const segVideo = useRef<HTMLVideoElement | null>(null);
@@ -532,20 +611,347 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
}, [playback, segment, cached, renderedOpen, deckPreview]);
// ---- audition -----------------------------------------------------------
- const play = useCallback(
- (from: number, to: number) => {
+ //
+ // EVERY BOUNDED RANGE PLAYS FROM DECODED AUDIO. A selection, an edge, a line
+ // of the transcript, the auto-audition: each is an AudioBufferSourceNode
+ // started at an exact offset and stopped at an exact time on the audio
+ // clock, so what you hear ends where the selection ends, to the sample. The
+ // <video> used to play these itself and was stopped from `timeupdate`, which
+ // fires every 15-250 ms -- the stop overran by up to a quarter of a second,
+ // and by a different amount every time, which is exactly what makes a cut
+ // between two words impossible to judge by ear. (The old HTML chooser had
+ // this right; it is the same technique.)
+ //
+ // The picture FOLLOWS, muted, and the playhead is read off the audio clock.
+ // When the decode fails the element plays instead, stopped per animation
+ // frame by remaining time, and the bench says so.
+
+ /** The decoded audio for the cached window: one decode per window. */
+ const decoded = useRef<Decoded | null>(null);
+ const decoding = useRef<{ key: string; p: Promise<Decoded> } | null>(null);
+ const actx = useRef<AudioContext | null>(null);
+ const session = useRef<Session | null>(null);
+ const playGen = useRef(0);
+ // Pauses the bench asked for itself and has not yet heard the event of. A
+ // `pause` event is a task, so it arrives AFTER the next playback has started:
+ // without the count, stopping one playback to start the next would read as
+ // somebody pausing the picture, and stop the new one.
+ const selfPauses = useRef(0);
+ const pauseVideo = useCallback((el: HTMLVideoElement) => {
+ if (el.paused) return;
+ selfPauses.current += 1;
+ el.pause();
+ }, []);
+ // Read by play() at the moment it schedules, so a mark moved since the last
+ // render is the one you hear -- without re-creating play() on every nudge.
+ const muteRef = useRef<number | null>(data.clip.muteFrom);
+ const [audio, setAudio] = useState<{ state: "idle" | "loading" | "ready" | "failed"; why?: string }>({
+ state: "idle",
+ });
+ const [playInfo, setPlayInfo] = useState<PlayInfo | null>(null);
+
+ /**
+ * The cached window's sound over (at least) `want`, decoded once.
+ *
+ * Through /api/report/audio -- ffmpeg's decode of the same file the player
+ * loads, as PCM -- and decoded here on an OfflineAudioContext, so no audio
+ * output is opened before somebody asks to hear something. A buffer belongs
+ * to no context and plays in the real one.
+ */
+ const ensureDecoded = useCallback(
+ async (want: { from: number; to: number }): Promise<Decoded> => {
+ if (!cached) throw new Error("nothing is cached for this clip");
+ const have = decoded.current;
+ if (have && have.name === cached.name && covers(have.span, want.from, want.to)) return have;
+ const span = decodeSpan({ from: cached.from, to: cached.to }, want);
+ const key = `${cached.name}|${span.from}|${span.to}`;
+ if (decoding.current?.key === key) return decoding.current.p;
+ const p = (async () => {
+ setAudio({ state: "loading" });
+ const r = await fetch(
+ `/api/report/audio?project=${encodeURIComponent(data.project)}&clip=${encodeURIComponent(clip.id)}` +
+ `&file=${encodeURIComponent(cached.name)}&from=${span.from}&to=${span.to}`,
+ );
+ if (!r.ok) {
+ const j = (await r.json().catch(() => null)) as { error?: string } | null;
+ throw new Error(j?.error ?? `the audio route answered ${r.status}`);
+ }
+ const from = Number(r.headers.get("x-audio-from") ?? span.from);
+ const bytes = await r.arrayBuffer();
+ const Offline =
+ typeof window !== "undefined" ? (window.OfflineAudioContext ?? null) : null;
+ if (!Offline) throw new Error("this browser has no Web Audio");
+ const buf = await new Offline(2, 1, 48000).decodeAudioData(bytes);
+ const d: Decoded = { name: cached.name, span: { from, to: from + buf.duration }, buf };
+ decoded.current = d;
+ setAudio({ state: "ready" });
+ return d;
+ })();
+ decoding.current = { key, p };
+ p.catch((e: unknown) => {
+ if (decoding.current?.p === p) decoding.current = null;
+ setAudio({ state: "failed", why: e instanceof Error ? e.message : String(e) });
+ });
+ return p;
+ },
+ [cached, data.project, clip.id],
+ );
+
+ // A different window -- a wider one after "fetch more", or the first after a
+ // fetch -- is different audio: drop the old decode and start the new one
+ // now, so the first play after it does not wait.
+ const cachedName = cached?.name ?? null;
+ useEffect(() => {
+ decoded.current = null;
+ decoding.current = null;
+ setAudio({ state: "idle" });
+ if (!cachedName) return;
+ ensureDecoded({ from: clip.start, to: clip.end }).catch(() => {
+ /* said in the bench; play() falls back to the element */
+ });
+ // ensureDecoded changes identity with `cached`, which is this effect's
+ // whole subject; keying on the NAME is what stops a refresh() that returns
+ // the same window from decoding it again.
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [cachedName]);
+
+ /** Stop whatever is playing, Web Audio or element, and let go of it. */
+ const stopPlayback = useCallback(() => {
+ const s = session.current;
+ if (!s) return;
+ session.current = null;
+ cancelAnimationFrame(s.raf);
+ for (const t of s.timers) clearTimeout(t);
+ if (s.node) {
+ s.node.onended = null;
+ try {
+ s.node.stop();
+ } catch {
+ /* never started, or already stopped */
+ }
+ s.node.disconnect();
+ s.gain?.disconnect();
+ }
+ const el = video.current;
+ if (el) {
+ pauseVideo(el);
+ el.muted = s.mutedBefore;
+ }
+ setPlayInfo((pi) => (pi && pi.state === "playing" ? { ...pi, state: "stopped" } : pi));
+ }, [pauseVideo]);
+
+ /**
+ * The fallback: the element, stopped per animation frame by REMAINING TIME.
+ *
+ * `timeupdate` was the old stop and is the overrun this replaces; a frame
+ * tick that stops once the end is under half a frame away is the best the
+ * element can do, and the bench says that this is what is playing.
+ */
+ const playElement = useCallback(
+ (from: number, to: number, gen: number) => {
const el = video.current;
if (!el || !cached) return;
+ const rate = playback.rate;
el.currentTime = Math.max(0, from - fetchStart);
- el.playbackRate = playback.rate;
- stopAt.current = to;
+ el.playbackRate = rate;
+ const s: Session = {
+ gen,
+ engine: "element",
+ node: null,
+ gain: null,
+ raf: 0,
+ timers: [],
+ from,
+ to,
+ rate,
+ t0: 0,
+ ctxStop: 0,
+ mutedBefore: el.muted,
+ };
+ session.current = s;
+ setPlayInfo({ engine: "element", state: "playing", from, to, rate, ctxStart: null, ctxStop: null, endedAt: null });
+ const stopNow = () => {
+ if (session.current !== s) return;
+ pauseVideo(el);
+ session.current = null;
+ cancelAnimationFrame(s.raf);
+ for (const t of s.timers) clearTimeout(t);
+ el.muted = s.mutedBefore;
+ setPlayInfo((pi) => (pi ? { ...pi, state: "stopped", endedAt: el.currentTime + fetchStart } : pi));
+ };
+ const tick = () => {
+ if (session.current !== s) return;
+ const t = el.currentTime + fetchStart;
+ setPlayhead(t);
+ // The mark, as near as a frame tick gets it.
+ const m = muteRef.current;
+ el.muted = s.mutedBefore || (m != null && t >= m);
+ if (!el.paused && elementShouldStop(t, to, rate)) {
+ stopNow();
+ return;
+ }
+ // Inside the last few frames, a timer for the REMAINING time: it lands
+ // between frames, where the next tick would land up to a frame late.
+ const remaining = (to - t) / rate;
+ if (!el.paused && remaining < 0.1 && !s.timers.length) {
+ s.timers.push(window.setTimeout(stopNow, remaining * 1000));
+ }
+ s.raf = requestAnimationFrame(tick);
+ };
+ s.raf = requestAnimationFrame(tick);
// A rejected play() is normal, not a bug: Chrome refuses unmuted audio on
// a document nobody has interacted with (a typed URL, a fresh tab), and
// an unhandled rejection in that case would be noise. The seek has
// already happened either way.
void el.play().catch(() => {});
},
- [cached, fetchStart, playback.rate],
+ [cached, fetchStart, playback.rate, pauseVideo],
+ );
+
+ const play = useCallback(
+ async (from: number, to: number) => {
+ const gen = (playGen.current += 1);
+ stopPlayback();
+ if (!cached || !(to > from)) return;
+ const rate = playback.rate;
+ // The context is opened HERE, inside the click or the key that asked:
+ // that gesture is what lets it start.
+ let ac = actx.current;
+ if (!ac && typeof window !== "undefined" && window.AudioContext) {
+ ac = new window.AudioContext();
+ actx.current = ac;
+ }
+ const resumed = ac && ac.state !== "running" ? ac.resume().catch(() => {}) : null;
+ let d: Decoded;
+ try {
+ if (!ac) throw new Error("this browser has no Web Audio");
+ d = await ensureDecoded({ from, to });
+ } catch {
+ if (gen === playGen.current) playElement(from, to, gen);
+ return;
+ }
+ if (gen !== playGen.current) return;
+ // Longer than one decode: the element, rather than a second decode in
+ // the middle of a click.
+ if (!covers(d.span, from, to)) {
+ playElement(from, to, gen);
+ return;
+ }
+ if (resumed) await Promise.race([resumed, new Promise((res) => setTimeout(res, 300))]);
+ // No gesture has ever reached this document, so the context may not
+ // start: the same refusal a muted-autoplay policy gives the element, and
+ // just as silent.
+ if (gen !== playGen.current || ac!.state !== "running") return;
+ const ctx = ac!;
+ const sch = bufferSchedule(d.span, d.buf.duration, from, to, rate);
+ if (!sch) return;
+
+ const node = ctx.createBufferSource();
+ node.buffer = d.buf;
+ node.playbackRate.value = rate;
+ const gain = ctx.createGain();
+ node.connect(gain);
+ gain.connect(ctx.destination);
+ // Ahead of now, so start and stop stay on the clock they were computed
+ // on: a start in the past begins late at the SAME offset.
+ const t0 = ctx.currentTime + START_LEAD;
+ const ctxStop = t0 + sch.wall;
+ const m = muteRamp(muteRef.current, from, from + sch.duration, rate);
+ if (m) {
+ if (m.at <= 0) gain.gain.setValueAtTime(0, t0);
+ else {
+ gain.gain.setValueAtTime(1, t0 + m.at);
+ gain.gain.linearRampToValueAtTime(0, t0 + m.at + m.fade);
+ }
+ }
+ // The stop is a TIME on the context's clock, not start()'s duration
+ // argument: that one is buffer content, and a clock time means the same
+ // thing at every speed.
+ node.start(t0, sch.offset);
+ node.stop(ctxStop);
+
+ const el = video.current;
+ const s: Session = {
+ gen,
+ engine: "webaudio",
+ node,
+ gain,
+ raf: 0,
+ timers: [],
+ from,
+ to: from + sch.duration,
+ rate,
+ t0,
+ ctxStop,
+ mutedBefore: el?.muted ?? false,
+ };
+ session.current = s;
+ setPlayInfo({
+ engine: "webaudio",
+ state: "playing",
+ from,
+ to: s.to,
+ rate,
+ ctxStart: t0,
+ ctxStop,
+ endedAt: null,
+ });
+
+ // The picture follows, muted. It is not what is being judged; a few
+ // milliseconds of drift between the two is not worth a sync loop.
+ if (el) {
+ el.muted = true;
+ el.playbackRate = rate;
+ el.currentTime = Math.max(0, from - fetchStart);
+ s.timers.push(
+ window.setTimeout(() => {
+ if (session.current === s) void el.play().catch(() => {});
+ }, START_LEAD * 1000),
+ );
+ }
+
+ node.onended = () => {
+ if (session.current !== s) return;
+ // How late the context clock reads at `ended`, mapped back onto the
+ // source: what the playback measured, for the bench's own record.
+ const late = ctx.currentTime - ctxStop;
+ session.current = null;
+ cancelAnimationFrame(s.raf);
+ for (const t of s.timers) clearTimeout(t);
+ node.disconnect();
+ gain.disconnect();
+ if (el) {
+ pauseVideo(el);
+ el.muted = s.mutedBefore;
+ }
+ setPlayhead(s.to);
+ setPlayInfo((pi) => (pi ? { ...pi, state: "stopped", endedAt: s.to + Math.max(0, late) * rate } : pi));
+ };
+
+ const tick = () => {
+ if (session.current !== s) return;
+ setPlayhead(playheadAt(from, t0, ctx.currentTime, rate, sch.duration));
+ s.raf = requestAnimationFrame(tick);
+ };
+ s.raf = requestAnimationFrame(tick);
+ },
+ [cached, fetchStart, playback.rate, ensureDecoded, playElement, stopPlayback, pauseVideo],
+ );
+
+ // A speed change mid-playback would leave the picture and the sound at two
+ // rates: stop, and the next play is at the new one.
+ useEffect(() => {
+ stopPlayback();
+ }, [playback.rate, stopPlayback]);
+
+ // Let go of the audio output with the bench.
+ useEffect(
+ () => () => {
+ stopPlayback();
+ void actx.current?.close().catch(() => {});
+ actx.current = null;
+ },
+ [stopPlayback],
);
// ---- auto-audition -------------------------------------------------------
@@ -579,20 +985,31 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
};
}, [playback.auto, cached, clip.id, clip.start, clip.end, play]);
+ // The element's own controls: unbounded "play from here", which stays on the
+ // element. Its clock moves the playhead only while nothing bounded is
+ // playing -- during a Web Audio playback the muted picture would otherwise
+ // fight the audio clock for it. Pausing the picture by hand stops the sound
+ // it is following.
useEffect(() => {
const el = video.current;
if (!el) return;
const tick = () => {
- const t = el.currentTime + fetchStart;
- setPlayhead(t);
- if (stopAt.current != null && t >= stopAt.current) {
- el.pause();
- stopAt.current = null;
+ if (!session.current) setPlayhead(el.currentTime + fetchStart);
+ };
+ const paused = () => {
+ if (selfPauses.current > 0) {
+ selfPauses.current -= 1;
+ return;
}
+ if (session.current) stopPlayback();
};
el.addEventListener("timeupdate", tick);
- return () => el.removeEventListener("timeupdate", tick);
- }, [fetchStart]);
+ el.addEventListener("pause", paused);
+ return () => {
+ el.removeEventListener("timeupdate", tick);
+ el.removeEventListener("pause", paused);
+ };
+ }, [fetchStart, stopPlayback, cachedName]);
// ---- the selection ------------------------------------------------------
//
@@ -655,6 +1072,12 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
setSel({ from: next.start, to: next.end });
setDirty(false);
}
+ // The mark goes back to what was STORED -- rounded, or cleared because
+ // the window no longer held it.
+ if (patch.muteFrom !== undefined) {
+ setMute(next.muteFrom);
+ muteRef.current = next.muteFrom;
+ }
// Re-sync only the fields this save carried, and from what the writer
// actually stored -- which is trimmed, rounded, or gone.
setDraft((d) => {
@@ -745,13 +1168,13 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
// longer contains, so the rule lives here rather than in each of them.
const windowMoved =
Math.abs(round2(sel.from) - clip.start) > 0.02 || Math.abs(round2(sel.to) - clip.end) > 0.02;
+ // The mark moved, set or cleared since the last save.
+ const muteMoved =
+ (mute == null) !== (clip.muteFrom == null) ||
+ (mute != null && clip.muteFrom != null && Math.abs(round2(mute) - clip.muteFrom) > 0.005);
+ const unsaved = dirty || muteMoved;
- const windowPatch = useCallback((): {
- start: number;
- end: number;
- cutStart?: string;
- cutEnd?: string;
- } => {
+ const windowPatch = useCallback((): WindowPatch => {
const start = round2(sel.from);
const end = round2(sel.to);
const cutOutside =
@@ -761,8 +1184,16 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
// The extent is the judgement being made right now; the cut was derived
// from a wider one and is no longer inside it. Clearing it in the SAME
// patch is what keeps the writer's rule and the screen agreeing.
- return cutOutside ? { start, end, cutStart: "", cutEnd: "" } : { start, end };
- }, [sel.from, sel.to, clip.cutStart, clip.cutEnd]);
+ const out: WindowPatch = cutOutside
+ ? { start, end, cutStart: "", cutEnd: "" }
+ : { start, end };
+ // The mute mark rides the same patch, by the same rule: a mark the new
+ // extent does not hold is cleared rather than refused.
+ const markOutside = mute != null && (mute < start - 0.02 || mute > end + 0.02);
+ if (markOutside) out.muteFrom = "";
+ else if (muteMoved) out.muteFrom = mute == null ? "" : round2(mute);
+ return out;
+ }, [sel.from, sel.to, clip.cutStart, clip.cutEnd, mute, muteMoved]);
// ---- the walk's verdict ---------------------------------------------------
//
@@ -776,7 +1207,7 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
// is a judgement about THAT window, and the advance would otherwise walk
// away from it -- so the edges go in the SAME patch as the verdict rather
// than needing `save window` pressed first. One write, one token.
- const win = windowMoved ? windowPatch() : null;
+ const win = windowMoved || muteMoved ? windowPatch() : null;
// A note survives a confirmation. It stops being a complaint and becomes
// what it now says it is: why this clip is here in the shape it is in.
const ok = await save({ verdict: "confirmed", ...(win ?? {}) });
@@ -788,7 +1219,7 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
: "saved — the window you moved was saved with it",
);
if (ok && data.next) router.push(`/browse/${data.project}/clip/${data.next}`);
- }, [save, router, data.project, data.next, windowMoved, windowPatch]);
+ }, [save, router, data.project, data.next, windowMoved, muteMoved, windowPatch]);
const rejectClip = useCallback(() => {
setNeedNote(true);
@@ -807,6 +1238,73 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
el.setSelectionRange(el.value.length, el.value.length);
}, [clip, save]);
+ // ---- the mute mark ---------------------------------------------------------
+ //
+ // A finale's last clip plays its picture to the end, but its ending sound is
+ // not wanted: `muteFrom` fades the sound out from a source second to the end
+ // of the clip. It is set BY EAR, at the last silence before that sound, so it
+ // is placed the ways an edge is -- at the playhead, by a click on the
+ // waveform, by a nudge -- and every playback here mutes at it, so a mark is
+ // heard before it is saved.
+
+ /** Re-aim the playback in flight at a new mark, from now. */
+ const retargetMute = useCallback((t: number | null) => {
+ const s = session.current;
+ const ctx = actx.current;
+ if (!s || s.engine !== "webaudio" || !s.gain || !ctx) return;
+ const now = ctx.currentTime;
+ const g = s.gain.gain;
+ g.cancelScheduledValues(now);
+ const pos = playheadAt(s.from, s.t0, now, s.rate, s.to - s.from);
+ const m = t == null ? null : muteRamp(t, pos, s.to, s.rate);
+ if (!m) {
+ g.setValueAtTime(1, now);
+ return;
+ }
+ if (m.at <= 0) {
+ // Already past the mark: fade out from here.
+ g.setValueAtTime(g.value, now);
+ g.linearRampToValueAtTime(0, now + MUTE_FADE / s.rate);
+ return;
+ }
+ g.setValueAtTime(1, now);
+ g.setValueAtTime(1, now + m.at);
+ g.linearRampToValueAtTime(0, now + m.at + m.fade);
+ }, []);
+
+ /**
+ * Put the mark at `t`, inside the selection. `audition` plays across it --
+ * three seconds of sound and two of what should now be silence -- the way a
+ * moved edge plays the edge it moved.
+ */
+ const placeMute = useCallback(
+ (t: number, audition: boolean) => {
+ const at = round2(Math.min(sel.to, Math.max(sel.from, t)));
+ setMute(at);
+ muteRef.current = at;
+ setMutePick(false);
+ if (audition) void play(Math.max(sel.from, at - 3), Math.min(sel.to, at + 2));
+ else retargetMute(at);
+ },
+ [sel.from, sel.to, play, retargetMute],
+ );
+
+ const clearMute = useCallback(() => {
+ setMute(null);
+ muteRef.current = null;
+ setMutePick(false);
+ retargetMute(null);
+ }, [retargetMute]);
+
+ /** `m`: where you are listening. Anywhere outside the selection is refused. */
+ const muteAtPlayhead = useCallback(() => {
+ if (playhead == null || playhead < sel.from - 0.02 || playhead > sel.to + 0.02) {
+ setNote("the playhead is not inside the selection — play to the spot, or pick it on the waveform");
+ return;
+ }
+ placeMute(playhead, false);
+ }, [playhead, sel.from, sel.to, placeMute]);
+
// ---- keyboard -----------------------------------------------------------
useEffect(() => {
const nudge = (which: "from" | "to", by: number) =>
@@ -841,12 +1339,14 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
case ">": nudge("to", step); break;
case " ":
e.preventDefault();
- play(sel.from, sel.to);
+ void play(sel.from, sel.to);
break;
case "r":
case "R":
setSel({ from: clip.start, to: clip.end });
setDirty(false);
+ setMute(clip.muteFrom);
+ muteRef.current = clip.muteFrom;
break;
// Walking the cut. Reviewing a whole video is nineteen clips in a row,
// and going back to the project page between each one is nineteen round
@@ -882,6 +1382,28 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
case "A":
choosePlayback((pb) => ({ ...pb, auto: !pb.auto }));
break;
+ // The mute mark: `m` at the playhead, `M` clears it, and `;` `'` move
+ // it like an edge (shift for 0.5 s), each playing across it.
+ case "m":
+ muteAtPlayhead();
+ break;
+ case "M":
+ clearMute();
+ break;
+ case ";":
+ case ":":
+ case "'":
+ case '"':
+ if (mute == null) {
+ setNote("no mute mark yet — m sets one at the playhead");
+ break;
+ }
+ placeMute(mute + (e.key === ";" || e.key === ":" ? -step : step), true);
+ break;
+ case "Escape":
+ if (!mutePick) return;
+ setMutePick(false);
+ break;
default:
return;
}
@@ -893,6 +1415,7 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
sel,
clip.start,
clip.end,
+ clip.muteFrom,
onSel,
play,
router,
@@ -902,6 +1425,11 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
confirmClip,
rejectClip,
choosePlayback,
+ mute,
+ mutePick,
+ muteAtPlayhead,
+ clearMute,
+ placeMute,
]);
const refresh = useCallback(async (): Promise<ClipBenchData | null> => {
@@ -1129,10 +1657,14 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
const saveWindow = useCallback(() => {
const patch = windowPatch();
void save(patch).then((ok) => {
- if (ok && patch.cutStart !== undefined)
- setNote("saved — the cut no longer fitted this window and was cleared");
+ if (!ok) return;
+ const cleared = [
+ patch.cutStart !== undefined ? "the cut" : null,
+ patch.muteFrom === "" && mute != null ? "the mute mark" : null,
+ ].filter(Boolean);
+ if (cleared.length) setNote(`saved — ${cleared.join(" and ")} no longer fitted this window and ${cleared.length > 1 ? "were" : "was"} cleared`);
});
- }, [windowPatch, save]);
+ }, [windowPatch, save, mute]);
// ---- the warnings --------------------------------------------------------
const endCue = cues.find((c) => sel.to >= c.start - 0.02 && sel.to <= c.end + 0.02) ?? null;
@@ -1373,29 +1905,38 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
unsaved — was {hms(clip.start)} – {hms(clip.end)}
</span>
)}
+ {muteMoved && (
+ <span data-mute-unsaved="" className="num text-[var(--color-dirty)]">
+ mute mark unsaved — was {clip.muteFrom == null ? "none" : hms(clip.muteFrom)}
+ </span>
+ )}
<span className="flex flex-wrap items-center gap-1.5">
<button
type="button"
+ data-save-window=""
className={buttonVariants({ variant: "primary", size: "sm" })}
- disabled={!dirty || !!busy}
+ disabled={!unsaved || !!busy}
onClick={saveWindow}
>
save window
</button>
<button
type="button"
+ data-play-selection=""
className={buttonVariants({ size: "sm" })}
- onClick={() => play(sel.from, sel.to)}
+ onClick={() => void play(sel.from, sel.to)}
>
play selection
</button>
<button
type="button"
className={buttonVariants({ size: "sm" })}
- disabled={!dirty}
+ disabled={!unsaved}
onClick={() => {
setSel({ from: clip.start, to: clip.end });
setDirty(false);
+ setMute(clip.muteFrom);
+ muteRef.current = clip.muteFrom;
}}
>
reset
@@ -1446,7 +1987,41 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
>
auto-audition {playback.auto ? "on" : "off"}
</button>
+ {/* What is playing, and what it measured. The attributes are the
+ bench's record of its last playback, on the audio clock --
+ the claim is "stops where the selection ends", and this is
+ where it can be checked. */}
+ <span
+ data-playback=""
+ data-audio-state={audio.state}
+ data-play-engine={playInfo?.engine ?? ""}
+ data-play-state={playInfo?.state ?? ""}
+ data-play-from={playInfo?.from ?? ""}
+ data-play-to={playInfo?.to ?? ""}
+ data-play-rate={playInfo?.rate ?? ""}
+ data-play-ctx-start={playInfo?.ctxStart ?? ""}
+ data-play-ctx-stop={playInfo?.ctxStop ?? ""}
+ data-play-ended-at={playInfo?.endedAt ?? ""}
+ className="micro"
+ title={
+ audio.state === "failed"
+ ? undefined
+ : "every bounded playback is the decoded audio, started and stopped on the audio clock: it ends where the selection ends, to the sample"
+ }
+ >
+ {audio.state === "ready"
+ ? "exact playback"
+ : audio.state === "loading"
+ ? "decoding the audio…"
+ : null}
+ </span>
</span>
+ {audio.state === "failed" && (
+ <span data-playback-fallback="" className="text-[11px] text-[var(--color-dirty)]">
+ exact playback unavailable — {audio.why}. The video plays instead, stopped by the
+ frame: up to half a frame either side of the end.
+ </span>
+ )}
{/* ---- EXTENT above, CUT here ----
The window row says how much of the recording is worth having.
This says what will actually play, and offers to derive it
@@ -1493,10 +2068,60 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
</button>
)}
</span>
+ {/* ---- the MUTE MARK ----
+ Beside the cut because it is the same kind of decision -- what
+ the finished clip does with its seconds -- and drafted like the
+ edges: heard at once, written by `save window` or `y`. */}
+ <span
+ data-mute={mute ?? ""}
+ className="flex flex-wrap items-center gap-1.5"
+ title="From the mark to the end of the clip the sound fades out and the picture plays on. Set it at the last silence before the ending sound."
+ >
+ {mute != null ? (
+ <span data-mute-from={mute} className="num text-[var(--color-text)]">
+ mute from {hms(mute)}{" "}
+ <span className="text-[var(--color-dim)]">
+ ({Math.max(0, sel.to - mute).toFixed(2)}s silent to the end)
+ </span>
+ </span>
+ ) : (
+ <span className="text-[var(--color-dim)]">sound to the end</span>
+ )}
+ <button
+ type="button"
+ data-mute-set=""
+ className={buttonVariants({ size: "sm" })}
+ title="put the mute mark at the playhead"
+ onClick={muteAtPlayhead}
+ >
+ mute from here
+ </button>
+ <button
+ type="button"
+ data-mute-pick={mutePick ? "armed" : ""}
+ aria-pressed={mutePick}
+ className={buttonVariants({ variant: mutePick ? "primary" : "outline", size: "sm" })}
+ title="the next click on the waveform places the mute mark (Esc cancels)"
+ onClick={() => setMutePick((v) => !v)}
+ >
+ {mutePick ? "click the waveform…" : "pick on waveform"}
+ </button>
+ {mute != null && (
+ <button
+ type="button"
+ data-mute-clear=""
+ className={buttonVariants({ size: "sm" })}
+ onClick={clearMute}
+ >
+ clear mute
+ </button>
+ )}
+ </span>
<span className="micro">
<kbd>[</kbd> <kbd>]</kbd> start · <kbd>,</kbd> <kbd>.</kbd> end — each plays the edge
it moved · <kbd>space</kbd> the whole selection · <kbd>R</kbd> reset · <kbd>-</kbd>{" "}
- <kbd>=</kbd> speed · <kbd>a</kbd> auto — shift for 0.5s
+ <kbd>=</kbd> speed · <kbd>a</kbd> auto · <kbd>m</kbd> mute from the playhead,{" "}
+ <kbd>;</kbd> <kbd>'</kbd> move it, <kbd>M</kbd> clear — shift for 0.5s
</span>
{busy && <span className="text-[var(--color-meter)]">{busy}</span>}
{note && (
@@ -1507,19 +2132,58 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
</div>
{/* ---- the instrument ---- */}
- <Waveform
- view={view}
- sel={sel}
- cand={{ from: clip.start, to: clip.end }}
- words={cues.map((c) => ({ start: c.start, end: c.end, w: c.text.slice(0, 24) }))}
- peaks={peaks}
- playhead={playhead}
- height={104}
- onSel={onSel}
- onReachEdge={() => {
- /* widening is a FETCH here, not a redraw -- see the button below */
- }}
- />
+ <div className="relative">
+ <Waveform
+ view={view}
+ sel={sel}
+ cand={{ from: clip.start, to: clip.end }}
+ words={cues.map((c) => ({ start: c.start, end: c.end, w: c.text.slice(0, 24) }))}
+ peaks={peaks}
+ playhead={playhead}
+ height={104}
+ onSel={onSel}
+ onReachEdge={() => {
+ /* widening is a FETCH here, not a redraw -- see the button below */
+ }}
+ />
+ {/* The mute mark over the waveform: a line where the sound stops
+ and a hatch over what is silent to the end of the selection.
+ In the interaction colour, like the handles -- it is an edit,
+ not a reading -- and never in the way of a drag. */}
+ {mute != null && mute >= view.from && mute <= view.to && (
+ <>
+ <div
+ data-mute-span=""
+ className="pointer-events-none absolute top-0 h-full [background:repeating-linear-gradient(135deg,color-mix(in_srgb,var(--color-dim)_22%,transparent)_0_3px,transparent_3px_7px)]"
+ style={{
+ left: pct(mute),
+ width: `calc(${pct(Math.max(mute, sel.to))} - ${pct(mute)})`,
+ }}
+ />
+ <div
+ data-mute-marker={mute}
+ className="pointer-events-none absolute top-0 h-full border-l-2 border-dashed border-[var(--color-sel)]"
+ style={{ left: pct(mute) }}
+ >
+ <span className="absolute left-1 top-0.5 rounded bg-[var(--color-panel-2)] px-1 font-mono text-[9px] leading-tight text-[var(--color-sel)]">
+ mute
+ </span>
+ </div>
+ </>
+ )}
+ {/* Armed: the next click places the mark rather than an edge. */}
+ {mutePick && (
+ <div
+ data-mute-pick-layer=""
+ className="absolute inset-0 cursor-crosshair rounded-md outline outline-1 outline-[var(--color-sel)]"
+ onPointerDown={(e) => {
+ const r = e.currentTarget.getBoundingClientRect();
+ const frac = Math.min(1, Math.max(0, (e.clientX - r.left) / Math.max(1, r.width)));
+ placeMute(view.from + frac * span, true);
+ }}
+ />
+ )}
+ </div>
{/* Offered at the edge of the cache, as always -- and also whenever
the rail has something to read past it. Having just read the next
@@ -1767,7 +2431,7 @@ export default function ClipBench({ data }: { data: ClipBenchData }) {
if (!atMaxPad) void fetchMore(padsForCue(c));
return;
}
- play(c.start, Math.min(c.end + 4, view.to));
+ void play(c.start, Math.min(c.end + 4, view.to));
}}
className={`flex w-full gap-2 px-2 py-1 text-left hover:bg-[color-mix(in_srgb,var(--color-sel)_10%,transparent)] ${
inSel ? "bg-[color-mix(in_srgb,var(--color-sel)_14%,transparent)]" : ""
diff --git a/umtool/components/projects/ClipBenchPage.tsx b/umtool/components/projects/ClipBenchPage.tsx
@@ -88,6 +88,11 @@ export default async function ClipBenchPage({
cutStart: entry.cutStart ?? null,
cutEnd: entry.cutEnd ?? null,
lockCut: !!entry.lockCut,
+ // The mute mark, from the manifest's own entry: readClipDetail's rows
+ // carry the fields it reads, and this is not one of them.
+ muteFrom:
+ ((manifest.timeline ?? []) as { id: string; muteFrom?: number }[]).find((e) => e.id === clipId)
+ ?.muteFrom ?? null,
verdict:
entry.verdict === "confirmed" || entry.verdict === "incorrect" ? entry.verdict : null,
// From the manifest's own entry: what the on-screen panel says over this
diff --git a/umtool/e2e/clip-bench.spec.ts b/umtool/e2e/clip-bench.spec.ts
@@ -57,6 +57,7 @@ const readClipIn = (project: string, id: string) => {
verdict?: string;
cutStart?: number;
cutEnd?: number;
+ muteFrom?: number;
}[];
};
return m.timeline.find((e) => e.id === id)!;
@@ -804,13 +805,265 @@ test("moving the end auditions the END", async ({ page }) => {
await page.locator("body").press(".");
// Clamped to the cached file, which ends at 9.00 -- a drag never downloads.
const to = Math.min(before.end + 0.05, 9);
- const t = await page
- .getByTestId("clip-video")
- .evaluate((el: HTMLVideoElement) => el.currentTime);
- // vid1_0.00-9.00 starts at 0.00, so file time IS source time here. The four
- // seconds ENDING on the new edge, not the four after the start.
- expect(t).toBeGreaterThan(to - 4 - 0.4);
- expect(t).toBeLessThan(to - 4 + 2);
+ // The four seconds ENDING on the new edge, not the four after the start --
+ // read from the bench's own record of what it scheduled, on the audio clock.
+ const playback = page.locator("[data-playback]");
+ await expect(playback).toHaveAttribute("data-play-engine", "webaudio", { timeout: 15_000 });
+ expect(Number(await playback.getAttribute("data-play-to"))).toBeCloseTo(to, 2);
+ expect(Number(await playback.getAttribute("data-play-from"))).toBeCloseTo(to - 4, 2);
+ // And the picture follows it there, muted: vid1_0.00-9.00 starts at 0.00,
+ // so file time IS source time.
+ await expect
+ .poll(() => page.getByTestId("clip-video").evaluate((el: HTMLVideoElement) => el.currentTime))
+ .toBeGreaterThan(to - 4 - 0.4);
+});
+
+// ---------------------------------------------------------------------------
+// Exact playback.
+//
+// Every bounded range plays from the DECODED audio with Web Audio, started and
+// stopped on the audio clock. The element it replaces was stopped from
+// `timeupdate`, which overran the end by up to a quarter of a second -- the
+// difference between a cut between two words and one into the next.
+//
+// The proof is the signal, not the bench's word for it: TAP (below) is an
+// AudioWorklet put between the bench's audio and the speakers that records the
+// first and last NON-ZERO frame it is handed, on the context's own frame
+// clock. The bench publishes when it scheduled the stop (`data-play-ctx-stop`);
+// the two must agree to within one render quantum (128 frames).
+//
+// vid1 is a 440 Hz tone with silences at 2.9-3.1 and 5.9-6.1, so a selection
+// ending at 5.00 ends INSIDE the tone: the last sound is the stop, not a
+// silence that happened to come first.
+// ---------------------------------------------------------------------------
+
+const TAP = `(() => {
+ const code = \`class Tap extends AudioWorkletProcessor {
+ constructor() { super(); this.first = -1; this.last = -1;
+ this.port.onmessage = (e) => {
+ if (e.data === "reset") { this.first = -1; this.last = -1; }
+ if (e.data === "read") this.port.postMessage({ first: this.first, last: this.last, sr: sampleRate });
+ };
+ }
+ process(inputs) {
+ const ch = inputs[0] && inputs[0][0];
+ if (ch) for (let i = 0; i < ch.length; i += 1) if (ch[i] !== 0) {
+ const f = currentFrame + i; if (this.first < 0) this.first = f; this.last = f;
+ }
+ return true;
+ }
+ }
+ registerProcessor("tap", Tap);\`;
+ const url = URL.createObjectURL(new Blob([code], { type: "application/javascript" }));
+ const Orig = window.AudioContext;
+ if (!Orig) return;
+ const conn = AudioNode.prototype.connect;
+ window.__tapRead = () => new Promise((res) => {
+ const n = window.__tapNode;
+ if (!n) return res(null);
+ n.port.onmessage = (e) => res(e.data);
+ n.port.postMessage("read");
+ });
+ window.__tapReset = () => window.__tapNode && window.__tapNode.port.postMessage("reset");
+ window.AudioContext = class extends Orig {
+ constructor(...a) {
+ super(...a);
+ const ctx = this;
+ ctx.audioWorklet.addModule(url).then(() => {
+ const n = new AudioWorkletNode(ctx, "tap", { outputChannelCount: [1] });
+ conn.call(n, ctx.destination);
+ ctx.__tap = n;
+ window.__tapNode = n;
+ for (const src of ctx.__pending || []) conn.call(src, n);
+ ctx.__pending = [];
+ });
+ }
+ };
+ AudioNode.prototype.connect = function (dest, ...rest) {
+ const r = conn.call(this, dest, ...rest);
+ const tap = this.context && this.context.__tap;
+ if (dest === this.context.destination && this !== tap) {
+ if (tap) conn.call(this, tap);
+ else (this.context.__pending = this.context.__pending || []).push(this);
+ }
+ return r;
+ };
+})();`;
+
+type Tap = { first: number; last: number; sr: number } | null;
+const tapRead = (page: import("@playwright/test").Page) =>
+ page.evaluate(() => (window as unknown as { __tapRead: () => Promise<Tap> }).__tapRead());
+const tapReset = (page: import("@playwright/test").Page) =>
+ page.evaluate(() => (window as unknown as { __tapReset: () => void }).__tapReset());
+
+/** The bench's record of its last playback, as numbers. */
+const lastPlay = async (page: import("@playwright/test").Page) => {
+ const p = page.locator("[data-playback]");
+ const n = async (k: string) => Number(await p.getAttribute(`data-play-${k}`));
+ return {
+ engine: await p.getAttribute("data-play-engine"),
+ from: await n("from"),
+ to: await n("to"),
+ rate: await n("rate"),
+ ctxStart: await n("ctx-start"),
+ ctxStop: await n("ctx-stop"),
+ };
+};
+
+/** Start a playback with `go` and wait until the bench says it has stopped. */
+const playThrough = async (page: import("@playwright/test").Page, go: () => Promise<void>) => {
+ const p = page.locator("[data-playback]");
+ await tapReset(page);
+ await go();
+ await expect(p).toHaveAttribute("data-play-state", "playing", { timeout: 10_000 });
+ await expect(p).toHaveAttribute("data-play-state", "stopped", { timeout: 20_000 });
+ // `ended` reaches the page a task after the audio thread stopped; the tap's
+ // reply is one more message behind it.
+ await page.waitForTimeout(100);
+ return { play: await lastPlay(page), tap: await tapRead(page) };
+};
+
+/** Put c01 back to its fixture window with no mute mark. */
+const resetC01 = async (request: import("@playwright/test").APIRequestContext) => {
+ const { token: t } = await token(request, "c01");
+ const r = await request.put("/api/report/window", {
+ data: { project: PROJECT, clip: "c01", start: 3, end: 6, muteFrom: "", token: t },
+ });
+ expect(r.ok()).toBeTruthy();
+};
+
+test("a play-selection stops within one audio render quantum of its end, at any speed", async ({
+ page,
+ request,
+}) => {
+ await resetC01(request);
+ await page.addInitScript(TAP);
+ await page.goto(bench("c01"));
+ await expect(page.locator("[data-playback]")).toHaveAttribute("data-audio-state", "ready", {
+ timeout: 15_000,
+ });
+ await keyboardLive(page);
+ // 6.00 -> 5.00 (shift-, is half a second): the end now lies in the tone.
+ // Each press auditions the end it moved, which also opens the audio output
+ // and puts the tap in the path.
+ await page.locator("body").press("Shift+Comma");
+ await page.locator("body").press("Shift+Comma");
+ await expect
+ .poll(async () => Number(await page.locator("[data-playback]").getAttribute("data-play-to")), {
+ timeout: 10_000,
+ })
+ .toBeCloseTo(5, 3);
+ await expect(page.locator("[data-playback]")).toHaveAttribute("data-play-state", "stopped", {
+ timeout: 10_000,
+ });
+
+ for (const rate of ["1", "2"]) {
+ await page.locator("[data-playback-rate]").selectOption(rate);
+ const { play, tap } = await playThrough(page, () => page.locator("[data-play-selection]").click());
+ expect(play.engine).toBe("webaudio");
+ expect(play.from).toBeCloseTo(3, 3);
+ expect(play.to).toBeCloseTo(5, 3);
+ expect(play.rate).toBe(Number(rate));
+ expect(tap, "the tap saw the bench's audio").not.toBeNull();
+ const { first, last, sr } = tap!;
+ expect(last).toBeGreaterThan(first);
+ const quantum = 128;
+ // The STOP: the last sound is the scheduled stop, to within a quantum.
+ const stopFrame = Math.round(play.ctxStop * sr);
+ expect(Math.abs(last + 1 - stopFrame), `stopped ${last + 1 - stopFrame} frames from the end at ${rate}x`).toBeLessThanOrEqual(quantum);
+ // And what played was the selection: two seconds of source, at any speed.
+ const playedSource = ((last + 1 - first) / sr) * Number(rate);
+ expect(Math.abs(playedSource - 2) * sr).toBeLessThanOrEqual(quantum * Number(rate) + 1);
+ }
+});
+
+test("the mute mark: picked on the waveform, heard at once, saved, shown, cleared", async ({
+ page,
+ request,
+}) => {
+ await resetC01(request);
+ await page.addInitScript(TAP);
+ await page.goto(bench("c01"));
+ const playback = page.locator("[data-playback]");
+ await expect(playback).toHaveAttribute("data-audio-state", "ready", { timeout: 15_000 });
+ await expect(page.locator("[data-mute]")).toHaveAttribute("data-mute", "");
+ await expect(page.locator("[data-mute-marker]")).toHaveCount(0);
+
+ // Armed, the next click on the waveform places the mark: at 5.00 of the
+ // cached 0.00-9.00, the tone between the two silences.
+ await page.locator("[data-mute-pick]").click();
+ await expect(page.locator("[data-mute-pick=armed]")).toBeVisible();
+ const layer = page.locator("[data-mute-pick-layer]");
+ const box = (await layer.boundingBox())!;
+ const { play, tap } = await playThrough(page, () =>
+ page.mouse.click(box.x + (box.width * 5) / 9, box.y + box.height / 2),
+ );
+ const marker = page.locator("[data-mute-marker]");
+ await expect(marker).toBeVisible();
+ const mark = Number(await marker.getAttribute("data-mute-marker"));
+ expect(mark).toBeGreaterThan(4.9);
+ expect(mark).toBeLessThan(5.1);
+ await expect(page.locator("[data-mute-pick-layer]")).toHaveCount(0);
+ await expect(page.locator("[data-mute-unsaved]")).toContainText("was none");
+
+ // The pick plays ACROSS the mark -- up to three seconds before it (here
+ // from the selection's start), to the end of the selection -- and the sound
+ // goes at the mark: the last non-zero frame is where the fade (0.05 s) runs
+ // out, not the 6.00 the tone runs on to.
+ expect(play.engine).toBe("webaudio");
+ expect(play.from).toBeCloseTo(Math.max(3, mark - 3), 2);
+ expect(play.to).toBeCloseTo(6, 2);
+ const { last, sr } = tap!;
+ const silentAt = play.ctxStart + (mark - play.from) + 0.05;
+ expect(Math.abs((last + 1) / sr - silentAt), "silent at the mark, not at the end").toBeLessThan(0.01);
+
+ // Saved by `save window`, like the edges; the manifest has it.
+ await page.locator("[data-save-window]").click();
+ await expect(page.locator("[data-bench-note]")).toContainText("saved");
+ expect(readClip("c01").muteFrom).toBeCloseTo(mark, 2);
+ await expect(page.locator("[data-mute-unsaved]")).toHaveCount(0);
+
+ // A reload shows the saved mark.
+ await page.reload();
+ await expect(page.locator("[data-mute-marker]")).toHaveAttribute("data-mute-marker", String(readClip("c01").muteFrom));
+ await expect(page.locator("[data-mute-from]")).toBeVisible();
+
+ // Moved like an edge, and cleared.
+ await keyboardLive(page);
+ await page.locator("body").press("Shift+Semicolon");
+ await expect(page.locator("[data-mute-marker]")).toHaveAttribute(
+ "data-mute-marker",
+ String(Number((readClip("c01").muteFrom! - 0.5).toFixed(2))),
+ );
+ await page.locator("[data-mute-clear]").click();
+ await expect(page.locator("[data-mute-marker]")).toHaveCount(0);
+ await expect(page.locator("[data-mute-unsaved]")).toBeVisible();
+ await page.locator("[data-save-window]").click();
+ await expect(page.locator("[data-bench-note]")).toContainText("saved");
+ expect(readClip("c01").muteFrom).toBeUndefined();
+
+ // The writer's rule, through the route: inside the clip, or refused.
+ const { token: t } = await token(request, "c01");
+ const out = await request.put("/api/report/window", {
+ data: { project: PROJECT, clip: "c01", muteFrom: 8, token: t },
+ });
+ expect(out.status()).toBe(400);
+ expect(((await out.json()) as { error: string }).error).toMatch(/must lie inside the clip 3–6/);
+});
+
+test("a decode that fails falls back to the element, and the bench says so", async ({ page }) => {
+ await page.route("**/api/report/audio**", (r) =>
+ r.fulfill({ status: 422, json: { error: "this file has no audio track" } }),
+ );
+ await page.goto(bench("c01"));
+ const fallback = page.locator("[data-playback-fallback]");
+ await expect(fallback).toBeVisible({ timeout: 15_000 });
+ await expect(fallback).toContainText("this file has no audio track");
+ await page.locator("[data-play-selection]").click();
+ await expect(page.locator("[data-playback]")).toHaveAttribute("data-play-engine", "element");
+ await expect
+ .poll(() => page.getByTestId("clip-video").evaluate((el: HTMLVideoElement) => !el.paused))
+ .toBe(true);
});
test("the playback speed is this browser's, and it survives a reload", async ({ page }) => {
@@ -841,6 +1094,11 @@ test("auto-audition plays the clip you walk onto", async ({ page, request }) =>
await page.locator("[data-clip-nav=next]").click();
await expect(page.locator("[data-bench=c04]")).toBeVisible();
+ // The whole clip, from the decoded audio -- and the picture with it.
+ const playback = page.locator("[data-playback]");
+ await expect(playback).toHaveAttribute("data-play-engine", "webaudio", { timeout: 15_000 });
+ expect(Number(await playback.getAttribute("data-play-from"))).toBeCloseTo(15, 2);
+ expect(Number(await playback.getAttribute("data-play-to"))).toBeCloseTo(18, 2);
await expect
.poll(
() => page.getByTestId("clip-video").evaluate((el: HTMLVideoElement) => !el.paused),
