"use client"; import AnchoredNotes from "@/components/notes/AnchoredNotes"; import GeneratedBanner from "@/components/notes/GeneratedBanner"; import { useSharedNotes, wroteNotes } from "@/components/notes/NotesProvider"; import { useCallback, useEffect, useRef, useState } from "react"; import Link from "next/link"; import { useRouter } from "next/navigation"; import Waveform, { type Peaks } from "@/components/Waveform"; import { badgeVariants } from "@/components/ui/badge"; import { buttonVariants } from "@/components/ui/button"; // The renderer's own line, imported rather than re-written. A preview that // disagrees with the header by a character is worth less than no preview: you // would only find out twenty minutes into a build. import { attributionLine } from "umtool-report-to-video/attribution"; import { MUTE_FADE, START_LEAD, bufferSchedule, covers, decodeSpan, elementShouldStop, muteRamp, playheadAt, } from "@/lib/report/playback.mjs"; import { DeckFrame, NeutralFrame, composePreview, midOf, onscreenValue, type DeckPreviewDoc, type DeckTexts, type Onscreen, } from "./OnscreenSection"; // --------------------------------------------------------------------------- // Editing a clip's window against the audio and the words. // // "How much context does this clip need" was a loop of hand-editing JSON, // re-running two CLIs and watching an mp4. This is that loop, in one place. // // EVERYTHING IS IN ABSOLUTE SOURCE SECONDS -- the manifest's, the cue file's, // the QR's. The cached file's own start (`fetchStart`) is the only relative // number in the whole component, and it exists solely to set video.currentTime. // The moment those two are allowed to mix is the moment a window is off by the // pad and nobody can see why. // // A DRAG NEVER DOWNLOADS. Dragging past the cached window clamps and offers a // button, because a handle that silently starts a 12-second network fetch is a // handle you stop trusting. // // THE EDGES AND THE ATTRIBUTION ARE ONE SITTING. Watching a clip is when you // find out both that it starts too late and that the header calls it by the // archive's title and the archive's upload date -- which for a VOD mirror is // years after the stream. So the fields that decide the burned-in line are here // beside the handles, with a live preview of exactly what ffmpeg will draw, and // a button to render this one clip and look at it. // --------------------------------------------------------------------------- type Cue = { start: number; end: number; text: string; endsSentence: boolean }; /** * Another clip in the cut from the SAME recording. * * The question this answers is the one you cannot answer from inside a single * clip: should this window be wider, or is what it is missing already in the * cut a few entries later? Computed server-side from the manifest -- see * siblingsOf() -- because array order IS the cut and nothing here should be * re-deriving that. */ type Sibling = { id: string; start: number; end: number; cite: number | null; quote: string; note: string | null; steps: number; where: string; side: "after" | "before" | "overlap"; gap: number | null; }; type Win = { name: string; from: number; to: number }; type Clip = { id: string; video: string; channel: string | null; start: number; end: number; cite: number | null; quote: string | null; note: string | null; /** What the REPORT got wrong about this clip. Never rendered. */ correction: string | null; /** The attribution overrides. Absent means "use the archived record's own". */ title: string | null; date: string | null; citeUrl: string | null; lock: boolean; lockStart: boolean; lockEnd: boolean; /** * The CUT: what the video actually plays, inside the reviewed extent. * * `start`/`end` is how much of this recording is worth having -- a judgement * about the material. The cut is a different question: the sentences the * quote is made of. Absent means "the whole extent plays", which is what * every manifest did before the two were separated. */ cutStart: number | null; cutEnd: number | null; /** The cut is deliberate; `resolve-windows --cut-to-quote` leaves it alone. */ lockCut: boolean; /** * The MUTE MARK, in source seconds: from here to the end of the clip the * sound fades out and the picture plays on. Set by ear, at the last silence * before a finale's ending sound. Absent means the clip plays with its sound. */ muteFrom: number | null; /** "confirmed" / "incorrect", or null for "nobody has looked at this yet". */ verdict: "confirmed" | "incorrect" | null; /** What the on-screen panel says over this clip. Absent means the auto text. */ onscreen: Onscreen | null; }; /** The on-screen fields as the inputs hold them. */ type OsDraft = { title: string; subtitle: string }; const osDraftOf = (o: Onscreen | null): OsDraft => ({ title: o?.title ?? "", subtitle: o?.subtitle ?? "" }); const sameOs = (a: OsDraft, b: OsDraft) => a.title.trim() === b.title.trim() && a.subtitle.trim() === b.subtitle.trim(); /** * Has anybody looked at this clip, and did they agree with the description? * * The same three answers lib/projects/report.mjs's clipVerdict() reads out of * the manifest, from the copy this component holds -- LEGACY entries included: * a `correction` with no verdict was written before `incorrect` existed and * means exactly that. */ const verdictOf = (c: Clip): "incorrect" | "confirmed" | "unreviewed" => { if (c.verdict === "confirmed" || c.verdict === "incorrect") return c.verdict; return (c.correction ?? "").trim() ? "incorrect" : "unreviewed"; }; /** * What the state line says. * * "confirmed · with note" is its own reading rather than a badge somewhere * else: a note on a good clip -- why its window moved, a caveat for the * writers -- must never be mistaken for a complaint about the clip. */ const verdictLabel = (c: Clip): string => { const v = verdictOf(c); if (v === "confirmed") return (c.correction ?? "").trim() ? "confirmed · with note" : "confirmed"; return v === "incorrect" ? "incorrect" : "not yet reviewed"; }; /** The five fields that decide the burned-in header and the QR. */ const ATTRIB = [ ["title", "title", "What the header calls this stream. Empty uses the archived record's own title, cleaned."], ["date", "date", "The stream's own date, YYYY-MM-DD. Empty uses the archived copy's upload date — which for a VOD mirror is often years later."], ["cite", "cite", "The second the header prints, in absolute source seconds. Empty uses the clip's start."], ["citeUrl", "citeUrl", "Where the QR points. Empty derives it from the archive. Set it when the clip is cut from a mirror that reads better."], ["quote", "quote", "The words this clip exists for. Not drawn on screen — it is what the cut is checked against."], ["correction", "correction", "On an incorrect clip: what the report got wrong — wrong speaker, wrong addressee, wrong date, read by the next report pass. On a confirmed one it is just a note: why the window moved, or something a reader should know. Never rendered."], ] as const; type AttribKey = (typeof ATTRIB)[number][0]; /** A field's SAVED value, as the input shows it. */ const attribValue = (c: Clip, k: AttribKey): string => k === "cite" ? (c.cite == null ? "" : String(c.cite)) : (c[k] ?? ""); const emptyDraft = (c: Clip) => Object.fromEntries(ATTRIB.map(([k]) => [k, attribValue(c, k)])) as Record; /** * A saved entry, back into the shape this component holds. * * Explicit rather than a spread: an empty value DELETES the key, so the entry * that comes back has no `title` at all — and `{...clip, ...entry}` would leave * the old one on screen as though the deletion had not happened. */ const fromEntry = (prev: Clip, e: Record): Clip => ({ ...prev, start: Number(e.start), end: Number(e.end), cite: e.cite == null ? null : Number(e.cite), quote: (e.quote as string) ?? null, note: (e.note as string) ?? null, correction: (e.correction as string) ?? null, title: (e.title as string) ?? null, date: (e.date as string) ?? null, citeUrl: (e.citeUrl as string) ?? null, lock: !!e.lock, lockStart: !!e.lockStart, lockEnd: !!e.lockEnd, cutStart: e.cutStart == null ? null : Number(e.cutStart), cutEnd: e.cutEnd == null ? null : Number(e.cutEnd), lockCut: !!e.lockCut, muteFrom: e.muteFrom == null ? null : Number(e.muteFrom), verdict: e.verdict === "confirmed" || e.verdict === "incorrect" ? e.verdict : null, onscreen: (e.onscreen as Onscreen | undefined) ?? null, }); export type ClipBenchData = { project: string; clip: Clip; view: { from: number; to: number }; windows: Win[]; proposed: { start: number; end: number } | null; endsSentence: boolean | null; noPunctuation: boolean; sourceDuration: number | null; segment: string | null; /** The built segment's mtime. In the URL, so a re-render busts the cache. */ segmentMtime: number | null; /** The archived record's own title (already cleaned) and upload date. */ sourceTitle: string | null; uploadDate: string | null; /** Who the header will name, already resolved server-side. Heads the line. */ sourceChannel: string | null; /** * Neighbours ON THE WALK, computed server-side: the nearest clip either side * that still needs judgement and is fetched. Null when there is none that * way, which is what "first" / "last" say. Not the neighbours in the cut -- * a clip with nothing to play is a dead end, and one already answered is the * round trip the walk exists to remove. */ prev: string | null; next: string | null; /** Where this clip sits in the cut, 1-based, and how long the cut is. */ index: number; total: number; /** * How many OTHER clips have been reviewed. The count this clip contributes is * computed here, so pressing `y` moves the progress reading without a reload * -- and without this component having to guess at the rest of the cut. */ reviewedOthers: number; /** * The walk's own reading: `ready` clips are fetched AND still need judgement, * out of `needing` that need it at all. The gap between the two numbers is * what is waiting on a download rather than on you. */ ready: number; needing: number; /** Is there a cached file holding this clip's window, end to end? */ fetched: boolean; cues: Cue[]; token: string | null; fetchPad: number; /** The widest pad `/api/report/fetch` will accept. */ maxPad: number; /** A /mix deep link for this clip's cached window, or null. Server-built. */ mixHref: string | null; /** Every other clip in the cut from this same recording, by start. */ siblings: Sibling[]; /** Is the cut built with the on-screen panel (`render.chrome`)? */ deckOn: boolean; /** The manifest's `generatedBy`: edits here are noted for the agent that runs it. */ generatedBy?: string | null; }; const hms = (t: number) => { const s = Math.max(0, t); const m = Math.floor(s / 60); const sec = s - m * 60; const h = Math.floor(m / 60); return h > 0 ? `${h}:${String(m % 60).padStart(2, "0")}:${sec.toFixed(2).padStart(5, "0")}` : `${m}:${sec.toFixed(2).padStart(5, "0")}`; }; const round2 = (n: number) => Number(n.toFixed(2)); // --------------------------------------------------------------------------- // Playback preferences. // // Per BROWSER, never per manifest: how fast somebody listens and whether a // clip plays itself on arrival are facts about the person at the desk, not // about the cut. localStorage, read inside a try/catch because a private // window throws on access rather than returning null. // --------------------------------------------------------------------------- const PLAYBACK_KEY = "umtool.bench.playback"; const RATES = [0.75, 1, 1.25, 1.5, 2] as const; type Playback = { rate: number; auto: boolean }; const PLAYBACK_DEFAULT: Playback = { rate: 1, auto: false }; const readPlayback = (): Playback => { try { const raw = localStorage.getItem(PLAYBACK_KEY); if (!raw) return PLAYBACK_DEFAULT; const j = JSON.parse(raw) as Partial; const rate = Number(j.rate); return { rate: RATES.includes(rate as (typeof RATES)[number]) ? rate : PLAYBACK_DEFAULT.rate, auto: !!j.auto, }; } catch { return PLAYBACK_DEFAULT; } }; /** How much of a moved edge to play: enough to hear the join, not the clip. */ const EDGE_AUDITION = 4; /** How far past the cached window the cue rail reads ahead, per step. */ const PEEK_STEP = 60; /** * Close enough that the gap is worth a sentence rather than a number. * * Under this, "the next 12 s are not in the cut" is a decision waiting to be * made -- widen this clip, or leave the hole because the next clip picks it * up. Over it the two clips are simply elsewhere in the same stream. */ const NEAR_SIBLING = 15; /** m:ss, for a list of positions rather than a pair of edges. */ const clock = (t: number) => { const s = Math.max(0, Math.floor(t)); const m = Math.floor(s / 60); return `${Math.floor(m / 60) > 0 ? `${Math.floor(m / 60)}:${String(m % 60).padStart(2, "0")}` : m}:${String(s % 60).padStart(2, "0")}`; }; /** What `save window` (and a confirmation that moved something) writes. */ type WindowPatch = { start: number; end: number; cutStart?: string; cutEnd?: string; /** A number to set the mark, "" to clear it; absent leaves it alone. */ muteFrom?: number | string; }; /** A decoded span of the cached window, placed on the source clock. */ type Decoded = { name: string; span: { from: number; to: number }; buf: AudioBuffer }; /** * What is playing, and what the last playback measured. * * On the page as data attributes (`data-play-*`), because "did it stop where * the selection ends" is the claim this bench now makes, and a spec has to be * able to check it against the audio clock rather than against a feeling. */ type PlayInfo = { engine: "webaudio" | "element"; state: "playing" | "stopped"; from: number; /** The scheduled end, in source seconds. */ to: number; rate: number; /** Context clock: when the source starts and when it is told to stop. */ ctxStart: number | null; ctxStop: number | null; /** * Where the playback had reached, in source seconds, when the page heard it * end: the element's own position at its pause, or the context clock at * `ended` -- which reaches the page a task later than the sound stopped, so * for Web Audio it is an upper bound and `ctxStop` is the stop itself. */ endedAt: number | null; }; /** The playback in flight. One at a time: a new one stops the last. */ type Session = { gen: number; engine: "webaudio" | "element"; node: AudioBufferSourceNode | null; gain: GainNode | null; raf: number; timers: number[]; from: number; to: number; rate: number; t0: number; ctxStop: number; /** The element's own mute, put back when the picture stops following. */ mutedBefore: boolean; }; export default function ClipBench({ data }: { data: ClipBenchData }) { // The project's notes: this clip's row notes, and the edit notes a save on a // generated manifest leaves (the save says so, and they are re-read). const notes = useSharedNotes({ project: data.project }); const [clip, setClip] = useState(data.clip); const [windows, setWindows] = useState(data.windows); const [cues, setCues] = useState(data.cues); const [proposed, setProposed] = useState(data.proposed); const [view, setView] = useState(data.view); const [sel, setSel] = useState({ from: data.clip.start, to: data.clip.end }); // The mute mark as drafted. Like the edges it is unsaved until `save window` // or `y`, and it is what the bench's own playback mutes at -- so a mark is // heard before it is written. const [mute, setMute] = useState(data.clip.muteFrom); // Armed: the next click on the waveform places the mark instead of moving // an edge. const [mutePick, setMutePick] = useState(false); const [peaks, setPeaks] = useState(null); // The cues AROUND the cached window: what is coming, read before paying for // the media. One request, widened only when somebody asks. const [peek, setPeek] = useState([]); // The score the resolver last reported for this clip. Not stored in the // manifest: it is a fact about a matching run, not about the cut. const [cutScore, setCutScore] = useState(null); const [peekPad, setPeekPad] = useState(PEEK_STEP); const [playhead, setPlayhead] = useState(null); const playheadRef = useRef(null); useEffect(() => { playheadRef.current = playhead; }, [playhead]); const [note, setNote] = useState(null); const [busy, setBusy] = useState(null); const [dirty, setDirty] = useState(false); const [draft, setDraft] = useState>(() => emptyDraft(data.clip)); // `x` was pressed and nothing has been typed yet: the box is REQUIRED until // it has something in it. An incorrect verdict with no note is a complaint // nobody can act on, so the bench asks for the note before it writes one. const [needNote, setNeedNote] = useState(false); // Defaults on the server and on the first paint, then whatever this browser // remembers. Reading localStorage during render would be a hydration // mismatch on the one prop that changes what you hear. const [playback, setPlayback] = useState(PLAYBACK_DEFAULT); const [segment, setSegment] = useState(data.segment); const [segmentMtime, setSegmentMtime] = useState(data.segmentMtime); // ---- the on-screen panel ------------------------------------------------ // Its two fields, the text the panel says with them empty (`osAuto`, from // the On-screen route), and a composed preview of the whole cut's panel, of // which this bench shows this clip's segment. const [os, setOs] = useState(() => osDraftOf(data.clip.onscreen)); const [osAuto, setOsAuto] = useState(null); const [osMax, setOsMax] = useState(48); const [deckPreview, setDeckPreview] = useState(null); const [deckError, setDeckError] = useState(null); // The rendered strip is folded; its overlay only exists while it is open. const [renderedOpen, setRenderedOpen] = useState(false); const [segT, setSegT] = useState(null); const [segPaused, setSegPaused] = useState(true); const [segDur, setSegDur] = useState(0); const router = useRouter(); const video = useRef(null); // A REF, not state. Two saves can be in flight -- tab out of `title` straight // into `date` and both blur handlers fire -- and the second must carry the // token the first was given back, which a re-render has not delivered yet. const token = useRef(data.token); // And they must not interleave: same read-modify-write, one clip. const saving = useRef>(Promise.resolve(true)); // `x` answers "no" by putting the cursor in the note, which is the answer. const correctionBox = useRef(null); const segVideo = useRef(null); const transcriptBox = useRef(null); // One auto-audition per clip, keyed by id: `canplay` fires again after a // seek, and a clip that replays itself every time you drag is unusable. const autoPlayed = useRef(null); // The widest cached file is the one the bench draws from: it is how much room // there is to drag before anything has to be fetched. const cached = windows[0] ?? null; const fetchStart = cached?.from ?? 0; const cachedTo = cached?.to ?? 0; // ---- peaks -------------------------------------------------------------- useEffect(() => { if (!cached) return; let live = true; void (async () => { const r = await fetch( `/api/report/peaks?project=${encodeURIComponent(data.project)}&clip=${encodeURIComponent(clip.id)}&file=${encodeURIComponent(cached.name)}`, { cache: "no-store" }, ); if (!r.ok || !live) return; setPeaks((await r.json()) as Peaks); })(); return () => { live = false; }; }, [cached, data.project, clip.id]); // Follow the playhead in the reading pane, and ONLY while something is // playing: scrolling the words out from under somebody who is reading ahead // would be worse than not following at all. useEffect(() => { if (playhead == null) return; const box = transcriptBox.current; if (!box) return; const row = box.querySelector(`[data-tcue="${cues.find((c) => playhead >= c.start && playhead < c.end)?.start ?? -1}"]`); if (row) row.scrollIntoView({ block: "nearest" }); }, [playhead, cues]); // ---- reading ahead -------------------------------------------------------- // // The rail used to stop where the CACHE stops, so "should I fetch more" could // only be answered by fetching more. Cues are text from the archive and cost // nothing next to media, so the rail reads past both edges and marks what is // not on disk. Deciding to spend a download is then a decision about words // you have already read. useEffect(() => { let live = true; void (async () => { const r = await fetch( `/api/report/cues?project=${encodeURIComponent(data.project)}&clip=${encodeURIComponent(clip.id)}` + `&from=${(view.from - peekPad).toFixed(2)}&to=${(view.to + peekPad).toFixed(2)}`, { cache: "no-store" }, ); if (!r.ok || !live) return; const j = (await r.json()) as { cues: Cue[] }; setPeek(j.cues ?? []); })(); return () => { live = false; }; }, [data.project, clip.id, view.from, view.to, peekPad]); // ---- playback preferences ------------------------------------------------ useEffect(() => setPlayback(readPlayback()), []); // ---- the on-screen preview ---------------------------------------------- // // Composed once per clip: the cut's whole panel, from which the strip and // the overlay show this clip's segment. Typing never recomposes -- the // fields are patched in by message, before anything is saved. useEffect(() => { if (!data.deckOn) return; let live = true; void (async () => { const [pv, table] = await Promise.all([ composePreview(data.project, null, {}, { posts: false }), fetch(`/api/report/onscreen?project=${encodeURIComponent(data.project)}`, { cache: "no-store" }) .then((r) => (r.ok ? r.json() : null)) .catch(() => null) as Promise<{ rows?: { id: string; auto: OsDraft }[]; maxChars?: number; } | null>, ]); if (!live) return; if ("error" in pv) setDeckError(pv.error); else setDeckPreview(pv); const row = table?.rows?.find((r) => r.id === clip.id); if (row) setOsAuto(row.auto); if (table?.maxChars) setOsMax(table.maxChars); })(); return () => { live = false; }; }, [data.deckOn, data.project, clip.id]); // The rendered segment drives the overlay's clock: its own time, plus where // its segment starts in the cut. useEffect(() => { const el = segVideo.current; if (!el || !renderedOpen) return; const tick = () => setSegT(el.currentTime); const state = () => setSegPaused(el.paused); const meta = () => { setSegDur(el.duration || 0); // A quarter in, not zero: at the segment's first frame the PREVIOUS // clip's title is still handing over, which reads as the wrong title. if (el.currentTime === 0 && el.duration) el.currentTime = el.duration / 4; }; el.addEventListener("timeupdate", tick); el.addEventListener("seeked", tick); el.addEventListener("play", state); el.addEventListener("pause", state); el.addEventListener("loadedmetadata", meta); if (el.readyState >= 1) meta(); return () => { el.removeEventListener("timeupdate", tick); el.removeEventListener("seeked", tick); el.removeEventListener("play", state); el.removeEventListener("pause", state); el.removeEventListener("loadedmetadata", meta); }; }, [segment, segmentMtime, renderedOpen, deckPreview]); /** * Change a preference AND remember it. Never an effect on `playback`. * * That was the bug: an effect that persisted state on every change also ran * on the first render, when the state is still the DEFAULT -- so mounting * the bench wrote `auto: false` over what the browser remembered, and in * StrictMode's second mount the loader read back the value the first mount * had just clobbered. The setting survived the page you changed it on and * died on the next clip, which is the worst possible shape for a bug like * this. Writing only where somebody actually chose something cannot do that. * (The write inside the updater is idempotent, which is what StrictMode's * double-invoke requires of it.) */ const choosePlayback = useCallback((fn: (pb: Playback) => Playback) => { setPlayback((pb) => { const next = fn(pb); try { localStorage.setItem(PLAYBACK_KEY, JSON.stringify(next)); } catch { /* a private window refuses to store; the session still works */ } return next; }); }, []); useEffect(() => { // BOTH players. Listening to the cut at 1.5x and then to the rendered // segment at 1x is two different clips as far as your ear is concerned. // // defaultPlaybackRate as well as playbackRate, and that is the whole bug: // the media load algorithm resets playbackRate TO defaultPlaybackRate, so // a rate set before the element finished loading its source was silently // back at 1x by the time anything played. const apply = (el: HTMLVideoElement | null) => { if (!el) return; el.defaultPlaybackRate = playback.rate; el.playbackRate = playback.rate; }; apply(video.current); apply(segVideo.current); const el = video.current; if (!el) return; const again = () => apply(el); el.addEventListener("loadedmetadata", again); return () => el.removeEventListener("loadedmetadata", again); }, [playback, segment, cached, renderedOpen, deckPreview]); // ---- audition ----------------------------------------------------------- // // EVERY BOUNDED RANGE PLAYS FROM DECODED AUDIO. A selection, an edge, a line // of the transcript, the auto-audition: each is an AudioBufferSourceNode // started at an exact offset and stopped at an exact time on the audio // clock, so what you hear ends where the selection ends, to the sample. The //