"use client"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { autoCrop, boxInFrame, clampBox, CORNER_PX, FACE_REASONS, OTHER_CODE, reasonText, scaleOf, SHOVE_FLOOR, type Box, type FaceBox, type FaceJudgement, type FaceVerdict, } from "@/lib/face-types"; // --------------------------------------------------------------------------- // THE FACE JUDGER. // // Several shipped cover corners carry stream UI rather than a face. The // bottom-left of the Super Mario RPG cover is the clearest: a strip of YouTube // chrome, a dark block, and the webcam inset's own red border, with Jeremy in // the right 60% of the picture. // // This is the um sorter's lesson on a different axis. That deck exists because // ten of 23 free-text notes said "cut off" and six were filed as `not an um` -- // clips rejected for their WINDOW rather than for their CONTENT. Here a corner // gets rejected for its CROP rather than for its FACE, and the fix is the same: // make the crop a parameter you can see and move, instead of an output you can // only accept or discard. // // So the page is one candidate at a time, like the um deck, and the crop is // draggable. Three things are stated rather than implied, because each of them // is a number somebody would otherwise have to guess at: // // THE SHOVE how far the automatic crop sat off the face to stay inside the // frame. On the SMRPG corner that is ~92px, and those 92px ARE // the YouTube chrome. Unmeasured it is a mystery; measured it is // the sentence that explains what went wrong. // THE SCALE 300px of output over the crop's own height. Half the corners on // record already upscale, so tightening a crop to exclude UI // trades contamination for softness -- and that trade is the // human's to make, not this component's to make quietly. // THE SPEND a rejection costs a source episode out of a pool the // unique-face rule never gives back. // // The crop box the deck draws is computed by autoCrop(), a pure // re-implementation of facecrop.py's maths -- so what you see beside your own // framing is what the CLI would actually do, on every drag, with no round trip. // --------------------------------------------------------------------------- export type FaceEntryView = { key: string; video: string; srcStart: number; frameAt: number; covers: string[]; accepted: string[]; onAccepted: boolean; crop: Box | null; link: string | null; }; type Detection = { face: FaceBox | null; auto: Box | null; /** facecrop.py's own box, so the deck can prove it is drawing the right one. */ pyAuto: Box | null; agrees: boolean; shove: number; frame: { w: number; h: number }; error?: string; }; const HANDLE = 10; // display px within which a pointer grabs a corner const MIN_SIDE = 40; // source px; below this the corner is unusable anyway const fmt = (n: number) => n.toFixed(2); const boxAttr = (b: Box | null) => (b ? `${b.x},${b.y},${b.w},${b.h}` : ""); export default function FaceDeck({ queue, initial, rejected, start, }: { queue: FaceEntryView[]; initial: Record; rejected: string[]; start: string | null; }) { const startAt = Math.max(0, queue.findIndex((e) => e.key === start)); const [idx, setIdx] = useState(startAt); const [faces, setFaces] = useState(initial); const [spent, setSpent] = useState(rejected); // The opening crop is computed for the FIRST RENDER, not applied by an effect // afterwards. Server-rendering it is what makes a recorded crop present in the // HTML rather than appearing a beat later: an effect-applied box is invisible // to anything reading the page before hydration -- which is a real reader // here, since the deck is linked to with ?key= from a note and from the bench. const opening = queue[startAt]; const first = opening ? (initial[opening.key] ?? null) : null; // Detections are held per key for the life of the page. A second pass over a // corner already looked at must not cost another second of python. const [dets, setDets] = useState>({}); const [loading, setLoading] = useState(false); const [crop, setCrop] = useState(first?.crop ?? opening?.crop ?? null); const [pending, setPending] = useState(first?.verdict ?? null); const [code, setCode] = useState(first?.code ?? null); const [text, setText] = useState(first?.text ?? ""); const [guess, setGuess] = useState(null); const [busy, setBusy] = useState(false); const [error, setError] = useState(null); const canvas = useRef(null); const preview = useRef(null); const image = useRef(null); // Both of these are keyed CACHES rather than plain flags, and they are refs // rather than state for one reason: React's dev-mode double-invoke runs every // effect twice, and a guard held in state has not updated by the second run. // A corner already fetched must not be fetched again -- not on a re-mount, not // on the way back through the queue, and above all not twice on arrival. const images = useRef>(new Map()); const asked = useRef>(new Set()); const drag = useRef<{ mode: "move" | "resize"; ax: number; ay: number; ox: number; oy: number } | null>(null); const entry = queue[idx] ?? null; const det = entry ? dets[entry.key] : undefined; const judged = entry ? faces[entry.key] : undefined; const frame = det?.frame ?? null; // ---- the frame ------------------------------------------------------------ // Requested at NATIVE resolution, so the canvas's pixels ARE source pixels and // a dragged box needs no mapping to become a crop facecrop.py can cut. const frameSrc = entry ? `/api/face/frame?video=${encodeURIComponent(entry.video)}&at=${entry.frameAt}` : null; const [ready, setReady] = useState(false); useEffect(() => { if (!frameSrc) return; let img = images.current.get(frameSrc); if (!img) { img = new Image(); images.current.set(frameSrc, img); img.src = frameSrc; // the ONE request for this corner's frame } if (img.complete && img.naturalWidth > 0) { image.current = img; setReady(true); return; } setReady(false); const shown = img; const onLoad = () => { image.current = shown; setReady(true); }; shown.addEventListener("load", onLoad); return () => shown.removeEventListener("load", onLoad); }, [frameSrc]); // ---- the detection -------------------------------------------------------- // NOTHING SPAWNS ON LOAD. The detector is ~0.94s of python and the queue is 22 // rows; detecting all of them to draw a list would be twenty seconds of work // for twenty-one pictures nobody is looking at yet. useEffect(() => { if (!entry || dets[entry.key] || asked.current.has(entry.key)) return; asked.current.add(entry.key); let live = true; setLoading(true); fetch(`/api/face/detect?video=${encodeURIComponent(entry.video)}&at=${entry.frameAt}`, { cache: "no-store", }) .then(async (r) => { const j = (await r.json()) as Detection & { error?: string }; if (!live) return; setDets((d) => ({ ...d, [entry.key]: j })); }) .catch(() => { if (live) { setDets((d) => ({ ...d, [entry.key]: { face: null, auto: null, pyAuto: null, agrees: true, shove: 0, frame: { w: 0, h: 0 }, error: "the detector did not answer", }, })); } }) .finally(() => live && setLoading(false)); return () => { live = false; }; }, [entry, dets]); // ---- what the crop starts as --------------------------------------------- // A judgement already made wins, then the box the manifest recorded, then the // automatic one. That order is the point of the tool: a human framing is never // re-derived out from under itself. useEffect(() => { if (!entry) return; setPending(judged?.verdict ?? null); setCode(judged?.code ?? null); setText(judged?.text ?? ""); setGuess(null); setError(null); setCrop(judged?.crop ?? entry.crop ?? det?.auto ?? null); // Deliberately keyed on the CORNER, not on the detection: re-running this // when the detection lands would throw away a box already being dragged. // eslint-disable-next-line react-hooks/exhaustive-deps }, [entry?.key]); // The one case where a late detection may set the box: there was no box at all. useEffect(() => { if (crop === null && det?.auto) setCrop(det.auto); }, [crop, det]); // ---- drawing -------------------------------------------------------------- const draw = useCallback(() => { const c = canvas.current; const img = image.current; if (!c || !img) return; c.width = img.naturalWidth; c.height = img.naturalHeight; const ctx = c.getContext("2d"); if (!ctx) return; ctx.drawImage(img, 0, 0); const stroke = (b: Box, colour: string, dash: number[], width: number) => { ctx.save(); ctx.setLineDash(dash); ctx.lineWidth = width; ctx.strokeStyle = colour; ctx.strokeRect(b.x + 0.5, b.y + 0.5, b.w, b.h); ctx.restore(); }; if (det?.face) stroke(det.face, "rgba(132,150,168,0.75)", [4, 4], 2); // WHAT THE CLI WOULD DO, always drawn, so the diff between the automatic // crop and the chosen one is visible rather than remembered. if (det?.auto) stroke(det.auto, "rgba(210,153,34,0.85)", [10, 6], 3); if (guess) stroke(guess, "rgba(86,212,196,0.9)", [2, 6], 3); if (crop) { stroke(crop, "#58a6ff", [], 4); ctx.save(); ctx.fillStyle = "#58a6ff"; for (const [hx, hy] of [ [crop.x, crop.y], [crop.x + crop.w, crop.y], [crop.x, crop.y + crop.h], [crop.x + crop.w, crop.y + crop.h], ]) { ctx.fillRect(hx - 7, hy - 7, 14, 14); } ctx.restore(); } const p = preview.current; if (p && crop) { p.width = CORNER_PX; p.height = CORNER_PX; const pctx = p.getContext("2d"); if (pctx) { pctx.clearRect(0, 0, CORNER_PX, CORNER_PX); // The SAME resample the corner gets: source box in, 300x300 out. pctx.drawImage(img, crop.x, crop.y, crop.w, crop.h, 0, 0, CORNER_PX, CORNER_PX); } } }, [crop, det, guess]); useEffect(() => { draw(); }, [draw, ready]); // ---- dragging ------------------------------------------------------------- // Square-locked, because CORNER_W and CORNER_H are both 300 and a rectangle // would be squashed rather than cropped on the way in. const toSource = (e: React.PointerEvent) => { const c = canvas.current; if (!c) return { x: 0, y: 0, k: 1 }; const r = c.getBoundingClientRect(); // The canvas is drawn at SOURCE resolution and displayed at whatever the // layout gives it, so this ratio is the only mapping in the component -- // and it maps display pixels to source pixels, never the other way. const k = r.width > 0 ? c.width / r.width : 1; return { x: (e.clientX - r.left) * k, y: (e.clientY - r.top) * k, k }; }; const onDown = (e: React.PointerEvent) => { if (!crop || !frame) return; const { x, y, k } = toSource(e); const near = HANDLE * k; const corners: [number, number, number, number][] = [ [crop.x, crop.y, crop.x + crop.w, crop.y + crop.h], [crop.x + crop.w, crop.y, crop.x, crop.y + crop.h], [crop.x, crop.y + crop.h, crop.x + crop.w, crop.y], [crop.x + crop.w, crop.y + crop.h, crop.x, crop.y], ]; for (const [cx, cy, ax, ay] of corners) { if (Math.abs(x - cx) <= near && Math.abs(y - cy) <= near) { drag.current = { mode: "resize", ax, ay, ox: 0, oy: 0 }; e.currentTarget.setPointerCapture(e.pointerId); return; } } if (x >= crop.x && x <= crop.x + crop.w && y >= crop.y && y <= crop.y + crop.h) { drag.current = { mode: "move", ax: 0, ay: 0, ox: x - crop.x, oy: y - crop.y }; e.currentTarget.setPointerCapture(e.pointerId); } }; const onMove = (e: React.PointerEvent) => { const d = drag.current; if (!d || !crop || !frame) return; const { x, y } = toSource(e); if (d.mode === "move") { setCrop(clampBox({ ...crop, x: x - d.ox, y: y - d.oy }, frame.w, frame.h)); return; } // The opposite corner stays put and the side follows the longer reach, so // the box grows the way a square selection is expected to. const side = Math.max(MIN_SIDE, Math.min(Math.abs(x - d.ax), Math.abs(y - d.ay))); const nx = x < d.ax ? d.ax - side : d.ax; const ny = y < d.ay ? d.ay - side : d.ay; setCrop(clampBox({ x: nx, y: ny, w: side, h: side }, frame.w, frame.h)); }; const onUp = () => { drag.current = null; }; // ---- the border guess ----------------------------------------------------- // Client-side, on exactly the pixels on screen -- no route, no python. It // scans outward from the face box for a sustained straight gradient, which is // what the edge of a webcam inset looks like: a border, a bezel, a hard cut // between a video feed and a browser window. // // OFFERED, NEVER APPLIED. A variance spike found the inset on one of three // test frames and failed wherever the screen share had video playing, so this // is a guess and it is labelled as one. Applying a guess automatically would // be the clamp's mistake again, with a nicer heuristic. const suggest = useCallback(() => { const c = canvas.current; const img = image.current; if (!c || !img || !frame) return; const ctx = c.getContext("2d"); if (!ctx) return; const face = det?.face ?? (crop ? { x: crop.x, y: crop.y, w: crop.w, h: crop.h } : null); if (!face) return; const W = c.width; const H = c.height; const data = ctx.getImageData(0, 0, W, H).data; const luma = (x: number, y: number) => { const i = ((y | 0) * W + (x | 0)) * 4; return 0.299 * data[i] + 0.587 * data[i + 1] + 0.114 * data[i + 2]; }; const fx0 = Math.max(0, face.x); const fy0 = Math.max(0, face.y); const fx1 = Math.min(W - 1, face.x + face.w); const fy1 = Math.min(H - 1, face.y + face.h); // How far out to look: an inset is a few face-widths at most, and searching // the whole frame would find the browser chrome instead of the bezel. const reach = Math.round(Math.max(face.w, face.h) * 2.2); /** The strongest sustained vertical edge in [from,to), scanning `dir`. */ const vertical = (from: number, to: number, dir: 1 | -1) => { let best = { x: dir === 1 ? to : from, score: 0 }; for (let x = from; dir === 1 ? x < to : x > to; x += dir) { if (x < 1 || x >= W - 1) continue; let s = 0; let n = 0; for (let y = fy0; y <= fy1; y += 2) { s += Math.abs(luma(x + 1, y) - luma(x - 1, y)); n += 1; } const score = n ? s / n : 0; if (score > best.score) best = { x, score }; } return best; }; const horizontal = (from: number, to: number, dir: 1 | -1) => { let best = { y: dir === 1 ? to : from, score: 0 }; for (let y = from; dir === 1 ? y < to : y > to; y += dir) { if (y < 1 || y >= H - 1) continue; let s = 0; let n = 0; for (let x = fx0; x <= fx1; x += 2) { s += Math.abs(luma(x, y + 1) - luma(x, y - 1)); n += 1; } const score = n ? s / n : 0; if (score > best.score) best = { y, score }; } return best; }; // A flat wall reads as a weak edge everywhere; below this there is nothing // to propose and saying so is better than proposing noise. const FLOOR = 12; const left = vertical(Math.max(1, fx0 - reach), fx0, 1); const right = vertical(Math.min(W - 2, fx1 + reach), fx1, -1); const top = horizontal(Math.max(1, fy0 - reach), fy0, 1); const bottom = horizontal(Math.min(H - 2, fy1 + reach), fy1, -1); if ([left.score, right.score, top.score, bottom.score].every((s) => s < FLOOR)) { setError("no inset border found in these pixels — crop it by eye"); return; } const x0 = left.score >= FLOOR ? left.x : fx0; const x1 = right.score >= FLOOR ? right.x : fx1; const y0 = top.score >= FLOOR ? top.y : fy0; const y1 = bottom.score >= FLOOR ? bottom.y : fy1; // Squared off, because the corner is square. The inset's SHORTER side wins: // a square that overflows the border would put back exactly the content the // border was found to exclude. const side = Math.max(MIN_SIDE, Math.min(x1 - x0, y1 - y0)); setGuess(clampBox({ x: x0, y: y0, w: side, h: side }, frame.w, frame.h)); setError(null); }, [crop, det, frame]); // ---- committing ----------------------------------------------------------- const commit = useCallback(async () => { if (!entry || !pending) return; setBusy(true); setError(null); try { const res = await fetch("/api/face", { method: "POST", headers: { "content-type": "application/json" }, cache: "no-store", body: JSON.stringify({ key: entry.key, verdict: pending, ...(code !== null ? { code } : {}), ...(text ? { text } : {}), // A REJECT carries no crop: there is nothing to frame, and storing one // would leave a framing behind that make-thumb would never reach. ...(pending !== "reject" && crop ? { crop } : {}), ...(det?.auto ? { auto: det.auto } : {}), }), }); const j = (await res.json()) as { error?: string; judgement?: FaceJudgement; rejected?: string[] }; if (!res.ok || !j.judgement) throw new Error(j.error || `HTTP ${res.status}`); setFaces((f) => ({ ...f, [entry.key]: j.judgement as FaceJudgement })); if (j.rejected) setSpent(j.rejected); // Straight to the next one still unjudged, the way the um deck moves on. const next = queue.findIndex((e, i) => i > idx && !faces[e.key] && e.key !== entry.key); setIdx(next >= 0 ? next : Math.min(queue.length - 1, idx + 1)); } catch (e) { setError(e instanceof Error ? e.message : String(e)); } finally { setBusy(false); } }, [entry, pending, code, text, crop, det, queue, idx, faces]); const resetCrop = useCallback(() => { setGuess(null); if (det?.auto) setCrop(det.auto); else if (entry?.crop) setCrop(entry.crop); }, [det, entry]); // ---- keyboard ------------------------------------------------------------- // Same shape as Deck.tsx and WordSelector.tsx: a pending verdict owns the // digits, so 1/2/3 pick the verdict and then the digits pick its reason. Two // meanings for one key is what the um deck already does, and the alternative // is a second row of letters nobody would remember. useEffect(() => { const onKey = (e: KeyboardEvent) => { const t = e.target as HTMLElement | null; if (t && (t.tagName === "INPUT" || t.tagName === "TEXTAREA")) return; const k = e.key; if (k === "Enter") { e.preventDefault(); if (pending) void commit(); return; } if (k === "Escape") { e.preventDefault(); setPending(null); setCode(null); return; } if (k === "j") { e.preventDefault(); setIdx((i) => Math.min(queue.length - 1, i + 1)); return; } if (k === "k") { e.preventDefault(); setIdx((i) => Math.max(0, i - 1)); return; } if (k === "r") { e.preventDefault(); resetCrop(); return; } if (k === "b") { e.preventDefault(); suggest(); return; } if (/^[0-9]$/.test(k)) { e.preventDefault(); // A verdict that wants a reason takes the digits next; `clean` never // does, so 1/2/3 keep meaning the verdict while it is selected. if (pending && pending !== "clean") { const n = k === "9" ? OTHER_CODE : Number(k); if (n === OTHER_CODE || (n >= 1 && n <= FACE_REASONS.length)) setCode(n); return; } const v = ({ "1": "clean", "2": "recrop", "3": "reject" } as const)[k as "1" | "2" | "3"]; if (v) { setPending(v); setCode(null); } } }; window.addEventListener("keydown", onKey); return () => window.removeEventListener("keydown", onKey); }, [pending, queue.length, commit, resetCrop, suggest]); // ---- readings ------------------------------------------------------------- const scale = crop ? scaleOf(crop) : 0; // Recomputed here from the face box rather than taken from the route, so what // is on screen is the pure function the CLI mirrors -- the same reason the // route recomputes it too. const local = det?.face && frame ? autoCrop(det.face, frame.w, frame.h) : null; const shove = local?.shove ?? det?.shove ?? 0; const moved = useMemo( () => (crop && det?.auto ? boxAttr(crop) !== boxAttr(det.auto) : false), [crop, det], ); const inFrame = crop && frame ? boxInFrame(crop, frame.w, frame.h) : true; if (!entry) { return (

No corners on record. make-thumb.mjs writes them; there is nothing to judge until it has run.

); } const done = Object.keys(faces).length; return (
{/* ---- the candidate ---------------------------------------------- */}
{entry.video} @{fmt(entry.srcStart)} {idx + 1} of {queue.length} {entry.onAccepted ? ( on the shipped {entry.accepted.join(" + ")} cover ) : ( candidate only — {entry.covers.join(", ")} )} {entry.link && ( the moment )} {judged && ( judged {judged.verdict} {judged.code ? ` — ${reasonText(judged.code, judged.text)}` : ""} )}
{!ready && (

fetching the frame at {fmt(entry.frameAt)}s…

)}

the corner, at the size it is cut

{/* THE READING, in the meter colour and never a verdict hue. It says what the resample is; whether that is acceptable is a judgement this component does not get to make. */} {crop && (

side {crop.w}px → {CORNER_PX}px = {scale.toFixed(2)}× {scale > 1 ? " — soft" : ""}

)} {!inFrame && (

that box is outside the frame

)}
{/* ---- what the automatic crop did ------------------------------- */}
{loading && !det && (

running the detector…

)} {det && !det.face && (

no face found at {fmt(entry.frameAt)}s — this corner cannot be reproduced automatically, so crop it by hand

)} {det?.face && !det.agrees && ( /* The dashed box would not be what gets cut. Loud, because every reading on this page assumes the two functions are one. */

the drawn crop {boxAttr(det.auto)} is NOT what facecrop.py would cut ( {boxAttr(det.pyAuto)}) — trust neither until they agree again

)} {det?.face && shove > SHOVE_FLOOR && (

the automatic crop sat {Math.round(shove)} px off the face to stay inside the frame — that displacement is what fills the corner with stream UI

)} {det?.face && shove <= SHOVE_FLOOR && (

the automatic crop is centred on the face ( {shove.toFixed(1)} px off) — whatever is wrong here is inside the inset, not at the frame edge

)} {moved && (

your crop differs from the automatic one — r puts it back

)} {guess && (

a guess at the inset border: {boxAttr(guess)} —{" "} {" "} or ignore it. The scan finds a straight edge, not a webcam.

)}
{/* ---- the verdict ------------------------------------------------ */}
{( [ ["clean", "1", "the crop is fine"], ["recrop", "2", "the face is fine, the box is not"], ["reject", "3", "spend the episode, use another"], ] as const ).map(([v, key, why]) => ( ))}
{pending && pending !== "clean" && (
why {FACE_REASONS.map((r, i) => ( ))} {code === OTHER_CODE && ( setText(e.target.value)} onKeyDown={(e) => e.stopPropagation()} placeholder="in your own words" aria-label="reason" data-reason-text className="w-72 rounded border border-[var(--color-line)] bg-[var(--color-panel-2)] px-2 py-0.5 font-mono text-[12px] text-[var(--color-text)] outline-none focus:border-[var(--color-sel)]" /> )}
)} {pending === "reject" && (

rejecting spends {entry.video} — the unique-face rule never gives an episode back, and {spent.length} {spent.length === 1 ? "is" : "are"} spent this way already

)} {error &&

{error}

}

j/k move · 1/2/3 verdict · digits pick a reason once a verdict is chosen · r reset · b{" "} suggest · ↵ commit

{/* ---- the queue --------------------------------------------------- */}

{queue.length} corners on record · {done} judged · {spent.length} episodes spent

    {queue.map((e, i) => { const j = faces[e.key]; return (
  • ); })}
); }