commit 08651344cc988856cc130bfa0b8f3f87ec2b70cf
parent c0774250e9661acd7e0e5181521cb56c6ee6327f
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 18 Aug 2026 10:54:19 -0400
The face judger: the queue, the store, and the crop maths
Several shipped cover corners carry stream UI rather than a face -- the
bottom-left of the Super Mario RPG cover is a strip of YouTube chrome, a dark
block and the webcam inset's own red border, with Jeremy in the right 60%.
The cause is measurable. facecrop.py's clamp preserves the crop's SIZE at the
cost of moving it OFF the face: a webcam inset is always near a frame edge, so
when the padded box overruns the edge the crop slides inward and fills with
whatever sits beside the webcam. On that corner the crop sits 93px off the face
centre, and that displacement IS the chrome. Four of the 22 corners on record
are displaced; the rest need an eye.
So the crop becomes a parameter you can see and move, instead of an output you
can only accept or discard. This is the queue and the store for that:
lib/face-types.ts client-safe shapes plus autoCrop(), a pure
re-implementation of facecrop.py's maths -- the same
relationship verifyUniqueFaces() has to accept-thumb.mjs.
It returns `shove`, so the clamp is a number on screen
rather than a mystery. scaleOf() reports the resample:
half the corners on record already upscale, and trading
softness for a tighter crop is the human's call.
lib/faces.ts the 22 corners on record, deduped by video@srcStart
(six appear on up to four covers) and accepted-cover ones
first; face-verdicts.json, which is OUR file and so gets
the read-inside-the-lock discipline; and the detector,
which stays facecrop.py because the corner that ships is
cut by the python one.
ThumbCorner grows an optional `crop`. Recording the box is the enabling change
-- it is discarded today, which is why two accepted corners cannot be
reproduced at all.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Diffstat:
3 files changed, 502 insertions(+), 1 deletion(-)
diff --git a/umtool/lib/face-types.ts b/umtool/lib/face-types.ts
@@ -0,0 +1,165 @@
+// The face-judging shapes AND the crop maths, with NO server imports.
+//
+// Same boundary rule as lib/note-types.ts and lib/phrase-types.ts, and for the
+// same reason: FaceDeck.tsx needs CORNER_PX, autoCrop() and the reason list as
+// VALUES at runtime. A value import is not erased, so one that reaches
+// lib/faces.ts -> node:fs would follow into the client chunk and fail the
+// build -- a break `tsc --noEmit` cannot see and only the bundler catches.
+
+/** CORNER_W / CORNER_H in make-thumb.mjs. The corner is square and 300px. */
+export const CORNER_PX = 300;
+
+/** facecrop.py's `side = max(fw, fh) * 2.2` -- a portrait, not a nose. */
+export const PAD = 2.2;
+
+/**
+ * How far off the face centre a crop has to sit before it is worth saying.
+ *
+ * Not zero, because facecrop.py truncates the crop origin to whole pixels, so
+ * a perfectly centred crop still reports up to half a pixel of displacement.
+ * Eight is comfortably above that and far below the 93px that put YouTube
+ * chrome in the bottom-left corner of the Super Mario RPG cover.
+ */
+export const SHOVE_FLOOR = 8;
+
+/** A rectangle in SOURCE pixels. Never in canvas or preview pixels. */
+export type Box = { x: number; y: number; w: number; h: number };
+
+export type FaceBox = Box & { score: number; t: number };
+
+export type FaceVerdict = "clean" | "recrop" | "reject";
+
+export type FaceJudgement = {
+ /** `<video>@<srcStart>` -- the SAME clip key notes, verdicts and state use. */
+ key: string;
+ video: string;
+ verdict: FaceVerdict;
+ code?: number;
+ text?: string;
+ frameAt: number;
+ /** The human's framing, in source pixels. */
+ crop?: Box;
+ /** What facecrop.py would have done, kept so the diff survives the session. */
+ auto?: Box;
+ at: string;
+};
+
+// ---------------------------------------------------------------------------
+// The reason codes.
+//
+// APPEND-ONLY and 1-based, exactly as song/reasons.mjs is and for the same
+// reason -- the number is what gets stored, so renumbering would rewrite
+// history. It is its OWN list rather than a slice of that one because these are
+// faults of the PICTURE, not of the clip: "stream UI in frame" is not a thing
+// that can be wrong with a sound, and "pop / plosive" is not a thing that can
+// be wrong with a crop.
+// ---------------------------------------------------------------------------
+
+export const FACE_REASONS = [
+ "stream UI in frame", // 1 the chrome, the chat, the inset's own border
+ "blink or mid-word", // 2
+ "motion blurred", // 3
+ "not Jer", // 4
+ "face too small", // 5
+];
+
+/** Free text, for whatever the fixed list does not cover. As reasons.mjs has it. */
+export const OTHER_CODE = 99;
+
+export const reasonText = (code: number | undefined, text?: string): string => {
+ if (code === OTHER_CODE) return text ?? "other";
+ if (!code) return "";
+ return FACE_REASONS[code - 1] ?? `reason ${code}`;
+};
+
+// ---------------------------------------------------------------------------
+// The crop, computed here as well as there.
+//
+// A pure re-implementation of facecrop.py's crop maths, exactly as
+// verifyUniqueFaces() is a pure re-implementation of accept-thumb.mjs's rule.
+// That is what lets the deck draw WHAT THE CLI WOULD DO beside WHAT YOU CHOSE,
+// on every drag, without a round trip -- and lets a test assert the two agree
+// without spawning python.
+//
+// It also returns `shove`, which is the whole point of the tool. The clamp on
+// the last two lines preserves the crop's SIZE at the cost of moving it OFF the
+// face: a webcam inset is always near a frame edge, so when the padded box
+// overruns the edge the crop slides inward and fills with whatever sits beside
+// the webcam. That displacement is not a rounding error, it is the YouTube
+// chrome. Measured, it is a number; unmeasured, it is a mystery.
+// ---------------------------------------------------------------------------
+
+export function autoCrop(
+ face: Box,
+ frameW: number,
+ frameH: number,
+ aspect = 1,
+): { box: Box; shove: number } {
+ const cx = face.x + face.w / 2;
+ const cy = face.y + face.h / 2;
+
+ let side = Math.max(face.w, face.h) * PAD;
+ side = Math.min(side, Math.min(frameW, frameH));
+
+ // The face box sets the HEIGHT; the width follows the aspect, so a 16:9 crop
+ // is a square one with more room either side rather than the same picture
+ // zoomed out.
+ let ch = side;
+ let cwid = side * aspect;
+ if (cwid > frameW) {
+ cwid = frameW;
+ ch = Math.min(ch, cwid / aspect);
+ }
+
+ // Math.trunc, not Math.round: python's int() truncates, and a crop that
+ // disagreed with the CLI by a pixel would make every comparison approximate.
+ const x = Math.trunc(Math.max(0, Math.min(frameW - cwid, cx - cwid / 2)));
+ const y = Math.trunc(Math.max(0, Math.min(frameH - ch, cy - ch / 2)));
+ const box: Box = { x, y, w: Math.trunc(cwid), h: Math.trunc(ch) };
+
+ return { box, shove: Math.hypot(box.x + box.w / 2 - cx, box.y + box.h / 2 - cy) };
+}
+
+/**
+ * What the corner is resampled by: 300px of output over the crop's own height.
+ *
+ * Below 1 the crop is being shrunk and detail is spare. Above 1 it is being
+ * BLOWN UP and the corner is softer than the source -- which is the price of
+ * cropping in past stream UI, and the price is the human's to pay or refuse.
+ * Half the corners on record already upscale, so this is not a rare warning.
+ */
+export const scaleOf = (box: Box): number => (box.h > 0 ? CORNER_PX / box.h : 0);
+
+export const keyOf = (video: string, srcStart: number): string =>
+ `${video}@${(+srcStart).toFixed(2)}`;
+
+/** Whole pixels, and inside the frame. A crop is cut with integer slices. */
+export function boxInFrame(box: Box, frameW: number, frameH: number): boolean {
+ const finite = [box.x, box.y, box.w, box.h].every((n) => Number.isFinite(n));
+ if (!finite) return false;
+ if (box.w < 1 || box.h < 1) return false;
+ return box.x >= 0 && box.y >= 0 && box.x + box.w <= frameW && box.y + box.h <= frameH;
+}
+
+/** Round to whole pixels and slide (never shrink, unless it must) into frame. */
+export function clampBox(box: Box, frameW: number, frameH: number): Box {
+ const w = Math.max(1, Math.min(frameW, Math.round(box.w)));
+ const h = Math.max(1, Math.min(frameH, Math.round(box.h)));
+ return {
+ x: Math.max(0, Math.min(frameW - w, Math.round(box.x))),
+ y: Math.max(0, Math.min(frameH - h, Math.round(box.y))),
+ w,
+ h,
+ };
+}
+
+export const isVerdict = (v: unknown): v is FaceVerdict =>
+ v === "clean" || v === "recrop" || v === "reject";
+
+export const isBox = (v: unknown): v is Box => {
+ if (!v || typeof v !== "object") return false;
+ const b = v as Record<string, unknown>;
+ return (["x", "y", "w", "h"] as const).every(
+ (k) => typeof b[k] === "number" && Number.isFinite(b[k] as number),
+ );
+};
diff --git a/umtool/lib/faces.ts b/umtool/lib/faces.ts
@@ -0,0 +1,326 @@
+import { existsSync } from "node:fs";
+import { execFile } from "node:child_process";
+import path from "node:path";
+import { promisify } from "node:util";
+import { archiveMomentUrl } from "./archive";
+import { sourceVideo } from "./clips";
+import { SONG_CODE, SONG_SCRATCH, stateFile } from "./paths";
+import { readJson, withStateLock, writeJsonAtomic } from "./state";
+import { readThumbAccepted, readThumbManifest, type ThumbDoc } from "./thumbs";
+import {
+ autoCrop,
+ boxInFrame,
+ isBox,
+ isVerdict,
+ keyOf,
+ OTHER_CODE,
+ type Box,
+ type FaceBox,
+ type FaceJudgement,
+} from "./face-types";
+
+export * from "./face-types";
+
+const run = promisify(execFile);
+
+// ---------------------------------------------------------------------------
+// The face judger's queue and its store.
+//
+// The queue is THE CORNERS ON RECORD -- every face that has ever been cut into
+// a cover, read out of the two thumb manifests. Not "every clip in the song":
+// judging faces nobody has shipped would be work with no consumer, and the
+// corners that ship today are the ones carrying stream UI today.
+//
+// The store is OURS. lib/thumbs.ts explains at length why this app never writes
+// thumb-manifest.json -- accept-thumb.mjs is its sole writer, with a bare
+// writeFileSync, and a second writer with different atomicity guarantees on the
+// authority file is not acceptable. The same rule holds here, so the deck
+// writes its own file and make-thumb.mjs READS it. The UI proposes; the CLI
+// decides. Because this file has only one writer with a lock, it gets the good
+// discipline lib/state.ts sets out: re-read inside the lock, write by rename.
+// ---------------------------------------------------------------------------
+
+export type FaceEntry = {
+ key: string;
+ video: string;
+ srcStart: number;
+ frameAt: number;
+ /** Every cover in the run log that cuts this face. Six corners appear on more than one. */
+ covers: string[];
+ /** Those of them that are ACCEPTED -- the covers that ship today. */
+ accepted: string[];
+ onAccepted: boolean;
+ /**
+ * The box make-thumb.mjs recorded, when it recorded one. Absent means "never
+ * recorded", not "centred" -- the same way AsrWord.conf's absence means
+ * unknown. Every corner cut before the manifest carried a box is in that
+ * state, which is why two accepted corners cannot be reproduced today.
+ */
+ crop: Box | null;
+ link: string | null;
+};
+
+/** Errors carry the status the route should answer with, as ShortlistError does. */
+export class FaceError extends Error {
+ constructor(
+ message: string,
+ readonly status: number,
+ ) {
+ super(message);
+ }
+}
+
+// ---------------------------------------------------------------------------
+// The queue.
+// ---------------------------------------------------------------------------
+
+const collect = (doc: ThumbDoc, into: Map<string, FaceEntry>, acceptedNames: Set<string>) => {
+ for (const [name, e] of Object.entries(doc.thumbs)) {
+ for (const c of e.corners ?? []) {
+ if (!c.video) continue;
+ const srcStart = c.srcStart ?? 0;
+ const key = keyOf(c.video, srcStart);
+ const hit = into.get(key);
+ if (hit) {
+ // The SAME face on a second cover is one queue entry, not two. Six of
+ // the 22 appear on up to four covers, and judging one four times would
+ // be three chances to disagree with yourself about one picture.
+ if (!hit.covers.includes(name)) hit.covers.push(name);
+ if (acceptedNames.has(name) && !hit.accepted.includes(name)) hit.accepted.push(name);
+ hit.onAccepted = hit.onAccepted || acceptedNames.has(name);
+ hit.crop = hit.crop ?? c.crop ?? null;
+ continue;
+ }
+ into.set(key, {
+ key,
+ video: c.video,
+ srcStart,
+ frameAt: c.frameAt ?? srcStart,
+ covers: [name],
+ accepted: acceptedNames.has(name) ? [name] : [],
+ onAccepted: acceptedNames.has(name),
+ crop: c.crop ?? null,
+ link: archiveMomentUrl(c.video, srcStart),
+ });
+ }
+ }
+};
+
+/**
+ * Every corner on record, deduped, with the ones that ship first.
+ *
+ * ACCEPTED FIRST is the whole ordering, and it is not cosmetic: a corner on an
+ * accepted cover is on YouTube, and a corner on a candidate is on nothing. The
+ * accepted manifest is walked as well as the run log for the same reason
+ * thumbView() walks it -- a cover can be accepted and no longer be in the run
+ * log, and it is still the one that is live.
+ */
+export async function faceQueue(): Promise<FaceEntry[]> {
+ const [generated, accepted] = await Promise.all([readThumbManifest(), readThumbAccepted()]);
+ const acceptedNames = new Set(Object.keys(accepted.thumbs));
+ const map = new Map<string, FaceEntry>();
+ collect(generated, map, acceptedNames);
+ collect(accepted, map, acceptedNames);
+ // Stable partition, not a sort on a derived score: within each half the order
+ // is the order the manifests give, which is cover by cover and corner by
+ // corner -- the order somebody would name them in.
+ const all = [...map.values()];
+ return [...all.filter((e) => e.onAccepted), ...all.filter((e) => !e.onAccepted)];
+}
+
+// ---------------------------------------------------------------------------
+// The store.
+// ---------------------------------------------------------------------------
+
+export type FaceFile = { version: 1; faces: Record<string, FaceJudgement> };
+
+export const FACE_VERDICTS = () => stateFile("face-verdicts.json");
+
+const blank = (): FaceFile => ({ version: 1, faces: {} });
+
+export async function readFaceVerdicts(): Promise<FaceFile> {
+ const j = await readJson<FaceFile>(FACE_VERDICTS(), blank());
+ if (!j || typeof j !== "object" || !j.faces || typeof j.faces !== "object") return blank();
+ return { version: 1, faces: j.faces };
+}
+
+/** One mutation, one lock, and the read happens INSIDE it. */
+export async function setFaceVerdict(j: FaceJudgement): Promise<FaceFile> {
+ let out: FaceFile = blank();
+ await withStateLock(async () => {
+ const f = await readFaceVerdicts();
+ f.faces[j.key] = j;
+ await writeJsonAtomic(FACE_VERDICTS(), f);
+ out = f;
+ });
+ return out;
+}
+
+/**
+ * The videos make-thumb.mjs will skip.
+ *
+ * Exported from here rather than computed in the script so the deck can show
+ * what it has spent -- a rejection costs a source episode out of a pool the
+ * unique-face rule never gives back, and that cost should be visible at the
+ * moment it is paid rather than discovered at the next build.
+ */
+export async function rejectedVideos(): Promise<string[]> {
+ const f = await readFaceVerdicts();
+ return [
+ ...new Set(
+ Object.values(f.faces)
+ .filter((j) => j.verdict === "reject")
+ .map((j) => j.video),
+ ),
+ ];
+}
+
+// ---------------------------------------------------------------------------
+// Detection, and the frame it happened on.
+//
+// facecrop.py is the detector for the CLI and it is the detector here too. A
+// second implementation -- an onnx runtime in node, a different model -- would
+// be a second answer to "where is the face", and the corner that ships is cut
+// by the python one.
+// ---------------------------------------------------------------------------
+
+export const FACEDET_PYTHON = () =>
+ process.env.FACEDET_PYTHON ?? path.join(SONG_SCRATCH, "facedet", "bin", "python");
+
+export const FACECROP_PY = () => path.join(SONG_CODE, "facecrop.py");
+
+export type Detection = {
+ face: FaceBox | null;
+ auto: Box | null;
+ shove: number;
+ frame: { w: number; h: number };
+};
+
+/** The frame size, memoised. Every episode is 1280x720, but none of this asks it to be. */
+const frames = new Map<string, { w: number; h: number }>();
+
+export async function probeFrame(video: string): Promise<{ w: number; h: number }> {
+ const hit = frames.get(video);
+ if (hit) return hit;
+ const file = sourceVideo(video);
+ if (!existsSync(file)) throw new FaceError(`no media for ${video}`, 404);
+ const { stdout } = await run("ffprobe", [
+ "-v", "error", "-select_streams", "v:0",
+ "-show_entries", "stream=width,height", "-of", "csv=p=0:s=x", file,
+ ]);
+ const [w, h] = stdout.trim().split("x").map(Number);
+ if (!Number.isFinite(w) || !Number.isFinite(h) || w < 1 || h < 1) {
+ throw new FaceError(`could not read the frame size of ${video}`, 500);
+ }
+ const size = { w, h };
+ frames.set(video, size);
+ return size;
+}
+
+/**
+ * Run the detector on the frames around `at` and report what it found.
+ *
+ * ~1.5 seconds, because it decodes five frames and runs YuNet on each. That
+ * cost is why this is its own route rather than part of the frame route: a
+ * still is ~80ms, and bundling them would make every navigation wait on python.
+ */
+export async function detectFace(video: string, at: number): Promise<Detection> {
+ const file = sourceVideo(video);
+ if (!existsSync(file)) throw new FaceError(`no media for ${video}`, 404);
+ const py = FACEDET_PYTHON();
+ if (!existsSync(py)) throw new FaceError("the facedet venv is not installed", 503);
+
+ let stdout: string;
+ try {
+ ({ stdout } = await run(py, [FACECROP_PY(), "--detect", file, String(at)], {
+ maxBuffer: 1 << 20,
+ }));
+ } catch {
+ throw new FaceError("the detector failed", 500);
+ }
+
+ const j = JSON.parse(stdout.trim()) as {
+ face: FaceBox | null;
+ frame: { w: number; h: number };
+ };
+ if (!j.face) return { face: null, auto: null, shove: 0, frame: j.frame };
+
+ // The AUTO CROP is recomputed here from the face box rather than taken from
+ // python's own answer. Not distrust: it is the assertion that the pure
+ // re-implementation the deck draws with is the same function as the one that
+ // cuts, checked on every single detection instead of once in a test.
+ const { box, shove } = autoCrop(j.face, j.frame.w, j.frame.h);
+ return { face: j.face, auto: box, shove, frame: j.frame };
+}
+
+// ---------------------------------------------------------------------------
+// What a POST is allowed to be.
+// ---------------------------------------------------------------------------
+
+export type FaceBody = {
+ key: string;
+ verdict: string;
+ code?: number;
+ text?: string;
+ crop?: Box;
+ auto?: Box;
+};
+
+export function isFaceBody(v: unknown): v is FaceBody {
+ if (!v || typeof v !== "object") return false;
+ const b = v as Record<string, unknown>;
+ if (typeof b.key !== "string" || !b.key) return false;
+ if (!isVerdict(b.verdict)) return false;
+ if (b.code !== undefined && (typeof b.code !== "number" || !Number.isFinite(b.code))) return false;
+ if (b.text !== undefined && typeof b.text !== "string") return false;
+ if (b.crop !== undefined && !isBox(b.crop)) return false;
+ if (b.auto !== undefined && !isBox(b.auto)) return false;
+ return true;
+}
+
+export const FACE_TEXT_LIMIT = 400;
+
+/**
+ * Validate a proposal against the queue and the real frame, then store it.
+ *
+ * An unknown key is a 404 because the queue is the corners on record and a
+ * judgement about a corner nobody cut has nothing to apply to. A crop outside
+ * the frame is a 400 because facecrop.py would slice an empty array out of the
+ * image and write a corner of nothing.
+ */
+export async function judgeFace(body: FaceBody): Promise<FaceJudgement> {
+ const entry = (await faceQueue()).find((e) => e.key === body.key);
+ if (!entry) throw new FaceError(`no corner called ${body.key}`, 404);
+
+ const j: FaceJudgement = {
+ key: entry.key,
+ video: entry.video,
+ verdict: body.verdict as FaceJudgement["verdict"],
+ frameAt: entry.frameAt,
+ at: new Date().toISOString(),
+ };
+ if (body.code !== undefined) j.code = Math.trunc(body.code);
+ if (body.text) j.text = body.text.slice(0, FACE_TEXT_LIMIT).replace(/\s+$/, "");
+ if (j.code === OTHER_CODE && !j.text) throw new FaceError("that reason needs the words", 400);
+
+ if (body.crop) {
+ const frame = await probeFrame(entry.video);
+ const crop = {
+ x: Math.round(body.crop.x),
+ y: Math.round(body.crop.y),
+ w: Math.round(body.crop.w),
+ h: Math.round(body.crop.h),
+ };
+ if (!boxInFrame(crop, frame.w, frame.h)) {
+ throw new FaceError(
+ `that crop is not inside the ${frame.w}x${frame.h} frame`,
+ 400,
+ );
+ }
+ j.crop = crop;
+ }
+ if (body.auto) j.auto = body.auto;
+
+ await setFaceVerdict(j);
+ return j;
+}
diff --git a/umtool/lib/thumbs.ts b/umtool/lib/thumbs.ts
@@ -22,7 +22,17 @@ import { archiveMomentUrl } from "./archive";
// proposal can be tested BEFORE the job runs rather than refused after.
// ---------------------------------------------------------------------------
-export type ThumbCorner = { video: string; srcStart?: number; frameAt?: number };
+// `crop` is the box the corner was actually cut from, in SOURCE pixels, and it
+// is OPTIONAL because most corners predate it being recorded at all. Its absence
+// means "never recorded", not "centred" -- the same way AsrWord.conf's absence
+// means unknown. make-thumb.mjs writes it now; the two accepted corners that
+// find no face at their recorded frameAt are what its absence costs.
+export type ThumbCorner = {
+ video: string;
+ srcStart?: number;
+ frameAt?: number;
+ crop?: { x: number; y: number; w: number; h: number };
+};
export type ThumbEntry = { out: string; bgAt?: number; corners?: ThumbCorner[] };
export type ThumbDoc = { version: number; thumbs: Record<string, ThumbEntry>; used?: string[] };
