import { existsSync } from "node:fs"; import { execFile } from "node:child_process"; import path from "node:path"; import { facecropPy, facedetPython } from "./tools.mjs"; import { promisify } from "node:util"; import { archiveMomentUrl } from "./archive"; import { sourceVideo } from "./clips"; import { stateFile } from "./paths"; import { readJson, withStateLock, writeJsonAtomic } from "./state"; import { readThumbAccepted, readThumbManifest, type ThumbDoc } from "./thumbs"; import { autoCrop, boxInFrame, isBox, isVerdict, keyOf, OTHER_CODE, type Box, type FaceBox, type FaceJudgement, } from "./face-types"; export * from "./face-types"; const run = promisify(execFile); // --------------------------------------------------------------------------- // The face judger's queue and its store. // // The queue is THE CORNERS ON RECORD -- every face that has ever been cut into // a cover, read out of the two thumb manifests. Not "every clip in the song": // judging faces nobody has shipped would be work with no consumer, and the // corners that ship today are the ones carrying stream UI today. // // The store is OURS. lib/thumbs.ts explains at length why this app never writes // thumb-manifest.json -- accept-thumb.mjs is its sole writer, with a bare // writeFileSync, and a second writer with different atomicity guarantees on the // authority file is not acceptable. The same rule holds here, so the deck // writes its own file and make-thumb.mjs READS it. The UI proposes; the CLI // decides. Because this file has only one writer with a lock, it gets the good // discipline lib/state.ts sets out: re-read inside the lock, write by rename. // --------------------------------------------------------------------------- export type FaceEntry = { key: string; video: string; srcStart: number; frameAt: number; /** Every cover in the run log that cuts this face. Six corners appear on more than one. */ covers: string[]; /** Those of them that are ACCEPTED -- the covers that ship today. */ accepted: string[]; onAccepted: boolean; /** * The box make-thumb.mjs recorded, when it recorded one. Absent means "never * recorded", not "centred" -- the same way AsrWord.conf's absence means * unknown. Every corner cut before the manifest carried a box is in that * state, which is why two accepted corners cannot be reproduced today. */ crop: Box | null; link: string | null; }; /** Errors carry the status the route should answer with, as ShortlistError does. */ export class FaceError extends Error { constructor( message: string, readonly status: number, ) { super(message); } } // --------------------------------------------------------------------------- // The queue. // --------------------------------------------------------------------------- const collect = (doc: ThumbDoc, into: Map, acceptedNames: Set) => { for (const [name, e] of Object.entries(doc.thumbs)) { for (const c of e.corners ?? []) { if (!c.video) continue; const srcStart = c.srcStart ?? 0; const key = keyOf(c.video, srcStart); const hit = into.get(key); if (hit) { // The SAME face on a second cover is one queue entry, not two. Six of // the 22 appear on up to four covers, and judging one four times would // be three chances to disagree with yourself about one picture. if (!hit.covers.includes(name)) hit.covers.push(name); if (acceptedNames.has(name) && !hit.accepted.includes(name)) hit.accepted.push(name); hit.onAccepted = hit.onAccepted || acceptedNames.has(name); hit.crop = hit.crop ?? c.crop ?? null; continue; } into.set(key, { key, video: c.video, srcStart, frameAt: c.frameAt ?? srcStart, covers: [name], accepted: acceptedNames.has(name) ? [name] : [], onAccepted: acceptedNames.has(name), crop: c.crop ?? null, link: archiveMomentUrl(c.video, srcStart), }); } } }; /** * Every corner on record, deduped, with the ones that ship first. * * ACCEPTED FIRST is the whole ordering, and it is not cosmetic: a corner on an * accepted cover is on YouTube, and a corner on a candidate is on nothing. The * accepted manifest is walked as well as the run log for the same reason * thumbView() walks it -- a cover can be accepted and no longer be in the run * log, and it is still the one that is live. */ export async function faceQueue(): Promise { const [generated, accepted] = await Promise.all([readThumbManifest(), readThumbAccepted()]); const acceptedNames = new Set(Object.keys(accepted.thumbs)); const map = new Map(); collect(generated, map, acceptedNames); collect(accepted, map, acceptedNames); // Stable partition, not a sort on a derived score: within each half the order // is the order the manifests give, which is cover by cover and corner by // corner -- the order somebody would name them in. const all = [...map.values()]; return [...all.filter((e) => e.onAccepted), ...all.filter((e) => !e.onAccepted)]; } // --------------------------------------------------------------------------- // The store. // --------------------------------------------------------------------------- export type FaceFile = { version: 1; faces: Record }; export const FACE_VERDICTS = () => stateFile("face-verdicts.json"); const blank = (): FaceFile => ({ version: 1, faces: {} }); export async function readFaceVerdicts(): Promise { const j = await readJson(FACE_VERDICTS(), blank()); if (!j || typeof j !== "object" || !j.faces || typeof j.faces !== "object") return blank(); return { version: 1, faces: j.faces }; } /** One mutation, one lock, and the read happens INSIDE it. */ export async function setFaceVerdict(j: FaceJudgement): Promise { let out: FaceFile = blank(); await withStateLock(async () => { const f = await readFaceVerdicts(); f.faces[j.key] = j; await writeJsonAtomic(FACE_VERDICTS(), f); out = f; }); return out; } /** * The videos make-thumb.mjs will skip. * * Exported from here rather than computed in the script so the deck can show * what it has spent -- a rejection costs a source episode out of a pool the * unique-face rule never gives back, and that cost should be visible at the * moment it is paid rather than discovered at the next build. */ export async function rejectedVideos(): Promise { const f = await readFaceVerdicts(); return [ ...new Set( Object.values(f.faces) .filter((j) => j.verdict === "reject") .map((j) => j.video), ), ]; } // --------------------------------------------------------------------------- // Detection, and the frame it happened on. // // facecrop.py is the detector for the CLI and it is the detector here too. A // second implementation -- an onnx runtime in node, a different model -- would // be a second answer to "where is the face", and the corner that ships is cut // by the python one. // --------------------------------------------------------------------------- // The paths live in lib/tools.mjs so `umtool doctor` probes the SAME python and // the same script this runs -- a doctor that checked a different interpreter // than the one that cuts would be worse than no doctor. export const FACEDET_PYTHON = facedetPython; export const FACECROP_PY = facecropPy; export type Detection = { face: FaceBox | null; /** What autoCrop() says -- the function the deck draws with. */ auto: Box | null; /** What facecrop.py itself said -- the function that CUTS. */ pyAuto: Box | null; /** Whether the two are the same box. If this is ever false the deck is lying. */ agrees: boolean; shove: number; frame: { w: number; h: number }; }; /** The frame size, memoised. Every episode is 1280x720, but none of this asks it to be. */ const frames = new Map(); export async function probeFrame(video: string): Promise<{ w: number; h: number }> { const hit = frames.get(video); if (hit) return hit; const file = sourceVideo(video); if (!existsSync(file)) throw new FaceError(`no media for ${video}`, 404); const { stdout } = await run("ffprobe", [ "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height", "-of", "csv=p=0:s=x", file, ]); const [w, h] = stdout.trim().split("x").map(Number); if (!Number.isFinite(w) || !Number.isFinite(h) || w < 1 || h < 1) { throw new FaceError(`could not read the frame size of ${video}`, 500); } const size = { w, h }; frames.set(video, size); return size; } /** * Run the detector on the frames around `at` and report what it found. * * ~1.5 seconds, because it decodes five frames and runs YuNet on each. That * cost is why this is its own route rather than part of the frame route: a * still is ~80ms, and bundling them would make every navigation wait on python. */ export async function detectFace(video: string, at: number): Promise { const file = sourceVideo(video); if (!existsSync(file)) throw new FaceError(`no media for ${video}`, 404); const py = FACEDET_PYTHON(); if (!existsSync(py)) throw new FaceError("the facedet venv is not installed", 503); let stdout: string; try { ({ stdout } = await run(py, [FACECROP_PY(), "--detect", file, String(at)], { maxBuffer: 1 << 20, })); } catch { throw new FaceError("the detector failed", 500); } const j = JSON.parse(stdout.trim()) as { face: FaceBox | null; auto: Box | null; frame: { w: number; h: number }; }; if (!j.face) { return { face: null, auto: null, pyAuto: null, agrees: true, shove: 0, frame: j.frame }; } // BOTH ANSWERS ARE RETURNED, and this is the point of the route carrying // `pyAuto` at all. // // autoCrop() is a pure re-implementation of facecrop.py's crop maths, and the // deck draws with it on every drag because a round trip per pixel would be // absurd. A re-implementation is only worth having if it is provably the same // function, so python's own box comes back beside it -- checked on every // single detection rather than once in a test, and available to a test that // then does not have to spawn python to make the comparison. const { box, shove } = autoCrop(j.face, j.frame.w, j.frame.h); const same = (a: Box | null, b: Box | null) => !!a && !!b && a.x === b.x && a.y === b.y && a.w === b.w && a.h === b.h; return { face: j.face, auto: box, pyAuto: j.auto ?? null, agrees: same(box, j.auto ?? null), shove, frame: j.frame, }; } // --------------------------------------------------------------------------- // What a POST is allowed to be. // --------------------------------------------------------------------------- export type FaceBody = { key: string; verdict: string; code?: number; text?: string; crop?: Box; auto?: Box; }; export function isFaceBody(v: unknown): v is FaceBody { if (!v || typeof v !== "object") return false; const b = v as Record; if (typeof b.key !== "string" || !b.key) return false; if (!isVerdict(b.verdict)) return false; if (b.code !== undefined && (typeof b.code !== "number" || !Number.isFinite(b.code))) return false; if (b.text !== undefined && typeof b.text !== "string") return false; if (b.crop !== undefined && !isBox(b.crop)) return false; if (b.auto !== undefined && !isBox(b.auto)) return false; return true; } export const FACE_TEXT_LIMIT = 400; /** * Validate a proposal against the queue and the real frame, then store it. * * An unknown key is a 404 because the queue is the corners on record and a * judgement about a corner nobody cut has nothing to apply to. A crop outside * the frame is a 400 because facecrop.py would slice an empty array out of the * image and write a corner of nothing. */ export async function judgeFace(body: FaceBody): Promise { const entry = (await faceQueue()).find((e) => e.key === body.key); if (!entry) throw new FaceError(`no corner called ${body.key}`, 404); const j: FaceJudgement = { key: entry.key, video: entry.video, verdict: body.verdict as FaceJudgement["verdict"], frameAt: entry.frameAt, at: new Date().toISOString(), }; if (body.code !== undefined) j.code = Math.trunc(body.code); if (body.text) j.text = body.text.slice(0, FACE_TEXT_LIMIT).replace(/\s+$/, ""); if (j.code === OTHER_CODE && !j.text) throw new FaceError("that reason needs the words", 400); if (body.crop) { const frame = await probeFrame(entry.video); const crop = { x: Math.round(body.crop.x), y: Math.round(body.crop.y), w: Math.round(body.crop.w), h: Math.round(body.crop.h), }; if (!boxInFrame(crop, frame.w, frame.h)) { throw new FaceError( `that crop is not inside the ${frame.w}x${frame.h} frame`, 400, ); } j.crop = crop; } if (body.auto) j.auto = body.auto; await setFaceVerdict(j); return j; }