import { test, expect, type APIRequestContext, type Page } from "@playwright/test"; import { readFileSync } from "node:fs"; import path from "node:path"; import { autoCrop, scaleOf, CORNER_PX, SHOVE_FLOOR, type Box } from "../lib/face-types"; import { caps, NO_SOURCE_MEDIA } from "./capabilities"; // The face judger, against the REAL episodes the fixture symlinks -- make-fixture // derives its cover corners from media/ precisely so the frame and detect routes // have something to serve. A synthetic 320x180 clip would exercise the plumbing // and prove nothing about the thing this page exists for, which is that a webcam // inset near a frame edge gets a crop shoved off the face. // // So no assertion here hardcodes a corner, a count or an episode id. Every one is // a SHAPE, a MECHANISM or an INVARIANT, derived at runtime from the manifests the // fixture actually wrote -- which matters more than usual here because deck.spec // accepts a cover mid-suite, and the accepted set is what the queue is ordered by. const CODE = () => path.join(process.cwd(), ".e2e-song", "code"); const readState = (name: string): T => JSON.parse(readFileSync(path.join(CODE(), name), "utf8")) as T; type Corner = { video: string; srcStart?: number; frameAt?: number; crop?: Box }; type Doc = { thumbs: Record }; type Entry = { key: string; video: string; srcStart: number; frameAt: number; covers: string[]; accepted: string[]; onAccepted: boolean; crop: Box | null; }; type Detection = { face: (Box & { score: number; t: number }) | null; auto: Box | null; pyAuto: Box | null; agrees: boolean; shove: number; frame: { w: number; h: number }; }; type FaceView = { queue: Entry[]; faces: Record; rejected: string[] }; const view = async (request: APIRequestContext): Promise => { const r = await request.get("/api/face"); expect(r.ok(), `GET /api/face -> ${r.status()}`).toBe(true); return (await r.json()) as FaceView; }; // Memoised for the whole FILE, not per test. // // A detection is ~0.94s of python decoding five frames and running YuNet on // each, and two tests here walk the entire queue. Twenty-two of those is a // solid twenty seconds of four-core load in the middle of a serial suite, and // the deck spec that runs after this one drags a mouse across a waveform and // measures what it selected. Load is not supposed to change that, but making // the suite carry twice the work it needs to is not the place to prove it. const seen = new Map(); const detect = async (request: APIRequestContext, e: Entry): Promise => { const hit = seen.get(e.key); if (hit) return hit; const r = await request.get( `/api/face/detect?video=${encodeURIComponent(e.video)}&at=${e.frameAt}`, ); expect(r.ok(), `detect ${e.key} -> ${r.status()}`).toBe(true); const d = (await r.json()) as Detection; seen.set(e.key, d); return d; }; const attr = async (page: Page, sel: string, name: string): Promise => (await page.locator(sel).first().getAttribute(name)) ?? ""; const parseBox = (s: string): Box => { const [x, y, w, h] = s.split(",").map(Number); return { x, y, w, h }; }; // --------------------------------------------------------------------------- // 1. The route itself. // --------------------------------------------------------------------------- // A STATIC segment inside app/browse/[song]/'s territory, the same trap // /browse/find and /browse/decisions both document. When it regresses the page // still renders perfectly on its own and only the URL stops working. test("/browse/faces is not swallowed by the [song] route", async ({ page }) => { const res = await page.goto("/browse/faces"); expect(res?.status()).toBe(200); await expect(page.getByRole("navigation", { name: "Breadcrumb" })).toContainText("faces"); await expect(page.locator("[data-face-deck]")).toBeVisible(); }); // The way in is the cover-art bench, not a tenth nav item -- AppNav is at nine // and says a tenth wraps the header on a laptop. test("the bench links to the judger and the nav does not", async ({ page }) => { await page.goto("/browse/alpha"); const link = page.locator("[data-judge-faces]").first(); await expect(link).toBeVisible(); const cover = (await link.getAttribute("data-judge-faces")) ?? ""; await expect(link).toHaveAttribute("href", `/browse/faces?cover=${encodeURIComponent(cover)}`); // Nine items, and faces is not one of them. const nav = page.getByRole("navigation").first(); await expect(nav.getByRole("link", { name: "faces", exact: true })).toHaveCount(0); // And the link lands on a corner that cover actually uses. await link.click(); await expect(page.locator("[data-face-key]")).toBeVisible(); const key = await attr(page, "[data-face-key]", "data-face-key"); const manifest = readState("thumb-manifest.json"); const keys = (manifest.thumbs[cover]?.corners ?? []).map( (c) => `${c.video}@${(c.srcStart ?? 0).toFixed(2)}`, ); expect(keys).toContain(key); }); // --------------------------------------------------------------------------- // 2. The queue is the corners on record, deduped, the shipped ones first. // --------------------------------------------------------------------------- test("the queue is every corner deduped, accepted covers first", async ({ request }) => { const manifest = readState("thumb-manifest.json"); const accepted = readState("thumb-accepted.json"); const acceptedNames = new Set(Object.keys(accepted.thumbs)); // Derived from the same files the app reads, never a hardcoded 22 -- or 11, // which is what the fixture happens to hold today. const entries: string[] = []; const onAccepted = new Set(); for (const [name, e] of Object.entries({ ...manifest.thumbs, ...accepted.thumbs })) { for (const c of e.corners ?? []) { const k = `${c.video}@${(c.srcStart ?? 0).toFixed(2)}`; entries.push(k); if (acceptedNames.has(name)) onAccepted.add(k); } } const distinct = new Set(entries); expect(distinct.size).toBeGreaterThan(0); // The fixture deliberately repeats corners across covers -- if it stopped, // this test would pass while checking nothing. expect(entries.length).toBeGreaterThan(distinct.size); const { queue } = await view(request); expect(queue.length).toBe(distinct.size); expect(new Set(queue.map((e) => e.key))).toEqual(distinct); // A corner on four covers is ONE entry that names all four. const shared = queue.find((e) => e.covers.length > 1); expect(shared, "no corner appears on more than one cover").toBeTruthy(); const flags = queue.map((e) => e.onAccepted); expect(new Set(queue.filter((e) => e.onAccepted).map((e) => e.key))).toEqual(onAccepted); // Every accepted-cover corner comes before every candidate-only one. expect(flags.lastIndexOf(true)).toBeLessThan( flags.indexOf(false) === -1 ? Number.MAX_SAFE_INTEGER : flags.indexOf(false), ); }); // --------------------------------------------------------------------------- // 3. The two implementations of one function. // --------------------------------------------------------------------------- // autoCrop() is a pure re-implementation of facecrop.py's crop maths, and the // deck draws with it on every drag because a round trip per pixel would be // absurd. A re-implementation is only worth having if it is provably the same // function -- so the detect route returns BOTH boxes and this compares them for // every corner in the queue, without the spec having to spawn python itself. test("autoCrop() is the same function as facecrop.py's crop maths", async ({ request }) => { test.slow(); // THE QUEUE IS BUILT FROM SOURCE VIDEOS. With no `media/` the deck is // empty, and every assertion here is about a frame in it. test.skip(!caps.media, NO_SOURCE_MEDIA); const { queue } = await view(request); let compared = 0; for (const e of queue) { const d = await detect(request, e); if (!d.face) continue; expect(d.pyAuto, `${e.key} detected a face but reported no crop`).toBeTruthy(); expect(d.agrees, `${e.key}: route says the two disagree`).toBe(true); // Recomputed here as well, so this is a THIRD evaluation of the same // arithmetic rather than the route grading its own homework. expect(autoCrop(d.face, d.frame.w, d.frame.h).box, `${e.key}`).toEqual(d.pyAuto); compared += 1; } expect(compared, "no corner in the queue found a face at all").toBeGreaterThan(0); }); // --------------------------------------------------------------------------- // 4. The clamp is REPORTED, not hidden. // --------------------------------------------------------------------------- // The invariant, on a face box built to sit against an edge. Pure, so it has a // true answer whatever the corpus holds: a 200px face whose centre is 40px from // the top cannot be centred in a 440px crop without leaving the frame, so the // crop must slide and the slide must be reported. test("autoCrop() reports the displacement it accepted", () => { const face = { x: 600, y: -60, w: 200, h: 200 }; const { box, shove } = autoCrop(face, 1280, 720); expect(box.y).toBe(0); expect(shove).toBeGreaterThan(SHOVE_FLOOR); // And a face in open ground is not displaced at all. const middle = autoCrop({ x: 540, y: 260, w: 200, h: 200 }, 1280, 720); expect(middle.shove).toBeLessThanOrEqual(SHOVE_FLOOR); }); // And the deck SAYS SO. This is the sentence that explains what went wrong on // the Super Mario RPG corner; a number computed and not shown would leave the // crop looking like an inexplicable choice rather than a measured mistake. test("the deck states the shove on a corner whose crop was displaced", async ({ page, request, }) => { // THE QUEUE IS BUILT FROM SOURCE VIDEOS. With no `media/` the deck is // empty, and every assertion here is about a frame in it. test.skip(!caps.media, NO_SOURCE_MEDIA); test.slow(); const { queue } = await view(request); let displaced: { key: string; shove: number } | null = null; for (const e of queue) { const d = await detect(request, e); if (d.face && d.shove > SHOVE_FLOOR) { displaced = { key: e.key, shove: d.shove }; break; } } expect(displaced, "no corner in the queue has a displaced crop").toBeTruthy(); await page.goto(`/browse/faces?key=${encodeURIComponent(displaced!.key)}`); await expect(page.locator("[data-shove-note]")).toBeVisible({ timeout: 20_000 }); await expect(page.locator("[data-shove-note]")).toContainText("off the face"); await expect(page.locator("[data-shove-note]")).toContainText( String(Math.round(displaced!.shove)), ); expect(Number(await attr(page, "[data-face-key]", "data-face-shove"))).toBeCloseTo( displaced!.shove, 0, ); }); // --------------------------------------------------------------------------- // 5. The scale is a reading, and it is the real one. // --------------------------------------------------------------------------- test("the scale on screen is 300 over the crop height", async ({ page, request }) => { // THE QUEUE IS BUILT FROM SOURCE VIDEOS. With no `media/` the deck is // empty, and every assertion here is about a frame in it. test.skip(!caps.media, NO_SOURCE_MEDIA); const { queue } = await view(request); const target = queue[0]; // A crop with a known height, chosen so the answer is not 1.00 by accident. const crop = { x: 100, y: 60, w: 450, h: 450 }; const post = await request.post("/api/face", { data: { key: target.key, verdict: "recrop", code: 1, crop }, }); expect(post.ok(), await post.text()).toBe(true); await page.goto(`/browse/faces?key=${encodeURIComponent(target.key)}`); // Read straight off the SERVER-RENDERED html, with no wait: the reading is // computed for the first render precisely so it does not appear a beat late. const shown = await attr(page, "[data-face-key]", "data-face-scale"); expect(shown).toBe((CORNER_PX / crop.h).toFixed(2)); expect(shown).toBe(scaleOf(crop).toFixed(2)); await expect(page.locator("[data-scale-note]")).toContainText(`${shown}×`); await expect(page.locator("[data-scale-note]")).toContainText(`${crop.w}px`); }); // --------------------------------------------------------------------------- // 6. A crop round-trips in SOURCE pixels. // --------------------------------------------------------------------------- // The units are the whole risk here. The canvas draws at the source's own // resolution precisely so a dragged box needs no conversion, and a crop that // came back scaled by the display width would be silently wrong rather than // obviously broken -- it would still be a square on a face, just the wrong one. test("a recorded crop survives a reload, in source pixels", async ({ page, request }) => { // THE QUEUE IS BUILT FROM SOURCE VIDEOS. With no `media/` the deck is // empty, and every assertion here is about a frame in it. test.skip(!caps.media, NO_SOURCE_MEDIA); const { queue } = await view(request); const target = queue[queue.length - 1]; const crop = { x: 812, y: 137, w: 331, h: 331 }; const post = await request.post("/api/face", { data: { key: target.key, verdict: "recrop", code: 1, crop }, }); expect(post.ok(), await post.text()).toBe(true); const saved = (await post.json()) as { judgement: { crop: Box; frameAt: number } }; expect(saved.judgement.crop).toEqual(crop); await page.goto(`/browse/faces?key=${encodeURIComponent(target.key)}`); expect(parseBox(await attr(page, "[data-face-key]", "data-crop"))).toEqual(crop); // And it is in the file make-thumb.mjs reads, not only in the response. const file = readState<{ faces: Record }>("face-verdicts.json"); expect(file.faces[target.key].crop).toEqual(crop); // The frame the canvas draws is the frame the crop is expressed in. const frame = await attr(page, "[data-face-key]", "data-frame"); if (frame) { const [w, h] = frame.split("x").map(Number); expect(crop.x + crop.w).toBeLessThanOrEqual(w); expect(crop.y + crop.h).toBeLessThanOrEqual(h); } }); // --------------------------------------------------------------------------- // 7. Nothing spawns on load. // --------------------------------------------------------------------------- // The detector is ~0.94s of python. Detecting the whole queue to draw a list of // buttons would be twenty seconds of work for pictures nobody is looking at, and // it would happen on every navigation. One candidate is focused, so exactly one // frame and one detection. test("only the focused candidate costs anything", async ({ page, request }) => { const { queue } = await view(request); expect(queue.length).toBeGreaterThan(2); const detects: string[] = []; const framesHit: string[] = []; page.on("request", (r) => { const u = new URL(r.url()); if (u.pathname === "/api/face/detect") detects.push(u.searchParams.get("video") ?? ""); if (u.pathname === "/api/face/frame") framesHit.push(u.searchParams.get("video") ?? ""); }); await page.goto("/browse/faces"); await expect(page.locator("[data-face-canvas]")).toBeVisible(); // The rows are all rendered -- this is not passing because the list is empty. expect(await page.locator("[data-face-row]").count()).toBe(queue.length); await expect .poll(() => detects.length, { timeout: 20_000, message: "no detection ran at all" }) .toBe(1); expect(framesHit.length).toBe(1); expect(detects[0]).toBe(queue[0].video); // Moving on costs exactly one more of each, and not one per row. await page.locator("body").press("j"); await expect .poll(() => detects.length, { timeout: 20_000 }) .toBe(2); expect(framesHit.length).toBe(2); expect(detects[1]).toBe(queue[1].video); // Going back costs NOTHING: the detection is held for the life of the page. await page.locator("body").press("k"); await page.waitForTimeout(1500); expect(detects.length).toBe(2); }); // --------------------------------------------------------------------------- // 8. A rejection is spent, and make-thumb.mjs is what spends it. // --------------------------------------------------------------------------- test("reject puts the video in the set make-thumb reads", async ({ request }) => { const { queue, rejected: before } = await view(request); // The LAST corner, so this never retires the episode an earlier test focused. const target = queue[queue.length - 1]; const post = await request.post("/api/face", { data: { key: target.key, verdict: "reject", code: 1 }, }); expect(post.ok(), await post.text()).toBe(true); const body = (await post.json()) as { rejected: string[] }; expect(body.rejected).toContain(target.video); expect(body.rejected.length).toBe(before.length + (before.includes(target.video) ? 0 : 1)); const after = await view(request); expect(after.rejected).toContain(target.video); // The FILE, read the way make-thumb.mjs reads it -- REJECTED is built from // exactly this, so a verdict that never reached the file would be a verdict // the next build silently ignores. const file = readState<{ faces: Record }>( "face-verdicts.json", ); expect(file.faces[target.key].verdict).toBe("reject"); expect(file.faces[target.key].video).toBe(target.video); // A reject carries no crop: there would be nothing to frame, and a stored one // would be a framing make-thumb can never reach. expect((file.faces[target.key] as { crop?: Box }).crop).toBeUndefined(); }); // --------------------------------------------------------------------------- // 9. What a proposal is not allowed to be. // --------------------------------------------------------------------------- test("a crop outside the frame is refused and an unknown corner is a 404", async ({ request, }) => { // THE QUEUE IS BUILT FROM SOURCE VIDEOS. With no `media/` the deck is // empty, and every assertion here is about a frame in it. test.skip(!caps.media, NO_SOURCE_MEDIA); const { queue } = await view(request); const target = queue[0]; const outside = await request.post("/api/face", { data: { key: target.key, verdict: "recrop", code: 1, crop: { x: 1200, y: 600, w: 400, h: 400 } }, }); expect(outside.status()).toBe(400); expect((await outside.json()).error).toContain("frame"); const zero = await request.post("/api/face", { data: { key: target.key, verdict: "recrop", code: 1, crop: { x: 10, y: 10, w: 0, h: 0 } }, }); expect(zero.status()).toBe(400); const unknown = await request.post("/api/face", { data: { key: "nosuchvideo@1.00", verdict: "clean" }, }); expect(unknown.status()).toBe(404); const nonsense = await request.post("/api/face", { data: { key: target.key, verdict: "maybe" } }); expect(nonsense.status()).toBe(400); // Free text is REQUIRED on the other-reason code, or the reason says nothing. const wordless = await request.post("/api/face", { data: { key: target.key, verdict: "reject", code: 99 }, }); expect(wordless.status()).toBe(400); // The media routes guard their own inputs rather than trusting the page. expect((await request.get("/api/face/frame?video=../etc/passwd&at=1")).status()).toBe(400); expect((await request.get("/api/face/detect?video=nosuchvideo&at=1")).status()).toBe(404); expect( (await request.get(`/api/face/frame?video=${target.video}&at=-5`)).status(), ).toBe(400); }); // --------------------------------------------------------------------------- // 10. The frame route serves the SOURCE's pixels. // --------------------------------------------------------------------------- // Native resolution is what makes the canvas's coordinates source coordinates, // which is what makes a dragged box a crop facecrop.py can cut without a ratio // in the middle. If this ever started scaling by default, every crop the deck // recorded would be wrong by that ratio and nothing on screen would say so. test("the frame comes back at the source's own size unless a width is asked for", async ({ request, }) => { // Same reason as the rest of this file's frame tests: no `media/`, no queue. test.skip(!caps.media, NO_SOURCE_MEDIA); const { queue } = await view(request); const e = queue[0]; const d = await detect(request, e); const native = await request.get( `/api/face/frame?video=${encodeURIComponent(e.video)}&at=${e.frameAt}`, ); expect(native.ok()).toBe(true); expect(native.headers()["content-type"]).toBe("image/jpeg"); const bytes = await native.body(); expect(jpegSize(bytes)).toEqual({ w: d.frame.w, h: d.frame.h }); const small = await request.get( `/api/face/frame?video=${encodeURIComponent(e.video)}&at=${e.frameAt}&w=320`, ); expect(jpegSize(await small.body()).w).toBe(320); }); /** The dimensions out of a JPEG's SOF marker -- no decoder, no dependency. */ function jpegSize(buf: Buffer): { w: number; h: number } { let i = 2; while (i < buf.length) { if (buf[i] !== 0xff) { i += 1; continue; } const marker = buf[i + 1]; // SOF0..SOF15, skipping the four that are not frame headers. if (marker >= 0xc0 && marker <= 0xcf && ![0xc4, 0xc8, 0xcc].includes(marker)) { return { h: buf.readUInt16BE(i + 5), w: buf.readUInt16BE(i + 7) }; } i += 2 + buf.readUInt16BE(i + 2); } throw new Error("no SOF marker in that jpeg"); }