Archilyzer · Source

archilyzer

Archilyzer
git clone https://archilyzer.pages.dev/source/archilyzer.git
Log | Files | Refs | README | LICENSE

commit c0774250e9661acd7e0e5181521cb56c6ee6327f
parent 6d1e54c9eadeb413ec869ab3d78a0f20e9594ac0
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date:   Tue, 18 Aug 2026 10:18:05 -0400

Test the console against the real corpus

The suite symlinks the REAL asr/ rather than synthesising one, because 300
files of genuine parakeet output is the only thing that exercises the chunk
seams, the multi-word tokens and the confidence spread this page is built
around. So no assertion here hardcodes a count: every one is a shape, a
mechanism, or an invariant with a true answer whatever the corpus holds,
derived at runtime from what the corpus actually returned. A test asserting
`the` is 23,081 hits would go red the first time an episode was re-cut, and
would have been testing the corpus rather than the code.

The one that earns its place most: EVERY ROW'S data-from equals
max(0, data-start - 0.35). That is the only thing standing between the lead-in
pad and someone simplifying trap 2 away, and the bug it prevents is silent --
a clipped first phoneme sounds like the archive is wrong, not like the window
is. Alongside it: the static segment is not swallowed by [song], no query
builds no index, paging is disjoint under one total, hit ids are word-keyed
(a time-keyed one would contain a `.`), adjacency is derived from a real hit's
own context, folding removes exactly what `collapsed` claims, no chunk-seam
twin survives a deduped result, nothing spawns an ffmpeg until `v`, the
audition asks for the window the row advertised, and a starred moment
round-trips with its window and query intact while a non-permutation reorder
is refused and changes nothing.

make-fixture seeds two piles it did not before. shortlist.json starts empty,
like every other pile -- inheriting real ones would mean a spec reordering a
list somebody is actually cutting from. suspect-sources.json was NEITHER
copied nor seeded, so the flagged-source mark was untestable: with no file the
set is empty and every assertion about it passed vacuously. It is DERIVED --
the first episode that actually says "the", so the filter has something to
hide -- and never copied, because the real file holds 116 human judgements
about who is speaking and a suite asserting against those would be asserting
against somebody's ear.

81 passed: the 67 that were already green, plus these 14.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Diffstat:
Mumtool/app/browse/find/page.tsx | 7+++++--
Mumtool/components/PhraseConsole.tsx | 2+-
Aumtool/e2e/find.spec.ts | 383+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mumtool/e2e/fixtures/make-fixture.mjs | 48++++++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 437 insertions(+), 3 deletions(-)

diff --git a/umtool/app/browse/find/page.tsx b/umtool/app/browse/find/page.tsx @@ -5,6 +5,7 @@ import ShortlistBar from "@/components/ShortlistBar"; import { episodeIds, isEpisode, + isFillerQuery, parsePhraseQuery, searchPhrases, MAX_TERMS, @@ -88,7 +89,9 @@ export default async function FindPage({ searchParams }: { searchParams: Promise const pages = result ? Math.max(1, Math.ceil(result.total / query.per)) : 1; const shown = result?.hits.length ?? 0; - const filler = result ? result.notes.some((n) => n.includes("ACOUSTICALLY")) : false; + // Ask the predicate, never the prose. Matching a word out of a note this file + // does not own is a link that disappears the next time the note is reworded. + const filler = isFillerQuery(query.terms); return ( <div className="flex h-full flex-col"> @@ -111,7 +114,7 @@ export default async function FindPage({ searchParams }: { searchParams: Promise /> {/* Carried so submitting the box does not silently drop the filters already applied. A GET form sends only its own fields. */} - {(["video", "conf", "clean", "dupes", "order", "per"] as const).map((k) => + {(["video", "conf", "clean", "dupes", "order", "per", "list"] as const).map((k) => raw.get(k) ? <input key={k} type="hidden" name={k} value={raw.get(k) as string} /> : null, )} <button diff --git a/umtool/components/PhraseConsole.tsx b/umtool/components/PhraseConsole.tsx @@ -354,7 +354,7 @@ export default function PhraseConsole({ </span> </div> - <p className="mt-1 text-[13px] leading-relaxed"> + <p className="mt-1 text-[13px] leading-relaxed" data-context> <span className="text-[var(--color-dim)]">{h.before} </span> <span className="font-mono font-medium tracking-wide text-[var(--color-text)]"> {h.text} diff --git a/umtool/e2e/find.spec.ts b/umtool/e2e/find.spec.ts @@ -0,0 +1,383 @@ +import { test, expect, type APIRequestContext, type Page } from "@playwright/test"; +import { readFileSync } from "node:fs"; +import path from "node:path"; + +// The phrase console, against the REAL ASR corpus -- the suite symlinks asr/ +// rather than synthesising one, because 300 files of genuine parakeet output is +// the only thing that exercises the chunk seams, the multi-word tokens and the +// confidence spread this page is built around. +// +// That means no assertion here may hardcode a count. Every one is a SHAPE, a +// MECHANISM, or an INVARIANT with a true answer whatever the corpus holds -- +// derived at runtime from what the corpus actually returned. A test that said +// `the` is 23,081 hits would go red the first time an episode was re-cut, and +// would have been testing the corpus rather than the code. + +const CHUNK_STEP = 237; +const CHUNK_OVERLAP = 3.5; +const LEAD_PAD = 0.35; + +type Hit = { + id: string; + video: string; + start: number; + from: number; + suspect: boolean; + after: string; + text: string; + confMin: number | null; +}; +type Result = { + hits: Hit[]; + total: number; + collapsed: number; + suppressed: number; +}; + +const find = async (request: APIRequestContext, qs: string): Promise<Result> => { + const r = await request.get(`/api/phrases?${qs}`); + expect(r.ok(), `GET /api/phrases?${qs} -> ${r.status()}`).toBe(true); + return (await r.json()) as Result; +}; + +/** The word the whole suite hangs off. Common enough to page, in every episode. */ +const COMMON = "the"; + +const attr = async (page: Page, sel: string, name: string): Promise<string> => + (await page.locator(sel).first().getAttribute(name)) ?? ""; + +// --------------------------------------------------------------------------- +// 1. The route itself. +// --------------------------------------------------------------------------- + +// A STATIC segment inside app/browse/[song]/'s territory. When this regresses +// the page still renders perfectly on its own and only the URL stops working, +// which is the failure mode that goes unnoticed longest. +test("/browse/find is not swallowed by the [song] route", async ({ page }) => { + const res = await page.goto("/browse/find"); + expect(res?.status()).toBe(200); + await expect(page.getByRole("navigation", { name: "Breadcrumb" })).toContainText("find"); + await expect(page.locator("#phrase-q")).toBeVisible(); +}); + +// --------------------------------------------------------------------------- +// 2. No query builds no index. +// --------------------------------------------------------------------------- + +// Building 105 MB of blob to draw an empty search box would be the whole cost +// of the feature paid for nothing. This reads what THIS RENDER built, not what +// happens to be in memory -- otherwise it would pass or fail on test order. +test("an empty console does not build the index", async ({ page }) => { + await page.goto("/browse/find"); + expect(await attr(page, "[data-find]", "data-corpus-built")).toBe("0"); + + await page.goto(`/browse/find?q=${COMMON}`); + expect(Number(await attr(page, "[data-find]", "data-corpus-built"))).toBeGreaterThan(0); +}); + +// --------------------------------------------------------------------------- +// 3-5. What a page of results is, and what a row promises. +// --------------------------------------------------------------------------- + +test("paging is honest: same total, disjoint pages, more held than shown", async ({ page }) => { + await page.goto(`/browse/find?q=${COMMON}`); + const total = await attr(page, "[data-find]", "data-total"); + const shown = await attr(page, "[data-find]", "data-shown"); + expect(Number(total)).toBeGreaterThan(Number(shown)); + + const ids1 = await page.locator("[data-hit]").evaluateAll((els) => + els.map((e) => e.getAttribute("data-hit")), + ); + + await page.goto(`/browse/find?q=${COMMON}&page=2`); + expect(await attr(page, "[data-find]", "data-total")).toBe(total); + const ids2 = await page.locator("[data-hit]").evaluateAll((els) => + els.map((e) => e.getAttribute("data-hit")), + ); + + expect(ids2.length).toBeGreaterThan(0); + const first = new Set(ids1); + expect(ids2.filter((id) => first.has(id))).toEqual([]); +}); + +// A hit's identity is `<video>#<wordIndex>`, never a timestamp: about one pair +// per 100k words shares a `start`, so a time-keyed id would collide and a +// starred pick would silently address a different word. A time-keyed id would +// also contain a `.`, which this pattern rejects. +test("every hit id is word-keyed and unique on the page", async ({ page }) => { + await page.goto(`/browse/find?q=${COMMON}`); + const ids = await page.locator("[data-hit]").evaluateAll((els) => + els.map((e) => e.getAttribute("data-hit") ?? ""), + ); + expect(ids.length).toBeGreaterThan(0); + for (const id of ids) expect(id).toMatch(/^[\w-]+#\d+$/); + expect(new Set(ids).size).toBe(ids.length); +}); + +// THE ONE TEST THAT CATCHES SOMEONE SIMPLIFYING TRAP 2 AWAY. +// +// Parakeet timestamps a word's START LATE by 25-290ms, so a preview that seeks +// to word.start clips the first phoneme -- and it does it quietly, sounding +// like the archive is wrong rather than like the window is. The pad is the fix +// and it has to be on EVERY row, clamped at zero for a hit near the top of an +// episode. +test("every row's preview window opens a lead-in before the word", async ({ page }) => { + await page.goto(`/browse/find?q=${COMMON}`); + const rows = await page.locator("[data-hit]").evaluateAll((els) => + els.map((e) => ({ + start: Number(e.getAttribute("data-start")), + from: Number(e.getAttribute("data-from")), + })), + ); + expect(rows.length).toBeGreaterThan(0); + for (const r of rows) { + expect(r.from).toBeCloseTo(Math.max(0, r.start - LEAD_PAD), 3); + } +}); + +// --------------------------------------------------------------------------- +// 6-8. The retrieval invariants, all derived from what the corpus returned. +// --------------------------------------------------------------------------- + +test("a two-word phrase is strictly adjacent, so it is rarer than its first word", async ({ + request, +}) => { + const one = await find(request, `q=${COMMON}`); + expect(one.hits.length).toBeGreaterThan(0); + + // The next word of a real hit's context, so the phrase is guaranteed to exist + // at least once. No hardcoded pair, no hardcoded count. + const next = one.hits[0].after.split(/\s+/)[0]; + expect(next, "the first hit has context after it").toBeTruthy(); + + const two = await find(request, `q=${encodeURIComponent(`${COMMON} ${next}`)}`); + expect(two.total).toBeGreaterThan(0); + expect(two.total).toBeLessThan(one.total); +}); + +// The fold is a STATED delta, never a silent truncation -- the same rule +// ROW_LIMIT follows. If `collapsed` ever stopped matching what went missing, +// the page would be quietly hiding hits while claiming a total. +test("folding chunk twins removes exactly what it says it removed", async ({ request }) => { + const raw = await find(request, `q=${COMMON}&dupes=0`); + const folded = await find(request, `q=${COMMON}&dupes=1`); + expect(raw.collapsed).toBe(0); + expect(folded.collapsed).toBeGreaterThan(0); + expect(raw.total - folded.total).toBe(folded.collapsed); +}); + +// The discriminator itself. In a deduped result no two hits in one episode may +// sit within 0.6s of each other with the later one inside a transcription seam: +// that combination IS the artefact. Genuine stutters live outside the band and +// must survive, which is why the rule is a conjunction and not a time window. +test("no chunk-seam twin survives a deduped result", async ({ request }) => { + const seen: { video: string; start: number }[] = []; + for (let p = 1; p <= 4; p += 1) { + const r = await find(request, `q=${COMMON}&per=200&page=${p}`); + if (!r.hits.length) break; + seen.push(...r.hits.map((h) => ({ video: h.video, start: h.start }))); + } + expect(seen.length).toBeGreaterThan(200); + + const bad = seen.filter((h, i) => { + if (i === 0) return false; + const prev = seen[i - 1]; + return ( + prev.video === h.video && h.start - prev.start < 0.6 && h.start % CHUNK_STEP < CHUNK_OVERLAP + ); + }); + expect(bad).toEqual([]); +}); + +// --------------------------------------------------------------------------- +// 9-10. What the page is allowed to spawn, and what it asks for. +// --------------------------------------------------------------------------- + +// /api/clip/[key]/video spawns an ffmpeg per window. 50 mounted players would +// be 50 processes on a navigation, on a machine that is usually also running a +// transcription -- so the picture mounts for the focused hit, after `v`, and +// never before. +test("no picture is decoded until it is asked for", async ({ page }) => { + const videoCalls: string[] = []; + page.on("request", (r) => { + if (/\/api\/clip\/.*\/video/.test(r.url())) videoCalls.push(r.url()); + }); + + await page.goto(`/browse/find?q=${COMMON}`); + await expect(page.locator("[data-hit]").first()).toBeVisible(); + expect(videoCalls).toEqual([]); + + // Focus a row that actually has media/ behind it, then ask for the picture. + const rows = page.locator("[data-hit]"); + let asked = false; + for (let i = 0; i < Math.min(5, await rows.count()); i += 1) { + await rows.nth(i).locator("[data-context]").click(); + const button = rows.nth(i).getByRole("button", { name: /^picture/ }); + const has = await button + .waitFor({ state: "visible", timeout: 3000 }) + .then(() => true) + .catch(() => false); + if (!has) continue; + await page.keyboard.press("v"); + await expect(rows.nth(i).locator("video")).toBeVisible(); + asked = true; + break; + } + expect(asked, "at least one of the first five hits has media/").toBe(true); + + // One player, one window, one ffmpeg. + expect(new Set(videoCalls).size).toBe(1); +}); + +// The preview reuses the clip route rather than growing a media route of its +// own, and it must ask for the window the row PROMISED -- the lead-in is only +// real if the request carries it. +test("the audition asks the clip route for the window the row advertises", async ({ page }) => { + const audio: URL[] = []; + page.on("request", (r) => { + if (/\/api\/clip\/.*\/audio/.test(r.url())) audio.push(new URL(r.url())); + }); + + await page.goto(`/browse/find?q=${COMMON}`); + const row = page.locator("[data-hit]").first(); + const from = await row.getAttribute("data-from"); + const video = await row.getAttribute("data-video"); + + await row.locator("[data-context]").click(); + await page.keyboard.press("Enter"); + await expect.poll(() => audio.length).toBeGreaterThan(0); + + expect(Number(audio[0].searchParams.get("from"))).toBeCloseTo(Number(from), 3); + // The key is a media ADDRESS -- `<video>@<start>` -- not the hit's identity. + expect(decodeURIComponent(audio[0].pathname)).toContain(`/api/clip/${video}@`); +}); + +// --------------------------------------------------------------------------- +// 11. The shortlist. +// --------------------------------------------------------------------------- + +test("a starred moment round-trips, and reorder is all-or-nothing", async ({ page, request }) => { + const name = `spec ${test.info().workerIndex}`; + const made = await request.post("/api/shortlist", { data: { op: "create", name } }); + expect(made.ok()).toBe(true); + const slug = (await made.json()).list.slug as string; + + await page.goto(`/browse/find?q=${COMMON}&list=${slug}`); + const rows = page.locator("[data-hit]"); + + // Star the first three, in the order they were starred -- which is the order + // the list must come back in, because the order IS the edit. + const starred: { id: string; start: number; from: number }[] = []; + for (let i = 0; i < 3; i += 1) { + const row = rows.nth(i); + await row.locator("[data-context]").click(); + await page.keyboard.press("s"); + await expect(row).toHaveAttribute("data-starred", "1"); + starred.push({ + id: (await row.getAttribute("data-hit")) ?? "", + start: Number(await row.getAttribute("data-start")), + from: Number(await row.getAttribute("data-from")), + }); + } + + const read = async () => + (await (await request.get(`/api/shortlist?list=${slug}`)).json()).list as { + picks: { id: string; start: number; from: number; text: string; q: string }[]; + }; + + // A pick is a SNAPSHOT: it carries the window and the query, so a + // re-transcription that moved the word can be DETECTED rather than silently + // followed. + let list = await read(); + expect(list.picks.map((p) => p.id)).toEqual(starred.map((s) => s.id)); + for (const [i, p] of list.picks.entries()) { + expect(p.start).toBeCloseTo(starred[i].start, 3); + expect(p.from).toBeCloseTo(starred[i].from, 3); + expect(p.q).toBe(COMMON); + expect(p.text.length).toBeGreaterThan(0); + } + + // A WHOLE PERMUTATION, and it is honoured exactly. + const reversed = [...starred].reverse().map((s) => s.id); + const ok = await request.post("/api/shortlist", { + data: { op: "reorder", list: slug, ids: reversed }, + }); + expect(ok.ok()).toBe(true); + list = await read(); + expect(list.picks.map((p) => p.id)).toEqual(reversed); + + // Anything that is not a permutation is a disagreement about what the list + // HOLDS, so it is refused rather than reconciled into an order nobody asked + // for -- and the list is left exactly as it was. + for (const ids of [reversed.slice(0, 2), [...reversed, "nope#1"], [reversed[0], reversed[0], reversed[1]]]) { + const bad = await request.post("/api/shortlist", { data: { op: "reorder", list: slug, ids } }); + expect(bad.status(), JSON.stringify(ids)).toBe(400); + } + expect((await read()).picks.map((p) => p.id)).toEqual(reversed); + + // Unstarring from the row takes it back off. + await rows.nth(0).locator("[data-context]").click(); + await page.keyboard.press("s"); + await expect(rows.nth(0)).toHaveAttribute("data-starred", "0"); + expect((await read()).picks.map((p) => p.id)).not.toContain(starred[0].id); + + expect((await request.get("/api/shortlist?list=does-not-exist")).status()).toBe(404); + expect( + (await request.post("/api/shortlist", { data: { op: "reorder", list: "does-not-exist", ids: [] } })) + .status(), + ).toBe(404); + + await request.post("/api/shortlist", { data: { op: "delete", list: slug } }); +}); + +// --------------------------------------------------------------------------- +// The flagged-source filter, which was untestable until make-fixture seeded it. +// --------------------------------------------------------------------------- + +// suspect-sources.json is DERIVED into the fixture, never copied -- the real +// file holds 116 human judgements about who is speaking. The spec reads the +// seed back rather than guessing which episode it names. +test("hiding flagged sources hides that episode and says how much it hid", async ({ request }) => { + const seed = JSON.parse( + readFileSync(path.join(process.cwd(), ".e2e-song", "code", "suspect-sources.json"), "utf8"), + ) as { k: string }[]; + expect(seed.length, "make-fixture seeded a flagged source").toBeGreaterThan(0); + const video = seed[0].k.slice(0, seed[0].k.lastIndexOf("@")); + + const scoped = await find(request, `q=${COMMON}&video=${video}`); + expect(scoped.total).toBeGreaterThan(0); + expect(scoped.hits.every((h) => h.suspect)).toBe(true); + + const clean = await find(request, `q=${COMMON}&clean=1`); + expect(clean.hits.every((h) => !h.suspect)).toBe(true); + // Stated, not silent: what the filter removed is reported as a number. + expect(clean.suppressed).toBeGreaterThan(0); + expect(clean.total).toBeLessThan((await find(request, `q=${COMMON}`)).total); +}); + +// --------------------------------------------------------------------------- +// The mechanism surface, and its refusals. +// --------------------------------------------------------------------------- + +test("/api/phrases refuses what it cannot answer", async ({ request }) => { + expect((await request.get("/api/phrases")).status()).toBe(400); + // Punctuation only: there is nothing searchable in it, and returning the + // whole corpus would be worse than saying so. + expect((await request.get("/api/phrases?q=...")).status()).toBe(400); + expect((await request.get("/api/phrases?q=a+b+c+d+e+f+g+h+i")).status()).toBe(400); + expect((await request.get(`/api/phrases?q=${COMMON}&per=500`)).status()).toBe(400); + expect((await request.get(`/api/phrases?q=${COMMON}&video=no-such-episode`)).status()).toBe(404); +}); + +// Trap 1, and the console has to say it out loud. The ASR barely transcribes +// fillers -- `um` is 2 occurrences in 623,079 tokens -- which is exactly why the +// palette was mined acoustically. The first search anyone runs will be this one, +// and without the note it reads as the tool being broken. +test("a filler query explains itself and points at /sort", async ({ page }) => { + await page.goto("/browse/find?q=um"); + await expect(page.locator("[data-note]").first()).toContainText("ACOUSTICALLY"); + await expect(page.getByRole("link", { name: /fillers are in \/sort/i })).toHaveAttribute( + "href", + "/sort", + ); +}); diff --git a/umtool/e2e/fixtures/make-fixture.mjs b/umtool/e2e/fixtures/make-fixture.mjs @@ -65,6 +65,9 @@ const seed = { "vocab.json": { words: [] }, // The undo stack starts empty, like every other pile. "journal.json": [], + // So do the shortlists. /browse/find writes here, and inheriting real ones + // would mean a spec reordering a list somebody is actually cutting from. + "shortlist.json": { version: 1, lists: {} }, }; for (const [f, v] of Object.entries(seed)) { writeFileSync(path.join(dest, "code", f), JSON.stringify(v, null, 1)); @@ -98,6 +101,50 @@ for (const d of ["wav48", "asr", "media"]) { if (existsSync(src)) symlinkSync(src, path.join(dest, "data", d)); } +// -- one FLAGGED SOURCE, derived from the symlinked asr/ ---------------------- +// +// suspect-sources.json was neither copied nor seeded before this, so the +// "flagged source" mark on /browse/find was untestable: with no file the set is +// empty and every assertion about it passes vacuously. +// +// It is DERIVED, never copied. The real file holds 116 real human judgements +// about which episodes carry a second speaker, and a suite that asserted +// against them would be asserting against somebody's ear. So this picks the +// first episode that actually says "the" -- so the flag has hits to hide -- and +// writes one key naming it. The spec reads this file back rather than guessing, +// which keeps the assertion exact whatever the corpus holds. +let flagged = null; +const asrDir = path.join(dest, "data", "asr"); +if (existsSync(asrDir)) { + for (const f of readdirSync(asrDir).filter((x) => x.endsWith(".json")).sort()) { + let words; + try { + words = JSON.parse(readFileSync(path.join(asrDir, f), "utf8")).words ?? []; + } catch { + continue; + } + const w = words.find((x) => String(x.w).toLowerCase().replace(/[^a-z]/g, "") === "the"); + if (!w) continue; + flagged = { video: f.slice(0, -5), start: +w.start }; + break; + } +} +writeFileSync( + path.join(dest, "code", "suspect-sources.json"), + JSON.stringify( + flagged + ? [ + { + k: `${flagged.video}@${flagged.start.toFixed(2)}`, + why: "seeded by make-fixture so the flagged-source filter has something to hide", + }, + ] + : [], + null, + 1, + ), +); + // -- the mix bench: a reports dir of its own, and two SYNTHESISED tracks ------- // // The bench RENDERS, and its default output directory is the real reports @@ -504,6 +551,7 @@ console.log(` loudness: deck wide.mp4 vs variants/wide-quiet.mp4, 10 dB apart`) console.log(` thumbs: alpha-c accepted, alpha-b free, alpha-d clashes on v1`); console.log(` plans: alpha (4 notes), alpha-v2 (+bass), alpha-body.json (depth 3), overlays.json (not a plan)`); console.log(` trim set: mk-hooks (10s stem, 2 hooks)`); +console.log(` flagged source: ${flagged ? flagged.video : "none — no asr/"}`); console.log(` SONG_CODE_DIR=${path.join(dest, "code")}`); console.log(` SONG_DIR=${path.join(dest, "data")}`); console.log(` SONG_REPORTS_DIR=${reports}`);