commit 3a648b6933cd84e87b22b76608f8377ae6730344
parent cb57555fac2ec278e24db1fb064d20aeaee4938a
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 21 Sep 2026 02:45:10 -0400
umtool e2e: no song data is a SKIP, not a red suite
The song project's 39 GB (`wav48/`, `asr/`, `media/`) is re-derivable from the
archive and deliberately not in the repo. `SONG_DIR`'s default is the job temp
dir the corpus was mined into, which on most machines no longer exists — and
`make-fixture.mjs` threw on its first readdir of it, taking the ENTIRE suite
down before the test server started, including the two dozen specs that have
nothing to do with the song project. Set `SONG_DIR` to a partial copy (this
machine has `cand2/` and none of the heavy dirs) and the fixture built fine but
empty, and twenty-odd specs went red.
Red that means "you are on a different laptop" is red people learn to ignore,
which is the only way a real failure gets missed.
`fixtures/song-capabilities.mjs` is the one rule, read by two callers.
`make-fixture.mjs` tolerates a missing SONG_DATA, writes
`fixture-capabilities.json` as the record, and says on STDERR which specs will
skip — the place somebody reading a CI log looks. `e2e/capabilities.ts` answers
the same question for the specs.
IT ANSWERS FROM THE SOURCE DIRECTORIES, NOT FROM THE FIXTURE'S JSON, and that
is the one non-obvious thing here: Playwright loads spec modules to build its
test list, and the fixture is built by a `webServer` command — so on a fresh
checkout the file is not there yet, and a helper that failed closed on its
absence would skip the whole song suite on a machine that HAS the data.
Three capabilities, because they are three different halves of the same absent
archive and a spec should say which it needs: `song` (cand2 + wav48 — a queue
with no candidates and a bench with no audio are the same failure from two
directions), `media` (the face deck reads source videos), `asr` (the phrase
console runs against the real corpus; synthesising one would defeat its point).
Skips added from an actual no-song-data run, not from guessing: browse (3, all
plan-derived), deck (2, same), faces (6, all need a frame), find (the whole
file — every assertion there derives its expected answer from what the corpus
returned).
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
9 files changed, 214 insertions(+), 5 deletions(-)
diff --git a/AGENTS.md b/AGENTS.md
@@ -28,9 +28,16 @@ SONG_DIR=~/reports/quartering-uh-song/data pnpm --filter umtool run e2e clip-ben
`SONG_DIR` is read by `umtool/song/paths.mjs` when `e2e/fixtures/make-fixture.mjs` builds
the fixture. The suite never *runs* against the real song dir — it reads it to derive an
-empty-state copy and to symlink the heavy audio — but without it the fixture build looks
-for that data at its original job-temp path and fails. umtool's suite is queued like every
-other.
+empty-state copy and to symlink the heavy audio. umtool's suite is queued like every other.
+
+**Without it the song-data specs SKIP; they no longer fail.** The 39 GB (`wav48/`, `asr/`,
+`media/`) is re-derivable from the archive and is not in the repo, so a machine that never
+had it built an empty fixture and went red — red that means "you are on a different laptop",
+which is the kind people learn to ignore. `make-fixture.mjs` now tolerates a missing
+`SONG_DIR`, writes `e2e/.e2e-song/fixture-capabilities.json` naming what it found, and the
+specs that judge a clip read it through `e2e/capabilities.ts` and skip themselves. A
+capability is a directory that exists, decided once by the builder — not re-guessed per
+spec.
See [WORKTREES.md](WORKTREES.md) for the port scheme, the queue, and the shared-data caveat.
diff --git a/umtool/docs/clip-bench.md b/umtool/docs/clip-bench.md
@@ -109,6 +109,20 @@ the one it replaces. A side that cannot grow says which reason it is — *"the
recording starts here"* or the pad cap — rather than offering a press that
cannot help. The message after a fetch names the side that moved.
+**And a third control, for when neither edge is the answer.** *"fetch whole
+source via editor"* posts `full: true`, which the editor answers by putting the
+ENTIRE recording in its saved-video store — not in `clips/`, because that is
+where big containers already live with a retention rule that leaves an
+explicitly-requested one alone. It skips the cached-window check on the way
+(that check asks "does a file hold this span", and a whole recording is not a
+span), and `clipWindowDirs` reads the pointer back so the build treats the
+container as the window `[0, duration]`: every clip of that video is then
+fetched at once, and re-cutting costs nothing. It is a separate, plainly
+labelled button rather than a smarter default because a recording can be
+gigabytes of somebody's bandwidth. There is no `UMTOOL_LOCAL_FETCH` twin — a
+machine with no editor has nowhere to put a full source that anything else
+would find.
+
**The words run as words, under the rail.** The rail is a timeline — it
squeezes each cue into a column a few characters wide, which is right for
dragging an edge against and useless for reading, and reading is why anybody
diff --git a/umtool/e2e/browse.spec.ts b/umtool/e2e/browse.spec.ts
@@ -1,4 +1,5 @@
import { test, expect } from "@playwright/test";
+import { caps, NO_SONG_DATA } from "./capabilities";
// The browse pages, against the synthesised videos/ tree in make-fixture.mjs:
//
@@ -158,6 +159,11 @@ test("promote refuses a variant that names no cut", async ({ request }) => {
});
test("provenance joins the plan to the archive with a 3s lead-in", async ({ page }) => {
+ // NEEDS A PLAN, AND A PLAN NEEDS A CANDIDATE. make-fixture writes
+ // `fixture-build.plan.json` from the first COPIED cand2 entry, and it copies
+ // one only when its wav is present — so without the song bulk data there is
+ // no plan, no provenance row, and nothing here to join.
+ test.skip(!caps.song, NO_SONG_DATA);
await page.goto("/browse/alpha/wide");
// The per-note table is collapsed by default -- 880 rows is the normal case,
// so it opens on demand.
@@ -178,6 +184,8 @@ test("provenance joins the plan to the archive with a 3s lead-in", async ({ page
});
test("the plan picker offers a choice rather than guessing", async ({ page }) => {
+ // One plan is written only when a candidate was copied — see above.
+ test.skip(!caps.song, NO_SONG_DATA);
await page.goto("/browse/alpha/wide");
await expect(page.locator("[data-plan='alpha.plan.json']")).toBeVisible();
});
@@ -391,6 +399,8 @@ test("notes attach to anything selectable, including things that do not exist ye
});
test("a mark resolves through the plan to the clip playing there", async ({ request }) => {
+ // "Through the plan" is the point, and the plan is candidate-derived.
+ test.skip(!caps.song, NO_SONG_DATA);
// alpha's clips.csv places its only note at songTime 1.500.
await request.post("/api/browse/notes", {
data: { song: "alpha", target: "moment:wide.mp4@2.00", note: "the kit is late here" },
diff --git a/umtool/e2e/capabilities.ts b/umtool/e2e/capabilities.ts
@@ -0,0 +1,48 @@
+import { SONG_DATA } from "../song/paths.mjs";
+import { songCapabilities } from "./fixtures/song-capabilities.mjs";
+
+// WHAT THIS MACHINE'S SONG DATA OFFERS, so a spec can skip instead of going red.
+//
+// The rule lives in `fixtures/song-capabilities.mjs` and is shared with
+// `make-fixture.mjs` — see its header for why the spec side answers from the
+// SOURCE directories rather than from the fixture's written JSON (Playwright
+// collects spec modules to build its test list, and the fixture is built by a
+// `webServer` command, so on a fresh checkout the file is not there yet).
+//
+// Read synchronously at module scope on purpose: `test.skip(cond, reason)` at
+// the top level of a file skips the whole file, and that has to be decided
+// while the module is being collected.
+
+export type FixtureCapabilities = {
+ songData: boolean;
+ cand2: boolean;
+ wav48: boolean;
+ asr: boolean;
+ media: boolean;
+ song: boolean;
+ songDir: string;
+};
+
+export const caps = songCapabilities(SONG_DATA) as FixtureCapabilities;
+
+// The one sentence, so several files cannot word it several ways. Playwright
+// prints it beside the skip, and it names the variable to set — "skipped" with
+// no reason is a spec nobody ever runs again.
+export const NO_SONG_DATA =
+ `needs the song project's bulk data (cand2/ + wav48/), which is not at ` +
+ `${caps.songDir}. Set SONG_DIR to a copy to run these.`;
+
+// The face deck reads SOURCE VIDEOS (`media/`), which is a different half of
+// the same absent 39 GB — a machine could in principle have one and not the
+// other, so the specs that need frames say which they need rather than
+// borrowing `song`.
+export const NO_SOURCE_MEDIA =
+ `needs the song project's source videos (media/), which are not at ` +
+ `${caps.songDir}. Set SONG_DIR to a copy to run these.`;
+
+// The phrase console runs against the REAL ASR corpus (`asr/`) — synthesising
+// one would defeat the point, since what it exercises is chunk seams,
+// multi-word tokens and a real confidence spread.
+export const NO_ASR_CORPUS =
+ `needs the song project's ASR corpus (asr/), which is not at ` +
+ `${caps.songDir}. Set SONG_DIR to a copy to run these.`;
diff --git a/umtool/e2e/deck.spec.ts b/umtool/e2e/deck.spec.ts
@@ -1,4 +1,5 @@
import { test, expect } from "@playwright/test";
+import { caps, NO_SONG_DATA } from "./capabilities";
import { readFileSync } from "node:fs";
import path from "node:path";
@@ -186,6 +187,11 @@ test("a media token that was never minted serves nothing", async ({ request }) =
// ---------------------------------------------------------------------------
test("the diff reports the seeded edits exactly", async ({ request }) => {
+ // BOTH PLANS ARE CANDIDATE-DERIVED. make-fixture writes them from the first
+ // COPIED cand2 entry, and it copies one only when its wav is present — so
+ // with no song bulk data there is nothing to diff, and the seeded edits this
+ // asserts on never existed.
+ test.skip(!caps.song, NO_SONG_DATA);
const r = await request.get(
"/api/browse/plandiff?song=alpha&a=alpha.plan.json&b=alpha-v2.plan.json",
);
@@ -282,6 +288,8 @@ test("accepting a disjoint cover runs the CLI and holds the rule", async ({ requ
// ---------------------------------------------------------------------------
test("the sources page rolls every plan up by episode", async ({ page }) => {
+ // "Every plan" is the candidate-derived ones — see above.
+ test.skip(!caps.song, NO_SONG_DATA);
await page.goto("/browse/sources");
await expect(page.locator("[data-source]").first()).toBeVisible();
diff --git a/umtool/e2e/faces.spec.ts b/umtool/e2e/faces.spec.ts
@@ -2,6 +2,7 @@ import { test, expect, type APIRequestContext, type Page } from "@playwright/tes
import { readFileSync } from "node:fs";
import path from "node:path";
import { autoCrop, scaleOf, CORNER_PX, SHOVE_FLOOR, type Box } from "../lib/face-types";
+import { caps, NO_SOURCE_MEDIA } from "./capabilities";
// The face judger, against the REAL episodes the fixture symlinks -- make-fixture
// derives its cover corners from media/ precisely so the frame and detect routes
@@ -167,6 +168,9 @@ test("the queue is every corner deduped, accepted covers first", async ({ reques
// every corner in the queue, without the spec having to spawn python itself.
test("autoCrop() is the same function as facecrop.py's crop maths", async ({ request }) => {
test.slow();
+ // THE QUEUE IS BUILT FROM SOURCE VIDEOS. With no `media/` the deck is
+ // empty, and every assertion here is about a frame in it.
+ test.skip(!caps.media, NO_SOURCE_MEDIA);
const { queue } = await view(request);
let compared = 0;
@@ -208,6 +212,9 @@ test("the deck states the shove on a corner whose crop was displaced", async ({
page,
request,
}) => {
+ // THE QUEUE IS BUILT FROM SOURCE VIDEOS. With no `media/` the deck is
+ // empty, and every assertion here is about a frame in it.
+ test.skip(!caps.media, NO_SOURCE_MEDIA);
test.slow();
const { queue } = await view(request);
let displaced: { key: string; shove: number } | null = null;
@@ -237,6 +244,9 @@ test("the deck states the shove on a corner whose crop was displaced", async ({
// ---------------------------------------------------------------------------
test("the scale on screen is 300 over the crop height", async ({ page, request }) => {
+ // THE QUEUE IS BUILT FROM SOURCE VIDEOS. With no `media/` the deck is
+ // empty, and every assertion here is about a frame in it.
+ test.skip(!caps.media, NO_SOURCE_MEDIA);
const { queue } = await view(request);
const target = queue[0];
@@ -266,6 +276,9 @@ test("the scale on screen is 300 over the crop height", async ({ page, request }
// came back scaled by the display width would be silently wrong rather than
// obviously broken -- it would still be a square on a face, just the wrong one.
test("a recorded crop survives a reload, in source pixels", async ({ page, request }) => {
+ // THE QUEUE IS BUILT FROM SOURCE VIDEOS. With no `media/` the deck is
+ // empty, and every assertion here is about a frame in it.
+ test.skip(!caps.media, NO_SOURCE_MEDIA);
const { queue } = await view(request);
const target = queue[queue.length - 1];
const crop = { x: 812, y: 137, w: 331, h: 331 };
@@ -378,6 +391,9 @@ test("reject puts the video in the set make-thumb reads", async ({ request }) =>
test("a crop outside the frame is refused and an unknown corner is a 404", async ({
request,
}) => {
+ // THE QUEUE IS BUILT FROM SOURCE VIDEOS. With no `media/` the deck is
+ // empty, and every assertion here is about a frame in it.
+ test.skip(!caps.media, NO_SOURCE_MEDIA);
const { queue } = await view(request);
const target = queue[0];
@@ -425,6 +441,8 @@ test("a crop outside the frame is refused and an unknown corner is a 404", async
test("the frame comes back at the source's own size unless a width is asked for", async ({
request,
}) => {
+ // Same reason as the rest of this file's frame tests: no `media/`, no queue.
+ test.skip(!caps.media, NO_SOURCE_MEDIA);
const { queue } = await view(request);
const e = queue[0];
const d = await detect(request, e);
diff --git a/umtool/e2e/find.spec.ts b/umtool/e2e/find.spec.ts
@@ -1,6 +1,15 @@
import { test, expect, type APIRequestContext, type Page } from "@playwright/test";
import { readFileSync } from "node:fs";
import path from "node:path";
+import { caps, NO_ASR_CORPUS } from "./capabilities";
+
+// THE WHOLE FILE SKIPS WITHOUT THE CORPUS. Everything below derives its
+// expected answer from what the corpus actually returned, so with no `asr/`
+// there is nothing to derive from: the console is empty and every assertion is
+// either vacuous or red. File-scope `test.skip` rather than ten per-test ones,
+// because "this file needs the ASR corpus" is one fact, and the header below
+// already says it.
+test.skip(!caps.asr, NO_ASR_CORPUS);
// The phrase console, against the REAL ASR corpus -- the suite symlinks asr/
// rather than synthesising one, because 300 files of genuine parakeet output is
diff --git a/umtool/e2e/fixtures/make-fixture.mjs b/umtool/e2e/fixtures/make-fixture.mjs
@@ -21,16 +21,28 @@ import {
import { spawnSync } from "node:child_process";
import path from "node:path";
import { SONG_DATA } from "../../song/paths.mjs";
+import { songCapabilities } from "./song-capabilities.mjs";
const CODE = path.resolve(path.dirname(new URL(import.meta.url).pathname), "..", "..", "song");
const dest = path.resolve(process.argv[2] ?? path.join(process.cwd(), ".e2e-song"));
+// A readdir that answers "nothing" instead of throwing. SONG_DATA's default is
+// the job temp dir the corpus was mined into, which on most machines no longer
+// exists — see the capabilities block below.
+const listDir = (p) => {
+ try {
+ return readdirSync(p);
+ } catch {
+ return [];
+ }
+};
+
rmSync(dest, { recursive: true, force: true });
mkdirSync(path.join(dest, "code"), { recursive: true });
mkdirSync(path.join(dest, "data", "cand2"), { recursive: true });
// -- code + state ------------------------------------------------------------
-for (const f of readdirSync(CODE)) {
+for (const f of listDir(CODE)) {
if (f.endsWith(".mjs") || f.endsWith(".py")) copyFileSync(path.join(CODE, f), path.join(dest, "code", f));
}
// Fresh, empty state: the suite asserts on counts, and inheriting 3,959
@@ -85,7 +97,13 @@ for (const f of ["dates.json", "titles.json", "order-model.json"]) {
writeFileSync(path.join(dest, "code", "um-manifest.json"), JSON.stringify({ version: 1, items: [] }, null, 1));
// -- a few candidates, from videos whose audio is actually present ------------
-const cands = readdirSync(path.join(SONG_DATA, "cand2")).filter((f) => f.endsWith(".json"));
+//
+// TOLERANT OF A MISSING SONG_DATA, and that is the whole point of the
+// capabilities file below. `SONG_DIR`'s default is the job temp dir the corpus
+// was mined into, which on most machines no longer exists — so this readdir
+// used to throw and take the entire suite down before the test server started,
+// including the two dozen specs that have nothing to do with the song project.
+const cands = listDir(path.join(SONG_DATA, "cand2")).filter((f) => f.endsWith(".json"));
let taken = 0;
for (const f of cands) {
if (taken >= 3) break;
@@ -122,6 +140,42 @@ if (existsSync(MODELS)) symlinkSync(MODELS, path.join(dest, "models"));
const FACEDET = path.join(path.dirname(SONG_DATA), "facedet");
if (existsSync(FACEDET)) symlinkSync(FACEDET, path.join(dest, "facedet"));
+// -- WHAT THIS FIXTURE ACTUALLY GOT -------------------------------------------
+//
+// The heavy inputs are SYMLINKED from SONG_DATA, and on a machine where that
+// directory is absent or partial the fixture builds fine and is simply empty:
+// no candidates (the loop above requires a wav per video), no asr, no media, no
+// face detector. Every spec that judges a clip then failed — and failed
+// LOUDLY, as a red suite, over a machine that never had the 39 GB rather than
+// over anything a change broke. Red that means "you are on a different laptop"
+// trains people to ignore red.
+//
+// So the fixture records what it found and the specs that need it skip
+// themselves. A capability is a DIRECTORY THAT EXISTS, checked here once, not a
+// guess made per spec — and `song` is the compound the clip specs actually
+// need: candidates to list AND audio to cut.
+const capabilities = {
+ ...songCapabilities(SONG_DATA),
+ // What actually landed, as opposed to what the source offered: a candidate
+ // is copied only when its wav is present, and the detector needs both halves.
+ copiedCandidates: taken,
+ facedet: existsSync(path.join(dest, "facedet")) && existsSync(path.join(dest, "models")),
+};
+writeFileSync(
+ path.join(dest, "fixture-capabilities.json"),
+ JSON.stringify(capabilities, null, 1) + "\n",
+);
+if (!capabilities.song) {
+ // ON STDERR, because a suite that skips part of itself must say why where
+ // somebody reading a CI log will look.
+ process.stderr.write(
+ `make-fixture: no song bulk data at ${SONG_DATA} ` +
+ `(cand2=${capabilities.cand2} wav48=${capabilities.wav48} asr=${capabilities.asr} ` +
+ `media=${capabilities.media}) — the song-data specs will SKIP. ` +
+ `Set SONG_DIR to a copy to run them.\n`,
+ );
+}
+
// -- one FLAGGED SOURCE, derived from the symlinked asr/ ----------------------
//
// suspect-sources.json was neither copied nor seeded before this, so the
diff --git a/umtool/e2e/fixtures/song-capabilities.mjs b/umtool/e2e/fixtures/song-capabilities.mjs
@@ -0,0 +1,41 @@
+import { existsSync } from "node:fs";
+import path from "node:path";
+
+// WHAT THE SONG PROJECT'S BULK DATA OFFERS THIS MACHINE — one rule, two
+// readers.
+//
+// The 39 GB (`wav48/`, `asr/`, `media/`) is re-derivable from the archive and
+// deliberately not in the repo. `SONG_DIR`'s default is the job temp dir the
+// corpus was mined into, which on most machines no longer exists, so
+// `make-fixture.mjs` builds an empty fixture and every spec that judges a clip
+// used to fail — LOUDLY, as a red suite, over a machine that never had the data
+// rather than over anything a change broke. Red that means "you are on a
+// different laptop" is red people learn to ignore.
+//
+// WHY THIS IS A SHARED MODULE AND NOT THE FIXTURE'S WRITTEN JSON ALONE.
+// `make-fixture.mjs` writes `fixture-capabilities.json` as the RECORD — it is
+// what an operator reads in a CI log — but a spec cannot depend on that file
+// existing when it is collected: Playwright loads the spec modules to build its
+// test list, and the fixture is built by a `webServer` command. On a fresh
+// checkout the file is simply not there yet, and a helper that failed closed on
+// that would skip the whole song suite on a machine that HAS the data. So both
+// sides answer from the same directories, here.
+export function songCapabilities(songData) {
+ const has = (d) => existsSync(path.join(songData, d));
+ const caps = {
+ songData: existsSync(songData),
+ // The candidate files — the population every clip list is built from.
+ cand2: has("cand2"),
+ // The audio. `make-fixture` copies a candidate only when its wav is
+ // present, so without this the queues are empty whatever cand2 holds.
+ wav48: has("wav48"),
+ asr: has("asr"),
+ media: has("media"),
+ songDir: songData,
+ };
+ // THE COMPOUND THE CLIP SPECS READ. A queue with no candidates and a bench
+ // with no audio are the same failure from two directions, so one flag names
+ // it.
+ caps.song = caps.cand2 && caps.wav48;
+ return caps;
+}
