commit 7b80437cd056f02d6449a9d75631250d9c1ae8b9
parent 0ac038738d4d921dd325c267e7f282d9b3680739
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 1 Oct 2026 16:52:41 -0400
umtool: storage tests; e2e cache in the fixture; a storage spec over a media root
lib/report/storage.test.mjs (20, in test:scripts): the env-derived roots in a
child process, ensureOutDir's five states, the media root never created, the
movers out and back with each cut point resumed, a failed copy leaving the
source alone, and the walk's clips/share-* skips. e2e: UMTOOL_CACHE_DIR is the
fixture's cache/ (app and CLI), never ~/.cache; make-fixture adds
storage-fixture, storage-fresh-fixture and a sibling media root;
storage.spec.ts moves out, builds through the link, unplugs the root, makes a
fresh link, and moves back.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
9 files changed, 602 insertions(+), 7 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -169,6 +169,8 @@ yarn-error.log*
# run alongside a dev server someone is judging clips in (Next refuses two for
# one project).
umtool/.e2e-song/
+# ...and the storage spec's media root, a sibling of it (UMTOOL_MEDIA_DIR).
+umtool/.e2e-song-media/
umtool/.next/
# Any alternate dist dir, not just the e2e one.
#
diff --git a/umtool/bin/umtool.mjs b/umtool/bin/umtool.mjs
@@ -425,8 +425,10 @@ async function cmdStorage() {
const dryRun = has("--dry-run");
const log = (m) => (json ? console.error(m) : console.log(m));
- if (!sub || sub === "status") {
- const refs = positional[1] ? [await pick(positional[1])] : await projectRefs();
+ if (sub !== "move-out" && sub !== "move-back") {
+ // `umtool storage`, `umtool storage <project>`, `umtool storage status [<project>]`.
+ const one = sub === "status" ? positional[1] : sub;
+ const refs = one ? [await pick(one)] : await projectRefs();
const rows = [];
for (const p of refs) rows.push({ id: p.id, ...(await outDirState(p.dir)) });
if (json) return out({ media: { path: MEDIA_ROOT, tiered: MEDIA_TIERED }, projects: rows });
@@ -438,8 +440,10 @@ async function cmdStorage() {
return;
}
- const move = sub === "move-out" ? moveDirToMedia : sub === "move-back" ? moveDirToLocal : null;
- if (!move) die(`unknown storage command "${sub}" — move-out or move-back`);
+ const move = sub === "move-out" ? moveDirToMedia : moveDirToLocal;
+ if (sub === "move-out" && !MEDIA_TIERED) {
+ die("UMTOOL_MEDIA_DIR is not set: there is no media root to move out/ to. Set it to a directory on the media drive (outside the reports root).");
+ }
const all = has("--all");
if (!all && !positional[1]) die(`which project? \`umtool storage ${sub} <project>\` or --all`);
const refs = all ? await projectRefs() : [await pick(positional[1])];
diff --git a/umtool/e2e/fixtures/make-fixture.mjs b/umtool/e2e/fixtures/make-fixture.mjs
@@ -1683,6 +1683,34 @@ copyFileSync(
path.join(DASH, "out", "clips-raw", "vid1_0.00-9.00.mp4"),
);
+// -- the media root (release 17) ------------------------------------------------
+//
+// storage.spec.ts moves storage-fixture's out/ to a media root and back, builds
+// through the link, and unplugs the root. Its CLI is given UMTOOL_MEDIA_DIR =
+// this directory; the APP is not (every other spec's out/ stays a directory).
+// A SIBLING of the fixture, not inside it: REPORTS_ROOT is the fixture root, and
+// a media root inside the tree it mirrors is refused. Reset here, every run.
+// storage-fixture dash-fixture's shape: a cached window, buildable offline
+// storage-fresh-fixture no out/ at all: the first writer makes the link
+const MEDIA = `${dest}-media`;
+rmSync(MEDIA, { recursive: true, force: true });
+mkdirSync(MEDIA, { recursive: true });
+for (const slug of ["storage-fixture", "storage-fresh-fixture"]) {
+ const dir = writeProject(
+ slug,
+ manifest(slug, "The Storage Fixture", { siteOrigin: "https://archive.example" }, [
+ { type: "clip", id: "c01", video: "vid1", start: 3.0, end: 6.0, cite: 3, section: 0, lock: true, quote: "and because" },
+ { type: "clip", id: "c02", video: "vid1", start: 9.0, end: 12.0, cite: 9, section: 0, lock: true, quote: "another whole sentence" },
+ ]),
+ );
+ if (slug !== "storage-fixture") continue;
+ mkdirSync(path.join(dir, "out", "clips-raw"), { recursive: true });
+ copyFileSync(
+ path.join(REPORT, "out", "clips-raw", "vid1_0.00-9.00.mp4"),
+ path.join(dir, "out", "clips-raw", "vid1_0.00-9.00.mp4"),
+ );
+}
+
console.log(`fixture at ${dest}`);
if (planned) console.log(` planned clip (used in a build): ${planned}`);
console.log(` videos/: alpha (4 cuts, 3 variants), beta (2 cuts), deck (1 cut, 2 variants)`);
@@ -1697,6 +1725,8 @@ console.log(` flagged source: ${flagged ? flagged.video : "none — no asr/"}`)
console.log(` SONG_CODE_DIR=${path.join(dest, "code")}`);
console.log(` SONG_DIR=${path.join(dest, "data")}`);
console.log(` SONG_REPORTS_DIR=${reports}`);
+console.log(` UMTOOL_CACHE_DIR=${path.join(dest, "cache")} (removed with the fixture; never ~/.cache)`);
+console.log(` storage spec media root: ${MEDIA} (UMTOOL_MEDIA_DIR on its CLI only; reset here)`);
console.log(` YTDLP_BIN=${path.join(BIN, "yt-dlp")} QRENCODE_BIN=${path.join(BIN, "qrencode")} HYPERFRAMES_BIN=${path.join(BIN, "hyperframes")}`);
console.log(` CHANNELS_DIR=${CHANNELS} (testchan/vid1 punctuated, vid2 not; vid3/vid4/vid5 for the editor fetch)`);
console.log(` projects: report-fixture (4 clips, 1 mid-sentence), no-origin-fixture,`);
diff --git a/umtool/e2e/projects.spec.ts b/umtool/e2e/projects.spec.ts
@@ -318,6 +318,8 @@ const cliEnv = {
SONG_REPORTS_DIR: path.join(FIXTURE, "reports"),
SONG_DIR: path.join(FIXTURE, "data"),
CHANNELS_DIR: path.join(FIXTURE, "channels"),
+ // The fixture's cache, as playwright.config.ts gives the app (never ~/.cache).
+ UMTOOL_CACHE_DIR: path.join(FIXTURE, "cache"),
};
const umtool = (args: string[]) =>
execFileSync("node", ["bin/umtool.mjs", ...args], { cwd: UMTOOL, encoding: "utf8", env: cliEnv });
@@ -415,7 +417,7 @@ test("deleting the index changes nothing but latency", async ({ request }) => {
// CACHE_DIR is documented as derived output, safe to delete at any time. This
// is that promise, tested.
- rmSync(path.join(FIXTURE, "data", ".cache", "umtool", "index"), {
+ rmSync(path.join(FIXTURE, "cache", "index"), {
recursive: true,
force: true,
});
diff --git a/umtool/e2e/report-longform.spec.ts b/umtool/e2e/report-longform.spec.ts
@@ -21,6 +21,8 @@ const cliEnv = {
SONG_REPORTS_DIR: path.join(FIXTURE, "reports"),
SONG_DIR: path.join(FIXTURE, "data"),
CHANNELS_DIR: path.join(FIXTURE, "channels"),
+ // The fixture's cache, as playwright.config.ts gives the app (never ~/.cache).
+ UMTOOL_CACHE_DIR: path.join(FIXTURE, "cache"),
YTDLP_BIN: path.join(FIXTURE, "bin", "yt-dlp"),
};
const umtool = (args: string[]) =>
diff --git a/umtool/e2e/storage.spec.ts b/umtool/e2e/storage.spec.ts
@@ -0,0 +1,180 @@
+import { test, expect, type APIRequestContext } from "@playwright/test";
+import { execFileSync, spawnSync } from "node:child_process";
+import { existsSync, lstatSync, readdirSync, readlinkSync, renameSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+// ---------------------------------------------------------------------------
+// A project's render scratch on a media root (release 17, slice U1).
+//
+// With UMTOOL_MEDIA_DIR set, a project's out/ is a link to the same
+// project-relative path under it: made by the first writer, or moved there by
+// `umtool storage move-out`. Only THIS spec's CLI is given the variable (the
+// media root is make-fixture's `<fixture>-media`, reset every run); the app is
+// not, which is the point of half of it -- a reader, and a build the app runs,
+// go through the link without knowing a media root exists.
+//
+// The tests run in order and hand the project's state on: moved out, built
+// through, unplugged, plugged back, moved back.
+// ---------------------------------------------------------------------------
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const UMTOOL = path.join(HERE, "..");
+const FIXTURE = path.join(UMTOOL, ".e2e-song");
+const MEDIA = `${FIXTURE}-media`;
+const UNPLUGGED = `${MEDIA}.unplugged`;
+const PROJECT = "reports/storage-fixture";
+const FRESH = "reports/storage-fresh-fixture";
+const dirOf = (id: string) => path.join(FIXTURE, id);
+const mirrorOf = (id: string) => path.join(MEDIA, id);
+
+const env = {
+ ...process.env,
+ SONG_REPORTS_DIR: path.join(FIXTURE, "reports"),
+ SONG_DIR: path.join(FIXTURE, "data"),
+ CHANNELS_DIR: path.join(FIXTURE, "channels"),
+ UMTOOL_CACHE_DIR: path.join(FIXTURE, "cache"),
+ YTDLP_BIN: path.join(FIXTURE, "bin", "yt-dlp"),
+ UMTOOL_MEDIA_DIR: MEDIA,
+};
+const umtool = (args: string[]) =>
+ JSON.parse(execFileSync("node", ["bin/umtool.mjs", ...args, "--json"], { cwd: UMTOOL, encoding: "utf8", env }));
+// The pipeline's first writer, as a build's step 1 runs it.
+const checkAvailability = (id: string) =>
+ spawnSync(
+ "node",
+ [
+ path.join(UMTOOL, "report-to-video", "check-availability.mjs"),
+ path.join(dirOf(id), "video.manifest.json"),
+ "--out",
+ path.join(dirOf(id), "out"),
+ "--allow-missing",
+ ],
+ { cwd: path.join(UMTOOL, "report-to-video"), encoding: "utf8", env },
+ );
+
+const isLink = (p: string) => existsSync(path.dirname(p)) && lstatSync(p, { throwIfNoEntry: false })?.isSymbolicLink() === true;
+const isRealDir = (p: string) => lstatSync(p, { throwIfNoEntry: false })?.isDirectory() === true;
+const parked = (id: string) => readdirSync(dirOf(id)).filter((n) => n.startsWith("out.moved-") || n === "out.incoming");
+
+async function projectRow(request: APIRequestContext, id: string) {
+ const j = (await (await request.get("/api/browse/projects")).json()) as {
+ projects: { id: string; state: string; facts: string[] }[];
+ };
+ return j.projects.find((p) => p.id === id);
+}
+
+test.describe.configure({ mode: "serial" });
+
+test.afterAll(() => {
+ // A failure mid-way must not leave the root unplugged for the next run's
+ // reader of this file -- make-fixture resets it anyway.
+ if (existsSync(UNPLUGGED) && !existsSync(MEDIA)) renameSync(UNPLUGGED, MEDIA);
+});
+
+test("move-out leaves a link to the media root, and readers see the same project", async ({ request }) => {
+ const out = path.join(dirOf(PROJECT), "out");
+ expect(isRealDir(out)).toBe(true);
+ const before = await projectRow(request, PROJECT);
+ expect(before).toBeTruthy();
+
+ // A dry run measures and changes nothing.
+ const dry = umtool(["storage", "move-out", PROJECT, "--dry-run"]);
+ expect(dry.results[0].state).toBe("would-move");
+ expect(dry.results[0].bytes).toBeGreaterThan(0);
+ expect(isRealDir(out)).toBe(true);
+ expect(existsSync(mirrorOf(PROJECT))).toBe(false);
+
+ const moved = umtool(["storage", "move-out", PROJECT]);
+ expect(moved.ok).toBe(true);
+ expect(moved.results[0].state).toBe("moved");
+ expect(isLink(out)).toBe(true);
+ expect(readlinkSync(out)).toBe(path.join(mirrorOf(PROJECT), "out"));
+ expect(existsSync(path.join(mirrorOf(PROJECT), "out", "clips-raw", "vid1_0.00-9.00.mp4"))).toBe(true);
+ expect(parked(PROJECT)).toEqual([]);
+
+ // The index and the page read <project>/out by path; the link changes nothing.
+ const after = await projectRow(request, PROJECT);
+ expect(after?.state).toBe(before?.state);
+ expect(after?.facts).toEqual(before?.facts);
+
+ // Again: nothing to do.
+ expect(umtool(["storage", "move-out", PROJECT]).results[0].state).toBe("already");
+ const status = umtool(["storage", PROJECT]);
+ expect(status.projects[0].state).toBe("link");
+});
+
+test("a build the app runs writes through the link, and leaves it a link", async ({ request }) => {
+ const start = await request.post("/api/report/build", { data: { project: PROJECT, preset: "fast" } });
+ expect(start.ok()).toBeTruthy();
+ const { job } = (await start.json()) as { job: { id: string } };
+ let state = "running";
+ for (let i = 0; i < 150 && state === "running"; i += 1) {
+ const j = (await (await request.get("/api/jobs")).json()) as { jobs: { id: string; state: string }[] };
+ state = j.jobs.find((x) => x.id === job.id)?.state ?? "running";
+ if (state === "running") await new Promise((r) => setTimeout(r, 200));
+ }
+ expect(state).toBe("done");
+
+ const out = path.join(dirOf(PROJECT), "out");
+ expect(isLink(out)).toBe(true);
+ expect(existsSync(path.join(mirrorOf(PROJECT), "out", "storage-fixture.mp4"))).toBe(true);
+ expect(existsSync(path.join(mirrorOf(PROJECT), "out", "availability.json"))).toBe(true);
+});
+
+test("an unplugged media root refuses loudly and materialises nothing", () => {
+ renameSync(MEDIA, UNPLUGGED);
+ try {
+ expect(umtool(["storage", PROJECT]).projects[0].state).toBe("dangling");
+
+ // A moved project: the link dangles, and the first writer says why.
+ const r = checkAvailability(PROJECT);
+ expect(r.status).not.toBe(0);
+ expect(r.stderr).toContain("is the media drive mounted?");
+ expect(isLink(path.join(dirOf(PROJECT), "out"))).toBe(true);
+
+ // A project with no out/ yet: the root is stat'd, never created.
+ const fresh = checkAvailability(FRESH);
+ expect(fresh.status).not.toBe(0);
+ expect(fresh.stderr).toContain("is not there");
+ expect(existsSync(path.join(dirOf(FRESH), "out"))).toBe(false);
+
+ // Neither recreated the root on the disk it was "on".
+ expect(existsSync(MEDIA)).toBe(false);
+
+ // The doctor says so, and exits 1.
+ const doctor = spawnSync("node", ["bin/umtool.mjs", "doctor", "--json"], { cwd: UMTOOL, encoding: "utf8", env });
+ expect(doctor.status).toBe(1);
+ const roots = JSON.parse(doctor.stdout).roots;
+ expect(roots.media.tiered).toBe(true);
+ expect(roots.media.problem).toContain("is not there");
+ expect(roots.cache.path).toBe(path.join(FIXTURE, "cache"));
+ } finally {
+ renameSync(UNPLUGGED, MEDIA);
+ }
+});
+
+test("the first writer of a project with no out/ makes the link", () => {
+ const r = checkAvailability(FRESH);
+ expect(r.status, r.stderr).toBe(0);
+ const out = path.join(dirOf(FRESH), "out");
+ expect(isLink(out)).toBe(true);
+ expect(readlinkSync(out)).toBe(path.join(mirrorOf(FRESH), "out"));
+ expect(existsSync(path.join(mirrorOf(FRESH), "out", "availability.json"))).toBe(true);
+});
+
+test("move-back makes out/ a real directory again and removes the media copy", () => {
+ const back = umtool(["storage", "move-back", PROJECT]);
+ expect(back.ok).toBe(true);
+ expect(back.results[0].state).toBe("moved");
+ const out = path.join(dirOf(PROJECT), "out");
+ expect(isRealDir(out)).toBe(true);
+ expect(existsSync(path.join(out, "storage-fixture.mp4"))).toBe(true);
+ expect(parked(PROJECT)).toEqual([]);
+ // The project's mirror is gone; the root, and the other project's, are not.
+ expect(existsSync(mirrorOf(PROJECT))).toBe(false);
+ expect(existsSync(MEDIA)).toBe(true);
+ expect(isLink(path.join(dirOf(FRESH), "out"))).toBe(true);
+
+ expect(umtool(["storage", "move-back", PROJECT]).results[0].state).toBe("already");
+});
diff --git a/umtool/lib/report/storage.test.mjs b/umtool/lib/report/storage.test.mjs
@@ -0,0 +1,370 @@
+// A project's render scratch on a media root (release 17): the roots in
+// lib/paths.mjs, ensureOutDir / ensureWriteDir, the two movers, and the walk's
+// skip rules for a media drive that is not there.
+//
+// Every call here names its own roots (`reportsRoot`, `mediaRoot`), so nothing
+// depends on, or touches, the process's REPORTS_ROOT. The env-derived roots are
+// tested in a child process, where the module is evaluated fresh.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { lstat, mkdir, mkdtemp, readFile, readdir, readlink, rename, rm, symlink, writeFile } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+import { mediaMirror } from "../paths.mjs";
+import { SKIP_DIRS, skipsDir } from "../projects/kinds.mjs";
+import { walkProjects } from "../projects/walk.mjs";
+import {
+ ensureOutDir,
+ ensureWriteDir,
+ mediaRootProblem,
+ moveDirToLocal,
+ moveDirToMedia,
+ outDirState,
+} from "./storage.mjs";
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+
+/** A reports root with one project, and a media root beside it (not inside). */
+async function world({ withOut = true } = {}) {
+ const base = await mkdtemp(path.join(tmpdir(), "umtool-storage-"));
+ const reportsRoot = path.join(base, "reports");
+ const mediaRoot = path.join(base, "media");
+ const projectDir = path.join(reportsRoot, "folder", "proj");
+ await mkdir(projectDir, { recursive: true });
+ await mkdir(mediaRoot);
+ await writeFile(path.join(projectDir, "video.manifest.json"), "{}\n");
+ if (withOut) {
+ await mkdir(path.join(projectDir, "out", "clips-raw"), { recursive: true });
+ await writeFile(path.join(projectDir, "out", "proj.mp4"), Buffer.alloc(4096, 1));
+ await writeFile(path.join(projectDir, "out", "clips-raw", "v_0-9.mp4"), Buffer.alloc(2048, 2));
+ }
+ const roots = { reportsRoot, mediaRoot };
+ const mirror = path.join(mediaRoot, "folder", "proj");
+ return { base, reportsRoot, mediaRoot, projectDir, roots, mirror, done: () => rm(base, { recursive: true, force: true }) };
+}
+
+const kind = async (p) => {
+ const st = await lstat(p).catch(() => null);
+ if (!st) return "missing";
+ return st.isSymbolicLink() ? "link" : st.isDirectory() ? "dir" : "file";
+};
+
+// ---------------------------------------------------------------------------
+// The roots
+// ---------------------------------------------------------------------------
+
+/** lib/paths.mjs's values under a given environment, evaluated fresh. */
+function rootsUnder(envPatch) {
+ const env = { ...process.env };
+ for (const k of ["UMTOOL_MEDIA_DIR", "UMTOOL_CACHE_DIR", "UMTOOL_INDEX_DIR", "XDG_CACHE_HOME", "MIX_ROOTS", "MIX_WRITE_ROOTS"]) delete env[k];
+ Object.assign(env, { REPORTS_DIR: "/r/reports", SONG_DIR: "/r/song-data-that-is-not-there" }, envPatch);
+ const code =
+ "const m = await import(process.argv[1]);" +
+ "console.log(JSON.stringify({ media: m.MEDIA_ROOT, tiered: m.MEDIA_TIERED, reports: m.REPORTS_ROOT, cache: m.CACHE_DIR," +
+ " old: m.OLD_CACHE_DIR, index: m.INDEX_DIR, read: m.READ_ROOTS, write: m.WRITE_ROOTS }));";
+ const out = execFileSync(process.execPath, ["--input-type=module", "-e", code, path.join(HERE, "..", "paths.mjs")], {
+ env,
+ encoding: "utf8",
+ });
+ return JSON.parse(out);
+}
+
+test("MEDIA_ROOT unset is REPORTS_ROOT: nothing tiered, no new root", () => {
+ const r = rootsUnder({});
+ assert.equal(r.media, "/r/reports");
+ assert.equal(r.tiered, false);
+ assert.equal(r.read.filter((p) => p === "/r/reports").length, 1);
+});
+
+test("UMTOOL_MEDIA_DIR is a READ root and never a write root", () => {
+ const r = rootsUnder({ UMTOOL_MEDIA_DIR: "/m/umtool" });
+ assert.equal(r.media, "/m/umtool");
+ assert.equal(r.tiered, true);
+ assert.ok(r.read.includes("/m/umtool"));
+ assert.ok(!r.write.includes("/m/umtool"));
+});
+
+test("CACHE_DIR: UMTOOL_CACHE_DIR, else XDG_CACHE_HOME/archilyzer/umtool, else ~/.cache — never SONG_DATA", () => {
+ assert.equal(rootsUnder({ UMTOOL_CACHE_DIR: "/c/u" }).cache, "/c/u");
+ assert.equal(rootsUnder({ XDG_CACHE_HOME: "/x" }).cache, "/x/archilyzer/umtool");
+ const plain = rootsUnder({ XDG_CACHE_HOME: "" });
+ assert.equal(plain.cache, path.join(homedir(), ".cache", "archilyzer", "umtool"));
+ assert.equal(plain.index, path.join(plain.cache, "index"));
+ assert.equal(plain.old, "/r/song-data-that-is-not-there/.cache/umtool");
+ assert.ok(!plain.cache.startsWith("/r/song-data"));
+});
+
+test("mediaMirror: the project-relative path under the media root, or null outside the reports root", () => {
+ const roots = { reportsRoot: "/r", mediaRoot: "/m" };
+ assert.equal(mediaMirror("/r/a/b", roots), "/m/a/b");
+ assert.equal(mediaMirror("/r", roots), null);
+ assert.equal(mediaMirror("/elsewhere/p", roots), null);
+ assert.equal(mediaMirror("/r-sibling/p", roots), null);
+});
+
+// ---------------------------------------------------------------------------
+// ensureOutDir / ensureWriteDir
+// ---------------------------------------------------------------------------
+
+test("ensureOutDir, not tiered: a plain directory, as every writer made it", async () => {
+ const w = await world({ withOut: false });
+ try {
+ const out = await ensureOutDir(w.projectDir, { reportsRoot: w.reportsRoot, mediaRoot: w.reportsRoot });
+ assert.equal(await kind(out), "dir");
+ assert.deepEqual(await readdir(w.mediaRoot), []);
+ } finally {
+ await w.done();
+ }
+});
+
+test("ensureOutDir, tiered and absent: the mirror is made under the media root and linked", async () => {
+ const w = await world({ withOut: false });
+ try {
+ const out = await ensureOutDir(w.projectDir, w.roots);
+ assert.equal(await kind(out), "link");
+ assert.equal(await readlink(out), path.join(w.mirror, "out"));
+ assert.equal(await kind(path.join(w.mirror, "out")), "dir");
+ assert.deepEqual(await outDirState(w.projectDir), { state: "link", target: path.join(w.mirror, "out") });
+ // Again: the link is kept as it is.
+ assert.equal(await ensureOutDir(w.projectDir, w.roots), out);
+ } finally {
+ await w.done();
+ }
+});
+
+test("ensureOutDir keeps an existing real out/ even when tiered (move-out moves it, not the writer)", async () => {
+ const w = await world();
+ try {
+ await ensureOutDir(w.projectDir, w.roots);
+ assert.equal(await kind(path.join(w.projectDir, "out")), "dir");
+ assert.deepEqual(await readdir(w.mediaRoot), []);
+ } finally {
+ await w.done();
+ }
+});
+
+test("ensureOutDir never creates the media root: an unplugged drive refuses and nothing is made", async () => {
+ const w = await world({ withOut: false });
+ try {
+ await rm(w.mediaRoot, { recursive: true });
+ await assert.rejects(ensureOutDir(w.projectDir, w.roots), /is not there — is its drive mounted\? Nothing was created/);
+ assert.equal(existsSync(w.mediaRoot), false);
+ assert.equal(await kind(path.join(w.projectDir, "out")), "missing");
+ } finally {
+ await w.done();
+ }
+});
+
+test("a dangling out link refuses loudly; nothing is materialised in its place or under it", async () => {
+ const w = await world({ withOut: false });
+ try {
+ await ensureOutDir(w.projectDir, w.roots);
+ await rename(w.mediaRoot, `${w.mediaRoot}.unplugged`);
+ assert.equal((await outDirState(w.projectDir)).state, "dangling");
+ await assert.rejects(ensureOutDir(w.projectDir, w.roots), /is the media drive mounted\?/);
+ await assert.rejects(ensureWriteDir(path.join(w.projectDir, "out", "sourced", "segments")), /is the media drive mounted\?/);
+ // And a plain recursive mkdir through the link fails too (ENOTDIR), making nothing.
+ await assert.rejects(mkdir(path.join(w.projectDir, "out", "clips-raw"), { recursive: true }));
+ assert.equal(await kind(path.join(w.projectDir, "out")), "link");
+ assert.equal(existsSync(w.mediaRoot), false);
+ } finally {
+ await w.done();
+ }
+});
+
+test("a media root inside the reports root (or around it) is refused", async () => {
+ const w = await world({ withOut: false });
+ try {
+ const inner = { reportsRoot: w.reportsRoot, mediaRoot: path.join(w.reportsRoot, "media") };
+ await mkdir(inner.mediaRoot);
+ assert.match(await mediaRootProblem(inner), /must be outside the reports root/);
+ assert.match(await mediaRootProblem({ reportsRoot: w.reportsRoot, mediaRoot: w.base }), /must be outside/);
+ assert.equal(await mediaRootProblem(w.roots), null);
+ assert.equal(await mediaRootProblem({ reportsRoot: w.reportsRoot, mediaRoot: w.reportsRoot }), null);
+ } finally {
+ await w.done();
+ }
+});
+
+test("ensureWriteDir makes a project's out/ through ensureOutDir before anything under it", async () => {
+ const w = await world({ withOut: false });
+ try {
+ // ensureWriteDir uses the process's roots; under the default (no
+ // UMTOOL_MEDIA_DIR in the test environment) it is a plain directory, the
+ // old behaviour. The tiered case is ensureOutDir's, tested above, and the
+ // storage e2e drives the pipeline scripts through it.
+ const deep = path.join(w.projectDir, "out", "sourced", "chrome", "deck-stills");
+ assert.equal(await ensureWriteDir(deep), deep);
+ assert.equal(await kind(deep), "dir");
+ // A directory that is not under any out/ is made as it always was.
+ const other = path.join(w.base, "elsewhere", "x");
+ await ensureWriteDir(other);
+ assert.equal(await kind(other), "dir");
+ } finally {
+ await w.done();
+ }
+});
+
+// ---------------------------------------------------------------------------
+// The movers
+// ---------------------------------------------------------------------------
+
+test("moveDirToMedia: copy, verify, link; the bytes are the same and nothing is left parked", async () => {
+ const w = await world();
+ try {
+ const logs = [];
+ const r = await moveDirToMedia(w.projectDir, "out", { ...w.roots, log: (m) => logs.push(m) });
+ assert.equal(r.state, "moved");
+ assert.equal(r.files, 2);
+ assert.equal(r.bytes, 4096 + 2048);
+ const out = path.join(w.projectDir, "out");
+ assert.equal(await kind(out), "link");
+ assert.equal(await readlink(out), path.join(w.mirror, "out"));
+ assert.deepEqual(await readFile(path.join(out, "proj.mp4")), Buffer.alloc(4096, 1));
+ assert.deepEqual((await readdir(w.projectDir)).sort(), ["out", "video.manifest.json"]);
+ assert.ok(logs.some((l) => l.includes("--delete")), "the mirror pass ran");
+ // Again: nothing to do.
+ assert.equal((await moveDirToMedia(w.projectDir, "out", w.roots)).state, "already");
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDirToMedia --dry-run measures and changes nothing", async () => {
+ const w = await world();
+ try {
+ const r = await moveDirToMedia(w.projectDir, "out", { ...w.roots, dryRun: true });
+ assert.equal(r.state, "would-move");
+ assert.equal(r.bytes, 6144);
+ assert.equal(await kind(path.join(w.projectDir, "out")), "dir");
+ assert.deepEqual(await readdir(w.mediaRoot), []);
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDirToMedia refuses without a media root, and on a link that is not its own", async () => {
+ const w = await world();
+ try {
+ await assert.rejects(
+ moveDirToMedia(w.projectDir, "out", { reportsRoot: w.reportsRoot, mediaRoot: w.reportsRoot }),
+ /UMTOOL_MEDIA_DIR is not set/,
+ );
+ await assert.rejects(moveDirToMedia(w.projectDir, "../x", w.roots), /not a project directory name/);
+ const other = path.join(w.base, "other");
+ await mkdir(other);
+ await symlink(other, path.join(w.projectDir, "clips"));
+ await assert.rejects(moveDirToMedia(w.projectDir, "clips", w.roots), /already a link, to .* not to/);
+ assert.equal((await moveDirToMedia(w.projectDir, "share-none", w.roots)).state, "absent");
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDirToMedia: a failed copy leaves the source untouched", async () => {
+ const w = await world();
+ try {
+ await assert.rejects(moveDirToMedia(w.projectDir, "out", { ...w.roots, rsyncBin: "false" }), /rsync failed/);
+ assert.equal(await kind(path.join(w.projectDir, "out")), "dir");
+ assert.deepEqual(await readFile(path.join(w.projectDir, "out", "proj.mp4")), Buffer.alloc(4096, 1));
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDirToMedia finishes a run cut between the park and the link", async () => {
+ const w = await world();
+ try {
+ // What a cut leaves: the verified copy on the media root, the source parked.
+ await mkdir(path.join(w.mirror), { recursive: true });
+ execFileSync("cp", ["-a", path.join(w.projectDir, "out"), path.join(w.mirror, "out")]);
+ await rename(path.join(w.projectDir, "out"), path.join(w.projectDir, "out.moved-20261001T000000Z"));
+ const r = await moveDirToMedia(w.projectDir, "out", w.roots);
+ assert.equal(r.state, "moved");
+ assert.equal(r.resumed, true);
+ assert.equal(await kind(path.join(w.projectDir, "out")), "link");
+ assert.deepEqual((await readdir(w.projectDir)).sort(), ["out", "video.manifest.json"]);
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDirToMedia removes a parked copy left by a cut after the link", async () => {
+ const w = await world();
+ try {
+ await moveDirToMedia(w.projectDir, "out", w.roots);
+ await mkdir(path.join(w.projectDir, "out.moved-20261001T000000Z"));
+ assert.equal((await moveDirToMedia(w.projectDir, "out", w.roots)).state, "already");
+ assert.deepEqual((await readdir(w.projectDir)).sort(), ["out", "video.manifest.json"]);
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDirToLocal: a real directory again, the media copy and its empty parents gone, the root kept", async () => {
+ const w = await world();
+ try {
+ await moveDirToMedia(w.projectDir, "out", w.roots);
+ const r = await moveDirToLocal(w.projectDir, "out", w.roots);
+ assert.equal(r.state, "moved");
+ assert.equal(r.bytes, 6144);
+ const out = path.join(w.projectDir, "out");
+ assert.equal(await kind(out), "dir");
+ assert.deepEqual(await readFile(path.join(out, "clips-raw", "v_0-9.mp4")), Buffer.alloc(2048, 2));
+ assert.deepEqual((await readdir(w.projectDir)).sort(), ["out", "video.manifest.json"]);
+ assert.equal(existsSync(path.join(w.mediaRoot, "folder")), false);
+ assert.equal(existsSync(w.mediaRoot), true);
+ assert.equal((await moveDirToLocal(w.projectDir, "out", w.roots)).state, "already");
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDirToLocal refuses a dangling link and finishes a cut rename", async () => {
+ const w = await world();
+ try {
+ await moveDirToMedia(w.projectDir, "out", w.roots);
+ await rename(w.mediaRoot, `${w.mediaRoot}.unplugged`);
+ await assert.rejects(moveDirToLocal(w.projectDir, "out", w.roots), /is the media drive mounted\? Nothing moved/);
+ assert.equal(await kind(path.join(w.projectDir, "out")), "link");
+ await rename(`${w.mediaRoot}.unplugged`, w.mediaRoot);
+
+ // A cut between removing the link and renaming the verified copy.
+ execFileSync("cp", ["-a", path.join(w.mirror, "out"), path.join(w.projectDir, "out.incoming")]);
+ await rm(path.join(w.projectDir, "out"));
+ const r = await moveDirToLocal(w.projectDir, "out", w.roots);
+ assert.equal(r.state, "moved");
+ assert.equal(r.resumed, true);
+ assert.equal(await kind(path.join(w.projectDir, "out")), "dir");
+ assert.equal(existsSync(path.join(w.mirror, "out")), false);
+ } finally {
+ await w.done();
+ }
+});
+
+// ---------------------------------------------------------------------------
+// The walk never descends what may sit on the media drive
+// ---------------------------------------------------------------------------
+
+test("the project walk skips out, clips and share-* (a link into an unplugged drive is never stat'd)", async () => {
+ assert.ok(SKIP_DIRS.has("out") && SKIP_DIRS.has("clips"));
+ assert.ok(skipsDir("share-emancipation") && skipsDir("clips") && !skipsDir("shares") && !skipsDir("project"));
+ const w = await world();
+ try {
+ for (const hidden of ["clips", "share-x"]) {
+ const d = path.join(w.reportsRoot, hidden, "inner");
+ await mkdir(d, { recursive: true });
+ await writeFile(path.join(d, "video.manifest.json"), "{}\n");
+ }
+ const ids = (await walkProjects(w.reportsRoot)).map((p) => p.id);
+ assert.deepEqual(ids, ["folder/proj"]);
+ } finally {
+ await w.done();
+ }
+});
diff --git a/umtool/next.config.ts b/umtool/next.config.ts
@@ -19,7 +19,8 @@ const nextConfig: NextConfig = {
// with "Can't resolve 'cbor-x'", which names a package nothing here uses.
serverExternalPackages: ["lmdb"],
// No route's trace may list the e2e fixture (.e2e-song, where
- // e2e/fixtures/make-fixture.mjs links the song data), the e2e server's own
+ // e2e/fixtures/make-fixture.mjs links the song data; .e2e-song-media, the
+ // storage spec's media root), the e2e server's own
// build directory (.next-e2e) or an env file: none is a run-time input. The
// clip-audio route's trace listed 1,704 such files (plans/release-15.md, slice
// UT). That was fixed at the call (lib/paths.mjs `cacheFile`); this is the
@@ -27,7 +28,7 @@ const nextConfig: NextConfig = {
// back to what the sibling routes list. scripts/next-build-trace.test.mjs
// reads the last build's traces back.
outputFileTracingExcludes: {
- "/*": ["./.e2e-song/**/*", "./.next-e2e/**/*", "./.env*"],
+ "/*": ["./.e2e-song/**/*", "./.e2e-song-media/**/*", "./.next-e2e/**/*", "./.env*"],
},
turbopack: {
// Same reasoning as editor/next.config.ts: Turbopack infers the workspace
diff --git a/umtool/playwright.config.ts b/umtool/playwright.config.ts
@@ -76,6 +76,10 @@ export default defineConfig({
// var. CHANNELS_DIR has to be said explicitly: it is where a report
// video's cue files live, and its default is the real 3 GB corpus.
`CHANNELS_DIR=${FIXTURE}/channels ` +
+ // The cache (the project index, posters, analyses) is no longer under
+ // SONG_DIR (release 17): its default is the user's ~/.cache, which a
+ // suite must never write. The fixture's own, rebuilt with it every run.
+ `UMTOOL_CACHE_DIR=${FIXTURE}/cache ` +
// Stub binaries, so a build spec is offline and deterministic. The
// pipeline already reads both as overrides; the fixture writes them.
`YTDLP_BIN=${FIXTURE}/bin/yt-dlp QRENCODE_BIN=${FIXTURE}/bin/qrencode ` +
