commit b8a6acb2cc043c6bfa72ebf43edfffdf49aecd67
parent 605cd6e51a17989d0f968ce4e7eae5bec8d7a321
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 1 Oct 2026 16:41:19 -0400
umtool: out/ through ensureOutDir; storage movers; doctor roots; storage CLI
lib/report/storage.mjs: ensureOutDir (a link into MEDIA_ROOT when tiered, a
directory when not, a loud refusal on a dangling link; the media root is
stat'd, never created), ensureWriteDir for the pipeline's writers,
moveDirToMedia / moveDirToLocal (copy, mirror toward the copy, verify, park,
link; dispatched on disk, so a cut run is finished by running it again).
build-video, check-availability, render-cards, compose-chrome and the deck
still make out/ through it; export says when out/ dangles. The project walk
skips clips/ and share-*; the mix picker follows a project's out link into
the media root. umtool doctor reports the roots and a leftover old cache;
umtool storage [move-out|move-back <project>|--all] [--dry-run].
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
11 files changed, 677 insertions(+), 13 deletions(-)
diff --git a/umtool/bin/umtool.mjs b/umtool/bin/umtool.mjs
@@ -27,6 +27,9 @@
// umtool new <slug> [--kind report-video] [--from <report.md>|<share URL>|<channel>/<id>]
// [--site-origin URL] [--seed chapters] [--brand archilyzer-media]
// umtool doctor [--json] exit 1 if the report pipeline is missing a tool
+// or the media root is set and not there
+// umtool storage [move-out|move-back <project>|--all] [--dry-run] [--json]
+// where each project's out/ lives; move it
// umtool snapshot <project> [--label L] copy the manifest into revisions/
// umtool diff <project> <snapshot> what changed since that snapshot
// umtool export <project> --format toc-bbcode|toc-markdown|description|chapters [--variant V]
@@ -60,6 +63,8 @@ import { buildSteps, checkSourcesSteps, PRESETS } from "../lib/report/driver.mjs
import { openIndex, signRecord } from "../lib/projects/index-db.mjs";
import { probeTools } from "../lib/tools.mjs";
import { scaffoldReportVideo } from "../lib/projects/scaffold.mjs";
+import { CACHE_DIR, INDEX_DIR, MEDIA_ROOT, MEDIA_TIERED, OLD_CACHE_DIR } from "../lib/paths.mjs";
+import { measureTree, mediaRootProblem, moveDirToLocal, moveDirToMedia, outDirState, pathState } from "../lib/report/storage.mjs";
const argv = process.argv.slice(2);
const cmd = argv.find((a) => !a.startsWith("-")) ?? "help";
@@ -338,11 +343,35 @@ function cmdKinds() {
}
}
+/**
+ * The roots, as the doctor reports them: where projects are read, where their
+ * out/ goes, where the cache is -- and a cache left where it used to live
+ * (under SONG_DATA, before release 17), which is derived and safe to delete
+ * once `umtool index` has rebuilt the new one.
+ */
+async function rootsReport() {
+ const isDir = async (p) => (await pathState(p)).kind === "dir";
+ const problem = await mediaRootProblem();
+ const oldPresent = OLD_CACHE_DIR !== CACHE_DIR && (await isDir(OLD_CACHE_DIR));
+ return {
+ ok: !problem,
+ reports: { path: REPORTS_ROOT, present: await isDir(REPORTS_ROOT) },
+ media: { path: MEDIA_ROOT, tiered: MEDIA_TIERED, present: await isDir(MEDIA_ROOT), problem },
+ cache: { path: CACHE_DIR, present: await isDir(CACHE_DIR), index: await isDir(INDEX_DIR) },
+ oldCache: oldPresent
+ ? { path: OLD_CACHE_DIR, present: true, bytes: (await measureTree(OLD_CACHE_DIR)).bytes }
+ : { path: OLD_CACHE_DIR, present: false },
+ };
+}
+
+const mb = (n) => `${(n / 1024 ** 2).toFixed(1)} MB`;
+
async function cmdDoctor() {
// The one command that shells out on purpose. Seven version flags, ~100 ms.
const r = await probeTools();
+ const roots = await rootsReport();
if (json) {
- out(r);
+ out({ ...r, roots });
} else {
for (const t of r.tools) {
const mark = t.present ? "ok " : t.required ? "MISSING" : "absent";
@@ -356,8 +385,92 @@ async function cmdDoctor() {
? "\nthe report pipeline can build here"
: "\nthe report pipeline is MISSING a tool it cannot run without",
);
+ console.log("\nroots");
+ console.log(` reports ${roots.reports.path}${roots.reports.present ? "" : " (not there)"}`);
+ console.log(
+ roots.media.tiered
+ ? ` media ${roots.media.path} — every project's out/ is linked here (UMTOOL_MEDIA_DIR)` +
+ (roots.media.problem ? `\n MISSING ${roots.media.problem}` : "")
+ : ` media = reports (UMTOOL_MEDIA_DIR unset): out/ stays in each project`,
+ );
+ console.log(
+ ` cache ${roots.cache.path}` +
+ (roots.cache.index ? "" : " (no index yet — `umtool index` builds it; everything works without)"),
+ );
+ if (roots.oldCache.present) {
+ console.log(
+ ` old cache ${roots.oldCache.path} ${mb(roots.oldCache.bytes ?? 0)} — the cache's old place, ` +
+ "no longer read; derived, safe to delete",
+ );
+ }
+ }
+ process.exit(r.ok && roots.ok ? 0 : 1);
+}
+
+// ---------------------------------------------------------------------------
+// storage: where each project's out/ lives, and moving it (release 17).
+//
+// umtool storage every project's out/: dir | link | DANGLING | none
+// umtool storage move-out <p>|--all out/ to the media root, a link left in its place
+// umtool storage move-back <p>|--all out/ back to a real directory in the project
+// --dry-run measure and say; change nothing
+//
+// The movers are lib/report/storage.mjs's, which `umtool` and the app share.
+// Run a move when nothing is building: the app's jobs live in its memory, so
+// this cannot see them -- the verify refuses when the tree keeps changing, but
+// a write in the last instant before the swap would be lost with the parked copy.
+// ---------------------------------------------------------------------------
+async function cmdStorage() {
+ const sub = positional[0];
+ const dryRun = has("--dry-run");
+ const log = (m) => (json ? console.error(m) : console.log(m));
+
+ if (!sub || sub === "status") {
+ const refs = positional[1] ? [await pick(positional[1])] : await projectRefs();
+ const rows = [];
+ for (const p of refs) rows.push({ id: p.id, ...(await outDirState(p.dir)) });
+ if (json) return out({ media: { path: MEDIA_ROOT, tiered: MEDIA_TIERED }, projects: rows });
+ console.log(MEDIA_TIERED ? `media root ${MEDIA_ROOT}` : "media root unset (UMTOOL_MEDIA_DIR): out/ stays in each project");
+ for (const r of rows) {
+ const what = { absent: "none", dir: "dir", link: "link", dangling: "DANGLING", other: "OTHER" }[r.state] ?? r.state;
+ console.log(`${what.padEnd(9)} ${r.id}${r.target ? ` -> ${r.target}` : ""}`);
+ }
+ return;
+ }
+
+ const move = sub === "move-out" ? moveDirToMedia : sub === "move-back" ? moveDirToLocal : null;
+ if (!move) die(`unknown storage command "${sub}" — move-out or move-back`);
+ const all = has("--all");
+ if (!all && !positional[1]) die(`which project? \`umtool storage ${sub} <project>\` or --all`);
+ const refs = all ? await projectRefs() : [await pick(positional[1])];
+
+ const results = [];
+ let failed = 0;
+ for (const p of refs) {
+ try {
+ const r = await move(p.dir, "out", { dryRun, log });
+ results.push({ id: p.id, ...r });
+ if (!json && r.state !== "absent") {
+ const size = r.bytes !== undefined ? ` ${r.files} file(s), ${mb(r.bytes)}` : "";
+ console.log(`${r.state.padEnd(12)} ${p.id}${size}`);
+ }
+ } catch (e) {
+ failed += 1;
+ results.push({ id: p.id, state: "failed", error: e?.message ?? String(e) });
+ if (!json) console.log(`${"FAILED".padEnd(12)} ${p.id}\n ${e?.message ?? e}`);
+ }
+ }
+ const bytes = results.reduce((n, r) => n + (r.bytes ?? 0), 0);
+ if (json) out({ ok: failed === 0, dryRun, bytes, results });
+ else {
+ const moved = results.filter((r) => r.state === "moved" || r.state === "would-move").length;
+ console.log(
+ `\n${moved} project(s) ${dryRun ? "would move" : "moved"}, ${mb(bytes)}` +
+ (failed ? `; ${failed} FAILED` : "") +
+ (dryRun ? " — dry run, nothing changed" : ""),
+ );
}
- process.exit(r.ok ? 0 : 1);
+ if (failed) process.exit(1);
}
async function cmdSnapshot() {
@@ -462,6 +575,10 @@ function usage() {
" umtool new <slug> [--from <report.md>|<share URL>|<channel>/<id>] [--site-origin URL] [--seed chapters]",
" [--brand archilyzer-media] render.brand: the report-to-video brand preset",
" umtool doctor [--json] exit 1 if the report pipeline is missing a tool",
+ " or the media root is set and not there; the roots, and a leftover old cache",
+ " umtool storage [<project>] where each project's out/ lives (dir, link, DANGLING)",
+ " umtool storage move-out|move-back <project>|--all [--dry-run]",
+ " out/ to the media root (UMTOOL_MEDIA_DIR) and back; run when nothing is building",
" umtool snapshot <project> [--label L] copy the manifest into revisions/",
" umtool diff <project> <snapshot> what changed since that snapshot",
" umtool export <project> --format toc-bbcode|toc-markdown|description|chapters [--variant V]",
@@ -488,6 +605,7 @@ const COMMANDS = {
folders: cmdFolders,
kinds: cmdKinds,
doctor: cmdDoctor,
+ storage: cmdStorage,
snapshot: cmdSnapshot,
diff: cmdDiff,
export: cmdExport,
diff --git a/umtool/lib/media.ts b/umtool/lib/media.ts
@@ -2,10 +2,10 @@ import { createHash } from "node:crypto";
import { spawn } from "node:child_process";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
-import { mkdir, readdir, readFile, rename, stat, writeFile } from "node:fs/promises";
+import { mkdir, readdir, readFile, realpath, rename, stat, writeFile } from "node:fs/promises";
import { existsSync } from "node:fs";
import path from "node:path";
-import { MEDIA_ROOTS, MIX_CACHE, REPORTS_ROOT, SONG_REPORTS, labelFor } from "./paths";
+import { MEDIA_ROOT, MEDIA_ROOTS, MEDIA_TIERED, MIX_CACHE, REPORTS_ROOT, SONG_REPORTS, inside, labelFor } from "./paths";
import { brightnessCurve, brightnessSteps } from "../song/flatness.mjs";
const run = promisify(execFile);
@@ -79,6 +79,19 @@ const SCRATCH_FILE = /^(poly-song-|polytmp-|seg_|i_|o_|ms\d?seg|out\.raw)/;
/** Below this is a fragment, a probe or a one-note extraction, not a track. */
const MIN_INTERESTING = 256 * 1024;
+/**
+ * A project's `out` linked to the media root (UMTOOL_MEDIA_DIR, release 17) is
+ * walked like the directory it replaced, so a tiered deliverable stays in the
+ * picker under its project. Only a link INTO the media root: every other link
+ * stays unfollowed, as it always was (SONG_DATA's 39 GB are links).
+ */
+async function isMediaLink(e: { isSymbolicLink(): boolean }, abs: string): Promise<boolean> {
+ if (!MEDIA_TIERED || !e.isSymbolicLink()) return false;
+ const real = await realpath(abs).catch(() => null);
+ if (!real || !inside(MEDIA_ROOT, real)) return false;
+ return stat(real).then((s) => s.isDirectory(), () => false);
+}
+
export type MediaRow = { path: string; label: string; size: number; mtimeMs: number };
/**
@@ -103,7 +116,7 @@ export async function listMediaUnder(root: string, maxDepth = 2, limit = 200): P
for (const e of entries) {
if (e.name.startsWith(".")) continue;
const abs = path.join(dir, e.name);
- if (e.isDirectory()) {
+ if (e.isDirectory() || (await isMediaLink(e, abs))) {
if (depth > 0 && !SCRATCH_DIR.test(e.name)) await walk(abs, depth - 1);
continue;
}
@@ -139,7 +152,7 @@ export async function listMedia(limit = 400): Promise<MediaRow[]> {
for (const e of entries) {
if (e.name.startsWith(".")) continue;
const abs = path.join(dir, e.name);
- if (e.isDirectory()) {
+ if (e.isDirectory() || (await isMediaLink(e, abs))) {
if (depth > 0 && !SCRATCH_DIR.test(e.name)) await walk(abs, depth - 1);
continue;
}
@@ -168,7 +181,13 @@ export async function listMedia(limit = 400): Promise<MediaRow[]> {
// to find nothing anybody would load, which is the opposite of the problem
// this is fixing.
const depthFor = (r: string) => (r === REPORTS_ROOT || r === SONG_REPORTS ? 2 : 1);
- for (const r of MEDIA_ROOTS) await walk(r, depthFor(r));
+ // The media root is reached through the projects' `out` links, under the
+ // project's own path; walked as a root as well, every tiered deliverable
+ // would be listed twice and the second copy would land in "other".
+ for (const r of MEDIA_ROOTS) {
+ if (MEDIA_TIERED && r === MEDIA_ROOT) continue;
+ await walk(r, depthFor(r));
+ }
out.sort((a, b) => b.mtimeMs - a.mtimeMs);
return out.slice(0, limit);
}
diff --git a/umtool/lib/projects/kinds.mjs b/umtool/lib/projects/kinds.mjs
@@ -47,8 +47,23 @@ export const SKIP_DIRS = new Set([
// never reaches inside one -- but a snapshot directory left behind by a
// deleted manifest must not read as a project either.
"revisions",
+ // A report's cut clips. Like out/, it may be a link to the media root
+ // (release 17), and the walk follows a link to a directory with a stat --
+ // which, on a media drive that is unplugged or stalled, is a hang or a
+ // miss, never a project.
+ "clips",
]);
+/**
+ * Name PREFIXES the walk never descends into, for the same reason as `clips`:
+ * `share-<x>/` (a deliver's zip and its staging) may be a link to the media
+ * root, and none of them can contain a project.
+ */
+export const SKIP_PREFIXES = ["share-"];
+
+/** Whether the walk skips a directory entry by its name. */
+export const skipsDir = (name) => SKIP_DIRS.has(name) || SKIP_PREFIXES.some((p) => name.startsWith(p));
+
const has = (names, n) => names.has(n);
const someMatch = (names, re) => [...names].some((n) => re.test(n));
diff --git a/umtool/lib/projects/walk.mjs b/umtool/lib/projects/walk.mjs
@@ -15,7 +15,7 @@
// project. The 3.1 GB is never touched.
import { readdir, readFile, realpath, stat } from "node:fs/promises";
import path from "node:path";
-import { RESERVED_BROWSE, SKIP_DIRS, detectKind } from "./kinds.mjs";
+import { RESERVED_BROWSE, detectKind, skipsDir } from "./kinds.mjs";
/** A single safe path segment: no separators, no traversal, no dotfiles. */
export const isSegment = (v) => /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(v) && !v.includes("..");
@@ -103,7 +103,7 @@ export async function walkProjects(root, { maxDepth = MAX_DEPTH } = {}) {
for (const e of entries) {
if (e.name.startsWith(".")) continue;
- if (SKIP_DIRS.has(e.name)) continue;
+ if (skipsDir(e.name)) continue;
let isDir = e.isDirectory();
if (!isDir && e.isSymbolicLink()) {
isDir = await stat(path.join(abs, e.name)).then((s) => s.isDirectory(), () => false);
diff --git a/umtool/lib/report/export.mjs b/umtool/lib/report/export.mjs
@@ -18,6 +18,7 @@ import { readFile, readdir, stat } from "node:fs/promises";
import path from "node:path";
import { DEFAULT_VARIANT, selectVariant, segmentOffsets } from "umtool-report-to-video/build-video";
import { citeUrlFor, channelFor, readAvailability, readManifest } from "../projects/report.mjs";
+import { outDirState } from "./storage.mjs";
export const EXPORT_FORMATS = ["toc-bbcode", "toc-markdown", "description", "chapters"];
@@ -69,6 +70,13 @@ const titleOf = (e, i) => e.chapter ?? (e.type === "clip" ? `${i + 1}. ${e.video
export async function chapterOffsets(dir, manifest, variant) {
const entries = manifest.timeline ?? [];
const outDir = path.join(dir, "out");
+ // Read through, never made: an export writes nothing under out/. But a link
+ // to a media root that is not mounted would read below as "no build", which
+ // sends somebody off to rebuild a cut that is sitting on an unplugged drive.
+ const out = await outDirState(dir);
+ if (out.state === "dangling") {
+ return { error: `out/ is a link to ${out.target}, which is not there — is the media drive mounted?` };
+ }
const ffmetaCandidates = [path.join(outDir, variant, "chapters.ffmeta"), path.join(outDir, "chapters.ffmeta")];
for (const file of ffmetaCandidates) {
const text = await readFile(file, "utf8").catch(() => null);
diff --git a/umtool/lib/report/onscreen.mjs b/umtool/lib/report/onscreen.mjs
@@ -15,7 +15,7 @@
//
// The preview never renders and never touches the build's project or cache:
// compose-chrome's `preview: true` writes out/<variant>/chrome/deck-preview/.
-import { mkdir, readFile, rm } from "node:fs/promises";
+import { readFile, rm } from "node:fs/promises";
import path from "node:path";
import { composeChrome } from "umtool-report-to-video/compose-chrome";
import { selectVariant } from "umtool-report-to-video/build-video";
@@ -38,6 +38,7 @@ import {
} from "../projects/report.mjs";
import { normalizePostPatches } from "./manifest.mjs";
import { deckPreviewDir, postsPreviewDir } from "./serve.mjs";
+import { ensureWriteDir } from "./storage.mjs";
/** The schedule document deck.mjs defines, built or estimated. */
/** @typedef {ReturnType<typeof estimateSchedule>} DeckSchedule */
@@ -430,7 +431,7 @@ export async function deckStill(project, variant, schedule, t) {
const want = deckPreviewDir(project.dir, variant);
const stills = path.join(outDir, "chrome", "deck-stills");
return serialised(want, async () => {
- await mkdir(stills, { recursive: true });
+ await ensureWriteDir(stills); // through ensureOutDir: out/ may be a link to the media root
const png = path.join(stills, `still-${process.pid}-${Math.random().toString(36).slice(2, 8)}.png`);
try {
/** @type {Record<string, unknown>} */
diff --git a/umtool/lib/report/storage.mjs b/umtool/lib/report/storage.mjs
@@ -0,0 +1,488 @@
+// Where a project's bulk lives: its render scratch (`out/`) on a media root, the
+// manifest and everything small beside it (release 17).
+//
+// lib/paths.mjs names the roots: REPORTS_ROOT holds every project; MEDIA_ROOT
+// (UMTOOL_MEDIA_DIR) holds the bulk; MEDIA_TIERED says they differ. A tiered
+// project's `out` is an ABSOLUTE SYMLINK to the same project-relative path
+// under MEDIA_ROOT, so every reader keeps opening `<project>/out/...` by path
+// and none of them knows the media drive exists.
+//
+// ensureOutDir the first writer's call: makes `out/` -- a link when
+// tiered, a directory when not -- and refuses loudly on a
+// dangling link instead of building a new tree beside it
+// moveDirToMedia an existing directory of the project to MEDIA_ROOT:
+// copy, mirror, verify, park, link, delete the parked copy
+// moveDirToLocal the reverse: back to a real directory in the project
+//
+// The movers take a NAME, not "out": `umtool storage move-out` moves `out/`
+// with them, and a project's deliverables (`clips/`, `share-*/`) are moved by
+// the same two calls behind the deliverables switch.
+//
+// Modelled on the editor's common/controller/relocateDir.ts, not imported from
+// it (umtool's scripts are plain .mjs under node, and that is a TypeScript
+// controller): the source is never touched until the copy verifies; `--delete`
+// only ever points at the copy under construction; every step past the copy is
+// dispatched on what is ON DISK, so a run cut at any point is finished by
+// running it again.
+//
+// THE MEDIA ROOT IS NEVER CREATED HERE. A media drive that is not mounted
+// leaves its mountpoint as an empty directory or no directory at all, and a
+// recursive mkdir would build the tree on the root filesystem and fill it. So
+// the root is stat'd first and must already be a directory; everything BELOW
+// it may be made with a recursive mkdir.
+//
+// Every path and fs call carries `turbopackIgnore`: lib/report/onscreen.mjs
+// imports this module, and the app's routes import that (plans/FACTS.md, "A
+// path joined from `process.cwd()` …").
+import { spawn } from "node:child_process";
+import { lstat, mkdir, readdir, readlink, rename, rm, rmdir, stat, statfs, symlink, unlink } from "node:fs/promises";
+import path from "node:path";
+import { MEDIA_ROOT, REPORTS_ROOT, inside, mediaMirror } from "../paths.mjs";
+
+/** The free space a move keeps on the volume it copies to, beyond the copy. */
+export const MOVE_MARGIN_BYTES = 1024 ** 3;
+
+/** The project directories the movers may move. One path segment, never a dot name. */
+const assertName = (name) => {
+ if (typeof name !== "string" || !name || name.startsWith(".") || /[\/\\\0]/.test(name)) {
+ throw new Error(`not a project directory name: ${JSON.stringify(name)}`);
+ }
+};
+
+/** The roots a call works against: the process's, unless a test names its own. */
+function rootsOf(opts = {}) {
+ const reportsRoot = path.resolve(/* turbopackIgnore: true */ opts.reportsRoot ?? REPORTS_ROOT);
+ const mediaRoot = path.resolve(/* turbopackIgnore: true */ opts.mediaRoot ?? MEDIA_ROOT);
+ return { reportsRoot, mediaRoot, tiered: mediaRoot !== reportsRoot };
+}
+
+/**
+ * What a path is RIGHT NOW. lstat, never stat: a dangling link -- the state an
+ * unmounted media drive leaves behind -- reads as absent through stat.
+ * @returns {Promise<{ kind: "missing" } | { kind: "dir" } | { kind: "link", target: string, targetIsDir: boolean } | { kind: "other" }>}
+ */
+export async function pathState(p) {
+ let st;
+ try {
+ st = await lstat(/* turbopackIgnore: true */ p);
+ } catch {
+ return { kind: "missing" };
+ }
+ if (st.isSymbolicLink()) {
+ const raw = await readlink(/* turbopackIgnore: true */ p).catch(() => "");
+ const target = path.resolve(/* turbopackIgnore: true */ path.dirname(/* turbopackIgnore: true */ p), raw);
+ const targetIsDir = await stat(/* turbopackIgnore: true */ p).then((s) => s.isDirectory(), () => false);
+ return { kind: "link", target, targetIsDir };
+ }
+ if (st.isDirectory()) return { kind: "dir" };
+ return { kind: "other" };
+}
+
+/**
+ * Why render scratch cannot go to the media root now, or null when it can (or
+ * when nothing is tiered). The root must already exist as a directory -- it is
+ * never created -- and must neither sit inside REPORTS_ROOT nor contain it (a
+ * mirror inside the tree it mirrors would be walked as projects).
+ */
+export async function mediaRootProblem(opts = {}) {
+ const { reportsRoot, mediaRoot, tiered } = rootsOf(opts);
+ if (!tiered) return null;
+ if (inside(reportsRoot, mediaRoot) || inside(mediaRoot, reportsRoot)) {
+ return `the media root ${mediaRoot} (UMTOOL_MEDIA_DIR) must be outside the reports root ${reportsRoot}, and must not contain it`;
+ }
+ const st = await stat(/* turbopackIgnore: true */ mediaRoot).catch(() => null);
+ if (!st) {
+ return `the media root ${mediaRoot} (UMTOOL_MEDIA_DIR) is not there — is its drive mounted? Nothing was created.`;
+ }
+ if (!st.isDirectory()) return `the media root ${mediaRoot} (UMTOOL_MEDIA_DIR) is not a directory`;
+ return null;
+}
+
+/**
+ * The state of a project's `out`, for a reader that wants to say WHY a build's
+ * files are not there rather than "no build": `dangling` is a link whose target
+ * is gone (the media drive is not mounted).
+ * @returns {Promise<{ state: "absent" | "dir" | "link" | "dangling" | "other", target?: string }>}
+ */
+export async function outDirState(projectDir) {
+ const s = await pathState(path.join(/* turbopackIgnore: true */ projectDir, "out"));
+ if (s.kind === "missing") return { state: "absent" };
+ if (s.kind === "link") return { state: s.targetIsDir ? "link" : "dangling", target: s.target };
+ return { state: s.kind };
+}
+
+/**
+ * Make sure `<projectDir>/out` exists, and return its path.
+ *
+ * a directory -> it, as it is (an untiered project, or one not moved yet)
+ * a link to a directory -> it
+ * a DANGLING link -> throws: the media drive is not there, and writing
+ * through the link would fail anyway -- but a
+ * recursive mkdir under it would fail with a
+ * message about some subdirectory, so this says
+ * what is actually wrong. Nothing is created.
+ * absent, tiered -> `<MEDIA_ROOT>/<project-relative>/out`, then the link
+ * absent, not tiered -> a directory, as every writer always made it
+ *
+ * A project outside REPORTS_ROOT (a hand-run script's `--out` somewhere else)
+ * is never tiered.
+ */
+export async function ensureOutDir(projectDir, opts = {}) {
+ const out = path.join(/* turbopackIgnore: true */ projectDir, "out");
+ const s = await pathState(out);
+ if (s.kind === "dir") return out;
+ if (s.kind === "link") {
+ if (s.targetIsDir) return out;
+ throw new Error(
+ `${out} is a link to ${s.target}, which is not there — is the media drive mounted? ` +
+ `Nothing was written, and nothing was created in its place.`,
+ );
+ }
+ if (s.kind === "other") throw new Error(`${out} exists and is not a directory`);
+
+ const roots = rootsOf(opts);
+ const mirror = roots.tiered ? mediaMirror(projectDir, roots) : null;
+ if (!mirror) {
+ await mkdir(/* turbopackIgnore: true */ out, { recursive: true });
+ return out;
+ }
+ const problem = await mediaRootProblem(roots);
+ if (problem) throw new Error(`cannot make ${out}: ${problem}`);
+ const target = path.join(/* turbopackIgnore: true */ mirror, "out");
+ // Recursive is safe here: the root itself was just seen to exist.
+ await mkdir(/* turbopackIgnore: true */ target, { recursive: true });
+ try {
+ await symlink(/* turbopackIgnore: true */ target, out, "dir");
+ } catch (e) {
+ // Two writers starting at once: whichever linked first won, and a link (or
+ // directory) that now resolves is as good as ours.
+ if (e?.code !== "EEXIST") throw e;
+ const again = await pathState(out);
+ if (again.kind === "dir" || (again.kind === "link" && again.targetIsDir)) return out;
+ throw e;
+ }
+ return out;
+}
+
+/**
+ * Make a directory a pipeline step writes into, and return it.
+ *
+ * When it is a project's `out` or lies under one (`out/<variant>`,
+ * `out/<variant>/chrome/deck-stills`), that `out` is made by ensureOutDir FIRST
+ * -- a link when tiered -- and only then is the rest made under it. A plain
+ * recursive mkdir of `out/<variant>/segments` was how every script made `out/`,
+ * and it would make a real directory where the link belongs. Any other
+ * directory (a hand-run script's `--out` somewhere else) is made as it always
+ * was.
+ */
+export async function ensureWriteDir(dir) {
+ let d = path.resolve(/* turbopackIgnore: true */ dir);
+ for (let i = 0; i < 4; i++) {
+ if (path.basename(/* turbopackIgnore: true */ d) === "out") {
+ await ensureOutDir(path.dirname(/* turbopackIgnore: true */ d));
+ break;
+ }
+ const up = path.dirname(/* turbopackIgnore: true */ d);
+ if (up === d) break;
+ d = up;
+ }
+ await mkdir(/* turbopackIgnore: true */ dir, { recursive: true });
+ return dir;
+}
+
+// ---------------------------------------------------------------------------
+// Measuring
+// ---------------------------------------------------------------------------
+
+/** Files and bytes under a directory. Follows no symlink. */
+export async function measureTree(dir) {
+ let bytes = 0;
+ let files = 0;
+ const stack = [dir];
+ while (stack.length) {
+ const cur = stack.pop();
+ let entries;
+ try {
+ entries = await readdir(/* turbopackIgnore: true */ cur, { withFileTypes: true });
+ } catch {
+ continue;
+ }
+ for (const e of entries) {
+ const p = path.join(/* turbopackIgnore: true */ cur, e.name);
+ if (e.isDirectory()) stack.push(p);
+ else if (e.isFile()) {
+ const st = await stat(/* turbopackIgnore: true */ p).catch(() => null);
+ if (st) {
+ bytes += st.size;
+ files += 1;
+ }
+ }
+ }
+ }
+ return { bytes, files };
+}
+
+async function freeBytes(dir) {
+ const s = await statfs(/* turbopackIgnore: true */ dir).catch(() => null);
+ return s ? Number(s.bavail) * Number(s.bsize) : null;
+}
+
+async function assertRoom(dir, bytes) {
+ const free = await freeBytes(dir);
+ if (free !== null && free < bytes + MOVE_MARGIN_BYTES) {
+ throw new Error(
+ `not enough room on ${dir}: ${gb(free)} free, the copy needs ${gb(bytes)} plus ${gb(MOVE_MARGIN_BYTES)} to spare. Nothing moved.`,
+ );
+ }
+}
+
+const gb = (n) => `${(n / 1024 ** 3).toFixed(2)} GB`;
+
+// ---------------------------------------------------------------------------
+// rsync
+// ---------------------------------------------------------------------------
+
+/** `rsync <args> <src>/ <dest>/` -- the trailing slashes copy the CONTENTS. */
+function rsync(bin, args, src, dest, log) {
+ return new Promise((resolve, reject) => {
+ const argv = [...args, `${src}/`, `${dest}/`];
+ log(`$ ${bin} ${argv.join(" ")}`);
+ const child = spawn(bin, argv, { stdio: ["ignore", "pipe", "pipe"] });
+ let output = "";
+ child.stdout.on("data", (c) => (output += c));
+ child.stderr.on("data", (c) => (output += c));
+ child.on("error", reject);
+ child.on("close", (code) => resolve({ code: code ?? 1, output }));
+ });
+}
+
+// What a dry run itemizes that is not a difference: rsync's own chatter.
+const driftLines = (output) =>
+ output
+ .split("\n")
+ .map((l) => l.trim())
+ .filter((l) => l && !l.startsWith("sending incremental") && !/^(sent|total size)/.test(l));
+
+/**
+ * COPY, MIRROR, VERIFY -- the source is not touched by any of it.
+ *
+ * 1. `rsync -a --partial`: the bytes, resumable.
+ * 2. `rsync -a --delete --info=del` toward the COPY: what changed on the
+ * source meanwhile is re-sent, what it no longer has leaves the copy
+ * (each removal in the log). Never pointed the other way: the copy is
+ * checked to be neither the source nor inside it, nor around it.
+ * 3. `rsync -a --dry-run --itemize-changes --delete` must list nothing, and
+ * the two trees must measure the same. One more mirror pass if the source
+ * moved under the check; a second difference refuses.
+ */
+async function copyMirrorVerify(src, dest, { rsyncBin, log }) {
+ if (inside(src, dest) || inside(dest, src)) {
+ throw new Error(`refusing to copy ${src} into ${dest}: one is inside the other. Nothing moved.`);
+ }
+ const copied = await rsync(rsyncBin, ["-a", "--partial"], src, dest, log);
+ if (copied.code !== 0) throw new Error(`rsync failed (exit ${copied.code}): ${copied.output.trim().split("\n").pop() ?? ""}`);
+ for (let pass = 0; ; pass++) {
+ const mirrored = await rsync(rsyncBin, ["-a", "--delete", "--info=del"], src, dest, log);
+ if (mirrored.output.trim()) log(mirrored.output.trim());
+ if (mirrored.code !== 0) throw new Error(`the mirror pass failed (exit ${mirrored.code}). The source has NOT been touched.`);
+ const check = await rsync(rsyncBin, ["-a", "--dry-run", "--itemize-changes", "--delete"], src, dest, () => {});
+ if (check.code !== 0) throw new Error(`the verify failed (exit ${check.code}). The source has NOT been touched.`);
+ const drift = driftLines(check.output);
+ if (!drift.length) break;
+ if (pass >= 1) {
+ throw new Error(
+ `the copy still differs from the source after a second mirror pass (${drift.length} item(s), first: ${drift[0]}) — ` +
+ `something is still writing into ${src}. Stop it and run the move again. The source has NOT been touched.`,
+ );
+ }
+ log(`the source changed during the copy (${drift.length} item(s)) — one more mirror pass`);
+ }
+ const [a, b] = await Promise.all([measureTree(src), measureTree(dest)]);
+ if (a.files !== b.files || a.bytes !== b.bytes) {
+ throw new Error(
+ `the copy does not measure the same: ${a.files} file(s)/${a.bytes} B against ${b.files}/${b.bytes} B. The source has NOT been touched.`,
+ );
+ }
+ return a;
+}
+
+// ---------------------------------------------------------------------------
+// The movers
+// ---------------------------------------------------------------------------
+
+const stamp = () => new Date().toISOString().replace(/[-:]/g, "").replace(/\.\d+Z$/, "Z");
+
+/** `<name>.moved-<stamp>` siblings: a move-out parked them and was cut before deleting. */
+async function parkedOf(projectDir, name) {
+ const names = await readdir(/* turbopackIgnore: true */ projectDir).catch(() => []);
+ return names
+ .filter((n) => n.startsWith(`${name}.moved-`))
+ .sort()
+ .map((n) => path.join(/* turbopackIgnore: true */ projectDir, n));
+}
+
+const defaults = (opts) => ({
+ rsyncBin: opts.rsyncBin ?? process.env.RSYNC_BIN ?? "rsync",
+ log: opts.log ?? (() => {}),
+ dryRun: !!opts.dryRun,
+});
+
+/**
+ * Move `<projectDir>/<name>` to the same project-relative path under the media
+ * root and leave an absolute link in its place.
+ *
+ * Dispatched on the disk, so running it again finishes a run that was cut:
+ *
+ * a link to the mirror -> "already" (and a parked copy left by a cut
+ * between the link and its deletion is removed)
+ * a link anywhere else -> refused; it is not this media root's
+ * absent, a parked copy -> the cut was between the park and the link
+ * (the copy had verified): link, delete the parked copy
+ * absent, nothing parked -> "absent", nothing to move
+ * a directory -> copy, mirror, verify, rename it to
+ * `<name>.moved-<stamp>`, link, delete the parked copy
+ *
+ * `dryRun` measures and changes nothing. Nothing in the project may be writing
+ * into `<name>` while it runs (the app's jobs are in its own memory, so a CLI
+ * caller cannot see them); the verify refuses if the tree keeps changing.
+ *
+ * @param {string} projectDir
+ * @param {string} name one path segment: "out", "clips", "share-<x>"
+ * @param {{ dryRun?: boolean, log?: (m: string) => void, rsyncBin?: string, reportsRoot?: string, mediaRoot?: string }} [opts]
+ * @returns {Promise<{ state: "moved" | "already" | "absent" | "would-move" | "would-finish", src: string, dest: string, bytes?: number, files?: number, resumed?: boolean }>}
+ */
+export async function moveDirToMedia(projectDir, name, opts = {}) {
+ assertName(name);
+ const { rsyncBin, log, dryRun } = defaults(opts);
+ const roots = rootsOf(opts);
+ if (!roots.tiered) {
+ throw new Error(`UMTOOL_MEDIA_DIR is not set: there is no media root to move ${name}/ to`);
+ }
+ const mirror = mediaMirror(projectDir, roots);
+ if (!mirror) throw new Error(`${projectDir} is not under the reports root ${roots.reportsRoot}`);
+ const src = path.join(/* turbopackIgnore: true */ projectDir, name);
+ const dest = path.join(/* turbopackIgnore: true */ mirror, name);
+ const s = await pathState(src);
+
+ if (s.kind === "link") {
+ if (s.target !== dest) {
+ throw new Error(`${src} is already a link, to ${s.target} — not to ${dest}. Nothing moved.`);
+ }
+ const parked = await parkedOf(projectDir, name);
+ if (!dryRun && s.targetIsDir) {
+ for (const p of parked) await rm(/* turbopackIgnore: true */ p, { recursive: true, force: true });
+ }
+ return { state: "already", src, dest };
+ }
+ if (s.kind === "other") throw new Error(`${src} is not a directory. Nothing moved.`);
+
+ if (s.kind === "missing") {
+ const parked = await parkedOf(projectDir, name);
+ if (!parked.length) return { state: "absent", src, dest };
+ if (parked.length > 1) {
+ throw new Error(`${src} is missing and there are ${parked.length} parked copies (${parked.join(", ")}) — settle them by hand`);
+ }
+ // A parked copy exists only after the copy verified, so the media side is
+ // complete -- provided it is reachable.
+ if ((await pathState(dest)).kind !== "dir") {
+ throw new Error(`${src} was parked at ${parked[0]}, but ${dest} is not there — is the media drive mounted? Nothing changed.`);
+ }
+ if (dryRun) return { state: "would-finish", src, dest };
+ log(`finishing a cut move: linking ${src} -> ${dest}`);
+ await symlink(/* turbopackIgnore: true */ dest, src, "dir");
+ await rm(/* turbopackIgnore: true */ parked[0], { recursive: true, force: true });
+ return { state: "moved", src, dest, resumed: true };
+ }
+
+ // A real directory: the move itself.
+ const problem = await mediaRootProblem(roots);
+ if (problem) throw new Error(`cannot move ${src}: ${problem}`);
+ const measured = await measureTree(src);
+ if (dryRun) return { state: "would-move", src, dest, ...measured };
+ await assertRoom(roots.mediaRoot, measured.bytes);
+ await mkdir(/* turbopackIgnore: true */ dest, { recursive: true });
+ const verified = await copyMirrorVerify(src, dest, { rsyncBin, log });
+ const parked = path.join(/* turbopackIgnore: true */ projectDir, `${name}.moved-${stamp()}`);
+ await rename(/* turbopackIgnore: true */ src, parked);
+ await symlink(/* turbopackIgnore: true */ dest, src, "dir");
+ await rm(/* turbopackIgnore: true */ parked, { recursive: true, force: true });
+ log(`moved ${src} -> ${dest} (${verified.files} file(s), ${gb(verified.bytes)})`);
+ return { state: "moved", src, dest, ...verified };
+}
+
+/**
+ * The reverse: `<projectDir>/<name>`, a link into the media root, becomes a
+ * real directory in the project again, and the media copy is deleted.
+ *
+ * a directory -> "already"
+ * absent, `<name>.incoming` -> the cut was between removing the link and
+ * renaming the verified copy: rename it, then
+ * delete the media copy when it can be named
+ * absent, nothing incoming -> "absent"
+ * a link whose target is gone -> refused: the drive is not mounted
+ * a link -> copy the target into `<name>.incoming`,
+ * mirror, verify, remove the link, rename,
+ * delete the media copy and any directories
+ * above it the move left empty (never the root)
+ *
+ * @param {string} projectDir
+ * @param {string} name
+ * @param {{ dryRun?: boolean, log?: (m: string) => void, rsyncBin?: string, reportsRoot?: string, mediaRoot?: string }} [opts]
+ * @returns {Promise<{ state: "moved" | "already" | "absent" | "would-move" | "would-finish", src: string, from?: string, bytes?: number, files?: number, resumed?: boolean }>}
+ */
+export async function moveDirToLocal(projectDir, name, opts = {}) {
+ assertName(name);
+ const { rsyncBin, log, dryRun } = defaults(opts);
+ const roots = rootsOf(opts);
+ const src = path.join(/* turbopackIgnore: true */ projectDir, name);
+ const incoming = path.join(/* turbopackIgnore: true */ projectDir, `${name}.incoming`);
+ const s = await pathState(src);
+
+ if (s.kind === "dir") return { state: "already", src };
+ if (s.kind === "other") throw new Error(`${src} is not a directory. Nothing moved.`);
+
+ if (s.kind === "missing") {
+ if ((await pathState(incoming)).kind !== "dir") return { state: "absent", src };
+ if (dryRun) return { state: "would-finish", src };
+ // `.incoming` outlives the link only once it verified.
+ log(`finishing a cut move: ${incoming} -> ${src}`);
+ await rename(/* turbopackIgnore: true */ incoming, src);
+ const mirror = roots.tiered ? mediaMirror(projectDir, roots) : null;
+ const from = mirror ? path.join(/* turbopackIgnore: true */ mirror, name) : undefined;
+ if (from) await dropMediaCopy(from, roots.mediaRoot);
+ return { state: "moved", src, from, resumed: true };
+ }
+
+ // A link.
+ const from = s.target;
+ if (!s.targetIsDir) {
+ throw new Error(`${src} is a link to ${from}, which is not there — is the media drive mounted? Nothing moved.`);
+ }
+ const measured = await measureTree(from);
+ if (dryRun) return { state: "would-move", src, from, ...measured };
+ await assertRoom(projectDir, measured.bytes);
+ await mkdir(/* turbopackIgnore: true */ incoming, { recursive: true });
+ const verified = await copyMirrorVerify(from, incoming, { rsyncBin, log });
+ await unlink(/* turbopackIgnore: true */ src); // the link, not what it points at
+ await rename(/* turbopackIgnore: true */ incoming, src);
+ await dropMediaCopy(from, roots.mediaRoot);
+ log(`moved ${from} -> ${src} (${verified.files} file(s), ${gb(verified.bytes)})`);
+ return { state: "moved", src, from, ...verified };
+}
+
+/**
+ * Delete a media copy that has been brought home, then every directory above
+ * it the move left empty, stopping at -- never removing -- the media root. A
+ * copy outside the media root (a link someone made by hand) is left alone.
+ */
+async function dropMediaCopy(from, mediaRoot) {
+ if (!inside(mediaRoot, from) || from === mediaRoot) return;
+ await rm(/* turbopackIgnore: true */ from, { recursive: true, force: true });
+ for (let d = path.dirname(/* turbopackIgnore: true */ from); d !== mediaRoot && inside(mediaRoot, d); d = path.dirname(/* turbopackIgnore: true */ d)) {
+ try {
+ await rmdir(/* turbopackIgnore: true */ d);
+ } catch {
+ break; // not empty: something else lives there
+ }
+ }
+}
diff --git a/umtool/report-to-video/build-video.mjs b/umtool/report-to-video/build-video.mjs
@@ -76,6 +76,7 @@ import {
cardWidth, contentWidth, reservedFooterHeight,
} from "./render-cards.mjs";
import { createCueSource, siteOriginFromManifest } from "./cues.mjs";
+import { ensureWriteDir } from "../lib/report/storage.mjs";
// The deck (`render.chrome`): its geometry, validation and schedule are pure
// and live in deck.mjs. This file only frames segments into its box and writes
// the schedule down -- it never has a copy of the arithmetic.
@@ -2616,6 +2617,11 @@ export async function buildVideo({ manifestPath, opts = {}, out, only, fetchOnly
const dirs = variantPaths(outRoot, manifest.slug, variant);
const outDir = dirs.dir;
+ // A project's out/ first, through ensureOutDir: with UMTOOL_MEDIA_DIR set it
+ // is a link to the media root, and the recursive mkdirs below would
+ // otherwise make it a real directory here. A dangling link refuses here,
+ // before a byte is fetched.
+ await ensureWriteDir(outRoot);
await mkdir(dirs.rawDir, { recursive: true });
for (const d of ["cards", "segments", "qr"]) {
await mkdir(path.join(outDir, d), { recursive: true });
diff --git a/umtool/report-to-video/check-availability.mjs b/umtool/report-to-video/check-availability.mjs
@@ -22,10 +22,11 @@
import { execFile } from "node:child_process";
import { promisify } from "node:util";
-import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { DEFAULT_CHANNELS_DIR } from "./cues.mjs";
+import { ensureWriteDir } from "../lib/report/storage.mjs";
// The per-platform yt-dlp args (Rumble's `--impersonate chrome`): the ONE table,
// in common, plain JS so bare `node` can load it.
import { platformArgsForUrl } from "yt-dlp-transcript-common/ytdlp/platformArgs.mjs";
@@ -149,7 +150,8 @@ export async function checkAvailability(manifestPath, { outDir, maxAgeDays = 0 }
}
const report = { manifest: path.resolve(manifestPath), checkedAt: new Date().toISOString(), sources };
- await mkdir(dir, { recursive: true });
+ // ensureWriteDir, not mkdir: a project's out/ may belong on the media root.
+ await ensureWriteDir(dir);
await writeFile(file, JSON.stringify(report, null, 2) + "\n", "utf8");
return { ...report, file };
}
diff --git a/umtool/report-to-video/compose-chrome.mjs b/umtool/report-to-video/compose-chrome.mjs
@@ -38,6 +38,7 @@ import path from "node:path";
import { ledgerTotals, dateKey } from "./ledger-totals.mjs";
import { selectVariant } from "./build-video.mjs";
+import { ensureWriteDir } from "../lib/report/storage.mjs";
import {
chromeCacheKey, deckLayout, frameCount, hyperframesCommand, postWindows, resolveDeck, sha256,
} from "./deck.mjs";
@@ -721,6 +722,10 @@ export async function composeChrome({
const manifest = selectVariant(JSON.parse(await readFile(manifestPath, "utf8")), variant);
// Absolute: the still is a file:// URL, and a relative one is no page at all.
const base = path.resolve(outDir ?? path.join(path.dirname(path.resolve(manifestPath)), "out", variant));
+ // The project's out/ through ensureOutDir before anything lands under it: a
+ // link to the media root when UMTOOL_MEDIA_DIR is set, and a loud refusal
+ // when that link dangles.
+ await ensureWriteDir(base);
from = Number(from ?? 0);
// The two regions drawn from the deck's schedule, and keyed by the render cache.
diff --git a/umtool/report-to-video/render-cards.mjs b/umtool/report-to-video/render-cards.mjs
@@ -38,6 +38,7 @@ import { brandFaces, brandManifest, brandSvgFace, childOpts } from "./brand.mjs"
import { BRAND_CARD_STYLES, renderBrandCard } from "./brand-cards.mjs";
import { FIRA_SANS, textWidth } from "./svg-faces.mjs";
import { deckOn, resolveDeck } from "./deck.mjs";
+import { ensureWriteDir } from "../lib/report/storage.mjs";
const execFileP = promisify(execFile);
@@ -1607,6 +1608,7 @@ async function main() {
const outDir = flag("--out") ?? path.join(path.dirname(path.resolve(manifestPath)), "out");
const only = flag("--only");
+ await ensureWriteDir(outDir); // a project's out/ may be a link to the media root
await mkdir(path.join(outDir, "cards"), { recursive: true });
const cards = manifest.timeline.filter(
