commit f653182813db446dfb88c882a734651e4dd507f9
parent e049b2351ceedc884c03196753eacb07a169c4da
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 1 Oct 2026 23:11:36 -0400
umtool: the deliverables switch — storage.deliverables, deliverableDir, umtool storage deliverables
video.manifest.json gains "storage": { "deliverables": "local" | "media" }
(absent = local), written only by manifest.mjs's updateStorage (a no-op is not
rewritten). storage.mjs: ensureOutDir's body generalised to a name
(ensureProjectDir); deliverableDir(project, name) makes clips/ or share-<x>/
where the switch says — a directory, or a link into the media mirror made by
the first writer; the media root never created; a dangling link, a cut move's
leftover, a bad value, or "media" in a process with no UMTOOL_MEDIA_DIR
refuse and create nothing. deliverableNames, deliverablesState,
deliverablesProblems for the readers; moveDeliverables moves clips/ and every
share-* with U1's movers, then sets the switch only when all of them are
there (idempotent; a cut move resumes). The leftover sentence names the
command that finishes a deliverable's move.
cut.mjs and buildShareBatch create their directory through deliverableDir;
a batch refuses while a deliverable dangles or a move was cut (a batch that
cannot see an earlier one would ship its clips again). listBatches and
sharedIdsIn follow links (a dangling batch is listed with no ids); a
leftover-shaped batch name is refused. umtool check: storage-unreachable
(blocking: out/, clips/ or a share-* link whose target is gone) and
storage-mismatch (open: a cut move, a directory against the switch, media
with no media root here); a bad value is manifest-invalid. umtool storage
deliverables <project> --to media|local [--dry-run] [--json], refused while
a pipeline process works in the project (now also apply-manifest.py and
build.py by cwd); umtool storage lists each report's deliverables. The mix
picker follows clips and share-* links into the media root as it does out.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
9 files changed, 935 insertions(+), 52 deletions(-)
diff --git a/umtool/bin/share-batch.mjs b/umtool/bin/share-batch.mjs
@@ -13,6 +13,7 @@
import process from "node:process";
import { resolveProject } from "../lib/projects/core.mjs";
import { buildShareBatch } from "../lib/report/deliver.mjs";
+import { isDeliverableName } from "../lib/report/storage.mjs";
const argv = process.argv.slice(2);
const val = (flag) => {
@@ -26,8 +27,10 @@ if (!projectArg || !name) {
console.error("usage: share-batch.mjs --project <id> --name <batch name>");
process.exit(2);
}
-if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name)) {
- console.error(`"${name}" is not a batch name: letters, digits, dot, dash and underscore`);
+if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name) || !isDeliverableName(`share-${name}`)) {
+ // The second test: `x.incoming` and `x.moved-<ts>` are what a cut move of
+ // share-x leaves behind (release 17), so a batch may not be called that.
+ console.error(`"${name}" is not a batch name: letters, digits, dot, dash and underscore, not ending .incoming or .moved-…`);
process.exit(2);
}
diff --git a/umtool/bin/umtool.mjs b/umtool/bin/umtool.mjs
@@ -30,12 +30,14 @@
// or the media root is set and not there
// umtool storage [move-out|move-back <project>|--all] [--dry-run] [--json]
// where each project's out/ lives; move it
+// umtool storage deliverables <project> --to media|local [--dry-run] [--json]
+// move clips/ and every share-*/, then set the switch
// umtool snapshot <project> [--label L] copy the manifest into revisions/
// umtool diff <project> <snapshot> what changed since that snapshot
// umtool export <project> --format toc-bbcode|toc-markdown|description|chapters [--variant V]
// umtool check-sources [<project>…] prints the re-check chain
import process from "node:process";
-import { readdirSync, readFileSync } from "node:fs";
+import { readdirSync, readFileSync, readlinkSync, realpathSync } from "node:fs";
import {
PROJECT_KINDS,
REPORTS_ROOT,
@@ -58,14 +60,23 @@ import { createSnapshot, listSnapshots, readSnapshot } from "../lib/report/snaps
import { diffManifests, formatChange } from "../lib/report/manifest-diff.mjs";
import { EXPORT_FORMATS, exportProject } from "../lib/report/export.mjs";
import path from "node:path";
-import { updateClip } from "../lib/report/manifest.mjs";
+import { updateClip, updateStorage } from "../lib/report/manifest.mjs";
import { hms } from "umtool-report-to-video/attribution";
import { buildSteps, checkSourcesSteps, PRESETS } from "../lib/report/driver.mjs";
import { openIndex, signRecord } from "../lib/projects/index-db.mjs";
import { probeTools } from "../lib/tools.mjs";
import { scaffoldReportVideo } from "../lib/projects/scaffold.mjs";
import { CACHE_DIR, INDEX_DIR, MEDIA_ROOT, MEDIA_TIERED, OLD_CACHE_DIR } from "../lib/paths.mjs";
-import { measureTree, mediaRootProblem, moveDirToLocal, moveDirToMedia, outDirState, pathState } from "../lib/report/storage.mjs";
+import {
+ deliverablesState,
+ measureTree,
+ mediaRootProblem,
+ moveDeliverables,
+ moveDirToLocal,
+ moveDirToMedia,
+ outDirState,
+ pathState,
+} from "../lib/report/storage.mjs";
const argv = process.argv.slice(2);
const cmd = argv.find((a) => !a.startsWith("-")) ?? "help";
@@ -416,6 +427,9 @@ async function cmdDoctor() {
// umtool storage every project's out/: dir | link | DANGLING | none
// umtool storage move-out <p>|--all out/ to the media root, a link left in its place
// umtool storage move-back <p>|--all out/ back to a real directory in the project
+// umtool storage deliverables <p> --to media|local
+// clips/ and every share-*/ to the media root or
+// back, then storage.deliverables in the manifest
// --dry-run measure and say; change nothing
//
// The movers are lib/report/storage.mjs's, which `umtool` and the app share.
@@ -431,6 +445,17 @@ async function cmdDoctor() {
*/
function pipelineProcessesFor(projectDir) {
const SCRIPTS = /(build-video|check-availability|render-cards|compose-chrome|verify-build|fetch-via-editor|resolve-windows|cut-from-cache|share-batch)\.mjs/;
+ // The report's OWN scripts (the Deliver panel's "apply rulings" and
+ // "rebuild"): apply-manifest.py deletes clips/ mp4s, build.py reads them.
+ // They are run with the project as their working directory and name no
+ // path, so they are found by script name and cwd.
+ const OWN = /(^|\/)(apply-manifest|build)\.py$/;
+ let realDir = projectDir;
+ try {
+ realDir = realpathSync(projectDir);
+ } catch {
+ /* gone: nothing can be running in it */
+ }
const pids = [];
let entries = [];
try {
@@ -446,8 +471,19 @@ function pipelineProcessesFor(projectDir) {
} catch {
continue;
}
- if (!args.some((a) => SCRIPTS.test(a))) continue;
- if (args.some((a) => a === projectDir || a.startsWith(projectDir + "/"))) pids.push(Number(pid));
+ if (args.some((a) => SCRIPTS.test(a))) {
+ if (args.some((a) => a === projectDir || a.startsWith(projectDir + "/"))) pids.push(Number(pid));
+ continue;
+ }
+ if (args.some((a) => OWN.test(a))) {
+ let cwd = "";
+ try {
+ cwd = readlinkSync(`/proc/${pid}/cwd`);
+ } catch {
+ continue;
+ }
+ if (cwd === projectDir || cwd === realDir) pids.push(Number(pid));
+ }
}
return pids;
}
@@ -457,17 +493,36 @@ async function cmdStorage() {
const dryRun = has("--dry-run");
const log = (m) => (json ? console.error(m) : console.log(m));
+ if (sub === "deliverables") return cmdStorageDeliverables(dryRun, log);
if (sub !== "move-out" && sub !== "move-back") {
// `umtool storage`, `umtool storage <project>`, `umtool storage status [<project>]`.
const one = sub === "status" ? positional[1] : sub;
const refs = one ? [await pick(one)] : await projectRefs();
const rows = [];
- for (const p of refs) rows.push({ id: p.id, ...(await outDirState(p.dir)) });
+ for (const p of refs) {
+ const row = { id: p.id, ...(await outDirState(p.dir)) };
+ // A report video's deliverables, when it has any or its switch is set.
+ const d = p.kind === "report-video" ? await deliverablesState(p.dir) : null;
+ if (d && (d.dirs.length || d.value !== undefined)) {
+ row.deliverables = { mode: d.mode, value: d.value, dirs: d.dirs };
+ }
+ rows.push(row);
+ }
if (json) return out({ media: { path: MEDIA_ROOT, tiered: MEDIA_TIERED }, projects: rows });
console.log(MEDIA_TIERED ? `media root ${MEDIA_ROOT}` : "media root unset (UMTOOL_MEDIA_DIR): out/ stays in each project");
+ const word = { absent: "none", dir: "dir", link: "link", dangling: "DANGLING", other: "OTHER" };
for (const r of rows) {
- const what = { absent: "none", dir: "dir", link: "link", dangling: "DANGLING", other: "OTHER" }[r.state] ?? r.state;
+ const what = word[r.state] ?? r.state;
console.log(`${what.padEnd(9)} ${r.id}${r.target ? ` -> ${r.target}` : ""}`);
+ if (r.deliverables) {
+ const d = r.deliverables;
+ console.log(
+ ` deliverables: ${d.mode ?? `INVALID ${JSON.stringify(d.value)}`}` +
+ (d.dirs.length
+ ? ` · ${d.dirs.map((x) => `${x.name} ${word[x.state] ?? x.state}${x.leftovers.length ? ` (+${x.leftovers.join(", ")})` : ""}`).join(" · ")}`
+ : ""),
+ );
+ }
}
return;
}
@@ -518,6 +573,64 @@ async function cmdStorage() {
if (failed) process.exit(1);
}
+/**
+ * `umtool storage deliverables <project> --to media|local [--dry-run]`: move
+ * clips/ and every share-* directory with the movers, then set storage.deliverables
+ * through the manifest's writer. Refused while a pipeline process -- a cut, a
+ * share batch, the report's own apply/build scripts -- works in the project.
+ *
+ * What the refusal cannot see: the app's in-process work (the app runs this
+ * as a JOB, and its one-job-at-a-time rule is what keeps its cuts and batches
+ * out), another machine writing over a network share, and a hand-run command
+ * that is none of those scripts. The movers' verify refuses a tree that keeps
+ * changing, but a write in the instant between the verify and the swap would
+ * be lost with the parked copy.
+ */
+async function cmdStorageDeliverables(dryRun, log) {
+ const to = val("--to");
+ if (to !== "media" && to !== "local") die("where to? `umtool storage deliverables <project> --to media|local`");
+ const p = await pick(positional[1]);
+ if (p.kind !== "report-video") die(`${p.id} is a ${p.kind} project — deliverables are a report video's`);
+ const busy = pipelineProcessesFor(p.dir);
+ if (busy.length) {
+ const msg = `${p.id}: a pipeline process is working in it (pid ${busy.join(", ")}) — nothing moved; run this when it has finished`;
+ if (json) out({ ok: false, busy: busy, error: msg });
+ else console.error(msg);
+ process.exit(1);
+ }
+ let r;
+ try {
+ r = await moveDeliverables(p.dir, to, {
+ dryRun,
+ log,
+ writeMode: (dir, mode) => updateStorage(dir, { deliverables: mode }),
+ });
+ } catch (e) {
+ if (json) out({ ok: false, error: e?.message ?? String(e) });
+ else console.error(e?.message ?? String(e));
+ process.exit(1);
+ }
+ if (json) out({ project: p.id, ...r });
+ else {
+ if (!r.results.length) console.log(`${p.id}: no clips/ and no share-*/ yet`);
+ for (const x of r.results) {
+ const size = x.bytes !== undefined ? ` ${x.files} file(s), ${mb(x.bytes)}` : "";
+ console.log(`${String(x.state === "failed" ? "FAILED" : x.state).padEnd(12)} ${x.name}${size}`);
+ if (x.error) console.log(` ${x.error}`);
+ if (x.mediaCopyLeft) console.log(` left in place: ${x.mediaCopyLeft} (not deleted; remove it by hand once checked)`);
+ }
+ const before = r.before ?? "local (unset)";
+ console.log(
+ dryRun
+ ? `\nstorage.deliverables: ${before} — would be ${to}; dry run, nothing changed`
+ : r.ok
+ ? `\nstorage.deliverables: ${r.written ? `${before} -> ${to}` : `${to} (unchanged)`}`
+ : `\nstorage.deliverables left at ${before}: ${r.results.filter((x) => x.state === "failed").length} FAILED — fix that and run it again`,
+ );
+ }
+ if (!r.ok) process.exit(1);
+}
+
async function cmdSnapshot() {
const p = await pick(positional[0]);
try {
@@ -623,6 +736,7 @@ function usage() {
" or the media root is set and not there; the roots, and a leftover old cache",
" umtool storage [<project>] where each project's out/ lives (dir, link, DANGLING)",
" umtool storage move-out|move-back <project>|--all [--dry-run]",
+ " umtool storage deliverables <project> --to media|local [--dry-run] clips/ + share-*/, then the switch",
" out/ to the media root (UMTOOL_MEDIA_DIR) and back; run when nothing is building",
" umtool snapshot <project> [--label L] copy the manifest into revisions/",
" umtool diff <project> <snapshot> what changed since that snapshot",
diff --git a/umtool/lib/media.ts b/umtool/lib/media.ts
@@ -85,11 +85,16 @@ const MIN_INTERESTING = 256 * 1024;
* picker under its project. Only a link INTO the media root: every other link
* stays unfollowed, as it always was (SONG_DATA's 39 GB are links).
*/
-/** The project directories that may be links to the media root (U2 adds deliverables). */
-const MEDIA_LINKS = new Set(["out"]);
+/**
+ * The project directories that may be links to the media root: render scratch
+ * (`out`), and a report's deliverables (`clips`, every `share-*`) once the
+ * project's switch has moved them (release 17, slice U2).
+ */
+const MEDIA_LINKS = new Set(["out", "clips"]);
+const isMediaLinkName = (name: string) => MEDIA_LINKS.has(name) || name.startsWith("share-");
async function isMediaLink(e: { name: string; isSymbolicLink(): boolean }, abs: string): Promise<boolean> {
- if (!MEDIA_TIERED || !e.isSymbolicLink() || !MEDIA_LINKS.has(e.name)) return false;
+ if (!MEDIA_TIERED || !e.isSymbolicLink() || !isMediaLinkName(e.name)) return false;
const real = await realpath(abs).catch(() => null);
if (!real || !inside(MEDIA_ROOT, real)) return false;
return stat(real).then((s) => s.isDirectory(), () => false);
diff --git a/umtool/lib/projects/report.mjs b/umtool/lib/projects/report.mjs
@@ -9,6 +9,7 @@ import { readdir, readFile, stat } from "node:fs/promises";
import path from "node:path";
import { DEFAULT_VARIANT, cachedWindowsFor } from "umtool-report-to-video/build-video";
import { rawCacheOf } from "../report/raw-cache.mjs";
+import { deliverablesState, finishCommand, outDirState, leftoversOf } from "../report/storage.mjs";
import { CHANNELS_DIR } from "../paths.mjs";
import { channelName, cleanTitle } from "umtool-report-to-video/attribution";
import { teaserTitle } from "umtool-report-to-video/deck";
@@ -762,6 +763,13 @@ export const REPORT_DECISION_KINDS = [
"claim-incoherent",
"stale-build",
"unbuilt",
+ // `out/`, `clips/` or a `share-*/` is a link whose target is not there: the
+ // media drive is not mounted (release 17). BLOCKING -- a build, a cut or a
+ // batch refuses at that directory, and a reader sees "nothing built/cut".
+ "storage-unreachable",
+ // Where the deliverables are disagrees with `storage.deliverables`, or a
+ // move was cut. `open`: nothing is lost, and one command settles it.
+ "storage-mismatch",
];
export async function reportDecisions(ctx, summary) {
@@ -1057,6 +1065,52 @@ export async function reportDecisions(ctx, summary) {
}
}
+ // --- where the bulk lives (release 17) -----------------------------------
+ // `out/` follows UMTOOL_MEDIA_DIR; the deliverables follow
+ // `storage.deliverables`. A link whose drive is not there reads as "never
+ // built" and "nothing cut" everywhere else, which is the sentence this
+ // exists to replace.
+ const outState = await outDirState(dir);
+ if (outState.state === "dangling") {
+ add("storage-unreachable", "out", `a link to ${outState.target}, which is not there — is the media drive mounted? Nothing can be built until it is`, "blocking");
+ }
+ const outLeft = await leftoversOf(dir, "out");
+ if (outLeft.length) {
+ add(
+ "storage-mismatch",
+ "out",
+ `a move of out/ was cut (left: ${outLeft.map((p) => path.basename(p)).join(", ")}) — \`${finishCommand("out", outLeft.some((p) => p.endsWith(".incoming"))).replace("<project>", id)}\` finishes it`,
+ "open",
+ );
+ }
+ const store = await deliverablesState(dir);
+ if (!store.mode) add("manifest-invalid", "storage.deliverables", `${store.error} — a cut or a batch refuses until it is`, "blocking");
+ for (const d of store.dirs) {
+ if (d.state === "dangling") {
+ add("storage-unreachable", d.name, `a link to ${d.target}, which is not there — is the media drive mounted? A cut or a batch refuses until it is`, "blocking");
+ }
+ if (d.leftovers.length) {
+ add(
+ "storage-mismatch",
+ d.name,
+ `a move of ${d.name}/ was cut (left: ${d.leftovers.join(", ")}) — \`${finishCommand(d.name, d.leftovers.some((l) => l.endsWith(".incoming"))).replace("<project>", id)}\` finishes it`,
+ "open",
+ );
+ } else if (store.mode === "media" && d.state === "dir") {
+ add("storage-mismatch", d.name, `a directory in the project, while storage.deliverables is media — \`umtool storage deliverables ${id} --to media\` moves it`, "open");
+ } else if (store.mode === "local" && store.value !== undefined && d.state === "link") {
+ add("storage-mismatch", d.name, `a link to ${d.target}, while storage.deliverables is local — \`umtool storage deliverables ${id} --to local\` brings it back`, "open");
+ }
+ }
+ if (store.mode === "media" && !store.tiered) {
+ add(
+ "storage-mismatch",
+ "storage.deliverables",
+ "media, and UMTOOL_MEDIA_DIR is not set here — a new clips/ or batch would be refused. Set it where umtool runs",
+ "open",
+ );
+ }
+
// --- the build -----------------------------------------------------------
if (s.build.stale) {
add(
diff --git a/umtool/lib/report/cut.mjs b/umtool/lib/report/cut.mjs
@@ -16,13 +16,14 @@
// pass, exported rather than copied, so the seconds this writes and the seconds
// the video renders are the same arithmetic.
import { execFile } from "node:child_process";
-import { mkdir, rename, rm } from "node:fs/promises";
+import { rename, rm } from "node:fs/promises";
import path from "node:path";
import { promisify } from "node:util";
import { FFMPEG_BIN, cutArgs } from "umtool-report-to-video/build-video";
import { clipsOf, readManifest } from "../projects/report.mjs";
import { ACCURATE_CUT_ARGS, probeSeconds } from "./encode.mjs";
import { projectCache } from "./serve.mjs";
+import { deliverableDir } from "./storage.mjs";
const execFileP = promisify(execFile);
@@ -93,8 +94,16 @@ export async function cutClipFromCache(
const want = end - start;
const a = Math.max(0, start - win.from);
const b = a + want;
- const dir = path.join(project.dir, CLIPS_DIR);
- await mkdir(dir, { recursive: true });
+ // Made where the project's deliverables switch says: a directory, or a link
+ // to the media root (release 17). A dangling link refuses here, before any
+ // ffmpeg runs, and nothing is made in its place. The tmp file below sits
+ // beside the final one either way, so the rename never crosses a volume.
+ let dir;
+ try {
+ dir = await deliverableDir(project.dir, CLIPS_DIR);
+ } catch (e) {
+ return { ok: false, id: clipId, reason: "storage", error: e instanceof Error ? e.message : String(e) };
+ }
const out = path.join(dir, `${clipId}.mp4`);
const tmp = path.join(dir, `.${clipId}.cutting.mp4`);
diff --git a/umtool/lib/report/deliver.mjs b/umtool/lib/report/deliver.mjs
@@ -22,6 +22,7 @@ import { FFMPEG_BIN } from "umtool-report-to-video/build-video";
import { citeUrlFor, clipVerdict, clipsOf, readManifest } from "../projects/report.mjs";
import { SHARE_PROFILES } from "./encode.mjs";
import { CLIPS_DIR } from "./cut.mjs";
+import { deliverableDir, deliverablesProblems, deliverablesState, isDeliverableName } from "./storage.mjs";
const execFileP = promisify(execFile);
@@ -123,9 +124,19 @@ export async function sharedIdsIn(dir) {
}
}
// And the files themselves, for a batch assembled before anyone wrote a list.
- const walk = async (d) => {
+ //
+ // A link to a directory is walked like one (release 17: a batch moved to the
+ // media root is a link, and so may be anything a person linked in), but only
+ // a few levels down -- a batch is `<variant>/<section>/<file>`, and a link
+ // that loops must not hang the panel.
+ const walk = async (d, depth = 0) => {
+ if (depth > 6) return;
for (const ent of await readdir(d, { withFileTypes: true }).catch(() => [])) {
- if (ent.isDirectory()) await walk(path.join(d, ent.name));
+ const p = path.join(d, ent.name);
+ const isDir =
+ ent.isDirectory() ||
+ (ent.isSymbolicLink() && (await stat(p).then((st) => st.isDirectory(), () => false)));
+ if (isDir) await walk(p, depth + 1);
else {
const m = /^([A-Za-z]{1,3}\d{1,3})_.*\.mp4$/.exec(ent.name);
if (m) ids.add(m[1]);
@@ -136,22 +147,41 @@ export async function sharedIdsIn(dir) {
return ids;
}
-/** The batches already in this project, newest name last. */
+/**
+ * The batches already in this project, newest name last.
+ *
+ * A batch may be a LINK to the media root (release 17: `umtool storage
+ * deliverables <p> --to media`), and is listed like the directory it replaced.
+ * A link whose drive is not there is listed too, as `dangling` with no ids:
+ * leaving it out would read as "never shipped" and the next batch would ship
+ * its clips again -- so a batch refuses while one dangles (buildShareBatch).
+ * A cut move's leftover (`share-x.moved-<ts>`, `share-x.incoming`) is not a
+ * batch.
+ */
export async function listBatches(projectDir) {
- const names = (await readdir(projectDir, { withFileTypes: true }).catch(() => []))
- .filter((e) => e.isDirectory() && e.name.startsWith(SHARE_PREFIX))
- .map((e) => e.name)
- .sort();
+ const entries = (await readdir(projectDir, { withFileTypes: true }).catch(() => [])).filter(
+ (e) => (e.isDirectory() || e.isSymbolicLink()) && e.name.startsWith(SHARE_PREFIX) && isDeliverableName(e.name),
+ );
+ const found = await Promise.all(
+ entries.map(async (e) => {
+ const dir = path.join(projectDir, e.name);
+ const isDir = e.isDirectory() || (await stat(dir).then((st) => st.isDirectory(), () => false));
+ // A symlink to a FILE named share-x is nothing of ours; a dangling one is.
+ if (!isDir && (await stat(dir).then(() => true, () => false))) return null;
+ return { name: e.name, dir, dangling: !isDir };
+ }),
+ );
+ const batches = found.filter((b) => b !== null).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
return Promise.all(
- names.map(async (name) => {
- const dir = path.join(projectDir, name);
- const ids = [...(await sharedIdsIn(dir))].sort();
+ batches.map(async ({ name, dir, dangling }) => {
+ const ids = dangling ? [] : [...(await sharedIdsIn(dir))].sort();
return {
name,
label: name.slice(SHARE_PREFIX.length),
dir,
ids,
- hasList: !!(await readFile(path.join(dir, "LIST.md"), "utf8").catch(() => null)),
+ dangling,
+ hasList: !dangling && !!(await readFile(path.join(dir, "LIST.md"), "utf8").catch(() => null)),
};
}),
);
@@ -233,6 +263,12 @@ export async function deliverStateOf(project, { manifest = null, entries = null
const batches = await listBatches(project.dir);
const shared = new Set(batches.flatMap((b) => b.ids));
const variants = await contentVariants(project.dir);
+ // Where the deliverables live, and what stops a cut or a batch being
+ // written now (release 17). A dangling clips/ reads as "nothing cut" above;
+ // these sentences are what keep that from turning into a re-cut of
+ // everything on the wrong drive.
+ const storage = await deliverablesState(project.dir);
+ const nextName = `batch-${new Date().toISOString().slice(0, 10)}`;
const rows = clips.map((e) => ({
id: e.id,
@@ -295,11 +331,17 @@ export async function deliverStateOf(project, { manifest = null, entries = null
incorrect: rows.filter((r) => r.verdict === "incorrect").map((r) => r.id),
},
candidates: candidates.map((r) => ({ id: r.id, section: r.section, file: r.file })),
- nextName: `batch-${new Date().toISOString().slice(0, 10)}`,
+ nextName,
variants,
incorrect: await incorrectCitations(project.dir, m, variants),
hasApplyScript: await exists(path.join(project.dir, "apply-manifest.py")),
hasBuildScript: await exists(path.join(project.dir, "build.py")),
+ storage: {
+ ...storage,
+ // The reasons a cut (into clips/) or a new batch would be refused.
+ cutBlocked: deliverablesProblems(storage, CLIPS_DIR),
+ shareBlocked: deliverablesProblems(storage, `${SHARE_PREFIX}${nextName}`),
+ },
};
}
@@ -356,10 +398,17 @@ export async function buildShareBatch(project, name, { log = console.log } = {})
const m = await readManifest(project.dir);
const clips = new Map(clipsOf(m).map((e) => [e.id, e]));
const headings = await sectionHeadings(project.dir);
+ // Refused while a deliverable cannot be read or written (release 17): a
+ // batch that cannot see an earlier batch would ship its clips again, and one
+ // that cannot see clips/ would ship nothing.
+ const blocked = deliverablesProblems(state.storage, `${SHARE_PREFIX}${name}`);
+ if (blocked.length) throw new Error(`share-${name} not built: ${blocked.join("; ")}`);
const chosen = state.candidates.map((c) => c.id);
if (!chosen.length) throw new Error("nothing to ship: every confirmed clip is already shared, or not cut yet");
- const root = path.join(project.dir, `${SHARE_PREFIX}${name}`);
+ // Made where the project's deliverables switch says: a directory, or a link
+ // to the media root. Everything below is made under it, through the link.
+ const root = await deliverableDir(project.dir, `${SHARE_PREFIX}${name}`);
const bySection = new Map();
for (const id of chosen) {
const e = clips.get(id);
diff --git a/umtool/lib/report/deliverables.test.mjs b/umtool/lib/report/deliverables.test.mjs
@@ -0,0 +1,356 @@
+// A report's deliverables (clips/, every share-*/) behind the per-project
+// switch (release 17, slice U2): `storage.deliverables` in the manifest,
+// deliverableDir for the writers, moveDeliverables for the move, and the
+// readers that must follow a moved batch.
+//
+// Every call names its own roots (`reportsRoot`, `mediaRoot`), as
+// storage.test.mjs does, so nothing touches the process's REPORTS_ROOT.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { lstat, mkdir, mkdtemp, readFile, readdir, readlink, rename, rm, stat, symlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import { listBatches, sharedIdsIn } from "./deliver.mjs";
+import { updateStorage } from "./manifest.mjs";
+import {
+ deliverableDir,
+ deliverableNames,
+ deliverablesMode,
+ deliverablesProblems,
+ deliverablesState,
+ isDeliverableName,
+ moveDeliverables,
+} from "./storage.mjs";
+
+/** A reports root with one report project, its deliverables, and a media root beside it. */
+async function world({ manifest = { title: "fixture", timeline: [] }, deliverables = true } = {}) {
+ const base = await mkdtemp(path.join(tmpdir(), "umtool-deliverables-"));
+ const reportsRoot = path.join(base, "reports");
+ const mediaRoot = path.join(base, "media");
+ const projectDir = path.join(reportsRoot, "folder", "proj");
+ await mkdir(projectDir, { recursive: true });
+ await mkdir(mediaRoot);
+ await writeFile(path.join(projectDir, "video.manifest.json"), JSON.stringify(manifest, null, 2) + "\n");
+ if (deliverables) {
+ await mkdir(path.join(projectDir, "clips"));
+ await writeFile(path.join(projectDir, "clips", "a01.mp4"), Buffer.alloc(3000, 1));
+ await mkdir(path.join(projectDir, "share-first", "orig", "A"), { recursive: true });
+ await writeFile(path.join(projectDir, "share-first", "orig", "A", "a01_2025-01-01_x.mp4"), Buffer.alloc(1000, 2));
+ await writeFile(path.join(projectDir, "share-first", "LIST.md"), "<!-- shared-ids: a01 -->\n");
+ }
+ const roots = { reportsRoot, mediaRoot };
+ const untiered = { reportsRoot, mediaRoot: reportsRoot };
+ const mirror = path.join(mediaRoot, "folder", "proj");
+ return { base, reportsRoot, mediaRoot, projectDir, roots, untiered, mirror, done: () => rm(base, { recursive: true, force: true }) };
+}
+
+const kind = async (p) => {
+ const st = await lstat(p).catch(() => null);
+ if (!st) return "missing";
+ return st.isSymbolicLink() ? "link" : st.isDirectory() ? "dir" : "file";
+};
+const readManifest = async (w) => JSON.parse(await readFile(path.join(w.projectDir, "video.manifest.json"), "utf8"));
+const writeMode = (dir, mode) => updateStorage(dir, { deliverables: mode });
+
+// ---------------------------------------------------------------------------
+// The switch, and the names
+// ---------------------------------------------------------------------------
+
+test("storage.deliverables: absent is local; the two values; anything else is an error", async () => {
+ const w = await world({ deliverables: false });
+ try {
+ assert.equal((await deliverablesMode(w.projectDir)).mode, "local");
+ await writeFile(path.join(w.projectDir, "video.manifest.json"), JSON.stringify({ storage: { deliverables: "media" } }));
+ assert.equal((await deliverablesMode(w.projectDir)).mode, "media");
+ await writeFile(path.join(w.projectDir, "video.manifest.json"), JSON.stringify({ storage: { deliverables: "platter" } }));
+ const bad = await deliverablesMode(w.projectDir);
+ assert.equal(bad.mode, null);
+ assert.match(bad.error, /"platter" — it is "local" or "media"/);
+ } finally {
+ await w.done();
+ }
+});
+
+test("updateStorage writes the switch, keeps the rest, and does not rewrite a no-op", async () => {
+ const w = await world({ manifest: { title: "kept", storage: { note: "kept too" } }, deliverables: false });
+ try {
+ await assert.rejects(updateStorage(w.projectDir, { deliverables: "platter" }), /"local" or "media"/);
+ const r = await updateStorage(w.projectDir, { deliverables: "media" });
+ assert.equal(r.changed, true);
+ const m = await readManifest(w);
+ assert.deepEqual(m.storage, { note: "kept too", deliverables: "media" });
+ assert.equal(m.title, "kept");
+ // The CLI's formatting: two-space indent and a trailing newline.
+ assert.match(await readFile(path.join(w.projectDir, "video.manifest.json"), "utf8"), /\n "storage": \{\n[\s\S]*\}\n$/);
+ const mtime = (await stat(path.join(w.projectDir, "video.manifest.json"))).mtimeMs;
+ const again = await updateStorage(w.projectDir, { deliverables: "media" });
+ assert.equal(again.changed, false);
+ assert.equal((await stat(path.join(w.projectDir, "video.manifest.json"))).mtimeMs, mtime);
+ } finally {
+ await w.done();
+ }
+});
+
+test("deliverable names: clips and share-*, never a leftover, a file or another directory", async () => {
+ assert.ok(isDeliverableName("clips") && isDeliverableName("share-first") && isDeliverableName("share-batch-2026.10.01"));
+ for (const no of ["out", "share-", "shares", "clips.moved-20261001T000000Z", "share-x.incoming", "../clips", "share-a/b"]) {
+ assert.equal(isDeliverableName(no), false, no);
+ }
+ const w = await world();
+ try {
+ await mkdir(path.join(w.projectDir, "share-b.moved-20261001T000000Z"));
+ await mkdir(path.join(w.projectDir, "share-c.incoming"));
+ await writeFile(path.join(w.projectDir, "share-file"), "not a directory");
+ await mkdir(path.join(w.projectDir, "revisions"));
+ const other = path.join(w.base, "elsewhere");
+ await mkdir(other);
+ await symlink(other, path.join(w.projectDir, "share-linked"));
+ // clips first; a batch present only as a leftover is named, so a move finishes it.
+ assert.deepEqual(await deliverableNames(w.projectDir), ["clips", "share-b", "share-c", "share-first", "share-linked"]);
+ } finally {
+ await w.done();
+ }
+});
+
+// ---------------------------------------------------------------------------
+// deliverableDir: what a cut and a batch call before they write
+// ---------------------------------------------------------------------------
+
+test("deliverableDir, local: a directory in the project, tiered or not", async () => {
+ const w = await world({ deliverables: false });
+ try {
+ const clips = await deliverableDir(w.projectDir, "clips", w.roots);
+ assert.equal(clips, path.join(w.projectDir, "clips"));
+ assert.equal(await kind(clips), "dir");
+ assert.deepEqual(await readdir(w.mediaRoot), []);
+ await assert.rejects(deliverableDir(w.projectDir, "out", w.roots), /not a deliverable directory/);
+ } finally {
+ await w.done();
+ }
+});
+
+test("deliverableDir, media: the first writer makes the mirror and the link; an existing directory is kept", async () => {
+ const w = await world({ manifest: { storage: { deliverables: "media" } }, deliverables: false });
+ try {
+ const share = await deliverableDir(w.projectDir, "share-next", w.roots);
+ assert.equal(await kind(share), "link");
+ assert.equal(await readlink(share), path.join(w.mirror, "share-next"));
+ // Written through the link, the file lands on the media root.
+ await mkdir(path.join(share, "orig"), { recursive: true });
+ assert.equal(await kind(path.join(w.mirror, "share-next", "orig")), "dir");
+ // Again: the link is it.
+ assert.equal(await deliverableDir(w.projectDir, "share-next", w.roots), share);
+ // A real clips/ is never replaced by a writer: moving it is the switch's job.
+ await mkdir(path.join(w.projectDir, "clips"));
+ assert.equal(await deliverableDir(w.projectDir, "clips", w.roots), path.join(w.projectDir, "clips"));
+ assert.equal(await kind(path.join(w.projectDir, "clips")), "dir");
+ } finally {
+ await w.done();
+ }
+});
+
+test("deliverableDir refuses, and creates nothing: no media root here, an unplugged root, a dangling link, a bad switch", async () => {
+ const w = await world({ manifest: { storage: { deliverables: "media" } }, deliverables: false });
+ try {
+ const clips = path.join(w.projectDir, "clips");
+ await assert.rejects(deliverableDir(w.projectDir, "clips", w.untiered), /UMTOOL_MEDIA_DIR is not set here.*--to local/);
+ assert.equal(await kind(clips), "missing");
+
+ await rename(w.mediaRoot, `${w.mediaRoot}.unplugged`);
+ await assert.rejects(deliverableDir(w.projectDir, "clips", w.roots), /is not there — is its drive mounted\? Nothing was created/);
+ assert.equal(await kind(clips), "missing");
+ assert.equal(await kind(w.mediaRoot), "missing");
+
+ await symlink(path.join(w.mirror, "clips"), clips);
+ await assert.rejects(deliverableDir(w.projectDir, "clips", w.roots), /is the media drive mounted\? Nothing was written/);
+ assert.equal(await kind(w.mediaRoot), "missing");
+ await rename(`${w.mediaRoot}.unplugged`, w.mediaRoot);
+ await rm(clips);
+
+ await writeFile(path.join(w.projectDir, "video.manifest.json"), JSON.stringify({ storage: { deliverables: "platter" } }));
+ await assert.rejects(deliverableDir(w.projectDir, "clips", w.roots), /"platter" — it is "local" or "media"/);
+ assert.equal(await kind(clips), "missing");
+ } finally {
+ await w.done();
+ }
+});
+
+test("deliverableDir refuses over a cut move's leftover and names the switch that finishes it", async () => {
+ const w = await world({ deliverables: false });
+ try {
+ await mkdir(path.join(w.projectDir, "clips.moved-20261001T000000Z"));
+ await assert.rejects(
+ deliverableDir(w.projectDir, "clips", w.roots),
+ /a move of clips\/ .* was cut .*umtool storage deliverables <project> --to media/,
+ );
+ assert.equal(await kind(path.join(w.projectDir, "clips")), "missing");
+ } finally {
+ await w.done();
+ }
+});
+
+// ---------------------------------------------------------------------------
+// moveDeliverables: the switch
+// ---------------------------------------------------------------------------
+
+test("moveDeliverables to media and back: every deliverable moves, then the switch is set", async () => {
+ const w = await world();
+ try {
+ const r = await moveDeliverables(w.projectDir, "media", { ...w.roots, writeMode });
+ assert.equal(r.ok, true);
+ assert.equal(r.written, true);
+ assert.equal(r.before, null);
+ assert.deepEqual(r.results.map((x) => [x.name, x.state]), [["clips", "moved"], ["share-first", "moved"]]);
+ for (const name of ["clips", "share-first"]) {
+ assert.equal(await kind(path.join(w.projectDir, name)), "link");
+ assert.equal(await readlink(path.join(w.projectDir, name)), path.join(w.mirror, name));
+ }
+ // A relative reference still resolves through the link.
+ assert.deepEqual(await readFile(path.join(w.projectDir, "clips", "a01.mp4")), Buffer.alloc(3000, 1));
+ assert.equal((await readManifest(w)).storage.deliverables, "media");
+ assert.equal((await readManifest(w)).title, "fixture");
+
+ // Idempotent: all "already", and the manifest is not rewritten.
+ const mtime = (await stat(path.join(w.projectDir, "video.manifest.json"))).mtimeMs;
+ const again = await moveDeliverables(w.projectDir, "media", { ...w.roots, writeMode });
+ assert.deepEqual(again.results.map((x) => x.state), ["already", "already"]);
+ assert.equal(again.written, false);
+ assert.equal((await stat(path.join(w.projectDir, "video.manifest.json"))).mtimeMs, mtime);
+
+ // The state readers see.
+ const st = await deliverablesState(w.projectDir, w.roots);
+ assert.equal(st.mode, "media");
+ assert.deepEqual(st.dirs.map((d) => [d.name, d.state]), [["clips", "link"], ["share-first", "link"]]);
+ assert.deepEqual(deliverablesProblems(st, "share-next"), []);
+
+ const back = await moveDeliverables(w.projectDir, "local", { ...w.roots, writeMode });
+ assert.equal(back.ok, true);
+ for (const name of ["clips", "share-first"]) assert.equal(await kind(path.join(w.projectDir, name)), "dir");
+ assert.equal((await readManifest(w)).storage.deliverables, "local");
+ // The project's mirror is gone, the media root is kept.
+ assert.equal(await kind(w.mirror), "missing");
+ assert.equal(await kind(w.mediaRoot), "dir");
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDeliverables: a dry run changes nothing; no media root or no manifest refuses", async () => {
+ const w = await world();
+ try {
+ const dry = await moveDeliverables(w.projectDir, "media", { ...w.roots, writeMode, dryRun: true });
+ assert.deepEqual(dry.results.map((x) => x.state), ["would-move", "would-move"]);
+ assert.equal(dry.results[0].bytes, 3000);
+ assert.equal(dry.written, false);
+ assert.equal(await kind(path.join(w.projectDir, "clips")), "dir");
+ assert.equal((await readManifest(w)).storage, undefined);
+ assert.deepEqual(await readdir(w.mediaRoot), []);
+
+ await assert.rejects(moveDeliverables(w.projectDir, "media", { ...w.untiered, writeMode }), /UMTOOL_MEDIA_DIR is not set/);
+ await assert.rejects(moveDeliverables(w.projectDir, "nowhere", { ...w.roots, writeMode }), /--to is "media" or "local"/);
+ await rm(path.join(w.projectDir, "video.manifest.json"));
+ await assert.rejects(moveDeliverables(w.projectDir, "media", { ...w.roots, writeMode }), /no video\.manifest\.json/);
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDeliverables: a project with nothing cut yet just sets the switch, and the first cut makes the link", async () => {
+ const w = await world({ deliverables: false });
+ try {
+ const r = await moveDeliverables(w.projectDir, "media", { ...w.roots, writeMode });
+ assert.deepEqual(r.results, []);
+ assert.equal(r.written, true);
+ const clips = await deliverableDir(w.projectDir, "clips", w.roots);
+ assert.equal(await kind(clips), "link");
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDeliverables: one failure leaves the switch where it was", async () => {
+ const w = await world();
+ try {
+ const r = await moveDeliverables(w.projectDir, "media", { ...w.roots, writeMode, rsyncBin: "false" });
+ assert.equal(r.ok, false);
+ assert.equal(r.written, false);
+ assert.ok(r.results.every((x) => x.state === "failed" && /rsync failed/.test(x.error)));
+ assert.equal((await readManifest(w)).storage, undefined);
+ assert.equal(await kind(path.join(w.projectDir, "clips")), "dir");
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDeliverables finishes a move cut between the park and the link", async () => {
+ const w = await world();
+ try {
+ // What a cut leaves for clips/: its verified copy on the media root, the source parked.
+ await mkdir(w.mirror, { recursive: true });
+ execFileSync("cp", ["-a", path.join(w.projectDir, "clips"), path.join(w.mirror, "clips")]);
+ await rename(path.join(w.projectDir, "clips"), path.join(w.projectDir, "clips.moved-20261001T000000Z"));
+ const st = await deliverablesState(w.projectDir, w.roots);
+ assert.deepEqual(st.dirs[0], { name: "clips", state: "absent", leftovers: ["clips.moved-20261001T000000Z"] });
+ assert.match(deliverablesProblems(st)[0], /a move of clips\/ was cut .*--to media/);
+
+ const r = await moveDeliverables(w.projectDir, "media", { ...w.roots, writeMode });
+ assert.equal(r.ok, true);
+ assert.equal(r.results[0].resumed, true);
+ assert.equal(await kind(path.join(w.projectDir, "clips")), "link");
+ assert.deepEqual((await readdir(w.projectDir)).sort(), ["clips", "share-first", "video.manifest.json", "video.manifest.json.bak"]);
+ } finally {
+ await w.done();
+ }
+});
+
+// ---------------------------------------------------------------------------
+// What a dangling deliverable stops, and the readers that follow a moved batch
+// ---------------------------------------------------------------------------
+
+test("deliverablesProblems: a dangling link stops everything; media without a root stops only a new directory", async () => {
+ const w = await world();
+ try {
+ await moveDeliverables(w.projectDir, "media", { ...w.roots, writeMode });
+ // The same project seen by a process with no media root: existing links
+ // still work, a NEW batch could not be made.
+ const here = await deliverablesState(w.projectDir, w.untiered);
+ assert.deepEqual(deliverablesProblems(here, "clips"), []);
+ assert.match(deliverablesProblems(here, "share-next")[0], /UMTOOL_MEDIA_DIR is not set in umtool's environment/);
+
+ await rename(w.mediaRoot, `${w.mediaRoot}.unplugged`);
+ const gone = await deliverablesState(w.projectDir, w.roots);
+ assert.deepEqual(gone.dirs.map((d) => d.state), ["dangling", "dangling"]);
+ const problems = deliverablesProblems(gone, "clips");
+ assert.equal(problems.length, 2);
+ assert.match(problems[0], /clips\/ is a link to .*, which is not there — is the media drive mounted\?/);
+ } finally {
+ await w.done();
+ }
+});
+
+test("listBatches and sharedIdsIn follow a moved batch; a dangling one is listed with no ids; a leftover is no batch", async () => {
+ const w = await world();
+ try {
+ await moveDeliverables(w.projectDir, "media", { ...w.roots, writeMode });
+ // A nested link inside a batch is followed too.
+ const extra = path.join(w.base, "extra");
+ await mkdir(extra);
+ await writeFile(path.join(extra, "b07_2025-02-02_y.mp4"), "x");
+ await symlink(extra, path.join(w.projectDir, "share-first", "small"));
+ assert.deepEqual([...(await sharedIdsIn(path.join(w.projectDir, "share-first")))].sort(), ["a01", "b07"]);
+
+ await mkdir(path.join(w.projectDir, "share-old.moved-20261001T000000Z"));
+ const batches = await listBatches(w.projectDir);
+ assert.deepEqual(batches.map((b) => [b.name, b.dangling, b.ids, b.hasList]), [["share-first", false, ["a01", "b07"], true]]);
+
+ await rename(w.mediaRoot, `${w.mediaRoot}.unplugged`);
+ const gone = await listBatches(w.projectDir);
+ assert.deepEqual(gone.map((b) => [b.name, b.dangling, b.ids]), [["share-first", true, []]]);
+ } finally {
+ await w.done();
+ }
+});
diff --git a/umtool/lib/report/manifest.mjs b/umtool/lib/report/manifest.mjs
@@ -32,6 +32,7 @@ import {
import { isCalendarDate } from "umtool-report-to-video/attribution";
import { normalizeOnscreen, validateChrome, validatePosts } from "umtool-report-to-video/deck";
import { parseMuteFrom } from "./playback.mjs";
+import { DELIVERABLES_MODES } from "./storage.mjs";
// Its own write queue, not lib/state.ts's.
//
@@ -707,3 +708,43 @@ export class PostsRefused extends Error {
this.errors = errors;
}
}
+
+
+// ---------------------------------------------------------------------------
+// STORAGE: where the project's deliverables live (release 17, slice U2).
+//
+// `"storage": { "deliverables": "local" | "media" }`, absent = local. Only
+// lib/report/storage.mjs's moveDeliverables calls this, and only once every
+// deliverable is where the value says -- a switch set before the move would
+// send the next cut to a drive the earlier ones are not on.
+// ---------------------------------------------------------------------------
+
+/**
+ * Set `storage.deliverables`. Any other key under `storage` is kept. When the
+ * file already says `deliverables`, nothing is written (no new mtime, so no
+ * open bench page's token goes stale over a no-op).
+ *
+ * @param {string} dir
+ * @param {{ deliverables: "local" | "media" }} patch
+ * @param {{ token?: string | null }} [opts]
+ * @returns {Promise<{ storage: Record<string, unknown>, token: string | null, changed: boolean }>}
+ */
+export async function updateStorage(dir, { deliverables } = {}, { token = null } = {}) {
+ if (!DELIVERABLES_MODES.includes(deliverables)) {
+ throw new Error(`storage.deliverables is "local" or "media", not ${JSON.stringify(deliverables)}`);
+ }
+ return withManifestLock(async () => {
+ const current = await manifestToken(dir);
+ if (token !== null && current !== token) throw new StaleToken(token, current);
+
+ const manifest = JSON.parse(await readFile(manifestFile(dir), "utf8"));
+ const storage =
+ manifest.storage && typeof manifest.storage === "object" && !Array.isArray(manifest.storage)
+ ? manifest.storage
+ : {};
+ if (storage.deliverables === deliverables) return { storage, token: current, changed: false };
+ manifest.storage = { ...storage, deliverables };
+ const nextToken = await writeManifestAtomic(dir, manifest);
+ return { storage: manifest.storage, token: nextToken, changed: true };
+ });
+}
diff --git a/umtool/lib/report/storage.mjs b/umtool/lib/report/storage.mjs
@@ -16,7 +16,10 @@
//
// The movers take a NAME, not "out": `umtool storage move-out` moves `out/`
// with them, and a project's deliverables (`clips/`, `share-*/`) are moved by
-// the same two calls behind the deliverables switch.
+// the same two calls behind the deliverables switch (slice U2, at the end of
+// this file): `storage.deliverables` in video.manifest.json says where they
+// live ("local", the default, or "media"), deliverableDir makes one where the
+// switch says, and moveDeliverables moves them all and then sets the switch.
//
// Modelled on the editor's common/controller/relocateDir.ts, not imported from
// it (umtool's scripts are plain .mjs under node, and that is a TypeScript
@@ -35,7 +38,7 @@
// imports this module, and the app's routes import that (plans/FACTS.md, "A
// path joined from `process.cwd()` …").
import { spawn } from "node:child_process";
-import { lstat, mkdir, readdir, readlink, rename, rm, rmdir, stat, statfs, symlink, unlink } from "node:fs/promises";
+import { lstat, mkdir, readFile, readdir, readlink, rename, rm, rmdir, stat, statfs, symlink, unlink } from "node:fs/promises";
import path from "node:path";
import { MEDIA_ROOT, REPORTS_ROOT, inside, mediaMirror } from "../paths.mjs";
@@ -128,51 +131,62 @@ export async function outDirState(projectDir) {
* is never tiered.
*/
export async function ensureOutDir(projectDir, opts = {}) {
- const out = path.join(/* turbopackIgnore: true */ projectDir, "out");
- const s = await pathState(out);
- if (s.kind === "dir") return out;
+ return ensureProjectDir(projectDir, "out", opts, (roots) => roots.tiered);
+}
+
+/**
+ * The one way a project directory that may live on the media root is made:
+ * `out` (tiered whenever UMTOOL_MEDIA_DIR is set) and a deliverable (tiered
+ * when the manifest says so). `tierFor(roots)` is asked only when the directory
+ * is absent, and may throw to refuse.
+ */
+async function ensureProjectDir(projectDir, name, opts, tierFor) {
+ const target = path.join(/* turbopackIgnore: true */ projectDir, name);
+ const s = await pathState(target);
+ if (s.kind === "dir") return target;
if (s.kind === "link") {
- if (s.targetIsDir) return out;
+ if (s.targetIsDir) return target;
throw new Error(
- `${out} is a link to ${s.target}, which is not there — is the media drive mounted? ` +
+ `${target} is a link to ${s.target}, which is not there — is the media drive mounted? ` +
`Nothing was written, and nothing was created in its place.`,
);
}
- if (s.kind === "other") throw new Error(`${out} exists and is not a directory`);
+ if (s.kind === "other") throw new Error(`${target} exists and is not a directory`);
- // A cut move leaves `out.moved-<ts>` or `out.incoming` beside a missing
- // `out`. Making a fresh, empty out/ there would let the next move-out mirror
+ // A cut move leaves `<name>.moved-<ts>` or `<name>.incoming` beside a missing
+ // `<name>`. Making a fresh, empty one there would let the next move mirror
// it over the complete media copy (review L5): refuse, and say how to finish.
- await assertNoLeftovers(projectDir, "out", "nothing was created");
+ await assertNoLeftovers(projectDir, name, "nothing was created");
const roots = rootsOf(opts);
- const mirror = roots.tiered ? mediaMirror(projectDir, roots) : null;
+ const tier = await tierFor(roots);
+ const mirror = tier && roots.tiered ? mediaMirror(projectDir, roots) : null;
if (!mirror) {
- await mkdir(/* turbopackIgnore: true */ out, { recursive: true });
- return out;
+ await mkdir(/* turbopackIgnore: true */ target, { recursive: true });
+ return target;
}
const problem = await mediaRootProblem(roots);
- if (problem) throw new Error(`cannot make ${out}: ${problem}`);
+ if (problem) throw new Error(`cannot make ${target}: ${problem}`);
// The project must exist before its mirror is made: otherwise the symlink
// fails and leaves an empty mirror on the media root (review N5).
// stat, not lstat: a project directory may itself be a link (the walk follows them).
if (!(await stat(/* turbopackIgnore: true */ projectDir).then((st) => st.isDirectory(), () => false))) {
- throw new Error(`cannot make ${out}: ${projectDir} is not a directory`);
+ throw new Error(`cannot make ${target}: ${projectDir} is not a directory`);
}
- const target = path.join(/* turbopackIgnore: true */ mirror, "out");
+ const dest = path.join(/* turbopackIgnore: true */ mirror, name);
// Recursive is safe here: the root itself was just seen to exist.
- await mkdir(/* turbopackIgnore: true */ target, { recursive: true });
+ await mkdir(/* turbopackIgnore: true */ dest, { recursive: true });
try {
- await symlink(/* turbopackIgnore: true */ target, out, "dir");
+ await symlink(/* turbopackIgnore: true */ dest, target, "dir");
} catch (e) {
// Two writers starting at once: whichever linked first won, and a link (or
// directory) that now resolves is as good as ours.
if (e?.code !== "EEXIST") throw e;
- const again = await pathState(out);
- if (again.kind === "dir" || (again.kind === "link" && again.targetIsDir)) return out;
+ const again = await pathState(target);
+ if (again.kind === "dir" || (again.kind === "link" && again.targetIsDir)) return target;
throw e;
}
- return out;
+ return target;
}
/**
@@ -362,11 +376,17 @@ async function assertNoLeftovers(projectDir, name, what, { beside = false } = {}
);
}
throw new Error(
- `a move of ${name}/ in ${projectDir} was cut (left: ${left.map((p) => path.basename(/* turbopackIgnore: true */ p)).join(", ")}) — ` +
- `run \`umtool storage ${left.some((p) => p.endsWith(".incoming")) ? "move-back" : "move-out"} <project>\` to finish it; ${what}`,
+ `a move of ${name}/ in ${projectDir} was cut (left: ${names}) — ` +
+ `run \`${finishCommand(name, left.some((p) => p.endsWith(".incoming")))}\` to finish it; ${what}`,
);
}
+/** The command that finishes a cut move of `<name>`: `out` has its own pair, a deliverable the switch. */
+export const finishCommand = (name, back) =>
+ name === "out"
+ ? `umtool storage ${back ? "move-back" : "move-out"} <project>`
+ : `umtool storage deliverables <project> --to ${back ? "local" : "media"}`;
+
const defaults = (opts) => ({
rsyncBin: opts.rsyncBin ?? process.env.RSYNC_BIN ?? "rsync",
log: opts.log ?? (() => {}),
@@ -567,3 +587,235 @@ async function dropMediaCopy(from, ownCopy, mediaRoot) {
}
return undefined;
}
+
+// ---------------------------------------------------------------------------
+// Deliverables: `clips/` and every `share-*/` (release 17, slice U2)
+//
+// Unlike out/, they do not follow UMTOOL_MEDIA_DIR on their own: they move per
+// project, by a switch -- `"storage": { "deliverables": "local" | "media" }` in
+// video.manifest.json (absent = local), written only through
+// lib/report/manifest.mjs's updateStorage. moveDeliverables moves what exists
+// with the two movers above and then sets the switch; deliverableDir is how a
+// writer (a cut, a share batch) makes a deliverable directory that does not
+// exist yet, where the switch says. A reader keeps opening
+// `<project>/clips/<id>.mp4` by path: through the link when it is one.
+// ---------------------------------------------------------------------------
+
+/** The two values of `storage.deliverables`. Absent is "local". */
+export const DELIVERABLES_MODES = ["local", "media"];
+
+/** The directory a project's cut clips live in (lib/report/cut.mjs's CLIPS_DIR). */
+const CLIPS = "clips";
+/** A share batch's directory prefix (lib/report/deliver.mjs's SHARE_PREFIX). */
+const SHARE = "share-";
+
+/** What a cut move leaves beside a deliverable: `<name>.moved-<stamp>` or `<name>.incoming`. */
+const LEFTOVER = /^(.+)\.(moved-[^/]*|incoming)$/;
+
+/** Is `name` a deliverable directory's name (and not a cut move's leftover)? */
+export const isDeliverableName = (name) =>
+ typeof name === "string" &&
+ !LEFTOVER.test(name) &&
+ (name === CLIPS || (name.startsWith(SHARE) && name.length > SHARE.length && !/[\/\\\0]/.test(name)));
+
+/**
+ * `storage.deliverables` of a parsed manifest.
+ * @returns {{ mode: "local" | "media" | null, value: unknown, error?: string }}
+ * `mode` null when the value is not one of the two (the error says so).
+ */
+export function deliverablesModeOf(manifest) {
+ const v = manifest?.storage?.deliverables;
+ if (v === undefined) return { mode: "local", value: undefined };
+ if (DELIVERABLES_MODES.includes(v)) return { mode: v, value: v };
+ return { mode: null, value: v, error: `storage.deliverables is ${JSON.stringify(v)} — it is "local" or "media"` };
+}
+
+/**
+ * `storage.deliverables`, read off the project's manifest. Read here (a plain
+ * JSON read) rather than through lib/projects/report.mjs, which this module's
+ * importers must not pull in; written only by manifest.mjs.
+ */
+export async function deliverablesMode(projectDir) {
+ const file = path.join(/* turbopackIgnore: true */ projectDir, "video.manifest.json");
+ const text = await readFile(/* turbopackIgnore: true */ file, "utf8").catch(() => null);
+ if (text === null) return { mode: "local", value: undefined, manifest: false };
+ try {
+ return { ...deliverablesModeOf(JSON.parse(text)), manifest: true };
+ } catch {
+ return { mode: null, value: undefined, manifest: true, error: `${file} is not valid JSON` };
+ }
+}
+
+/**
+ * Make sure the deliverable directory `<projectDir>/<name>` exists, and return
+ * its path. What a cut (`clips`) and a share batch (`share-<x>`) call before
+ * they write.
+ *
+ * a directory, or a link to one -> it, whatever the switch says (a move is
+ * what changes where an existing one lives)
+ * a DANGLING link -> refused, as ensureOutDir refuses: nothing
+ * is created in its place
+ * absent, the switch "local" -> a directory in the project
+ * absent, the switch "media" -> `<MEDIA_ROOT>/<project-relative>/<name>`
+ * and a link to it -- or refused when there
+ * is no media root here (UMTOOL_MEDIA_DIR
+ * unset in this process), it is not there,
+ * or the project is outside the reports
+ * root. Never silently local: a batch made
+ * on the wrong drive is a split nobody chose.
+ * a cut move's leftover beside it -> refused, naming the command that finishes it
+ *
+ * @param {string} projectDir
+ * @param {string} name "clips" or "share-<x>"
+ * @param {{ mode?: "local" | "media", reportsRoot?: string, mediaRoot?: string }} [opts]
+ * `mode` overrides the manifest's switch (tests).
+ */
+export async function deliverableDir(projectDir, name, opts = {}) {
+ if (!isDeliverableName(name)) throw new Error(`not a deliverable directory: ${JSON.stringify(name)}`);
+ return ensureProjectDir(projectDir, name, opts, async (roots) => {
+ const m = opts.mode ? { mode: opts.mode } : await deliverablesMode(projectDir);
+ if (!m.mode) throw new Error(`cannot make ${path.join(/* turbopackIgnore: true */ projectDir, name)}: ${m.error}`);
+ if (m.mode === "local") return false;
+ const target = path.join(/* turbopackIgnore: true */ projectDir, name);
+ if (!roots.tiered) {
+ throw new Error(
+ `cannot make ${target}: this project keeps its deliverables on the media root (storage.deliverables: media), ` +
+ `and UMTOOL_MEDIA_DIR is not set here — set it, or bring them back with ` +
+ `\`umtool storage deliverables <project> --to local\`. Nothing was created.`,
+ );
+ }
+ if (!mediaMirror(projectDir, roots)) {
+ throw new Error(`cannot make ${target}: ${projectDir} is not under the reports root ${roots.reportsRoot}, so it has no place on the media root`);
+ }
+ return true;
+ });
+}
+
+/**
+ * Every deliverable a project has, by name: `clips` and each `share-*` that is
+ * a directory or a link -- and the name of any whose cut move left only a
+ * leftover (`clips.moved-<ts>` with no `clips`), so a move finishes it.
+ * `clips` first, then the batches by name.
+ */
+export async function deliverableNames(projectDir) {
+ const entries = await readdir(/* turbopackIgnore: true */ projectDir, { withFileTypes: true }).catch(() => []);
+ const names = new Set();
+ for (const e of entries) {
+ const left = LEFTOVER.exec(e.name);
+ if (left && e.isDirectory() && isDeliverableName(left[1])) names.add(left[1]);
+ else if (isDeliverableName(e.name) && (e.isDirectory() || e.isSymbolicLink())) names.add(e.name);
+ }
+ return [...names].sort((a, b) => (a === CLIPS ? -1 : b === CLIPS ? 1 : a.localeCompare(b)));
+}
+
+/**
+ * Where a project's deliverables are, for the bench, `umtool storage` and
+ * `umtool check`. One lstat per name, and one stat through each link (which is
+ * what a reader does anyway).
+ *
+ * @returns {Promise<{ mode: "local" | "media" | null, value: unknown, error?: string,
+ * tiered: boolean, mediaRoot: string | null,
+ * dirs: Array<{ name: string, state: "absent" | "dir" | "link" | "dangling" | "other", target?: string, leftovers: string[] }> }>}
+ */
+export async function deliverablesState(projectDir, opts = {}) {
+ const roots = rootsOf(opts);
+ const m = await deliverablesMode(projectDir);
+ const dirs = [];
+ for (const name of await deliverableNames(projectDir)) {
+ const s = await pathState(path.join(/* turbopackIgnore: true */ projectDir, name));
+ const state = s.kind === "missing" ? "absent" : s.kind === "link" ? (s.targetIsDir ? "link" : "dangling") : s.kind;
+ const leftovers = (await leftoversOf(projectDir, name)).map((p) => path.basename(/* turbopackIgnore: true */ p));
+ dirs.push({ name, state, ...(s.kind === "link" ? { target: s.target } : {}), leftovers });
+ }
+ return {
+ mode: m.mode,
+ value: m.value,
+ ...(m.error ? { error: m.error } : {}),
+ tiered: roots.tiered,
+ mediaRoot: roots.tiered ? roots.mediaRoot : null,
+ dirs,
+ };
+}
+
+/**
+ * What stops a deliverable being WRITTEN now, as sentences, from a
+ * deliverablesState: a link whose drive is not there, a cut move's leftovers,
+ * and -- for a directory that does not exist yet -- a switch this process
+ * cannot honour. `name` narrows it to one directory (a cut asks for `clips`,
+ * a batch for its own `share-<x>`); every dangling link and leftover counts
+ * whatever the name, because a batch reads `clips/` and every earlier batch.
+ *
+ * @param {{ mode: string | null, error?: string, tiered: boolean,
+ * dirs: Array<{ name: string, state: string, target?: string, leftovers: string[] }> }} state
+ * @param {string | null} [name]
+ * @returns {string[]}
+ */
+export function deliverablesProblems(state, name = null) {
+ const out = [];
+ for (const d of state.dirs) {
+ if (d.state === "dangling") {
+ out.push(`${d.name}/ is a link to ${d.target}, which is not there — is the media drive mounted?`);
+ }
+ if (d.leftovers.length) {
+ out.push(
+ `a move of ${d.name}/ was cut (left: ${d.leftovers.join(", ")}) — ` +
+ `\`${finishCommand(d.name, d.leftovers.some((l) => l.endsWith(".incoming")))}\` finishes it`,
+ );
+ }
+ }
+ const exists = name && state.dirs.some((d) => d.name === name && d.state !== "absent");
+ if (name && !exists) {
+ if (!state.mode) out.push(state.error ?? "storage.deliverables is not \"local\" or \"media\"");
+ else if (state.mode === "media" && !state.tiered) {
+ out.push("this project keeps its deliverables on the media root (storage.deliverables: media), and UMTOOL_MEDIA_DIR is not set in umtool's environment");
+ }
+ }
+ return out;
+}
+
+/**
+ * The switch: move every deliverable to `to` ("media" or "local") with the
+ * movers, then set `storage.deliverables` -- only when every one of them is
+ * where it should be. Idempotent: a second run finds each "already" there and
+ * writes nothing (the manifest is not rewritten when the switch already says
+ * `to`). A run that was cut is finished by running it again, as the movers
+ * finish theirs.
+ *
+ * Nothing may be cutting or sharing into the project while it runs. The
+ * callers check (the app's one-job-at-a-time registry, the CLI's process
+ * scan); the movers' verify refuses a tree that keeps changing.
+ *
+ * @param {string} projectDir
+ * @param {"local" | "media"} to
+ * @param {{ writeMode: (dir: string, mode: "local" | "media") => Promise<unknown>,
+ * dryRun?: boolean, log?: (m: string) => void, rsyncBin?: string, reportsRoot?: string, mediaRoot?: string }} opts
+ * `writeMode` is manifest.mjs's updateStorage -- passed in, so this module
+ * (which the app's routes import) does not pull the manifest writer's imports.
+ */
+export async function moveDeliverables(projectDir, to, opts) {
+ if (!DELIVERABLES_MODES.includes(to)) throw new Error(`--to is "media" or "local", not ${JSON.stringify(to)}`);
+ const { dryRun } = defaults(opts);
+ const roots = rootsOf(opts);
+ const m = await deliverablesMode(projectDir);
+ if (!m.manifest) throw new Error(`${projectDir} has no video.manifest.json — deliverables are a report video's`);
+ if (to === "media" && !roots.tiered) {
+ throw new Error("UMTOOL_MEDIA_DIR is not set: there is no media root to move deliverables to. Nothing moved.");
+ }
+ const move = to === "media" ? moveDirToMedia : moveDirToLocal;
+ const results = [];
+ for (const name of await deliverableNames(projectDir)) {
+ try {
+ results.push({ name, ...(await move(projectDir, name, opts)) });
+ } catch (e) {
+ results.push({ name, state: "failed", error: e instanceof Error ? e.message : String(e) });
+ }
+ }
+ const failed = results.filter((r) => r.state === "failed").length;
+ const before = m.value ?? null;
+ let written = false;
+ if (!failed && !dryRun && m.value !== to) {
+ await opts.writeMode(projectDir, to);
+ written = true;
+ }
+ return { ok: failed === 0, to, before, written, dryRun, results };
+}
