commit 418ae1b1cacca5b3280e847b0ca39121ca298d95
parent 30f0378ec99dccc061b9b9365f547195ca6b8414
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 2 Oct 2026 00:17:16 -0400
umtool: review H1, N1 — the busy scan sees a cut or batch by --project; check flags a media link under no key
H1: the app runs cut-from-cache.mjs and share-batch.mjs as `--project <id>`,
which the scan (absolute project path in an argument) never matched, so a
shell-run `umtool storage deliverables` (or move-out/back) could move clips/
and share-* while the app cut or shared into them. The scan moves to
lib/report/busy.mjs (CLI-only: a literal /proc path stays out of the app's
bundle) and also counts a pipeline script whose --project value equals the
project's id or name, or resolves against the process's cwd to the project
directory — whole values, never a prefix. The CLI comment and docs say what
it covers and what it cannot see (a hand-run command that is none of these
scripts, another machine).
N1: absent means local, so a clips/ or share-* link into the media root
under no storage key is storage-mismatch too, as under an explicit "local".
Tests (deliverables.test.mjs, +4): namesProject's whole-value cases; a dummy
process carrying `cut-from-cache.mjs --project <id>` is seen and
`<id>-other` is not; the CLI refuses as busy with the pid and moves once
only `<id>-other` runs; check on a media link under no key, local, media.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
6 files changed, 262 insertions(+), 74 deletions(-)
diff --git a/umtool/bin/umtool.mjs b/umtool/bin/umtool.mjs
@@ -37,7 +37,6 @@
// umtool export <project> --format toc-bbcode|toc-markdown|description|chapters [--variant V]
// umtool check-sources [<project>…] prints the re-check chain
import process from "node:process";
-import { readdirSync, readFileSync, readlinkSync, realpathSync } from "node:fs";
import {
PROJECT_KINDS,
REPORTS_ROOT,
@@ -64,6 +63,7 @@ import { updateClip, updateStorage } from "../lib/report/manifest.mjs";
import { hms } from "umtool-report-to-video/attribution";
import { buildSteps, checkSourcesSteps, PRESETS } from "../lib/report/driver.mjs";
import { openIndex, signRecord } from "../lib/projects/index-db.mjs";
+import { pipelineProcessesFor } from "../lib/report/busy.mjs";
import { probeTools } from "../lib/tools.mjs";
import { scaffoldReportVideo } from "../lib/projects/scaffold.mjs";
import { CACHE_DIR, INDEX_DIR, MEDIA_ROOT, MEDIA_TIERED, OLD_CACHE_DIR } from "../lib/paths.mjs";
@@ -433,61 +433,14 @@ async function cmdDoctor() {
// --dry-run measure and say; change nothing
//
// The movers are lib/report/storage.mjs's, which `umtool` and the app share.
-// Run a move when nothing is building: the app's jobs live in its memory, so
-// this cannot see them -- the verify refuses when the tree keeps changing, but
-// a write in the last instant before the swap would be lost with the parked copy.
+// A project is skipped (move-out/back) or refused (deliverables) while a
+// pipeline process works in it -- lib/report/busy.mjs says what that scan sees
+// (the app's build steps, cuts and share batches, which are all processes, and
+// the report's own scripts) and what it cannot (a hand-run command that is none
+// of them, another machine). The verify refuses a tree that keeps changing, but
+// a write it cannot see, in the last instant before the swap, would be lost
+// with the parked copy.
// ---------------------------------------------------------------------------
-/**
- * The pids of report-pipeline processes whose command line names this project
- * (its manifest, its out/, or the directory itself). Linux /proc; elsewhere,
- * none. Cheap and coarse: it sees the pipeline's scripts, not the app's
- * in-process deck previews.
- */
-function pipelineProcessesFor(projectDir) {
- const SCRIPTS = /(build-video|check-availability|render-cards|compose-chrome|verify-build|fetch-via-editor|resolve-windows|cut-from-cache|share-batch)\.mjs/;
- // The report's OWN scripts (the Deliver panel's "apply rulings" and
- // "rebuild"): apply-manifest.py deletes clips/ mp4s, build.py reads them.
- // They are run with the project as their working directory and name no
- // path, so they are found by script name and cwd.
- const OWN = /(^|\/)(apply-manifest|build)\.py$/;
- let realDir = projectDir;
- try {
- realDir = realpathSync(projectDir);
- } catch {
- /* gone: nothing can be running in it */
- }
- const pids = [];
- let entries = [];
- try {
- entries = readdirSync("/proc").filter((n) => /^\d+$/.test(n));
- } catch {
- return pids;
- }
- for (const pid of entries) {
- if (Number(pid) === process.pid) continue;
- let args;
- try {
- args = readFileSync(`/proc/${pid}/cmdline`, "utf8").split("\0");
- } catch {
- continue;
- }
- if (args.some((a) => SCRIPTS.test(a))) {
- if (args.some((a) => a === projectDir || a.startsWith(projectDir + "/"))) pids.push(Number(pid));
- continue;
- }
- if (args.some((a) => OWN.test(a))) {
- let cwd = "";
- try {
- cwd = readlinkSync(`/proc/${pid}/cwd`);
- } catch {
- continue;
- }
- if (cwd === projectDir || cwd === realDir) pids.push(Number(pid));
- }
- }
- return pids;
-}
-
async function cmdStorage() {
const sub = positional[0];
const dryRun = has("--dry-run");
@@ -540,7 +493,7 @@ async function cmdStorage() {
for (const p of refs) {
// The app's jobs live in its memory, but the pipeline runs as processes:
// one whose command line names this project is building it now.
- const busy = pipelineProcessesFor(p.dir);
+ const busy = pipelineProcessesFor(p);
if (busy.length) {
results.push({ id: p.id, state: "busy", pids: busy });
if (!json) console.log(`${"busy".padEnd(12)} ${p.id} — a pipeline process is writing it (pid ${busy.join(", ")}); skipped`);
@@ -576,22 +529,25 @@ async function cmdStorage() {
/**
* `umtool storage deliverables <project> --to media|local [--dry-run]`: move
* clips/ and every share-* directory with the movers, then set storage.deliverables
- * through the manifest's writer. Refused while a pipeline process -- a cut, a
- * share batch, the report's own apply/build scripts -- works in the project.
+ * through the manifest's writer. Refused while a pipeline process works in the
+ * project (lib/report/busy.mjs): a build step naming its directory, a cut or a
+ * share batch whose `--project` is this project's id, name or directory, the
+ * report's own apply/build scripts by their working directory.
*
- * What the refusal cannot see: the app's in-process work (the app runs this
- * as a JOB, and its one-job-at-a-time rule is what keeps its cuts and batches
- * out), another machine writing over a network share, and a hand-run command
- * that is none of those scripts. The movers' verify refuses a tree that keeps
- * changing, but a write in the instant between the verify and the swap would
- * be lost with the parked copy.
+ * What the refusal cannot see: a hand-run command that is none of those
+ * scripts (an ffmpeg writing into clips/), and another machine writing over a
+ * network share. The app's own jobs are processes and are seen; run from the
+ * bench, this is itself a job, so the app's one-job-at-a-time rule keeps its
+ * cuts and batches out as well. The movers' verify refuses a tree that keeps
+ * changing, but a write it cannot see, in the instant between the verify and
+ * the swap, would be lost with the parked copy.
*/
async function cmdStorageDeliverables(dryRun, log) {
const to = val("--to");
if (to !== "media" && to !== "local") die("where to? `umtool storage deliverables <project> --to media|local`");
const p = await pick(positional[1]);
if (p.kind !== "report-video") die(`${p.id} is a ${p.kind} project — deliverables are a report video's`);
- const busy = pipelineProcessesFor(p.dir);
+ const busy = pipelineProcessesFor(p);
if (busy.length) {
const msg = `${p.id}: a pipeline process is working in it (pid ${busy.join(", ")}) — nothing moved; run this when it has finished`;
if (json) out({ ok: false, busy: busy, error: msg });
diff --git a/umtool/docs/cli.md b/umtool/docs/cli.md
@@ -28,7 +28,7 @@ decisions inbox cannot disagree about what is wrong with one.
| `doctor [--json]` | which tools are on this machine and where the roots are; **exit 1** if the report pipeline is missing one, or `UMTOOL_MEDIA_DIR` is set and not there |
| `storage [<project>] [--json]` | where each project's `out/` lives: dir, link, DANGLING, none — and a report's deliverables (`clips/`, `share-*/`) and its switch |
| `storage move-out\|move-back <project>\|--all [--dry-run] [--json]` | `out/` to the media root (a link left behind) and back — copied, mirrored, verified first; run when nothing is building |
-| `storage deliverables <project> --to media\|local [--dry-run] [--json]` | `clips/` and every `share-*/` to the media root and back, then `storage.deliverables` in the manifest; refused while a cut, a batch or the report's own scripts run in the project; **exit 1** on any failure (the switch is then left as it was) |
+| `storage deliverables <project> --to media\|local [--dry-run] [--json]` | `clips/` and every `share-*/` to the media root and back, then `storage.deliverables` in the manifest; refused while a build step, a cut or a share batch (by `--project` id, name or directory, as the app starts them) or the report's own scripts run in the project — a hand-run command that is none of these is not seen; **exit 1** on any failure (the switch is then left as it was) |
| `snapshot <project> [--label L]` | copy the manifest into `revisions/` |
| `diff <project> <snapshot>` | added / removed / moved / window / retyped, by entry id |
| `export <project> --format toc-bbcode\|toc-markdown\|description\|chapters [--variant V]` | the posting artifacts, from the build's chapter offsets |
diff --git a/umtool/docs/folders.md b/umtool/docs/folders.md
@@ -56,10 +56,15 @@ which runs it as a job) calls that — after every directory has moved:
`clips/` and each `share-*/` with the same copy-mirror-verify-swap movers, then
sets the switch. Run again, it finds each one `already` there and rewrites
nothing; a cut move is finished by running it again. It refuses while a
- pipeline process works in the project — a cut, a share batch, the report's own
- `apply-manifest.py` or `build.py`. It cannot see the app's in-process work
- (the bench runs the move as a job, and the one-job-at-a-time rule keeps cuts
- and batches out), a hand-run command, or another machine.
+ pipeline process works in the project (`lib/report/busy.mjs`, Linux `/proc`):
+ a build step whose arguments name the project's directory; a cut
+ (`cut-from-cache.mjs`) or share batch (`share-batch.mjs`) whose `--project`
+ is the project's id, its name, or a path that resolves to its directory —
+ whole values, never a prefix — which is how the app starts them; and the
+ report's own `apply-manifest.py` or `build.py`, by working directory. It
+ cannot see a hand-run command that is none of those scripts (an `ffmpeg` into
+ `clips/`) or another machine. From the bench the move is itself a job, so the
+ app's one-job-at-a-time rule keeps its cuts and batches out too.
- A directory that does not exist yet is made where the switch says, by the
writer: a cut makes `clips/`, a batch its `share-<name>/`
(`storage.mjs` `deliverableDir`). Under `"media"` that is a link to a new
diff --git a/umtool/lib/projects/report.mjs b/umtool/lib/projects/report.mjs
@@ -10,7 +10,7 @@ import path from "node:path";
import { DEFAULT_VARIANT, cachedWindowsFor } from "umtool-report-to-video/build-video";
import { rawCacheOf } from "../report/raw-cache.mjs";
import { deliverablesState, finishCommand, outDirState, leftoversOf } from "../report/storage.mjs";
-import { CHANNELS_DIR } from "../paths.mjs";
+import { CHANNELS_DIR, inside } from "../paths.mjs";
import { channelName, cleanTitle } from "umtool-report-to-video/attribution";
import { teaserTitle } from "umtool-report-to-video/deck";
@@ -1098,8 +1098,21 @@ export async function reportDecisions(ctx, summary) {
);
} else if (store.mode === "media" && d.state === "dir") {
add("storage-mismatch", d.name, `a directory in the project, while storage.deliverables is media — \`umtool storage deliverables ${id} --to media\` moves it`, "open");
- } else if (store.mode === "local" && store.value !== undefined && d.state === "link") {
- add("storage-mismatch", d.name, `a link to ${d.target}, while storage.deliverables is local — \`umtool storage deliverables ${id} --to local\` brings it back`, "open");
+ } else if (
+ store.mode === "local" &&
+ d.state === "link" &&
+ // Absent means local, so a link INTO the media root under no key at all
+ // is the same disagreement (review N1). A hand-made link elsewhere under
+ // no key is nobody's decision to second-guess; under an explicit
+ // "local", any link is.
+ (store.value !== undefined || (store.mediaRoot && d.target && inside(store.mediaRoot, d.target)))
+ ) {
+ add(
+ "storage-mismatch",
+ d.name,
+ `a link to ${d.target}, while storage.deliverables is ${store.value === undefined ? "unset (local)" : "local"} — \`umtool storage deliverables ${id} --to local\` brings it back, \`--to media\` records it`,
+ "open",
+ );
}
}
if (store.mode === "media" && !store.tiered) {
diff --git a/umtool/lib/report/busy.mjs b/umtool/lib/report/busy.mjs
@@ -0,0 +1,96 @@
+// Which report-pipeline processes are working in a project RIGHT NOW (release
+// 17): what `umtool storage move-out|move-back|deliverables` refuses over.
+//
+// Linux /proc; elsewhere, none. Imported by the CLI only (bin/umtool.mjs) --
+// never by anything the app bundles: a literal /proc path is a directory to
+// Turbopack.
+//
+// WHAT IT SEES, and how:
+// - the pipeline's node scripts (SCRIPTS), when an argument is the project
+// directory or a path under it -- how the build steps name a project
+// (`<project>/video.manifest.json`, `--out <project>/out`);
+// - the same scripts when their `--project` value names THIS project: equal
+// to its id, equal to its name, or resolving against the process's own
+// working directory to the project directory. Whole values, never a
+// prefix (`elfpire-eva-2` is not `elfpire-eva`). This is how the app runs
+// a cut (`cut-from-cache.mjs --project <id>`) and a share batch
+// (`share-batch.mjs --project <id>`). A name matched by a same-named
+// project elsewhere reads busy too, which only ever refuses;
+// - the report's OWN scripts (apply-manifest.py, build.py), which name no
+// path: by script name and working directory.
+// WHAT IT CANNOT SEE: the app's in-process work (none of it writes clips/ or
+// share-*; the app's own jobs are kept apart by its one-job-at-a-time rule),
+// a hand-run command that is none of these scripts (an ffmpeg into clips/),
+// and anything on another machine.
+import { readdirSync, readFileSync, readlinkSync, realpathSync } from "node:fs";
+import path from "node:path";
+
+export const PIPELINE_SCRIPTS =
+ /(build-video|check-availability|render-cards|compose-chrome|verify-build|fetch-via-editor|resolve-windows|cut-from-cache|share-batch)\.mjs/;
+const OWN_SCRIPTS = /(^|\/)(apply-manifest|build)\.py$/;
+
+/**
+ * Does one process's argv (and working directory) name this project?
+ * Pure: the /proc reads are the caller's.
+ * @param {string[]} args
+ * @param {string | null} cwd the process's working directory, when readable
+ * @param {{ dir: string, realDir?: string, id?: string, name?: string }} project
+ */
+export function namesProject(args, cwd, project) {
+ const dirs = [...new Set([project.dir, project.realDir ?? project.dir])];
+ const underDir = (a) => dirs.some((d) => a === d || a.startsWith(d + "/"));
+ if (args.some((a) => PIPELINE_SCRIPTS.test(a))) {
+ if (args.some(underDir)) return true;
+ for (let i = 0; i < args.length - 1; i++) {
+ if (args[i] !== "--project") continue;
+ const v = args[i + 1];
+ if (!v) continue;
+ if (v === project.id || v === project.name) return true;
+ if (cwd && dirs.includes(path.resolve(cwd, v))) return true;
+ }
+ return false;
+ }
+ if (args.some((a) => OWN_SCRIPTS.test(a))) return !!cwd && dirs.includes(cwd);
+ return false;
+}
+
+/**
+ * The pids of pipeline processes working in `project` (see the top of the file).
+ * @param {{ dir: string, id?: string, name?: string }} project
+ * @param {{ procRoot?: string, selfPid?: number }} [opts]
+ * @returns {number[]}
+ */
+export function pipelineProcessesFor(project, { procRoot = "/proc", selfPid = process.pid } = {}) {
+ let realDir = project.dir;
+ try {
+ realDir = realpathSync(project.dir);
+ } catch {
+ /* gone: nothing can be running in it */
+ }
+ const ref = { ...project, realDir };
+ const pids = [];
+ let entries = [];
+ try {
+ entries = readdirSync(procRoot).filter((n) => /^\d+$/.test(n));
+ } catch {
+ return pids;
+ }
+ for (const pid of entries) {
+ if (Number(pid) === selfPid) continue;
+ let args;
+ try {
+ args = readFileSync(`${procRoot}/${pid}/cmdline`, "utf8").split("\0").filter((a) => a !== "");
+ } catch {
+ continue;
+ }
+ if (!args.some((a) => PIPELINE_SCRIPTS.test(a) || OWN_SCRIPTS.test(a))) continue;
+ let cwd = null;
+ try {
+ cwd = readlinkSync(`${procRoot}/${pid}/cwd`);
+ } catch {
+ /* another user's process, or gone */
+ }
+ if (namesProject(args, cwd, ref)) pids.push(Number(pid));
+ }
+ return pids;
+}
diff --git a/umtool/lib/report/deliverables.test.mjs b/umtool/lib/report/deliverables.test.mjs
@@ -8,12 +8,14 @@
//
// Run with: pnpm test:scripts
import assert from "node:assert/strict";
-import { execFileSync } from "node:child_process";
+import { execFileSync, spawn, spawnSync } from "node:child_process";
import { lstat, mkdir, mkdtemp, readFile, readdir, readlink, rename, rm, stat, symlink, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import test from "node:test";
+import { fileURLToPath } from "node:url";
+import { namesProject, pipelineProcessesFor } from "./busy.mjs";
import { listBatches, sharedIdsIn } from "./deliver.mjs";
import { updateStorage } from "./manifest.mjs";
import {
@@ -354,3 +356,119 @@ test("listBatches and sharedIdsIn follow a moved batch; a dangling one is listed
await w.done();
}
});
+
+// ---------------------------------------------------------------------------
+// The busy scan (lib/report/busy.mjs) and the CLI's refusal (review H1), and
+// `umtool check` on a link under no key (review N1)
+// ---------------------------------------------------------------------------
+
+
+const UMTOOL_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
+
+test("namesProject: a script names a project by path, or by --project id, name or cwd-relative dir — whole values only", () => {
+ const p = { dir: "/r/folder/proj", id: "folder/proj", name: "proj" };
+ const cut = (v, cwd = "/umtool") => namesProject(["node", "/umtool/bin/cut-from-cache.mjs", "--project", v, "--clip", "a01"], cwd, p);
+ assert.equal(cut("folder/proj"), true);
+ assert.equal(cut("proj"), true);
+ assert.equal(cut("/r/folder/proj"), true);
+ assert.equal(cut("../proj", "/r/folder/other"), true);
+ for (const no of ["folder/proj-other", "folder/pro", "proj-2", "folder", "/r/folder/proj2"]) assert.equal(cut(no), false, no);
+ assert.equal(namesProject(["node", "share-batch.mjs", "--project", "folder/proj", "--name", "x"], null, p), true);
+ // A build step names the manifest by path.
+ assert.equal(namesProject(["node", "build-video.mjs", "/r/folder/proj/video.manifest.json"], null, p), true);
+ assert.equal(namesProject(["node", "build-video.mjs", "/r/folder/proj-2/video.manifest.json"], null, p), false);
+ // The report's own scripts, by working directory.
+ assert.equal(namesProject(["python3", "apply-manifest.py"], "/r/folder/proj", p), true);
+ assert.equal(namesProject(["python3", "build.py"], "/r/folder/proj-2", p), false);
+ // Anything else is not a pipeline process, whatever it names.
+ assert.equal(namesProject(["ffmpeg", "-i", "/r/folder/proj/clips/a.mp4"], null, p), false);
+});
+
+/** A process that does nothing for a while, with `cut-from-cache.mjs --project <v>` in its argv. */
+function dummyCut(value) {
+ const child = spawn(process.execPath, ["-e", "setTimeout(() => {}, 20000)", "cut-from-cache.mjs", "--project", value], {
+ stdio: "ignore",
+ });
+ return child;
+}
+
+test("pipelineProcessesFor sees a cut the app started with --project <id>, and not <id>-other", async () => {
+ const w = await world();
+ const mine = dummyCut("folder/proj");
+ const other = dummyCut("folder/proj-other");
+ try {
+ await new Promise((r) => setTimeout(r, 300));
+ const pids = pipelineProcessesFor({ dir: w.projectDir, id: "folder/proj", name: "proj" });
+ assert.ok(pids.includes(mine.pid), "the cut is seen");
+ assert.ok(!pids.includes(other.pid), "a project whose id merely starts with this one's is not");
+ } finally {
+ mine.kill();
+ other.kill();
+ await w.done();
+ }
+});
+
+/** The CLI against a temp reports root, with no media root unless given. */
+function cli(w, args, extra = {}) {
+ return spawnSync(process.execPath, [path.join(UMTOOL_DIR, "bin", "umtool.mjs"), ...args, "--json"], {
+ cwd: UMTOOL_DIR,
+ encoding: "utf8",
+ env: {
+ ...process.env,
+ REPORTS_DIR: w.reportsRoot,
+ SONG_DIR: path.join(w.base, "no-song-data"),
+ UMTOOL_CACHE_DIR: path.join(w.base, "cache"),
+ UMTOOL_MEDIA_DIR: "",
+ ...extra,
+ },
+ });
+}
+
+test("umtool storage deliverables refuses, as busy, while a cut runs in the project", async () => {
+ const w = await world();
+ const ls = cli(w, ["ls"]);
+ assert.equal(ls.status, 0, ls.stderr);
+ const id = JSON.parse(ls.stdout)[0].id;
+ const mine = dummyCut(id);
+ try {
+ await new Promise((r) => setTimeout(r, 300));
+ const r = cli(w, ["storage", "deliverables", id, "--to", "local"]);
+ assert.equal(r.status, 1);
+ const j = JSON.parse(r.stdout);
+ assert.ok(j.busy.includes(mine.pid), r.stdout);
+ assert.match(j.error, /a pipeline process is working in it .* nothing moved/);
+ assert.equal((await readManifest(w)).storage, undefined);
+ } finally {
+ mine.kill();
+ }
+ const other = dummyCut(`${id}-other`);
+ try {
+ await new Promise((r) => setTimeout(r, 300));
+ const r = cli(w, ["storage", "deliverables", id, "--to", "local"]);
+ assert.equal(r.status, 0, r.stderr + r.stdout);
+ assert.equal((await readManifest(w)).storage.deliverables, "local");
+ } finally {
+ other.kill();
+ await w.done();
+ }
+});
+
+test("umtool check: a link into the media root under no key is a mismatch, as under an explicit local", async () => {
+ const w = await world({ deliverables: false });
+ try {
+ await mkdir(path.join(w.mirror, "clips"), { recursive: true });
+ await symlink(path.join(w.mirror, "clips"), path.join(w.projectDir, "clips"));
+ const id = JSON.parse(cli(w, ["ls"]).stdout)[0].id;
+ const kinds = (extra) =>
+ JSON.parse(cli(w, ["check", id], extra).stdout)
+ .decisions.filter((d) => d.kind.startsWith("storage-"))
+ .map((d) => [d.kind, d.target, d.severity]);
+ assert.deepEqual(kinds({ UMTOOL_MEDIA_DIR: w.mediaRoot }), [["storage-mismatch", "clips", "open"]]);
+ await updateStorage(w.projectDir, { deliverables: "local" });
+ assert.deepEqual(kinds({ UMTOOL_MEDIA_DIR: w.mediaRoot }), [["storage-mismatch", "clips", "open"]]);
+ await updateStorage(w.projectDir, { deliverables: "media" });
+ assert.deepEqual(kinds({ UMTOOL_MEDIA_DIR: w.mediaRoot }), []);
+ } finally {
+ await w.done();
+ }
+});
