// Where a project's bulk lives: its render scratch (`out/`) on a media root, the // manifest and everything small beside it (release 17). // // lib/paths.mjs names the roots: REPORTS_ROOT holds every project; MEDIA_ROOT // (UMTOOL_MEDIA_DIR) holds the bulk; MEDIA_TIERED says they differ. A tiered // project's `out` is an ABSOLUTE SYMLINK to the same project-relative path // under MEDIA_ROOT, so every reader keeps opening `/out/...` by path // and none of them knows the media drive exists. // // ensureOutDir the first writer's call: makes `out/` -- a link when // tiered, a directory when not -- and refuses loudly on a // dangling link instead of building a new tree beside it // moveDirToMedia an existing directory of the project to MEDIA_ROOT: // copy, mirror, verify, park, link, delete the parked copy // moveDirToLocal the reverse: back to a real directory in the project // // The movers take a NAME, not "out": `umtool storage move-out` moves `out/` // with them, and a project's deliverables (`clips/`, `share-*/`) are moved by // the same two calls behind the deliverables switch (slice U2, at the end of // this file): `storage.deliverables` in video.manifest.json says where they // live ("local", the default, or "media"), deliverableDir makes one where the // switch says, and moveDeliverables moves them all and then sets the switch. // // Modelled on the editor's common/controller/relocateDir.ts, not imported from // it (umtool's scripts are plain .mjs under node, and that is a TypeScript // controller): the source is never touched until the copy verifies; `--delete` // only ever points at the copy under construction; every step past the copy is // dispatched on what is ON DISK, so a run cut at any point is finished by // running it again. // // THE MEDIA ROOT IS NEVER CREATED HERE. A media drive that is not mounted // leaves its mountpoint as an empty directory or no directory at all, and a // recursive mkdir would build the tree on the root filesystem and fill it. So // the root is stat'd first and must already be a directory; everything BELOW // it may be made with a recursive mkdir. // // Every path and fs call carries `turbopackIgnore`: lib/report/onscreen.mjs // imports this module, and the app's routes import that (plans/FACTS.md, "A // path joined from `process.cwd()` …"). import { spawn } from "node:child_process"; import { lstat, mkdir, readFile, readdir, readlink, rename, rm, rmdir, stat, statfs, symlink, unlink } from "node:fs/promises"; import path from "node:path"; import { MEDIA_ROOT, REPORTS_ROOT, inside, mediaMirror } from "../paths.mjs"; /** The free space a move keeps on the volume it copies to, beyond the copy. */ export const MOVE_MARGIN_BYTES = 1024 ** 3; /** The project directories the movers may move. One path segment, never a dot name. */ const assertName = (name) => { if (typeof name !== "string" || !name || name.startsWith(".") || /[\/\\\0]/.test(name)) { throw new Error(`not a project directory name: ${JSON.stringify(name)}`); } }; /** The roots a call works against: the process's, unless a test names its own. */ function rootsOf(opts = {}) { const reportsRoot = path.resolve(/* turbopackIgnore: true */ opts.reportsRoot ?? REPORTS_ROOT); const mediaRoot = path.resolve(/* turbopackIgnore: true */ opts.mediaRoot ?? MEDIA_ROOT); return { reportsRoot, mediaRoot, tiered: mediaRoot !== reportsRoot }; } /** * What a path is RIGHT NOW. lstat, never stat: a dangling link -- the state an * unmounted media drive leaves behind -- reads as absent through stat. * @returns {Promise<{ kind: "missing" } | { kind: "dir" } | { kind: "link", target: string, targetIsDir: boolean } | { kind: "other" }>} */ export async function pathState(p) { let st; try { st = await lstat(/* turbopackIgnore: true */ p); } catch { return { kind: "missing" }; } if (st.isSymbolicLink()) { const raw = await readlink(/* turbopackIgnore: true */ p).catch(() => ""); const target = path.resolve(/* turbopackIgnore: true */ path.dirname(/* turbopackIgnore: true */ p), raw); const targetIsDir = await stat(/* turbopackIgnore: true */ p).then((s) => s.isDirectory(), () => false); return { kind: "link", target, targetIsDir }; } if (st.isDirectory()) return { kind: "dir" }; return { kind: "other" }; } /** * Why render scratch cannot go to the media root now, or null when it can (or * when nothing is tiered). The root must already exist as a directory -- it is * never created -- and must neither sit inside REPORTS_ROOT nor contain it (a * mirror inside the tree it mirrors would be walked as projects). */ export async function mediaRootProblem(opts = {}) { const { reportsRoot, mediaRoot, tiered } = rootsOf(opts); if (!tiered) return null; if (inside(reportsRoot, mediaRoot) || inside(mediaRoot, reportsRoot)) { return `the media root ${mediaRoot} (UMTOOL_MEDIA_DIR) must be outside the reports root ${reportsRoot}, and must not contain it`; } const st = await stat(/* turbopackIgnore: true */ mediaRoot).catch(() => null); if (!st) { return `the media root ${mediaRoot} (UMTOOL_MEDIA_DIR) is not there — is its drive mounted? Nothing was created.`; } if (!st.isDirectory()) return `the media root ${mediaRoot} (UMTOOL_MEDIA_DIR) is not a directory`; return null; } /** * The state of a project's `out`, for a reader that wants to say WHY a build's * files are not there rather than "no build": `dangling` is a link whose target * is gone (the media drive is not mounted). * @returns {Promise<{ state: "absent" | "dir" | "link" | "dangling" | "other", target?: string }>} */ export async function outDirState(projectDir) { const s = await pathState(path.join(/* turbopackIgnore: true */ projectDir, "out")); if (s.kind === "missing") return { state: "absent" }; if (s.kind === "link") return { state: s.targetIsDir ? "link" : "dangling", target: s.target }; return { state: s.kind }; } /** * Make sure `/out` exists, and return its path. * * a directory -> it, as it is (an untiered project, or one not moved yet) * a link to a directory -> it * a DANGLING link -> throws: the media drive is not there, and writing * through the link would fail anyway -- but a * recursive mkdir under it would fail with a * message about some subdirectory, so this says * what is actually wrong. Nothing is created. * absent, tiered -> `//out`, then the link * absent, not tiered -> a directory, as every writer always made it * * A project outside REPORTS_ROOT (a hand-run script's `--out` somewhere else) * is never tiered. */ export async function ensureOutDir(projectDir, opts = {}) { return ensureProjectDir(projectDir, "out", opts, (roots) => roots.tiered); } /** * The one way a project directory that may live on the media root is made: * `out` (tiered whenever UMTOOL_MEDIA_DIR is set) and a deliverable (tiered * when the manifest says so). `tierFor(roots)` is asked only when the directory * is absent, and may throw to refuse. */ async function ensureProjectDir(projectDir, name, opts, tierFor) { const target = path.join(/* turbopackIgnore: true */ projectDir, name); const s = await pathState(target); if (s.kind === "dir") return target; if (s.kind === "link") { if (s.targetIsDir) return target; throw new Error( `${target} is a link to ${s.target}, which is not there — is the media drive mounted? ` + `Nothing was written, and nothing was created in its place.`, ); } if (s.kind === "other") throw new Error(`${target} exists and is not a directory`); // A cut move leaves `.moved-` or `.incoming` beside a missing // ``. Making a fresh, empty one there would let the next move mirror // it over the complete media copy (review L5): refuse, and say how to finish. await assertNoLeftovers(projectDir, name, "nothing was created"); const roots = rootsOf(opts); const tier = await tierFor(roots); const mirror = tier && roots.tiered ? mediaMirror(projectDir, roots) : null; if (!mirror) { await mkdir(/* turbopackIgnore: true */ target, { recursive: true }); return target; } const problem = await mediaRootProblem(roots); if (problem) throw new Error(`cannot make ${target}: ${problem}`); // The project must exist before its mirror is made: otherwise the symlink // fails and leaves an empty mirror on the media root (review N5). // stat, not lstat: a project directory may itself be a link (the walk follows them). if (!(await stat(/* turbopackIgnore: true */ projectDir).then((st) => st.isDirectory(), () => false))) { throw new Error(`cannot make ${target}: ${projectDir} is not a directory`); } const dest = path.join(/* turbopackIgnore: true */ mirror, name); // Recursive is safe here: the root itself was just seen to exist. await mkdir(/* turbopackIgnore: true */ dest, { recursive: true }); try { await symlink(/* turbopackIgnore: true */ dest, target, "dir"); } catch (e) { // Two writers starting at once: whichever linked first won, and a link (or // directory) that now resolves is as good as ours. if (e?.code !== "EEXIST") throw e; const again = await pathState(target); if (again.kind === "dir" || (again.kind === "link" && again.targetIsDir)) return target; throw e; } return target; } /** * Make a directory a pipeline step writes into, and return it. * * When it is a project's `out` or lies under one (`out/`, * `out//chrome/deck-stills`), that `out` is made by ensureOutDir FIRST * -- a link when tiered -- and only then is the rest made under it. A plain * recursive mkdir of `out//segments` was how every script made `out/`, * and it would make a real directory where the link belongs. Any other * directory (a hand-run script's `--out` somewhere else) is made as it always * was. */ export async function ensureWriteDir(dir) { let d = path.resolve(/* turbopackIgnore: true */ dir); for (let i = 0; i < 4; i++) { if (path.basename(/* turbopackIgnore: true */ d) === "out") { await ensureOutDir(path.dirname(/* turbopackIgnore: true */ d)); break; } const up = path.dirname(/* turbopackIgnore: true */ d); if (up === d) break; d = up; } await mkdir(/* turbopackIgnore: true */ dir, { recursive: true }); return dir; } // --------------------------------------------------------------------------- // Measuring // --------------------------------------------------------------------------- /** Files and bytes under a directory. Follows no symlink. */ export async function measureTree(dir) { let bytes = 0; let files = 0; const stack = [dir]; while (stack.length) { const cur = stack.pop(); let entries; try { entries = await readdir(/* turbopackIgnore: true */ cur, { withFileTypes: true }); } catch { continue; } for (const e of entries) { const p = path.join(/* turbopackIgnore: true */ cur, e.name); if (e.isDirectory()) stack.push(p); else if (e.isFile()) { const st = await stat(/* turbopackIgnore: true */ p).catch(() => null); if (st) { bytes += st.size; files += 1; } } } } return { bytes, files }; } async function freeBytes(dir) { const s = await statfs(/* turbopackIgnore: true */ dir).catch(() => null); return s ? Number(s.bavail) * Number(s.bsize) : null; } async function assertRoom(dir, bytes) { const free = await freeBytes(dir); if (free !== null && free < bytes + MOVE_MARGIN_BYTES) { throw new Error( `not enough room on ${dir}: ${gb(free)} free, the copy needs ${gb(bytes)} plus ${gb(MOVE_MARGIN_BYTES)} to spare. Nothing moved.`, ); } } const gb = (n) => `${(n / 1024 ** 3).toFixed(2)} GB`; // --------------------------------------------------------------------------- // rsync // --------------------------------------------------------------------------- /** `rsync / /` -- the trailing slashes copy the CONTENTS. */ function rsync(bin, args, src, dest, log) { return new Promise((resolve, reject) => { const argv = [...args, `${src}/`, `${dest}/`]; log(`$ ${bin} ${argv.join(" ")}`); const child = spawn(bin, argv, { stdio: ["ignore", "pipe", "pipe"] }); let output = ""; child.stdout.on("data", (c) => (output += c)); child.stderr.on("data", (c) => (output += c)); child.on("error", reject); child.on("close", (code) => resolve({ code: code ?? 1, output })); }); } // What a dry run itemizes that is not a difference: rsync's own chatter. const driftLines = (output) => output .split("\n") .map((l) => l.trim()) .filter((l) => l && !l.startsWith("sending incremental") && !/^(sent|total size)/.test(l)); /** * COPY, MIRROR, VERIFY -- the source is not touched by any of it. * * 1. `rsync -a --partial`: the bytes, resumable. * 2. `rsync -a --delete --info=del` toward the COPY: what changed on the * source meanwhile is re-sent, what it no longer has leaves the copy * (each removal in the log). Never pointed the other way: the copy is * checked to be neither the source nor inside it, nor around it. * 3. `rsync -a --dry-run --itemize-changes --delete` must list nothing, and * the two trees must measure the same. One more mirror pass if the source * moved under the check; a second difference refuses. */ async function copyMirrorVerify(src, dest, { rsyncBin, log }) { if (inside(src, dest) || inside(dest, src)) { throw new Error(`refusing to copy ${src} into ${dest}: one is inside the other. Nothing moved.`); } const copied = await rsync(rsyncBin, ["-a", "--partial"], src, dest, log); if (copied.code !== 0) throw new Error(`rsync failed (exit ${copied.code}): ${copied.output.trim().split("\n").pop() ?? ""}`); for (let pass = 0; ; pass++) { const mirrored = await rsync(rsyncBin, ["-a", "--delete", "--info=del"], src, dest, log); if (mirrored.output.trim()) log(mirrored.output.trim()); if (mirrored.code !== 0) throw new Error(`the mirror pass failed (exit ${mirrored.code}). The source has NOT been touched.`); const check = await rsync(rsyncBin, ["-a", "--dry-run", "--itemize-changes", "--delete"], src, dest, () => {}); if (check.code !== 0) throw new Error(`the verify failed (exit ${check.code}). The source has NOT been touched.`); const drift = driftLines(check.output); if (!drift.length) break; if (pass >= 1) { throw new Error( `the copy still differs from the source after a second mirror pass (${drift.length} item(s), first: ${drift[0]}) — ` + `something is still writing into ${src}. Stop it and run the move again. The source has NOT been touched.`, ); } log(`the source changed during the copy (${drift.length} item(s)) — one more mirror pass`); } const [a, b] = await Promise.all([measureTree(src), measureTree(dest)]); if (a.files !== b.files || a.bytes !== b.bytes) { throw new Error( `the copy does not measure the same: ${a.files} file(s)/${a.bytes} B against ${b.files}/${b.bytes} B. The source has NOT been touched.`, ); } return a; } // --------------------------------------------------------------------------- // The movers // --------------------------------------------------------------------------- const stamp = () => new Date().toISOString().replace(/[-:]/g, "").replace(/\.\d+Z$/, "Z"); /** `.moved-` siblings: a move-out parked them and was cut before deleting. */ async function parkedOf(projectDir, name) { const names = await readdir(/* turbopackIgnore: true */ projectDir).catch(() => []); return names .filter((n) => n.startsWith(`${name}.moved-`)) .sort() .map((n) => path.join(/* turbopackIgnore: true */ projectDir, n)); } /** `.incoming`, when a move-back left it (a cut between its copy and its rename). */ async function incomingOf(projectDir, name) { const p = path.join(/* turbopackIgnore: true */ projectDir, `${name}.incoming`); return (await pathState(p)).kind === "dir" ? p : null; } /** Every leftover of a cut move of ``: parked copies and an incoming copy. */ export async function leftoversOf(projectDir, name) { const incoming = await incomingOf(projectDir, name); return [...(await parkedOf(projectDir, name)), ...(incoming ? [incoming] : [])]; } /** * Refuse while a cut move of `` has left something behind. A writer that * made a fresh `/` there, and a move that then mirrored it over the * complete copy, would lose everything but the leftover nobody names. */ async function assertNoLeftovers(projectDir, name, what, { beside = false } = {}) { const left = await leftoversOf(projectDir, name); if (!left.length) return; const names = left.map((p) => path.basename(/* turbopackIgnore: true */ p)).join(", "); if (beside) { // A real `/` AND a leftover: running a move again would only refuse // again (re-review R1). Only a person can say which one to keep. throw new Error( `${name}/ and ${names} both exist in ${projectDir}: a move of ${name}/ was cut, and something made a fresh ${name}/ since. ` + `The leftover holds the moved data. Keep one and remove the other by hand, then run the move; ${what}`, ); } throw new Error( `a move of ${name}/ in ${projectDir} was cut (left: ${names}) — ` + `run \`${finishCommand(name, left.some((p) => p.endsWith(".incoming")))}\` to finish it; ${what}`, ); } /** The command that finishes a cut move of ``: `out` has its own pair, a deliverable the switch. */ export const finishCommand = (name, back) => name === "out" ? `umtool storage ${back ? "move-back" : "move-out"} ` : `umtool storage deliverables --to ${back ? "local" : "media"}`; const defaults = (opts) => ({ rsyncBin: opts.rsyncBin ?? process.env.RSYNC_BIN ?? "rsync", log: opts.log ?? (() => {}), dryRun: !!opts.dryRun, }); /** * Move `/` to the same project-relative path under the media * root and leave an absolute link in its place. * * Dispatched on the disk, so running it again finishes a run that was cut: * * a link to the mirror -> "already" (and a parked copy left by a cut * between the link and its deletion is removed) * a link anywhere else -> refused; it is not this media root's * absent, a parked copy -> the cut was between the park and the link * (the copy had verified): link, delete the parked copy * absent, nothing parked -> "absent", nothing to move * a directory -> copy, mirror, verify, rename it to * `.moved-`, link, delete the parked copy * a directory + a leftover -> refused: a parked copy or `.incoming` * beside a real directory means a writer made a * fresh one after a cut move (review L5) * absent + `.incoming` -> refused: a cut move-back is finished by move-back * * `dryRun` measures and changes nothing. Nothing in the project may be writing * into `` while it runs (the app's jobs are in its own memory, so a CLI * caller cannot see them); the verify refuses if the tree keeps changing. * * @param {string} projectDir * @param {string} name one path segment: "out", "clips", "share-" * @param {{ dryRun?: boolean, log?: (m: string) => void, rsyncBin?: string, reportsRoot?: string, mediaRoot?: string }} [opts] * @returns {Promise<{ state: "moved" | "already" | "absent" | "would-move" | "would-finish", src: string, dest: string, bytes?: number, files?: number, resumed?: boolean }>} */ export async function moveDirToMedia(projectDir, name, opts = {}) { assertName(name); const { rsyncBin, log, dryRun } = defaults(opts); const roots = rootsOf(opts); if (!roots.tiered) { throw new Error(`UMTOOL_MEDIA_DIR is not set: there is no media root to move ${name}/ to`); } const mirror = mediaMirror(projectDir, roots); if (!mirror) throw new Error(`${projectDir} is not under the reports root ${roots.reportsRoot}`); const src = path.join(/* turbopackIgnore: true */ projectDir, name); const dest = path.join(/* turbopackIgnore: true */ mirror, name); const s = await pathState(src); if (s.kind === "link") { if (s.target !== dest) { throw new Error(`${src} is already a link, to ${s.target} — not to ${dest}. Nothing moved.`); } const parked = await parkedOf(projectDir, name); if (!dryRun && s.targetIsDir) { for (const p of parked) await rm(/* turbopackIgnore: true */ p, { recursive: true, force: true }); } return { state: "already", src, dest }; } if (s.kind === "other") throw new Error(`${src} is not a directory. Nothing moved.`); if (s.kind === "missing") { const parked = await parkedOf(projectDir, name); if (!parked.length) { // A cut move-BACK is finished by move-back, never overtaken by a move-out. if (await incomingOf(projectDir, name)) await assertNoLeftovers(projectDir, name, "nothing moved"); return { state: "absent", src, dest }; } if (parked.length > 1) { throw new Error(`${src} is missing and there are ${parked.length} parked copies (${parked.join(", ")}) — settle them by hand`); } // A parked copy exists only after the copy verified, so the media side is // complete -- provided it is reachable. if ((await pathState(dest)).kind !== "dir") { throw new Error(`${src} was parked at ${parked[0]}, but ${dest} is not there — is the media drive mounted? Nothing changed.`); } if (dryRun) return { state: "would-finish", src, dest }; log(`finishing a cut move: linking ${src} -> ${dest}`); await symlink(/* turbopackIgnore: true */ dest, src, "dir"); await rm(/* turbopackIgnore: true */ parked[0], { recursive: true, force: true }); return { state: "moved", src, dest, resumed: true }; } // A real directory: the move itself -- unless a cut move left something // behind, in which case this directory is a writer's fresh one. await assertNoLeftovers(projectDir, name, "nothing moved", { beside: true }); const problem = await mediaRootProblem(roots); if (problem) throw new Error(`cannot move ${src}: ${problem}`); const measured = await measureTree(src); if (dryRun) return { state: "would-move", src, dest, ...measured }; await assertRoom(roots.mediaRoot, measured.bytes); await mkdir(/* turbopackIgnore: true */ dest, { recursive: true }); const verified = await copyMirrorVerify(src, dest, { rsyncBin, log }); const parked = path.join(/* turbopackIgnore: true */ projectDir, `${name}.moved-${stamp()}`); await rename(/* turbopackIgnore: true */ src, parked); await symlink(/* turbopackIgnore: true */ dest, src, "dir"); await rm(/* turbopackIgnore: true */ parked, { recursive: true, force: true }); log(`moved ${src} -> ${dest} (${verified.files} file(s), ${gb(verified.bytes)})`); return { state: "moved", src, dest, ...verified }; } /** * The reverse: `/`, a link into the media root, becomes a * real directory in the project again, and the media copy is deleted. * * a directory -> "already" * absent, `.incoming` -> the cut was between removing the link and * renaming the verified copy: rename it, and * report (never delete) the project's mirror * absent, nothing incoming -> "absent" * a link whose target is gone -> refused: the drive is not mounted * a link -> copy the target into `.incoming`, * mirror, verify, remove the link, rename, * then delete the copy only when it is this * project's own mirror under a tiered media * root (and the directories above it the move * left empty, never the root); any other * target is left and reported (mediaCopyLeft) * a parked `.moved-*` -> refused: a cut move-out is finished first * a directory + a leftover -> refused (a writer's fresh directory) * * @param {string} projectDir * @param {string} name * @param {{ dryRun?: boolean, log?: (m: string) => void, rsyncBin?: string, reportsRoot?: string, mediaRoot?: string }} [opts] * @returns {Promise<{ state: "moved" | "already" | "absent" | "would-move" | "would-finish", src: string, from?: string, bytes?: number, files?: number, resumed?: boolean }>} */ export async function moveDirToLocal(projectDir, name, opts = {}) { assertName(name); const { rsyncBin, log, dryRun } = defaults(opts); const roots = rootsOf(opts); const src = path.join(/* turbopackIgnore: true */ projectDir, name); const incoming = path.join(/* turbopackIgnore: true */ projectDir, `${name}.incoming`); const s = await pathState(src); const mirror = roots.tiered ? mediaMirror(projectDir, roots) : null; const ownCopy = mirror ? path.join(/* turbopackIgnore: true */ mirror, name) : null; if (s.kind === "dir") { await assertNoLeftovers(projectDir, name, "nothing moved", { beside: true }); // A move-back cut after its rename leaves the media copy behind (review // L4): report it, never delete it blind. const left = ownCopy && (await pathState(ownCopy)).kind === "dir" ? ownCopy : undefined; return { state: "already", src, ...(left ? { mediaCopyLeft: left } : {}) }; } if (s.kind === "other") throw new Error(`${src} is not a directory. Nothing moved.`); // A cut move-OUT is finished by move-out first. if ((await parkedOf(projectDir, name)).length) await assertNoLeftovers(projectDir, name, "nothing moved"); if (s.kind === "missing") { if ((await pathState(incoming)).kind !== "dir") return { state: "absent", src }; if (dryRun) return { state: "would-finish", src }; // `.incoming` outlives the link only once it verified. log(`finishing a cut move: ${incoming} -> ${src}`); await rename(/* turbopackIgnore: true */ incoming, src); // The link is gone, so what was copied cannot be told from the disk: the // project's own mirror is reported, never deleted (review L3). const left = ownCopy && (await pathState(ownCopy)).kind === "dir" ? ownCopy : undefined; return { state: "moved", src, resumed: true, ...(left ? { mediaCopyLeft: left } : {}) }; } // A link. const from = s.target; if (!s.targetIsDir) { throw new Error(`${src} is a link to ${from}, which is not there — is the media drive mounted? Nothing moved.`); } const measured = await measureTree(from); if (dryRun) return { state: "would-move", src, from, ...measured }; await assertRoom(projectDir, measured.bytes); await mkdir(/* turbopackIgnore: true */ incoming, { recursive: true }); const verified = await copyMirrorVerify(from, incoming, { rsyncBin, log }); await unlink(/* turbopackIgnore: true */ src); // the link, not what it points at await rename(/* turbopackIgnore: true */ incoming, src); const left = await dropMediaCopy(from, ownCopy, roots.mediaRoot); if (left) log(`left ${left} in place: it is not this project's own copy on the media root`); log(`moved ${from} -> ${src} (${verified.files} file(s), ${gb(verified.bytes)})`); return { state: "moved", src, from, ...verified, ...(left ? { mediaCopyLeft: left } : {}) }; } /** * Delete a media copy that has been brought home, then every directory above * it the move left empty, stopping at -- never removing -- the media root. * * ONLY this project's own mirror (`ownCopy`, null when nothing is tiered), and * only when the copy that came home IS that mirror (review L3). Without * UMTOOL_MEDIA_DIR the "media root" would be the reports root itself, and a * hand-made `out` link into another project's real out/ would be deleted after * the copy. Anything else is left where it is, and its path returned so the * caller can say so. * @returns {Promise} the path left in place, if any */ async function dropMediaCopy(from, ownCopy, mediaRoot) { if (!ownCopy || from !== ownCopy || !inside(mediaRoot, from) || from === mediaRoot) return from; if ((await pathState(from)).kind !== "dir") return undefined; await rm(/* turbopackIgnore: true */ from, { recursive: true, force: true }); for (let d = path.dirname(/* turbopackIgnore: true */ from); d !== mediaRoot && inside(mediaRoot, d); d = path.dirname(/* turbopackIgnore: true */ d)) { try { await rmdir(/* turbopackIgnore: true */ d); } catch { break; // not empty: something else lives there } } return undefined; } // --------------------------------------------------------------------------- // Deliverables: `clips/` and every `share-*/` (release 17, slice U2) // // Unlike out/, they do not follow UMTOOL_MEDIA_DIR on their own: they move per // project, by a switch -- `"storage": { "deliverables": "local" | "media" }` in // video.manifest.json (absent = local), written only through // lib/report/manifest.mjs's updateStorage. moveDeliverables moves what exists // with the two movers above and then sets the switch; deliverableDir is how a // writer (a cut, a share batch) makes a deliverable directory that does not // exist yet, where the switch says. A reader keeps opening // `/clips/.mp4` by path: through the link when it is one. // --------------------------------------------------------------------------- /** The two values of `storage.deliverables`. Absent is "local". */ export const DELIVERABLES_MODES = ["local", "media"]; /** The directory a project's cut clips live in (lib/report/cut.mjs's CLIPS_DIR). */ const CLIPS = "clips"; /** A share batch's directory prefix (lib/report/deliver.mjs's SHARE_PREFIX). */ const SHARE = "share-"; /** What a cut move leaves beside a deliverable: `.moved-` or `.incoming`. */ const LEFTOVER = /^(.+)\.(moved-[^/]*|incoming)$/; /** Is `name` a deliverable directory's name (and not a cut move's leftover)? */ export const isDeliverableName = (name) => typeof name === "string" && !LEFTOVER.test(name) && (name === CLIPS || (name.startsWith(SHARE) && name.length > SHARE.length && !/[\/\\\0]/.test(name))); /** * `storage.deliverables` of a parsed manifest. * @returns {{ mode: "local" | "media" | null, value: unknown, error?: string }} * `mode` null when the value is not one of the two (the error says so). */ export function deliverablesModeOf(manifest) { const v = manifest?.storage?.deliverables; if (v === undefined) return { mode: "local", value: undefined }; if (DELIVERABLES_MODES.includes(v)) return { mode: v, value: v }; return { mode: null, value: v, error: `storage.deliverables is ${JSON.stringify(v)} — it is "local" or "media"` }; } /** * `storage.deliverables`, read off the project's manifest. Read here (a plain * JSON read) rather than through lib/projects/report.mjs, which this module's * importers must not pull in; written only by manifest.mjs. */ export async function deliverablesMode(projectDir) { const file = path.join(/* turbopackIgnore: true */ projectDir, "video.manifest.json"); const text = await readFile(/* turbopackIgnore: true */ file, "utf8").catch(() => null); if (text === null) return { mode: "local", value: undefined, manifest: false }; try { return { ...deliverablesModeOf(JSON.parse(text)), manifest: true }; } catch { return { mode: null, value: undefined, manifest: true, error: `${file} is not valid JSON` }; } } /** * Make sure the deliverable directory `/` exists, and return * its path. What a cut (`clips`) and a share batch (`share-`) call before * they write. * * a directory, or a link to one -> it, whatever the switch says (a move is * what changes where an existing one lives) * a DANGLING link -> refused, as ensureOutDir refuses: nothing * is created in its place * absent, the switch "local" -> a directory in the project * absent, the switch "media" -> `//` * and a link to it -- or refused when there * is no media root here (UMTOOL_MEDIA_DIR * unset in this process), it is not there, * or the project is outside the reports * root. Never silently local: a batch made * on the wrong drive is a split nobody chose. * a cut move's leftover beside it -> refused, naming the command that finishes it * * @param {string} projectDir * @param {string} name "clips" or "share-" * @param {{ mode?: "local" | "media", reportsRoot?: string, mediaRoot?: string }} [opts] * `mode` overrides the manifest's switch (tests). */ export async function deliverableDir(projectDir, name, opts = {}) { if (!isDeliverableName(name)) throw new Error(`not a deliverable directory: ${JSON.stringify(name)}`); return ensureProjectDir(projectDir, name, opts, async (roots) => { const m = opts.mode ? { mode: opts.mode } : await deliverablesMode(projectDir); if (!m.mode) throw new Error(`cannot make ${path.join(/* turbopackIgnore: true */ projectDir, name)}: ${m.error}`); if (m.mode === "local") return false; const target = path.join(/* turbopackIgnore: true */ projectDir, name); if (!roots.tiered) { throw new Error( `cannot make ${target}: this project keeps its deliverables on the media root (storage.deliverables: media), ` + `and UMTOOL_MEDIA_DIR is not set here — set it, or bring them back with ` + `\`umtool storage deliverables --to local\`. Nothing was created.`, ); } if (!mediaMirror(projectDir, roots)) { throw new Error(`cannot make ${target}: ${projectDir} is not under the reports root ${roots.reportsRoot}, so it has no place on the media root`); } return true; }); } /** * Every deliverable a project has, by name: `clips` and each `share-*` that is * a directory or a link -- and the name of any whose cut move left only a * leftover (`clips.moved-` with no `clips`), so a move finishes it. * `clips` first, then the batches by name. */ export async function deliverableNames(projectDir) { const entries = await readdir(/* turbopackIgnore: true */ projectDir, { withFileTypes: true }).catch(() => []); const names = new Set(); for (const e of entries) { const left = LEFTOVER.exec(e.name); if (left && e.isDirectory() && isDeliverableName(left[1])) names.add(left[1]); else if (isDeliverableName(e.name) && (e.isDirectory() || e.isSymbolicLink())) names.add(e.name); } return [...names].sort((a, b) => (a === CLIPS ? -1 : b === CLIPS ? 1 : a.localeCompare(b))); } /** * Where a project's deliverables are, for the bench, `umtool storage` and * `umtool check`. One lstat per name, and one stat through each link (which is * what a reader does anyway). * * @returns {Promise<{ mode: "local" | "media" | null, value: unknown, error?: string, * tiered: boolean, mediaRoot: string | null, * dirs: Array<{ name: string, state: "absent" | "dir" | "link" | "dangling" | "other", target?: string, leftovers: string[] }> }>} */ export async function deliverablesState(projectDir, opts = {}) { const roots = rootsOf(opts); const m = await deliverablesMode(projectDir); const dirs = []; for (const name of await deliverableNames(projectDir)) { const s = await pathState(path.join(/* turbopackIgnore: true */ projectDir, name)); const state = s.kind === "missing" ? "absent" : s.kind === "link" ? (s.targetIsDir ? "link" : "dangling") : s.kind; const leftovers = (await leftoversOf(projectDir, name)).map((p) => path.basename(/* turbopackIgnore: true */ p)); dirs.push({ name, state, ...(s.kind === "link" ? { target: s.target } : {}), leftovers }); } return { mode: m.mode, value: m.value, ...(m.error ? { error: m.error } : {}), tiered: roots.tiered, mediaRoot: roots.tiered ? roots.mediaRoot : null, dirs, }; } /** * What stops a deliverable being WRITTEN now, as sentences, from a * deliverablesState: a link whose drive is not there, a cut move's leftovers, * and -- for a directory that does not exist yet -- a switch this process * cannot honour. `name` narrows it to one directory (a cut asks for `clips`, * a batch for its own `share-`); every dangling link and leftover counts * whatever the name, because a batch reads `clips/` and every earlier batch. * * @param {{ mode: string | null, error?: string, tiered: boolean, * dirs: Array<{ name: string, state: string, target?: string, leftovers: string[] }> }} state * @param {string | null} [name] * @returns {string[]} */ export function deliverablesProblems(state, name = null) { const out = []; for (const d of state.dirs) { if (d.state === "dangling") { out.push(`${d.name}/ is a link to ${d.target}, which is not there — is the media drive mounted?`); } if (d.leftovers.length) { out.push( `a move of ${d.name}/ was cut (left: ${d.leftovers.join(", ")}) — ` + `\`${finishCommand(d.name, d.leftovers.some((l) => l.endsWith(".incoming")))}\` finishes it`, ); } } const exists = name && state.dirs.some((d) => d.name === name && d.state !== "absent"); if (name && !exists) { if (!state.mode) out.push(state.error ?? "storage.deliverables is not \"local\" or \"media\""); else if (state.mode === "media" && !state.tiered) { out.push("this project keeps its deliverables on the media root (storage.deliverables: media), and UMTOOL_MEDIA_DIR is not set in umtool's environment"); } } return out; } /** * The switch: move every deliverable to `to` ("media" or "local") with the * movers, then set `storage.deliverables` -- only when every one of them is * where it should be. Idempotent: a second run finds each "already" there and * writes nothing (the manifest is not rewritten when the switch already says * `to`). A run that was cut is finished by running it again, as the movers * finish theirs. * * Nothing may be cutting or sharing into the project while it runs. The * callers check (the app's one-job-at-a-time registry, the CLI's process * scan); the movers' verify refuses a tree that keeps changing. * * @param {string} projectDir * @param {"local" | "media"} to * @param {{ writeMode: (dir: string, mode: "local" | "media") => Promise, * dryRun?: boolean, log?: (m: string) => void, rsyncBin?: string, reportsRoot?: string, mediaRoot?: string }} opts * `writeMode` is manifest.mjs's updateStorage -- passed in, so this module * (which the app's routes import) does not pull the manifest writer's imports. */ export async function moveDeliverables(projectDir, to, opts) { if (!DELIVERABLES_MODES.includes(to)) throw new Error(`--to is "media" or "local", not ${JSON.stringify(to)}`); const { dryRun } = defaults(opts); const roots = rootsOf(opts); const m = await deliverablesMode(projectDir); if (!m.manifest) throw new Error(`${projectDir} has no video.manifest.json — deliverables are a report video's`); if (to === "media" && !roots.tiered) { throw new Error("UMTOOL_MEDIA_DIR is not set: there is no media root to move deliverables to. Nothing moved."); } const move = to === "media" ? moveDirToMedia : moveDirToLocal; const results = []; for (const name of await deliverableNames(projectDir)) { try { results.push({ name, ...(await move(projectDir, name, opts)) }); } catch (e) { results.push({ name, state: "failed", error: e instanceof Error ? e.message : String(e) }); } } const failed = results.filter((r) => r.state === "failed").length; const before = m.value ?? null; let written = false; if (!failed && !dryRun && m.value !== to) { await opts.writeMode(projectDir, to); written = true; } return { ok: failed === 0, to, before, written, dryRun, results }; }