// Where things live, in plain ESM so the CLI and the app share one definition. // // This is the same trick song/paths.mjs plays: the values that decide which // directory is read and which is written must not be able to differ between // `umtool ls` and the page it is supposed to describe. lib/paths.ts re-exports // everything here with types; nothing computes a root twice. import { existsSync, realpathSync } from "node:fs"; import { lstat, realpath } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { SONG_DATA, SONG_REPORTS } from "../song/paths.mjs"; // SONG_REPORTS -- the um-song deliverables tree, and BROWSE_ROOT's parent -- is // defined in song/paths.mjs beside SONG_DATA, because the song scripts that // record paths relative to it (make-thumb, accept-thumb) import only siblings. export { SONG_DATA, SONG_REPORTS }; // Derived output (sliced mp3s, waveform peaks, the project index, posters, the // mix bench's analyses). Not in the repo, and safe to delete at any time. // // It used to live under SONG_DATA (`/.cache/umtool`), which tied // every project's index and every report's derived files to wherever the song // project's 39 GB happened to sit -- a report-only machine, or one whose song // data is on a drive that is not mounted, had its cache follow it there // (release 17). It is a cache, so it goes where caches go: // `UMTOOL_CACHE_DIR`, else `$XDG_CACHE_HOME/archilyzer/umtool` (an empty // XDG_CACHE_HOME is unset, as common/lib/paths.ts reads it), else // `~/.cache/archilyzer/umtool`. Nothing is migrated: the first `umtool index` // rebuilds the index there, and every other file is re-made on demand. // OLD_CACHE_DIR is only for `umtool doctor`, which reports a leftover one. const XDG_CACHE = process.env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache"); export const CACHE_DIR = path.resolve( /* turbopackIgnore: true */ process.env.UMTOOL_CACHE_DIR || path.join(/* turbopackIgnore: true */ XDG_CACHE, "archilyzer", "umtool"), ); export const OLD_CACHE_DIR = path.join(/* turbopackIgnore: true */ SONG_DATA, ".cache", "umtool"); /** * A file in CACHE_DIR, by name. A route names its cache files through this * rather than joining CACHE_DIR itself. Turbopack reads a path it can see as a * pattern of files to trace, in the join and in every fs call its value * reaches, and CACHE_DIR is unknown to it (an env var or the home directory). * So `path.join(CACHE_DIR, `${stamp}.${asMp3 ? "mp3" : "wav"}`)` in the * clip-audio route was a pattern that reached into umtool's dot-directories: * that route's trace listed the e2e fixture, the e2e server's build directory * and `.env.local` (plans/release-15.md, slice UT). A value returned by a * function from another module is opaque to it, so a call site traces nothing. */ export function cacheFile(name) { return path.join(/* turbopackIgnore: true */ CACHE_DIR, name); } // Render scratch: the body render and the cut background sit in the job temp // dir ABOVE SONG_DATA, not inside it, because render-poly.mjs writes them next // to its logs. export const SONG_SCRATCH = path.dirname(SONG_DATA); // --------------------------------------------------------------------------- // REPORTS_ROOT -- the tree every PROJECT hangs off. // // ~/reports holds an um-song project tree AND six report videos AND a report // with no video yet. Before this, only the um-song subtree was reachable: six // of seven projects resolved to null in the mix bench, because SONG_REPORTS was // the widest root there was. // // The e2e default is dirname(SONG_REPORTS_DIR) rather than a new env var, so a // fixture that sets SONG_REPORTS_DIR=/reports gets REPORTS_ROOT= // for free and stays confined. The walk skips `data`, which is where // that fixture symlinks 39 GB of audio. // --------------------------------------------------------------------------- export const REPORTS_ROOT = path.resolve( process.env.REPORTS_DIR ?? (process.env.SONG_REPORTS_DIR ? path.dirname(SONG_REPORTS) : path.join(os.homedir(), "reports")), ); const dedupe = (list) => [...new Set(list.map((p) => path.resolve(p)))]; // --------------------------------------------------------------------------- // MEDIA_ROOT -- where a project's RENDER SCRATCH (`out/`) lives (release 17). // // A report project's manifest, revisions/, notes and sources are small text and // stay under REPORTS_ROOT. Its `out/` -- fetched windows, segments, the // deliverable, ~18 of the 20 GB in ~/reports -- is bulk that can be re-made, // and belongs on a media drive. With UMTOOL_MEDIA_DIR set, a project's `out` is // an absolute SYMLINK to the same project-relative path under it: // // ///out -> ///out // // made by the first writer (lib/report/storage.mjs ensureOutDir) or moved there // by `umtool storage move-out`. Every reader keeps opening `/out/...` // by path; the link is the only place the media drive is named. // // UNSET, MEDIA_ROOT is REPORTS_ROOT, MEDIA_TIERED is false, and nothing changes: // `out/` is a plain directory in the project, as it always was. // // MEDIA_ROOT is READABLE -- so a client may name a media file by its real path // (one taken through a project's `out` link lands under it) to the mix bench's // /api/mix/{media,track} -- and never WRITABLE by a client-named path: what a // render may write to is still WRITE_ROOTS, judged lexically, so a write to // `/out/...` is judged by the project's place, never the link's target. // --------------------------------------------------------------------------- export const MEDIA_ROOT = path.resolve( /* turbopackIgnore: true */ process.env.UMTOOL_MEDIA_DIR || REPORTS_ROOT, ); /** True when render scratch goes to a media root of its own. */ export const MEDIA_TIERED = MEDIA_ROOT !== REPORTS_ROOT; /** * Where `abs` (a path under REPORTS_ROOT) is mirrored under MEDIA_ROOT, or null * when it is not under REPORTS_ROOT -- a project somewhere else is never tiered. * Pure: it never touches the disk. The roots are parameters so a test can name * its own; the defaults are the process's. */ export function mediaMirror(abs, { reportsRoot = REPORTS_ROOT, mediaRoot = MEDIA_ROOT } = {}) { const rel = path.relative( /* turbopackIgnore: true */ path.resolve(/* turbopackIgnore: true */ reportsRoot), path.resolve(/* turbopackIgnore: true */ abs), ); if (rel === "" || rel.startsWith("..") || path.isAbsolute(rel)) return null; return path.join(/* turbopackIgnore: true */ path.resolve(/* turbopackIgnore: true */ mediaRoot), rel); } // --------------------------------------------------------------------------- // READ vs WRITE, and why they are two lists. // // resolveInRoots() guards both what may be OPENED and what may be RENDERED TO. // Those were the same list, which meant widening the read root to reach report // videos would in the same stroke have made every report's out/ a legal render // target -- a 46 MB deliverable that cost an hour of fetches, one typo from // being overwritten by /api/mix/render. // // So: reports become readable and mixable, and NOTHING new becomes writable. // A mix of a report clip still lands in SONG_REPORTS, and a hand-typed path // outside the write set is refused exactly as it was before. // // SONG_REPORTS stays FIRST. It is a subdirectory of REPORTS_ROOT, so whichever // comes first decides every relative label -- and putting REPORTS_ROOT first // would silently rewrite every existing `videos//wide.mp4` label into // `quartering-uh-song/videos//wide.mp4`. labelFor and resolveInRoots read // the same ordered list, which is what keeps a label a round-trip. // --------------------------------------------------------------------------- // --------------------------------------------------------------------------- // The CORPUS is readable, and only readable. // // The editor fetches a clip window into channels//data//clips/, which // is the point: one managed, provenanced fetch that every tool reuses. But // /api/report/raw resolves the file it serves through resolveInRoots, so // without the corpus in READ_ROOTS a window that is right there 400s as // "outside the roots" and the bench plays nothing. // // It does NOT join WRITE_ROOTS, and that is the whole reason the two lists // exist: the corpus is real production data, and nothing here should be able to // render over it. // // Same env var the cue reader already uses (lib/projects/report.mjs // GLOBAL_CHANNELS_DIR, which now returns this one), so a fixture that confines // one confines both. // // With neither CHANNELS_DIR nor TRANSCRIPTS_DIR set, it is the checkout's own // transcripts/channels -- the corpus a plain checkout would have. It used to be // an absolute path in one machine's home directory, which every other clone // silently looked for and never found. // --------------------------------------------------------------------------- /** * The checkout this runs in: the nearest directory at or above `start` holding * pnpm-workspace.yaml, or `start`'s parent when there is none. * * Walked from the CWD, not from import.meta.url: the Next app imports this * module (eleven API routes), and Turbopack rewrites module URLs into * .next/server/chunks, so a path derived from one points at the build output * -- the SONG_CODE rule in lib/paths.ts. `next dev`, `next start` and the CLI * all run with cwd inside the checkout (the app at umtool/, which is what the * fallback assumes). report-to-video/cues.mjs keeps its import.meta.url walk: * only its CLI entry points (build-video, resolve-windows) read that default. * * EVERY PATH OP ON A cwd-DERIVED VALUE CARRIES `turbopackIgnore`. Turbopack * evaluates `process.cwd()` statically as the project, so an un-annotated * `path.join(REPO_ROOT, "transcripts", "channels")` became a DIRECTORY ASSET * REFERENCE: `next build` walked the whole corpus (hundreds of GB, `data/` * symlinked to another drive) and was OOM-killed, or died on the first symlink * out of the root. A worktree with no transcripts/ builds fine, which is how it * shipped. The comment is Turbopack's per-expression opt-out (the form its own * "whole project was traced" warning advises; the Next docs list the comment * for import(), require(), require.resolve() and new Worker() only); the values * at run time are unchanged. scripts/next-build-trace.test.mjs holds the line. */ export function findRepoRoot(start) { let dir = path.resolve(/* turbopackIgnore: true */ start); for (;;) { if (existsSync(path.join(/* turbopackIgnore: true */ dir, "pnpm-workspace.yaml"))) return dir; const up = path.dirname(/* turbopackIgnore: true */ dir); if (up === dir) return path.resolve(/* turbopackIgnore: true */ start, ".."); dir = up; } } /** * The checkout a umtool CLI belongs to -- `/umtool/bin/.mjs` as the * entry script (`process.argv[1]`) -- or null for anything else (the Next * server, a test runner, report-to-video's own CLIs). * * A CLI is run from wherever its user stands: `node ~/…/umtool/bin/umtool.mjs * notes --all` from a report workspace under REPORTS_DIR has no * pnpm-workspace.yaml above its cwd, so the cwd walk fell back to the cwd's * parent and SITES_DIR (and CHANNELS_DIR) pointed at nothing. The script's own * path names the checkout it is from. Only the bin/ entries: the server keeps * the cwd walk, for the reason findRepoRoot gives. * * @param {string | undefined} entry * @returns {string | null} */ export function cliRepoRoot(entry) { if (!entry || !/[\\/]umtool[\\/]bin[\\/][^\\/]+\.mjs$/.test(entry)) return null; let real; try { real = realpathSync(/* turbopackIgnore: true */ entry); } catch { return null; } // /umtool/bin/x.mjs -> , when is a checkout. const repo = path.resolve(/* turbopackIgnore: true */ path.dirname(/* turbopackIgnore: true */ real), "..", ".."); return existsSync(path.join(/* turbopackIgnore: true */ repo, "pnpm-workspace.yaml")) ? repo : null; } export const REPO_ROOT = cliRepoRoot(process.argv[1]) ?? findRepoRoot(process.cwd()); export const CHANNELS_DIR = path.resolve( /* turbopackIgnore: true */ process.env.CHANNELS_DIR ?? (process.env.TRANSCRIPTS_DIR ? path.join(/* turbopackIgnore: true */ process.env.TRANSCRIPTS_DIR, "channels") : path.join(/* turbopackIgnore: true */ REPO_ROOT, "transcripts", "channels")), ); // The SITES -- `transcripts/sites//`, each a site.json and its reports // (`reports//report.json`, `video.mp4`, `poster.jpg`). Resolved the way // CHANNELS_DIR is, and the way common/lib/paths.ts resolves its sitesDir, so // one `SITES_DIR` confines both. Readable only; the ONE file in it umtool may // write is a report's notes.json, and only through isCorpusNotesFile below. export const SITES_DIR = path.resolve( /* turbopackIgnore: true */ process.env.SITES_DIR ?? (process.env.TRANSCRIPTS_DIR ? path.join(/* turbopackIgnore: true */ process.env.TRANSCRIPTS_DIR, "sites") : path.join(/* turbopackIgnore: true */ REPO_ROOT, "transcripts", "sites")), ); export const READ_ROOTS = dedupe( process.env.MIX_ROOTS ? process.env.MIX_ROOTS.split(":").filter(Boolean) : [SONG_REPORTS, REPORTS_ROOT, SONG_DATA, SONG_SCRATCH, CHANNELS_DIR, MEDIA_ROOT, SITES_DIR], ); /** A site id or a report id: one lowercase url-safe segment (common/lib/report/schema.ts REPORT_ID_RE). */ export const SEGMENT_RE = /^[a-z0-9][a-z0-9-]{0,63}$/; export const NOTES_FILENAME = "notes.json"; /** * The ONE corpus path umtool may write: `SITES_DIR//reports//notes.json`. * * Lexically: exactly four segments under SITES_DIR, the second `reports`, the * site and report ids in the report-id grammar, the file named notes.json, and * no `..` or doubled separator anywhere. Then on disk: the report directory's * REAL path must be the lexical one under the real SITES_DIR -- a report dir * that is a symlink out of its site (or a site dir that is one) is refused -- * and it must already exist: a note never creates a report directory. A * notes.json that exists and is not a plain file (a symlink) is refused too. * * WRITE_ROOTS is untouched: nothing else in the corpus becomes writable. * * @param {string} abs * @param {{ sitesDir?: string }} [opts] * @returns {Promise} */ export async function isCorpusNotesFile(abs, { sitesDir = SITES_DIR } = {}) { if (typeof abs !== "string" || !path.isAbsolute(abs) || abs.includes("\0")) return false; if (path.resolve(/* turbopackIgnore: true */ abs) !== abs) return false; const root = path.resolve(/* turbopackIgnore: true */ sitesDir); const rel = path.relative(/* turbopackIgnore: true */ root, abs); if (!rel || rel.startsWith("..") || path.isAbsolute(rel)) return false; const parts = rel.split(path.sep); if (parts.length !== 4) return false; const [site, reports, report, name] = parts; if (!SEGMENT_RE.test(site) || reports !== "reports" || !SEGMENT_RE.test(report) || name !== NOTES_FILENAME) { return false; } const [realRoot, realDir] = await Promise.all([ realpath(/* turbopackIgnore: true */ root).catch(() => null), realpath(/* turbopackIgnore: true */ path.dirname(/* turbopackIgnore: true */ abs)).catch(() => null), ]); if (!realRoot || !realDir) return false; if (realDir !== path.join(/* turbopackIgnore: true */ realRoot, site, "reports", report)) return false; const st = await lstat(/* turbopackIgnore: true */ abs).catch(() => null); return !st || st.isFile(); } /** `SITES_DIR//reports//notes.json`, or null for a bad id. */ export function corpusNotesFile(site, report, { sitesDir = SITES_DIR } = {}) { if (!SEGMENT_RE.test(String(site)) || !SEGMENT_RE.test(String(report))) return null; return path.join(/* turbopackIgnore: true */ path.resolve(/* turbopackIgnore: true */ sitesDir), site, "reports", report, NOTES_FILENAME); } export const WRITE_ROOTS = dedupe( process.env.MIX_WRITE_ROOTS ? process.env.MIX_WRITE_ROOTS.split(":").filter(Boolean) : [SONG_REPORTS, SONG_SCRATCH], ); /** Back-compat alias. Every existing caller means "may this be read". */ export const MEDIA_ROOTS = READ_ROOTS; /** Analysis caches for the mix bench (envelopes, brightness curves). */ export const MIX_CACHE = path.join(CACHE_DIR, "mix"); /** Where the project index lives. Under CACHE_DIR, so a fixture gets its own. */ export const INDEX_DIR = process.env.UMTOOL_INDEX_DIR ? path.resolve(process.env.UMTOOL_INDEX_DIR) : path.join(CACHE_DIR, "index"); export const inside = (root, abs) => abs === root || abs.startsWith(root + path.sep); const rootsFor = (mode) => (mode === "write" ? WRITE_ROOTS : READ_ROOTS); /** * Resolve a client-supplied media path to an absolute one inside a known root, * or null. Relative paths are tried against each root in order, so the UI can * pass the short label it displays. * * It does not stat, so a relative path binds to the FIRST root it could live * under whether or not it is there. That is survivable because every path that * crosses the wire from a picker or a project link is absolute; a relative one * is a display label being handed back, and those were produced by labelFor * against this same ordered list. */ export function resolveInRoots(p, mode = "read") { if (!p) return null; const roots = rootsFor(mode); const candidates = path.isAbsolute(p) ? [path.resolve(p)] : roots.map((r) => path.resolve(r, p)); for (const abs of candidates) { if (roots.some((r) => inside(r, abs))) return abs; } return null; } /** The shortest root-relative label for an absolute path, for display. */ export function labelFor(abs) { for (const r of READ_ROOTS) { if (inside(r, abs)) return path.relative(r, abs) || path.basename(abs); } return abs; }