commit e049b2351ceedc884c03196753eacb07a169c4da
parent a210dace521bb77bc17e488119c7275a44fbdef1
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 1 Oct 2026 22:56:33 -0400
Merge r17/umtool-media-root (release 17 slice U1) — umtool's render scratch goes to a media root: UMTOOL_MEDIA_DIR makes a project's out/ a link into it (made by the first writer; never the root itself; a dangling link refuses), the cache moves to the user cache dir, umtool storage move-out|move-back move existing out/ trees with a copy-mirror-verify-swap mover, doctor reports the roots, the walk never descends an unmounted drive; reviewed SHIP after fixes
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
26 files changed, 1855 insertions(+), 23 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -169,6 +169,8 @@ yarn-error.log*
# run alongside a dev server someone is judging clips in (Next refuses two for
# one project).
umtool/.e2e-song/
+# ...and the storage spec's media root, a sibling of it (UMTOOL_MEDIA_DIR).
+umtool/.e2e-song-media/
umtool/.next/
# Any alternate dist dir, not just the e2e one.
#
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,8 @@
# Changelog
## [Unreleased]
+- **umtool can keep each report's render folder on a media drive.** With `UMTOOL_MEDIA_DIR` set, in umtool's environment (restart umtool after setting it), to a directory inside that drive, a report project's `out/` (its fetched windows, segments and finished video) is a link to the same path under that directory: a project's first build makes it there, and `umtool storage move-out <project>` (or `--all`) moves an existing one, copying it, checking the copy and only then leaving the link; `--dry-run` says how much would move, and `umtool storage move-back` brings one home. The manifest, its revisions, notes and sources stay where they are, and nothing in umtool reads a project differently. When the drive is not mounted, a build or source check refuses and says so instead of starting a new folder on the main disk; umtool never creates the media directory itself. `umtool storage` lists where each project's `out/` is. With `UMTOOL_MEDIA_DIR` unset nothing changes.
+- **umtool's cache moves to `~/.cache/archilyzer/umtool`** (`$XDG_CACHE_HOME/archilyzer/umtool` when that is set, or `UMTOOL_CACHE_DIR`). It was inside the song project's data folder, so it followed that folder onto whatever drive it was on. Run `umtool index` once after updating to rebuild the project index in its new place; umtool works without it, only slower, and the rest of the cache is remade as it is needed. `umtool doctor` now also shows the reports, media and cache folders, and the old cache folder while it is still there; it can be deleted.
- **umtool's report videos keep every clip's sound on its picture.** In a crossfaded cut each clip's audio was placed by the audio's own length and its picture by the picture's, and an encoded clip's audio is routinely a few to twenty milliseconds shorter or longer than its video, so the sound drifted further ahead clip by clip: by the end of a seventeen-clip cut it was a third of a second early, and two seconds on one with title and sources cards. Each clip's sound is now padded or trimmed to exactly its picture's length before the crossfade. Every crossfaded report video changes when it is rebuilt, and is in sync; a hard-cut video was not affected.
- **umtool's report videos can wear an on-screen deck: one panel under the footage for the whole cut, with a pip timeline, a title per clip, its source and date, and its QR.** A report manifest whose `render` says `"chrome": { "engine": "hyperframes", "layout": "deck" }` scales the footage into a box above a 190 px panel (both sizes are settings) and draws, over the whole cut, one unlabelled pip per clip on a track that fills as the cut plays, the clip's own title from `onscreen.title`, a subtitle naming the recording and its date (the channel too when the cut spans more than one; `onscreen.subtitle` replaces it), and the clip's QR. At each clip change the marker travels to the next pip and the title, subtitle and QR hand over; over a card the panel slides away and comes back after. The citation header, the corner QR and the section footer are not drawn on such a cut, and chapters take the clip's on-screen title. Every setting (sizes, spacing, date format, what the subtitle names, whether cards keep the panel, the motion's timings) is in `render.chrome.deck` and checked when it is saved; an unknown or out-of-range one is refused with a sentence saying why. The panel is rendered once per cut by HyperFrames (pinned to 0.8.24; `HYPERFRAMES_PKG` or `HYPERFRAMES_BIN` override it) and reused until its text or settings change. `build-video.mjs --chrome-only` redraws it over the built segments without rebuilding or fetching anything, `--no-chrome` builds the framed cut without it, and `--chrome-preview <at> <dur>` renders a short window. In umtool, the report page has an **On-screen** section — a switch, the settings, a table of every entry's title and subtitle with the automatic subtitle as its placeholder and a character counter, a live preview with a scrubber, a true still, **Re-render on-screen** and the built video — and the clip bench has on-screen title and subtitle fields with the panel previewed over the clip. The deck changes nothing, byte for byte, in a cut whose manifest has no `render.chrome`.
- **A report cut that wears the on-screen deck can show posts — Bluesky or X statements — as cards over the footage.** A report manifest's `posts` list (each with its platform, handle, date, words and link) is drawn near the end of the clip each post belongs with: the clip whose recording most closely precedes it by date, unless the post names one with `attachTo`; `hide` leaves one out. A clip's posts appear four seconds apart and stack down a column at the frame's top right; as the first appears, the footage eases aside (to 86 % of its box, at the far side) to make room, and the clip's last frame is held, in silence, for 2.5 seconds so the last post can be read; then they all leave together in the change to the next clip, which comes in at the normal size. When the column is full the oldest slide up and out. Each card slides in from the edge of the frame and flares in the deck's accent as it lands; it has an accent rail down its edge and shows the post's date, a platform label ("Bluesky" or "X") beside `@handle`, its words in paragraphs up to seven lines with an ellipsis, and a QR of the post's link, in the deck's colours and faces. The hold and the move are made where the cut is joined, not in a clip, so `--chrome-only` changes them without rebuilding one; chapters and the deck's timing count the hold. The timing, the hold (`hold`, 0 turns it off), the move (`shift`: its scale and seconds, or `false`), the column's side, width and inset, the QR size and the line limit are settings under `render.chrome.deck.posts`, and a bad post or setting is refused with a sentence before a build fetches anything. Only the seconds the cards are up are rendered, one short sequence per clip, cached like the deck; `--chrome-only`, `--chrome-preview` and a hard-cut cut lay them as they lay the deck, and `--no-chrome` draws neither — though it still holds and moves the footage, which are part of the cut rather than the chrome. A first post that appears inside the hold still moves the footage, and a hold is a whole number of frames. `posts` changes nothing in a cut that has none, and without the deck it is not drawn at all.
diff --git a/plans/release-17.md b/plans/release-17.md
@@ -323,4 +323,219 @@ hand; a dirent `isFile()` filter over a video dir hides it."** The `.relocating.
## Record
+### Slice U1, as shipped — umtool's render scratch goes to a media root (2026-10-01)
+
+Branch `r17/umtool-media-root` off `main` `7f4901f1`, `main` `90bd8384` (the deck/posts-room merge)
+merged in mid-slice, worktree `~/Projects/homepage-social-visible` (`pnpm wt list` block #11: editor
+4101, test 4111, export 4110), one Opus implementer. Scratch files `U1-*` in the job's `tmp`. The
+ruling is the plan's: render scratch (`out/`) goes to a media root by default; deliverables move per
+project by a switch (slice U2); manifests, `revisions/`, the caches and the cue cache stay put.
+
+**What it does.**
+- **Two roots, one new knob.** `umtool/lib/paths.mjs`: `MEDIA_ROOT = UMTOOL_MEDIA_DIR || REPORTS_ROOT`,
+ `MEDIA_TIERED` (they differ), `mediaMirror(abs, roots?)` (a path under `REPORTS_ROOT` → the same
+ relative path under `MEDIA_ROOT`, null outside; pure). `MEDIA_ROOT` joins `READ_ROOTS`, never
+ `WRITE_ROOTS`. Unset, nothing changes: `out/` is a directory in the project, and `READ_ROOTS` dedupes
+ it away. Every path op carries `turbopackIgnore`.
+- **The cache leaves `SONG_DATA`.** `CACHE_DIR = UMTOOL_CACHE_DIR || $XDG_CACHE_HOME/archilyzer/umtool`
+ (an empty `XDG_CACHE_HOME` is unset, as `common/lib/paths.ts` reads it; default `~/.cache`).
+ `INDEX_DIR`, `MIX_CACHE`, the posters, loudness and clip audio follow it. `OLD_CACHE_DIR`
+ (`<SONG_DATA>/.cache/umtool`) is named only for the doctor. Nothing is migrated: the index is
+ rebuilt by `umtool index` and everything else is remade on demand. The cue cache
+ (`REPORT_CACHE_DIR`, `report-to-video/cues.mjs`) is untouched.
+- **`umtool/lib/report/storage.mjs`** (new; modelled on `common/controller/relocateDir.ts`, not
+ importing it):
+ - `ensureOutDir(projectDir, roots?)`: a real `out/` → kept; a link to a directory → kept; a
+ **dangling link → refused** ("… is a link to …, which is not there — is the media drive mounted?
+ Nothing was written, and nothing was created in its place."); absent and tiered →
+ `mkdir -p <mirror>/out` and an absolute `symlink`; absent and not tiered → `mkdir` as before. The
+ media root itself is **stat'd and never created** (`mediaRootProblem`: missing, not a directory, or
+ inside/around `REPORTS_ROOT`). A project outside `REPORTS_ROOT` is never tiered. EEXIST from a
+ concurrent first writer is accepted when the winner resolves.
+ - `ensureWriteDir(dir)`: a directory a pipeline step writes into; when it is a project's `out` or up
+ to four levels under one, that `out` goes through `ensureOutDir` first, then `mkdir -p`.
+ - `moveDirToMedia(projectDir, name, opts)` / `moveDirToLocal(...)`: by NAME (`out` now; `clips`,
+ `share-*` for U2). Copy (`rsync -a --partial`), mirror toward the copy only (`-a --delete
+ --info=del`; refused when source and copy contain one another), verify (`--dry-run
+ --itemize-changes --delete` empty, one more mirror pass on a difference, a second refuses; equal
+ counts/bytes), then park (`<name>.moved-<ts>`), link, delete the parked copy. Space check on the
+ destination's volume (bytes + 1 GB). Every state is dispatched on the disk, so a cut run is
+ finished by running it again: a link to the mirror → `already` (a leftover parked copy removed);
+ absent with one parked copy → link and delete it; the reverse uses `<name>.incoming`, and after
+ the rename deletes the media copy and every directory above it the move left empty, never the
+ root. `dryRun` measures and changes nothing.
+ - `outDirState`, `pathState`, `measureTree` for readers and the CLI.
+- **Call sites.** `build-video.mjs` (`outRoot`, before any fetch), `check-availability.mjs` (its one
+ write), `render-cards.mjs` (CLI `--out`), `compose-chrome.mjs` (its `out/<variant>` base),
+ `lib/report/onscreen.mjs` (`deckStill`'s scratch) all make `out/` through `ensureWriteDir`.
+ `lib/report/export.mjs` reads only: it now says "out/ is a link to …, which is not there — is the
+ media drive mounted?" instead of "no build" when the link dangles. tmp-then-rename sites (`cut.mjs`,
+ the clip route) are untouched.
+- **The walk.** `kinds.mjs` `SKIP_DIRS` adds `clips` (`out` was already there) and `SKIP_PREFIXES =
+ ["share-"]`, read through `skipsDir(name)` by `walk.mjs`, so the project walk never stats a link
+ into a drive that is not there. The mix picker (`lib/media.ts`) follows a project's `out` link when
+ it points INTO the media root (so a tiered deliverable stays in the picker under its project) and
+ does not walk `MEDIA_ROOT` as a root of its own (it would list every tiered file twice).
+- **CLI.** `umtool doctor` adds `roots` (JSON) / a "roots" block: reports, media (tiered or "=
+ reports"), cache (and whether an index exists), and the old cache with its size while it is there;
+ it exits 1 when the media root is set and missing (the tools' `ok` keeps its meaning). `umtool
+ storage [<project>]` lists every project's `out` (dir, link, DANGLING, none); `umtool storage
+ move-out|move-back <project>|--all [--dry-run] [--json]` runs the movers, one line per project and a
+ total; move-out without `UMTOOL_MEDIA_DIR` refuses once.
+- **e2e env.** The app server and the specs' CLIs get `UMTOOL_CACHE_DIR=<fixture>/cache`
+ (`playwright.config.ts`, `projects.spec.ts`, `report-longform.spec.ts`; the index-deletion spec
+ now removes `cache/index`), so no run writes `~/.cache`. `make-fixture.mjs` adds
+ `storage-fixture` (a cached window, buildable offline), `storage-fresh-fixture` (no `out/`) and the
+ media root `umtool/.e2e-song-media/`, a sibling of the fixture (inside it would be inside
+ `REPORTS_ROOT`, which is refused), reset every run; `.gitignore` and umtool's trace excludes name
+ it. `storage.spec.ts` (new, 5): move-out (dry run first; the index's state and facts unchanged
+ through the link; again → already); **a build the app runs writes through the link and leaves it a
+ link** (the app has no `UMTOOL_MEDIA_DIR` at all); the root renamed away → `storage` says
+ dangling, `check-availability` refuses with the drive sentence on the moved project and with "is
+ not there" on the fresh one, no `out` made, the root not recreated, `doctor` exits 1; the first
+ writer of the fresh project makes the link; move-back → a real `out/`, the project's mirror gone,
+ the root and the other project's mirror kept.
+
+**Commits**
+
+| Commit | What |
+|---|---|
+| `65a3d146` | `umtool:` `MEDIA_ROOT`, `MEDIA_TIERED`, `mediaMirror`; `MEDIA_ROOT` in `READ_ROOTS`; `CACHE_DIR` from `UMTOOL_CACHE_DIR` / `XDG_CACHE_HOME`; `OLD_CACHE_DIR` |
+| `47d6d1b4` | `umtool:` `lib/report/storage.mjs`; the five writers through `ensureWriteDir`; export's dangling sentence; `clips` + `share-*` skips; the picker follows `out` links into the media root; `doctor` roots; `umtool storage` |
+| `4c2cd0c7` | merge of `main` `90bd8384` (the deck/posts-room branch: `build-video.mjs`, `make-fixture.mjs` and more) — one conflict, `export.mjs`'s imports, both kept |
+| `cb08e57a` | `umtool:` `storage.test.mjs` (20); the e2e cache in the fixture; the storage fixtures, media root and `storage.spec.ts`; `umtool storage <project>` status |
+| `efef56b3` | `umtool:` `docs/folders.md` (`MEDIA_ROOT`, `CACHE_DIR`), `docs/cli.md`; two `[Unreleased]` bullets in `editor/CHANGELOG.md` |
+| this commit | `plans:` this section |
+
+#### Gates (logs `$T/U1-*`)
+
+- **tsc** (all workspaces) clean at `47d6d1b4`, at the merge `4c2cd0c7` and at `cb08e57a`.
+- **common:** 2,484/2,484 (300 s, under load). **Editor unit:** 109/109. Neither touched; run on the
+ merged tree.
+- **test:scripts:** 390 tests (the merged `main`'s 370 + `storage.test.mjs`'s 20): 386 passed, 1
+ skipped, 3 failed, then 385/2/3 on a rerun — the three are `queue-lock.test.mjs` timing cases, a
+ different three each time, at a load average of 27–47 (other implementers' suites and builds);
+ `node --test scripts/queue-lock.test.mjs` alone: **11/11**. `storage.test.mjs` **20/20**.
+ `next-build-trace.test.mjs` is in it: **10/10** after each capped build below (its second skip on
+ the rerun is the staleness rule: the baseline run's `git checkout` of `main`'s umtool, below, gave
+ the modules new mtimes after the build).
+- **The capped umtool build with the corpus linked** (`ln -sT <primary>/transcripts transcripts`,
+ 76 channels visible through it; `systemd-run --scope -p MemoryMax=5G -p MemorySwapMax=0`, `timeout
+ -s KILL 240`, the link removed after), at `cb08e57a`: **exit 0, 53 s, 0.83 GB peak**; and once more
+ with `UMTOOL_MEDIA_DIR` set to a scratch directory: **exit 0, 53 s, 0.83 GB**. The two builds'
+ `.nft.json` entries (39,658 each, every route) are **identical** (`diff` empty); none names
+ `transcripts`, the scratch media root or `.e2e-song`. (The worktree carries an old `transcripts/`
+ directory — an `index.mdb` — which `ln -sT` refuses to replace: the script sets it aside for the
+ build and puts it back. A first attempt that did not was stopped before it counted.) The capped
+ editor build was not run: no editor code changed (only `editor/CHANGELOG.md`).
+- **Numbers tool:** none.
+- **umtool e2e** (`SONG_DIR=~/reports/quartering-uh-song/data pnpm --filter umtool run e2e …` from
+ the worktree root; the fixture found song data, `cand2`, `wav48` and `media`, no `asr`, no face
+ detector, so `find.spec`'s 14 skip):
+
+ | Run | At | Specs | Result |
+ |---|---|---|---|
+ | 1 | `efef56b3` | `storage`, `projects`, `report-longform`, `dashboard` | 37 passed, 6 failed, 10.3 min — `storage.spec` **5/5**; the six (`dashboard` ×3, `projects` ×3) are `page.goto: net::ERR_ABORTED` and 30 s timeouts at a load average of 47, and all six pass in run 2 |
+ | 2 | `efef56b3` | the full suite (21 files) | **214 passed**, 17 failed, 14 skipped, 13.8 min of tests (48 min with 34 min in the queue) — `faces` ×4 (`/api/face/detect` 503: no detector here), `triage` ×9 (no `asr` here), `browse:241`, `mix:166`, `mix:201`, `usage:112` |
+ | 3 | `main` `90bd8384`'s umtool, checked out into the worktree and restored after | `browse`, `faces`, `mix`, `triage`, `usage` | 48 passed, 15 failed, 7.2 min — the same `faces` ×4 and `triage` ×9, plus `browse:15`/`:34` (30 s timeouts) |
+ | 4 | `efef56b3` | the same five | 49 passed, 14 failed, 3.2 min — `faces` ×4 and `triage` ×9 as on `main`; `browse:241`, `mix:166`, `mix:201` pass; `usage:112` fails again |
+ | 5 | `efef56b3` | `usage` | **7 passed**, 0 failed, 19.6 s |
+
+ So against `main` on this machine: the `faces` and `triage` failures are the machine's (both
+ missing capabilities fail rather than skip — on `main` too); `mix:166`/`:201` and `browse:241`
+ fail only after the whole suite (the corpus window an earlier spec fetched for `vid1` wins the
+ picker's lookup), and pass in isolation on both; `usage:112` ("confirming the drop writes it
+ through") failed twice when it ran right after the failing `triage` specs on this branch, passed
+ once in that position on `main`, and passes alone — the verdict path it drives reads no cache and
+ no `out/`. Left to the reviewer as an order/timing question, not changed.
+
+#### Found and left
+
+- **Open question 2 — the four `*.mp4` near the project roots** (measured in `~/reports`, depth ≤ 2,
+ outside any `out/`): `kirsche-pippa/latest-contact-2026-06-20.mp4` (7.4 MB) is a cited clip fetched
+ through the MCP's `fetch_clip`, with its `.provenance.json` beside it — the sweep report's evidence,
+ a deliverable of a project that has no `out/`; `quartering-uh-song/jer-metalslug-bg.mp4` (51.5 MB)
+ and `quartering-uh-song/pokemon-no-music-recording.mp4` (3.1 MB) are song-project INPUTS (a song
+ spec's `background.path` names such a file relative to a media root, `song/spec.mjs`); `clips/
+ tim-pool-…mp4` (23.6 MB) is a loose cut at the reports root, in no project. None is render scratch:
+ all four are left untouched, and none is under U2's `clips/` or `share-*/`.
+- **What move-out would move today:** `umtool storage move-out --all --dry-run` against `~/reports`
+ (a scratch media root): **10 projects, 10.6 GB** (quartering-diet 5.0 GB, ferret-rescue 1.5 GB,
+ quartering-employee-count 1.2 GB, elfpire-eva 1.1 GB, …) — less than the plan's "≈ 18 of the 20
+ GB": the rest of `~/reports` is song data and loose files, not project `out/`s.
+- **Rollout, once:** set `UMTOOL_MEDIA_DIR` (the live umtool's environment) to a directory that
+ exists on the media drive, outside `~/reports`; restart umtool; run `umtool index` (the index is
+ rebuilt under `~/.cache/archilyzer/umtool`; until then everything works, slower); `umtool doctor`
+ shows the roots and the old cache (8.7 MB here), which can then be deleted; `umtool storage
+ move-out --all` (when nothing is building) moves the existing `out/`s.
+- **A dangling `out` reads as "no build" to the summary readers** (`lib/projects/report.mjs`'s
+ `stat0(out)`, `readAvailability`): only `export` and the writers say "is the media drive mounted?".
+ `umtool storage` and `umtool doctor` name it. `umtool check` learning it is U2's (plan: "`umtool
+ check` learns the two values").
+- **For U2:** `lib/report/deliver.mjs` `listBatches` filters `isDirectory()` on the project's dirents,
+ so a `share-*` that is a link would vanish from it; `sharedIdsIn`'s walk likewise does not follow a
+ link. The movers take any one-segment name and return `{ state, src, dest|from, bytes, files }`.
+- **A CLI move cannot see the app's jobs** (they live in its memory): the verify refuses when the tree
+ keeps changing, but a write in the instant between the verify and the park would be deleted with
+ the parked copy. The CLI says "run when nothing is building"; U2's bench button runs in the app and
+ can check.
+- **The worktree's stray `transcripts/`** (an `index.mdb` from 2026-09-28) is the trap the rules
+ describe; left in place.
+
+#### Deviations from the plan
+
+- `ensureOutDir` is not called in `driver.mjs`: its step builders are synchronous, are unit-tested with
+ a fake project directory, and only build argv; the call is in the scripts those steps run
+ (`build-video.mjs`, `check-availability.mjs`) through `ensureWriteDir`, which also covers a
+ hand-run script and the three other writers the plan did not list (`render-cards.mjs`,
+ `compose-chrome.mjs`, `onscreen.mjs`).
+- `export.mjs` writes nothing under `out/`, so it does not create it; it reports a dangling link instead.
+- `umtool storage move-back` and the plain `umtool storage [<project>]` listing were added beside
+ `move-out`: the e2e needs the way back, and an operator needs to see which projects moved.
+- `lib/media.ts` (not in the plan) follows `out` links into the media root and skips the root as its
+ own: without it every moved deliverable fell out of the mix picker.
+
+`[Unreleased]` (`editor/CHANGELOG.md`): "umtool can keep each report's render folder on a media
+drive." and "umtool's cache moves to `~/.cache/archilyzer/umtool`".
+
+#### Review (SHIP AFTER FIXES) and the fixes
+
+| Finding | Fix |
+|---|---|
+| F1 — the e2e app and the spec CLIs spread the shell's environment, so a shell exporting `UMTOOL_MEDIA_DIR` would put every fixture build's `out/` on the real media drive | `683e0a0f`: `UMTOOL_MEDIA_DIR=` (empty = unset under `\|\|`) in the webServer command; `UMTOOL_MEDIA_DIR: ""` in the `projects`, `report-longform` and `dashboard` CLI envs (`dashboard`'s doctor also gets the fixture cache); `storage.spec` keeps its own |
+| L1 — `doctor --json`'s `ok` was the tools' verdict while the exit status also counted the roots | `683e0a0f`: `ok = tools && roots`, `toolsOk` = the tools alone, `roots.ok` kept |
+| L2 — a CLI move cannot see the app's jobs | `683e0a0f`: `move-out`/`move-back` (one project or `--all`) skip, as `busy` with the pids, any project a running pipeline script (`build-video`, `check-availability`, `render-cards`, `compose-chrome`, `verify-build`, `fetch-via-editor`, `resolve-windows`, `cut-from-cache`, `share-batch`) names on its command line (`/proc/*/cmdline`; none elsewhere). It does not see the app's in-process deck previews; U2's in-app button can ask the app's jobs |
+| L3 — `dropMediaCopy` deleted any target inside "the media root", which untiered is the reports root | `a6926e17`: deletes only the project's own mirror, only when tiered and only when that is what came home; anything else is left and returned as `mediaCopyLeft` (the CLI prints "left in place … remove it by hand once checked"). A resumed move-back (the link already gone) reports the mirror, never deletes it |
+| L4 — a move-back cut after its rename orphaned the media copy silently | `a6926e17`: "already" reports the project's mirror as `mediaCopyLeft` while it exists |
+| L5 — a cut move's leftovers were not a guard | `a6926e17`: while `out.moved-*` or `out.incoming` exists, `ensureOutDir` (absent `out`) and both movers' directory branches refuse, naming the leftover and the move that finishes it; move-out refuses a lone `.incoming`, move-back a parked copy. `683e0a0f`: `folders.md` — the media root is a directory inside the drive, never the mountpoint; the leftovers rule |
+| N1 — the `paths.mjs` comment named the wrong reason for `READ_ROOTS` | `683e0a0f`: it names `/api/mix/{media,track}` and the lexical write check |
+| N2 — the walk did not skip `*.moved-*`/`*.incoming` | `683e0a0f`: `skipsDir` does |
+| N3 — `isMediaLink` realpathed every link it met | `683e0a0f`, `45cf9946`: only an entry named `out` (U2 adds its names to `MEDIA_LINKS`) |
+| N4 — the changelog bullet | `683e0a0f`: "in umtool's environment (restart umtool after setting it), to a directory inside that drive" |
+| N5 — an empty mirror for a project that does not exist | `a6926e17`: `ensureOutDir` checks the project directory first |
+
+`683e0a0f` left `report-to-video`'s tsc red (the `isMediaLink` parameter type); `45cf9946` restored it.
+
+**Re-review (SHIP AFTER FIXES, no further round):**
+- R1 — a real `out/` beside an `out.moved-*`/`out.incoming` sent each move to the other, which refused again → `4862ec4c`: both movers say both exist, that the leftover holds the moved data, to keep one and remove the other by hand, then run the move; `folders.md` says the same; the leftovers unit case asserts it for both movers.
+- R2 — `ensureOutDir`'s project check used `lstat`, refusing a project directory that is itself a link → `4862ec4c`: `stat`; a unit case links a project in.
+- N6 — the walk's leftover skip matched any `*.incoming`/`*.moved-*` folder → `4862ec4c`: only `out`, `clips` or `share-*` followed by one.
+- Gates: umtool and `report-to-video` tsc clean, all workspaces clean; `storage.test.mjs` 24/24; `test:scripts` 394: 392 passed, 2 skipped (LIVE, and `next-build-trace`'s staleness skip — `storage.mjs` changed after the last build; its path ops gained one `stat`, with `turbopackIgnore`), 0 failed.
+
+**Gates after the fixes:** tsc clean at `45cf9946`. `storage.test.mjs` 24/24 (+4: the untiered
+hand-made link, a tiered foreign link, the leftovers guard both ways, the ghost project; the
+resumed-move-back case now expects the mirror reported and kept). `test:scripts` **394: 393 passed, 1
+skipped (LIVE), 0 failed**, `next-build-trace` passing against a fresh build. Capped umtool build with
+the corpus linked (76 channels; the fixes touch `storage.mjs`'s path ops): exit 0, 39 s, 0.83 GB.
+umtool e2e at `45cf9946`: `storage`, `projects`, `report-longform`, `dashboard` — **42 passed**, 1 failed, 2.3 min (after 45 min in the queue; `storage.spec` 5/5) — the one is `dashboard:14`, the run's first test, a 30 s timeout on the cold first page; `dashboard.spec.ts` alone right after: **7 passed**, 0 failed, 42 s.
+
+**Still left (follow-ups):** the project summary (`lib/projects/report.mjs`) stats `out/<slug>.mp4`
+and reads `out/availability.json` through the link, so a **stalled** media drive blocks those reads,
+libuv's threadpool and the project list, and `availability.json` — small hot text — now lives on the
+media tier; a dangling link still reads as "no build" there (`umtool check` learning it is U2's). The
+`usage.spec:112` order question (after the `triage` specs, at load) is for a quiet-machine run of
+`triage.spec.ts usage.spec.ts` at integration. If a song project ever grows an `out/` and is moved, a
+mix render into it lands on the media root through the link (the write check is lexical; that matches
+the semantics).
+
## Rollout
diff --git a/umtool/bin/umtool.mjs b/umtool/bin/umtool.mjs
@@ -27,11 +27,15 @@
// umtool new <slug> [--kind report-video] [--from <report.md>|<share URL>|<channel>/<id>]
// [--site-origin URL] [--seed chapters] [--brand archilyzer-media]
// umtool doctor [--json] exit 1 if the report pipeline is missing a tool
+// or the media root is set and not there
+// umtool storage [move-out|move-back <project>|--all] [--dry-run] [--json]
+// where each project's out/ lives; move it
// umtool snapshot <project> [--label L] copy the manifest into revisions/
// umtool diff <project> <snapshot> what changed since that snapshot
// umtool export <project> --format toc-bbcode|toc-markdown|description|chapters [--variant V]
// umtool check-sources [<project>…] prints the re-check chain
import process from "node:process";
+import { readdirSync, readFileSync } from "node:fs";
import {
PROJECT_KINDS,
REPORTS_ROOT,
@@ -60,6 +64,8 @@ import { buildSteps, checkSourcesSteps, PRESETS } from "../lib/report/driver.mjs
import { openIndex, signRecord } from "../lib/projects/index-db.mjs";
import { probeTools } from "../lib/tools.mjs";
import { scaffoldReportVideo } from "../lib/projects/scaffold.mjs";
+import { CACHE_DIR, INDEX_DIR, MEDIA_ROOT, MEDIA_TIERED, OLD_CACHE_DIR } from "../lib/paths.mjs";
+import { measureTree, mediaRootProblem, moveDirToLocal, moveDirToMedia, outDirState, pathState } from "../lib/report/storage.mjs";
const argv = process.argv.slice(2);
const cmd = argv.find((a) => !a.startsWith("-")) ?? "help";
@@ -338,11 +344,37 @@ function cmdKinds() {
}
}
+/**
+ * The roots, as the doctor reports them: where projects are read, where their
+ * out/ goes, where the cache is -- and a cache left where it used to live
+ * (under SONG_DATA, before release 17), which is derived and safe to delete
+ * once `umtool index` has rebuilt the new one.
+ */
+async function rootsReport() {
+ const isDir = async (p) => (await pathState(p)).kind === "dir";
+ const problem = await mediaRootProblem();
+ const oldPresent = OLD_CACHE_DIR !== CACHE_DIR && (await isDir(OLD_CACHE_DIR));
+ return {
+ ok: !problem,
+ reports: { path: REPORTS_ROOT, present: await isDir(REPORTS_ROOT) },
+ media: { path: MEDIA_ROOT, tiered: MEDIA_TIERED, present: await isDir(MEDIA_ROOT), problem },
+ cache: { path: CACHE_DIR, present: await isDir(CACHE_DIR), index: await isDir(INDEX_DIR) },
+ oldCache: oldPresent
+ ? { path: OLD_CACHE_DIR, present: true, bytes: (await measureTree(OLD_CACHE_DIR)).bytes }
+ : { path: OLD_CACHE_DIR, present: false },
+ };
+}
+
+const mb = (n) => `${(n / 1024 ** 2).toFixed(1)} MB`;
+
async function cmdDoctor() {
// The one command that shells out on purpose. Seven version flags, ~100 ms.
const r = await probeTools();
+ const roots = await rootsReport();
if (json) {
- out(r);
+ // `ok` is what the exit status says (a script gates on either); the tools'
+ // own verdict stays readable as toolsOk.
+ out({ ...r, ok: r.ok && roots.ok, toolsOk: r.ok, roots });
} else {
for (const t of r.tools) {
const mark = t.present ? "ok " : t.required ? "MISSING" : "absent";
@@ -356,8 +388,134 @@ async function cmdDoctor() {
? "\nthe report pipeline can build here"
: "\nthe report pipeline is MISSING a tool it cannot run without",
);
+ console.log("\nroots");
+ console.log(` reports ${roots.reports.path}${roots.reports.present ? "" : " (not there)"}`);
+ console.log(
+ roots.media.tiered
+ ? ` media ${roots.media.path} — every project's out/ is linked here (UMTOOL_MEDIA_DIR)` +
+ (roots.media.problem ? `\n MISSING ${roots.media.problem}` : "")
+ : ` media = reports (UMTOOL_MEDIA_DIR unset): out/ stays in each project`,
+ );
+ console.log(
+ ` cache ${roots.cache.path}` +
+ (roots.cache.index ? "" : " (no index yet — `umtool index` builds it; everything works without)"),
+ );
+ if (roots.oldCache.present) {
+ console.log(
+ ` old cache ${roots.oldCache.path} ${mb(roots.oldCache.bytes ?? 0)} — the cache's old place, ` +
+ "no longer read; derived, safe to delete",
+ );
+ }
+ }
+ process.exit(r.ok && roots.ok ? 0 : 1);
+}
+
+// ---------------------------------------------------------------------------
+// storage: where each project's out/ lives, and moving it (release 17).
+//
+// umtool storage every project's out/: dir | link | DANGLING | none
+// umtool storage move-out <p>|--all out/ to the media root, a link left in its place
+// umtool storage move-back <p>|--all out/ back to a real directory in the project
+// --dry-run measure and say; change nothing
+//
+// The movers are lib/report/storage.mjs's, which `umtool` and the app share.
+// Run a move when nothing is building: the app's jobs live in its memory, so
+// this cannot see them -- the verify refuses when the tree keeps changing, but
+// a write in the last instant before the swap would be lost with the parked copy.
+// ---------------------------------------------------------------------------
+/**
+ * The pids of report-pipeline processes whose command line names this project
+ * (its manifest, its out/, or the directory itself). Linux /proc; elsewhere,
+ * none. Cheap and coarse: it sees the pipeline's scripts, not the app's
+ * in-process deck previews.
+ */
+function pipelineProcessesFor(projectDir) {
+ const SCRIPTS = /(build-video|check-availability|render-cards|compose-chrome|verify-build|fetch-via-editor|resolve-windows|cut-from-cache|share-batch)\.mjs/;
+ const pids = [];
+ let entries = [];
+ try {
+ entries = readdirSync("/proc").filter((n) => /^\d+$/.test(n));
+ } catch {
+ return pids;
+ }
+ for (const pid of entries) {
+ if (Number(pid) === process.pid) continue;
+ let args;
+ try {
+ args = readFileSync(`/proc/${pid}/cmdline`, "utf8").split("\0");
+ } catch {
+ continue;
+ }
+ if (!args.some((a) => SCRIPTS.test(a))) continue;
+ if (args.some((a) => a === projectDir || a.startsWith(projectDir + "/"))) pids.push(Number(pid));
+ }
+ return pids;
+}
+
+async function cmdStorage() {
+ const sub = positional[0];
+ const dryRun = has("--dry-run");
+ const log = (m) => (json ? console.error(m) : console.log(m));
+
+ if (sub !== "move-out" && sub !== "move-back") {
+ // `umtool storage`, `umtool storage <project>`, `umtool storage status [<project>]`.
+ const one = sub === "status" ? positional[1] : sub;
+ const refs = one ? [await pick(one)] : await projectRefs();
+ const rows = [];
+ for (const p of refs) rows.push({ id: p.id, ...(await outDirState(p.dir)) });
+ if (json) return out({ media: { path: MEDIA_ROOT, tiered: MEDIA_TIERED }, projects: rows });
+ console.log(MEDIA_TIERED ? `media root ${MEDIA_ROOT}` : "media root unset (UMTOOL_MEDIA_DIR): out/ stays in each project");
+ for (const r of rows) {
+ const what = { absent: "none", dir: "dir", link: "link", dangling: "DANGLING", other: "OTHER" }[r.state] ?? r.state;
+ console.log(`${what.padEnd(9)} ${r.id}${r.target ? ` -> ${r.target}` : ""}`);
+ }
+ return;
+ }
+
+ const move = sub === "move-out" ? moveDirToMedia : moveDirToLocal;
+ if (sub === "move-out" && !MEDIA_TIERED) {
+ die("UMTOOL_MEDIA_DIR is not set: there is no media root to move out/ to. Set it to a directory on the media drive (outside the reports root).");
+ }
+ const all = has("--all");
+ if (!all && !positional[1]) die(`which project? \`umtool storage ${sub} <project>\` or --all`);
+ const refs = all ? await projectRefs() : [await pick(positional[1])];
+
+ const results = [];
+ let failed = 0;
+ for (const p of refs) {
+ // The app's jobs live in its memory, but the pipeline runs as processes:
+ // one whose command line names this project is building it now.
+ const busy = pipelineProcessesFor(p.dir);
+ if (busy.length) {
+ results.push({ id: p.id, state: "busy", pids: busy });
+ if (!json) console.log(`${"busy".padEnd(12)} ${p.id} — a pipeline process is writing it (pid ${busy.join(", ")}); skipped`);
+ continue;
+ }
+ try {
+ const r = await move(p.dir, "out", { dryRun, log });
+ results.push({ id: p.id, ...r });
+ if (!json && r.state !== "absent") {
+ const size = r.bytes !== undefined ? ` ${r.files} file(s), ${mb(r.bytes)}` : "";
+ console.log(`${r.state.padEnd(12)} ${p.id}${size}`);
+ if (r.mediaCopyLeft) console.log(` left in place: ${r.mediaCopyLeft} (not deleted; remove it by hand once checked)`);
+ }
+ } catch (e) {
+ failed += 1;
+ results.push({ id: p.id, state: "failed", error: e?.message ?? String(e) });
+ if (!json) console.log(`${"FAILED".padEnd(12)} ${p.id}\n ${e?.message ?? e}`);
+ }
+ }
+ const bytes = results.reduce((n, r) => n + (r.bytes ?? 0), 0);
+ if (json) out({ ok: failed === 0, dryRun, bytes, results });
+ else {
+ const moved = results.filter((r) => r.state === "moved" || r.state === "would-move").length;
+ console.log(
+ `\n${moved} project(s) ${dryRun ? "would move" : "moved"}, ${mb(bytes)}` +
+ (failed ? `; ${failed} FAILED` : "") +
+ (dryRun ? " — dry run, nothing changed" : ""),
+ );
}
- process.exit(r.ok ? 0 : 1);
+ if (failed) process.exit(1);
}
async function cmdSnapshot() {
@@ -462,6 +620,10 @@ function usage() {
" umtool new <slug> [--from <report.md>|<share URL>|<channel>/<id>] [--site-origin URL] [--seed chapters]",
" [--brand archilyzer-media] render.brand: the report-to-video brand preset",
" umtool doctor [--json] exit 1 if the report pipeline is missing a tool",
+ " or the media root is set and not there; the roots, and a leftover old cache",
+ " umtool storage [<project>] where each project's out/ lives (dir, link, DANGLING)",
+ " umtool storage move-out|move-back <project>|--all [--dry-run]",
+ " out/ to the media root (UMTOOL_MEDIA_DIR) and back; run when nothing is building",
" umtool snapshot <project> [--label L] copy the manifest into revisions/",
" umtool diff <project> <snapshot> what changed since that snapshot",
" umtool export <project> --format toc-bbcode|toc-markdown|description|chapters [--variant V]",
@@ -488,6 +650,7 @@ const COMMANDS = {
folders: cmdFolders,
kinds: cmdKinds,
doctor: cmdDoctor,
+ storage: cmdStorage,
snapshot: cmdSnapshot,
diff: cmdDiff,
export: cmdExport,
diff --git a/umtool/docs/cli.md b/umtool/docs/cli.md
@@ -25,7 +25,9 @@ decisions inbox cannot disagree about what is wrong with one.
| `build <project> [--preset preview\|fast\|final] [--only ID]` | **prints** the chain |
| `index [--rebuild] [--prune] [--since MS] [--json]` | the cache |
| `new <slug> [--from <report.md>\|<share URL>\|<channel>/<id>] [--site-origin URL] [--seed chapters]` | scaffold |
-| `doctor [--json]` | which tools are on this machine; **exit 1** if the report pipeline is missing one |
+| `doctor [--json]` | which tools are on this machine and where the roots are; **exit 1** if the report pipeline is missing one, or `UMTOOL_MEDIA_DIR` is set and not there |
+| `storage [<project>] [--json]` | where each project's `out/` lives: dir, link, DANGLING, none |
+| `storage move-out\|move-back <project>\|--all [--dry-run] [--json]` | `out/` to the media root (a link left behind) and back — copied, mirrored, verified first; run when nothing is building |
| `snapshot <project> [--label L]` | copy the manifest into `revisions/` |
| `diff <project> <snapshot>` | added / removed / moved / window / retyped, by entry id |
| `export <project> --format toc-bbcode\|toc-markdown\|description\|chapters [--variant V]` | the posting artifacts, from the build's chapter offsets |
@@ -36,7 +38,8 @@ projects answering to one name is reported, never resolved by picking one.
## Environment
-`REPORTS_DIR`, `SONG_REPORTS_DIR`, `SONG_DIR`, `CHANNELS_DIR`, `UMTOOL_INDEX_DIR`
+`REPORTS_DIR`, `SONG_REPORTS_DIR`, `SONG_DIR`, `CHANNELS_DIR`, `UMTOOL_INDEX_DIR`,
+`UMTOOL_CACHE_DIR`, `UMTOOL_MEDIA_DIR`
— which is how it is tested against the e2e fixture. The path defaults
(`lib/paths.mjs`, `song/paths.mjs`):
@@ -45,6 +48,8 @@ projects answering to one name is reported, never resolved by picking one.
| `SONG_DIR` | `~/.local/share/archilyzer/song`, through its realpath — a symlink there is the supported way to keep the data where it is |
| `SONG_REPORTS_DIR` | `~/reports/quartering-uh-song` |
| `REPORTS_DIR` | `~/reports` (the parent of `SONG_REPORTS_DIR` when that is set) |
+| `UMTOOL_MEDIA_DIR` | unset = `REPORTS_DIR`: `out/` stays in each project. Set, each project's `out` is a link to the same path under it ([folders.md](folders.md)) |
+| `UMTOOL_CACHE_DIR` | `$XDG_CACHE_HOME/archilyzer/umtool`, else `~/.cache/archilyzer/umtool` (it was `<SONG_DIR>/.cache/umtool`) |
| `CHANNELS_DIR` | `$TRANSCRIPTS_DIR/channels`, else the checkout's `transcripts/channels` (found by walking up from the cwd to `pnpm-workspace.yaml`) |
| `VIDEO_ROOT` (`song/spec.mjs`, `song/video-dir.mjs`) | `~/reports/quartering-uh-song/videos` |
diff --git a/umtool/docs/folders.md b/umtool/docs/folders.md
@@ -9,6 +9,42 @@ environment variable.
`SONG_REPORTS` (the um-song deliverables) keeps its exact previous default and is
now a *subdirectory* of `REPORTS_ROOT` rather than the widest root there is.
+## `MEDIA_ROOT`
+
+Where a project's render scratch lives (release 17). `UMTOOL_MEDIA_DIR`, else
+`REPORTS_ROOT` — and then nothing is different: `out/` is a directory in the
+project. Set, a project's `out` is an absolute link to the same project-relative
+path under it (`<REPORTS_ROOT>/a/b/out -> <MEDIA_ROOT>/a/b/out`), made by the
+first writer (`lib/report/storage.mjs` `ensureOutDir`) or by
+`umtool storage move-out`. The manifest, `revisions/`, notes and sources stay put.
+
+- It must already exist, outside `REPORTS_ROOT`: umtool never creates it, so an
+ unmounted drive is a loud refusal ("is the media drive mounted?"), never a new
+ tree on the main disk. A dangling `out` link refuses the same way.
+- Make it a directory **inside** the drive (`<mount>/umtool`), never the
+ mountpoint itself: a mountpoint that stays behind as an empty directory when
+ the drive is unmounted passes the check, and the first build of a new project
+ would make its tree on the main disk.
+- A cut move leaves `out.moved-<stamp>` or `out.incoming` beside the project's
+ `out`. While one exists, no writer makes a new `out/` and both moves refuse,
+ naming it: run the move it names again to finish it. When a real `out/` exists
+ beside the leftover too (a writer made a fresh one after the cut), the moves
+ refuse and say so: the leftover holds the moved data — keep one, remove the
+ other by hand, then run the move.
+- It is a READ root (a realpath through the link lands under it), never a write root.
+- The walk skips `out`, `clips` and `share-*`, so it never stats a link into a
+ drive that is not there.
+- `umtool storage` lists every project's `out` (dir, link, DANGLING, none);
+ `umtool doctor` names the roots and exits 1 when this one is set and missing.
+
+## `CACHE_DIR`
+
+`UMTOOL_CACHE_DIR`, else `$XDG_CACHE_HOME/archilyzer/umtool`, else
+`~/.cache/archilyzer/umtool`: the project index, posters, the mix bench's
+analyses, sliced audio. Derived, safe to delete. Until release 17 it was
+`<SONG_DIR>/.cache/umtool`; `umtool doctor` reports a leftover one, and the first
+`umtool index` rebuilds the index in the new place.
+
## The walk
Two rules do almost all the work.
diff --git a/umtool/e2e/dashboard.spec.ts b/umtool/e2e/dashboard.spec.ts
@@ -51,7 +51,14 @@ test("umtool doctor reports a deliberately bad path as absent, and exits 1", ()
stdout = execFileSync("node", ["bin/umtool.mjs", "doctor", "--json"], {
cwd: UMTOOL,
encoding: "utf8",
- env: { ...process.env, YTDLP_BIN: path.join(FIXTURE, "bin", "definitely-not-here"), QRENCODE_BIN: path.join(FIXTURE, "bin", "qrencode") },
+ env: {
+ ...process.env,
+ YTDLP_BIN: path.join(FIXTURE, "bin", "definitely-not-here"),
+ QRENCODE_BIN: path.join(FIXTURE, "bin", "qrencode"),
+ // The fixture's roots, never this shell's media root or cache (empty = unset).
+ UMTOOL_MEDIA_DIR: "",
+ UMTOOL_CACHE_DIR: path.join(FIXTURE, "cache"),
+ },
});
} catch (e) {
const err = e as { status: number; stdout: string };
diff --git a/umtool/e2e/fixtures/make-fixture.mjs b/umtool/e2e/fixtures/make-fixture.mjs
@@ -1683,6 +1683,34 @@ copyFileSync(
path.join(DASH, "out", "clips-raw", "vid1_0.00-9.00.mp4"),
);
+// -- the media root (release 17) ------------------------------------------------
+//
+// storage.spec.ts moves storage-fixture's out/ to a media root and back, builds
+// through the link, and unplugs the root. Its CLI is given UMTOOL_MEDIA_DIR =
+// this directory; the APP is not (every other spec's out/ stays a directory).
+// A SIBLING of the fixture, not inside it: REPORTS_ROOT is the fixture root, and
+// a media root inside the tree it mirrors is refused. Reset here, every run.
+// storage-fixture dash-fixture's shape: a cached window, buildable offline
+// storage-fresh-fixture no out/ at all: the first writer makes the link
+const MEDIA = `${dest}-media`;
+rmSync(MEDIA, { recursive: true, force: true });
+mkdirSync(MEDIA, { recursive: true });
+for (const slug of ["storage-fixture", "storage-fresh-fixture"]) {
+ const dir = writeProject(
+ slug,
+ manifest(slug, "The Storage Fixture", { siteOrigin: "https://archive.example" }, [
+ { type: "clip", id: "c01", video: "vid1", start: 3.0, end: 6.0, cite: 3, section: 0, lock: true, quote: "and because" },
+ { type: "clip", id: "c02", video: "vid1", start: 9.0, end: 12.0, cite: 9, section: 0, lock: true, quote: "another whole sentence" },
+ ]),
+ );
+ if (slug !== "storage-fixture") continue;
+ mkdirSync(path.join(dir, "out", "clips-raw"), { recursive: true });
+ copyFileSync(
+ path.join(REPORT, "out", "clips-raw", "vid1_0.00-9.00.mp4"),
+ path.join(dir, "out", "clips-raw", "vid1_0.00-9.00.mp4"),
+ );
+}
+
console.log(`fixture at ${dest}`);
if (planned) console.log(` planned clip (used in a build): ${planned}`);
console.log(` videos/: alpha (4 cuts, 3 variants), beta (2 cuts), deck (1 cut, 2 variants)`);
@@ -1697,6 +1725,8 @@ console.log(` flagged source: ${flagged ? flagged.video : "none — no asr/"}`)
console.log(` SONG_CODE_DIR=${path.join(dest, "code")}`);
console.log(` SONG_DIR=${path.join(dest, "data")}`);
console.log(` SONG_REPORTS_DIR=${reports}`);
+console.log(` UMTOOL_CACHE_DIR=${path.join(dest, "cache")} (removed with the fixture; never ~/.cache)`);
+console.log(` storage spec media root: ${MEDIA} (UMTOOL_MEDIA_DIR on its CLI only; reset here)`);
console.log(` YTDLP_BIN=${path.join(BIN, "yt-dlp")} QRENCODE_BIN=${path.join(BIN, "qrencode")} HYPERFRAMES_BIN=${path.join(BIN, "hyperframes")}`);
console.log(` CHANNELS_DIR=${CHANNELS} (testchan/vid1 punctuated, vid2 not; vid3/vid4/vid5 for the editor fetch)`);
console.log(` projects: report-fixture (4 clips, 1 mid-sentence), no-origin-fixture,`);
diff --git a/umtool/e2e/projects.spec.ts b/umtool/e2e/projects.spec.ts
@@ -318,6 +318,10 @@ const cliEnv = {
SONG_REPORTS_DIR: path.join(FIXTURE, "reports"),
SONG_DIR: path.join(FIXTURE, "data"),
CHANNELS_DIR: path.join(FIXTURE, "channels"),
+ // The fixture's cache, as playwright.config.ts gives the app (never ~/.cache).
+ UMTOOL_CACHE_DIR: path.join(FIXTURE, "cache"),
+ // Never the real media root, whatever this shell exports (empty = unset).
+ UMTOOL_MEDIA_DIR: "",
};
const umtool = (args: string[]) =>
execFileSync("node", ["bin/umtool.mjs", ...args], { cwd: UMTOOL, encoding: "utf8", env: cliEnv });
@@ -415,7 +419,7 @@ test("deleting the index changes nothing but latency", async ({ request }) => {
// CACHE_DIR is documented as derived output, safe to delete at any time. This
// is that promise, tested.
- rmSync(path.join(FIXTURE, "data", ".cache", "umtool", "index"), {
+ rmSync(path.join(FIXTURE, "cache", "index"), {
recursive: true,
force: true,
});
diff --git a/umtool/e2e/report-longform.spec.ts b/umtool/e2e/report-longform.spec.ts
@@ -21,6 +21,10 @@ const cliEnv = {
SONG_REPORTS_DIR: path.join(FIXTURE, "reports"),
SONG_DIR: path.join(FIXTURE, "data"),
CHANNELS_DIR: path.join(FIXTURE, "channels"),
+ // The fixture's cache, as playwright.config.ts gives the app (never ~/.cache).
+ UMTOOL_CACHE_DIR: path.join(FIXTURE, "cache"),
+ // Never the real media root, whatever this shell exports (empty = unset).
+ UMTOOL_MEDIA_DIR: "",
YTDLP_BIN: path.join(FIXTURE, "bin", "yt-dlp"),
};
const umtool = (args: string[]) =>
diff --git a/umtool/e2e/storage.spec.ts b/umtool/e2e/storage.spec.ts
@@ -0,0 +1,180 @@
+import { test, expect, type APIRequestContext } from "@playwright/test";
+import { execFileSync, spawnSync } from "node:child_process";
+import { existsSync, lstatSync, readdirSync, readlinkSync, renameSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+// ---------------------------------------------------------------------------
+// A project's render scratch on a media root (release 17, slice U1).
+//
+// With UMTOOL_MEDIA_DIR set, a project's out/ is a link to the same
+// project-relative path under it: made by the first writer, or moved there by
+// `umtool storage move-out`. Only THIS spec's CLI is given the variable (the
+// media root is make-fixture's `<fixture>-media`, reset every run); the app is
+// not, which is the point of half of it -- a reader, and a build the app runs,
+// go through the link without knowing a media root exists.
+//
+// The tests run in order and hand the project's state on: moved out, built
+// through, unplugged, plugged back, moved back.
+// ---------------------------------------------------------------------------
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const UMTOOL = path.join(HERE, "..");
+const FIXTURE = path.join(UMTOOL, ".e2e-song");
+const MEDIA = `${FIXTURE}-media`;
+const UNPLUGGED = `${MEDIA}.unplugged`;
+const PROJECT = "reports/storage-fixture";
+const FRESH = "reports/storage-fresh-fixture";
+const dirOf = (id: string) => path.join(FIXTURE, id);
+const mirrorOf = (id: string) => path.join(MEDIA, id);
+
+const env = {
+ ...process.env,
+ SONG_REPORTS_DIR: path.join(FIXTURE, "reports"),
+ SONG_DIR: path.join(FIXTURE, "data"),
+ CHANNELS_DIR: path.join(FIXTURE, "channels"),
+ UMTOOL_CACHE_DIR: path.join(FIXTURE, "cache"),
+ YTDLP_BIN: path.join(FIXTURE, "bin", "yt-dlp"),
+ UMTOOL_MEDIA_DIR: MEDIA,
+};
+const umtool = (args: string[]) =>
+ JSON.parse(execFileSync("node", ["bin/umtool.mjs", ...args, "--json"], { cwd: UMTOOL, encoding: "utf8", env }));
+// The pipeline's first writer, as a build's step 1 runs it.
+const checkAvailability = (id: string) =>
+ spawnSync(
+ "node",
+ [
+ path.join(UMTOOL, "report-to-video", "check-availability.mjs"),
+ path.join(dirOf(id), "video.manifest.json"),
+ "--out",
+ path.join(dirOf(id), "out"),
+ "--allow-missing",
+ ],
+ { cwd: path.join(UMTOOL, "report-to-video"), encoding: "utf8", env },
+ );
+
+const isLink = (p: string) => existsSync(path.dirname(p)) && lstatSync(p, { throwIfNoEntry: false })?.isSymbolicLink() === true;
+const isRealDir = (p: string) => lstatSync(p, { throwIfNoEntry: false })?.isDirectory() === true;
+const parked = (id: string) => readdirSync(dirOf(id)).filter((n) => n.startsWith("out.moved-") || n === "out.incoming");
+
+async function projectRow(request: APIRequestContext, id: string) {
+ const j = (await (await request.get("/api/browse/projects")).json()) as {
+ projects: { id: string; state: string; facts: string[] }[];
+ };
+ return j.projects.find((p) => p.id === id);
+}
+
+test.describe.configure({ mode: "serial" });
+
+test.afterAll(() => {
+ // A failure mid-way must not leave the root unplugged for the next run's
+ // reader of this file -- make-fixture resets it anyway.
+ if (existsSync(UNPLUGGED) && !existsSync(MEDIA)) renameSync(UNPLUGGED, MEDIA);
+});
+
+test("move-out leaves a link to the media root, and readers see the same project", async ({ request }) => {
+ const out = path.join(dirOf(PROJECT), "out");
+ expect(isRealDir(out)).toBe(true);
+ const before = await projectRow(request, PROJECT);
+ expect(before).toBeTruthy();
+
+ // A dry run measures and changes nothing.
+ const dry = umtool(["storage", "move-out", PROJECT, "--dry-run"]);
+ expect(dry.results[0].state).toBe("would-move");
+ expect(dry.results[0].bytes).toBeGreaterThan(0);
+ expect(isRealDir(out)).toBe(true);
+ expect(existsSync(mirrorOf(PROJECT))).toBe(false);
+
+ const moved = umtool(["storage", "move-out", PROJECT]);
+ expect(moved.ok).toBe(true);
+ expect(moved.results[0].state).toBe("moved");
+ expect(isLink(out)).toBe(true);
+ expect(readlinkSync(out)).toBe(path.join(mirrorOf(PROJECT), "out"));
+ expect(existsSync(path.join(mirrorOf(PROJECT), "out", "clips-raw", "vid1_0.00-9.00.mp4"))).toBe(true);
+ expect(parked(PROJECT)).toEqual([]);
+
+ // The index and the page read <project>/out by path; the link changes nothing.
+ const after = await projectRow(request, PROJECT);
+ expect(after?.state).toBe(before?.state);
+ expect(after?.facts).toEqual(before?.facts);
+
+ // Again: nothing to do.
+ expect(umtool(["storage", "move-out", PROJECT]).results[0].state).toBe("already");
+ const status = umtool(["storage", PROJECT]);
+ expect(status.projects[0].state).toBe("link");
+});
+
+test("a build the app runs writes through the link, and leaves it a link", async ({ request }) => {
+ const start = await request.post("/api/report/build", { data: { project: PROJECT, preset: "fast" } });
+ expect(start.ok()).toBeTruthy();
+ const { job } = (await start.json()) as { job: { id: string } };
+ let state = "running";
+ for (let i = 0; i < 150 && state === "running"; i += 1) {
+ const j = (await (await request.get("/api/jobs")).json()) as { jobs: { id: string; state: string }[] };
+ state = j.jobs.find((x) => x.id === job.id)?.state ?? "running";
+ if (state === "running") await new Promise((r) => setTimeout(r, 200));
+ }
+ expect(state).toBe("done");
+
+ const out = path.join(dirOf(PROJECT), "out");
+ expect(isLink(out)).toBe(true);
+ expect(existsSync(path.join(mirrorOf(PROJECT), "out", "storage-fixture.mp4"))).toBe(true);
+ expect(existsSync(path.join(mirrorOf(PROJECT), "out", "availability.json"))).toBe(true);
+});
+
+test("an unplugged media root refuses loudly and materialises nothing", () => {
+ renameSync(MEDIA, UNPLUGGED);
+ try {
+ expect(umtool(["storage", PROJECT]).projects[0].state).toBe("dangling");
+
+ // A moved project: the link dangles, and the first writer says why.
+ const r = checkAvailability(PROJECT);
+ expect(r.status).not.toBe(0);
+ expect(r.stderr).toContain("is the media drive mounted?");
+ expect(isLink(path.join(dirOf(PROJECT), "out"))).toBe(true);
+
+ // A project with no out/ yet: the root is stat'd, never created.
+ const fresh = checkAvailability(FRESH);
+ expect(fresh.status).not.toBe(0);
+ expect(fresh.stderr).toContain("is not there");
+ expect(existsSync(path.join(dirOf(FRESH), "out"))).toBe(false);
+
+ // Neither recreated the root on the disk it was "on".
+ expect(existsSync(MEDIA)).toBe(false);
+
+ // The doctor says so, and exits 1.
+ const doctor = spawnSync("node", ["bin/umtool.mjs", "doctor", "--json"], { cwd: UMTOOL, encoding: "utf8", env });
+ expect(doctor.status).toBe(1);
+ const roots = JSON.parse(doctor.stdout).roots;
+ expect(roots.media.tiered).toBe(true);
+ expect(roots.media.problem).toContain("is not there");
+ expect(roots.cache.path).toBe(path.join(FIXTURE, "cache"));
+ } finally {
+ renameSync(UNPLUGGED, MEDIA);
+ }
+});
+
+test("the first writer of a project with no out/ makes the link", () => {
+ const r = checkAvailability(FRESH);
+ expect(r.status, r.stderr).toBe(0);
+ const out = path.join(dirOf(FRESH), "out");
+ expect(isLink(out)).toBe(true);
+ expect(readlinkSync(out)).toBe(path.join(mirrorOf(FRESH), "out"));
+ expect(existsSync(path.join(mirrorOf(FRESH), "out", "availability.json"))).toBe(true);
+});
+
+test("move-back makes out/ a real directory again and removes the media copy", () => {
+ const back = umtool(["storage", "move-back", PROJECT]);
+ expect(back.ok).toBe(true);
+ expect(back.results[0].state).toBe("moved");
+ const out = path.join(dirOf(PROJECT), "out");
+ expect(isRealDir(out)).toBe(true);
+ expect(existsSync(path.join(out, "storage-fixture.mp4"))).toBe(true);
+ expect(parked(PROJECT)).toEqual([]);
+ // The project's mirror is gone; the root, and the other project's, are not.
+ expect(existsSync(mirrorOf(PROJECT))).toBe(false);
+ expect(existsSync(MEDIA)).toBe(true);
+ expect(isLink(path.join(dirOf(FRESH), "out"))).toBe(true);
+
+ expect(umtool(["storage", "move-back", PROJECT]).results[0].state).toBe("already");
+});
diff --git a/umtool/lib/media.ts b/umtool/lib/media.ts
@@ -2,10 +2,10 @@ import { createHash } from "node:crypto";
import { spawn } from "node:child_process";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
-import { mkdir, readdir, readFile, rename, stat, writeFile } from "node:fs/promises";
+import { mkdir, readdir, readFile, realpath, rename, stat, writeFile } from "node:fs/promises";
import { existsSync } from "node:fs";
import path from "node:path";
-import { MEDIA_ROOTS, MIX_CACHE, REPORTS_ROOT, SONG_REPORTS, labelFor } from "./paths";
+import { MEDIA_ROOT, MEDIA_ROOTS, MEDIA_TIERED, MIX_CACHE, REPORTS_ROOT, SONG_REPORTS, inside, labelFor } from "./paths";
import { brightnessCurve, brightnessSteps } from "../song/flatness.mjs";
const run = promisify(execFile);
@@ -79,6 +79,22 @@ const SCRATCH_FILE = /^(poly-song-|polytmp-|seg_|i_|o_|ms\d?seg|out\.raw)/;
/** Below this is a fragment, a probe or a one-note extraction, not a track. */
const MIN_INTERESTING = 256 * 1024;
+/**
+ * A project's `out` linked to the media root (UMTOOL_MEDIA_DIR, release 17) is
+ * walked like the directory it replaced, so a tiered deliverable stays in the
+ * picker under its project. Only a link INTO the media root: every other link
+ * stays unfollowed, as it always was (SONG_DATA's 39 GB are links).
+ */
+/** The project directories that may be links to the media root (U2 adds deliverables). */
+const MEDIA_LINKS = new Set(["out"]);
+
+async function isMediaLink(e: { name: string; isSymbolicLink(): boolean }, abs: string): Promise<boolean> {
+ if (!MEDIA_TIERED || !e.isSymbolicLink() || !MEDIA_LINKS.has(e.name)) return false;
+ const real = await realpath(abs).catch(() => null);
+ if (!real || !inside(MEDIA_ROOT, real)) return false;
+ return stat(real).then((s) => s.isDirectory(), () => false);
+}
+
export type MediaRow = { path: string; label: string; size: number; mtimeMs: number };
/**
@@ -103,7 +119,7 @@ export async function listMediaUnder(root: string, maxDepth = 2, limit = 200): P
for (const e of entries) {
if (e.name.startsWith(".")) continue;
const abs = path.join(dir, e.name);
- if (e.isDirectory()) {
+ if (e.isDirectory() || (await isMediaLink(e, abs))) {
if (depth > 0 && !SCRATCH_DIR.test(e.name)) await walk(abs, depth - 1);
continue;
}
@@ -139,7 +155,7 @@ export async function listMedia(limit = 400): Promise<MediaRow[]> {
for (const e of entries) {
if (e.name.startsWith(".")) continue;
const abs = path.join(dir, e.name);
- if (e.isDirectory()) {
+ if (e.isDirectory() || (await isMediaLink(e, abs))) {
if (depth > 0 && !SCRATCH_DIR.test(e.name)) await walk(abs, depth - 1);
continue;
}
@@ -168,7 +184,13 @@ export async function listMedia(limit = 400): Promise<MediaRow[]> {
// to find nothing anybody would load, which is the opposite of the problem
// this is fixing.
const depthFor = (r: string) => (r === REPORTS_ROOT || r === SONG_REPORTS ? 2 : 1);
- for (const r of MEDIA_ROOTS) await walk(r, depthFor(r));
+ // The media root is reached through the projects' `out` links, under the
+ // project's own path; walked as a root as well, every tiered deliverable
+ // would be listed twice and the second copy would land in "other".
+ for (const r of MEDIA_ROOTS) {
+ if (MEDIA_TIERED && r === MEDIA_ROOT) continue;
+ await walk(r, depthFor(r));
+ }
out.sort((a, b) => b.mtimeMs - a.mtimeMs);
return out.slice(0, limit);
}
diff --git a/umtool/lib/paths.mjs b/umtool/lib/paths.mjs
@@ -14,9 +14,25 @@ import { SONG_DATA, SONG_REPORTS } from "../song/paths.mjs";
// record paths relative to it (make-thumb, accept-thumb) import only siblings.
export { SONG_DATA, SONG_REPORTS };
-// Derived output (sliced mp3s, waveform peaks, the project index). Lives with
-// the data, not in the repo, and is safe to delete at any time.
-export const CACHE_DIR = path.join(SONG_DATA, ".cache", "umtool");
+// Derived output (sliced mp3s, waveform peaks, the project index, posters, the
+// mix bench's analyses). Not in the repo, and safe to delete at any time.
+//
+// It used to live under SONG_DATA (`<SONG_DIR>/.cache/umtool`), which tied
+// every project's index and every report's derived files to wherever the song
+// project's 39 GB happened to sit -- a report-only machine, or one whose song
+// data is on a drive that is not mounted, had its cache follow it there
+// (release 17). It is a cache, so it goes where caches go:
+// `UMTOOL_CACHE_DIR`, else `$XDG_CACHE_HOME/archilyzer/umtool` (an empty
+// XDG_CACHE_HOME is unset, as common/lib/paths.ts reads it), else
+// `~/.cache/archilyzer/umtool`. Nothing is migrated: the first `umtool index`
+// rebuilds the index there, and every other file is re-made on demand.
+// OLD_CACHE_DIR is only for `umtool doctor`, which reports a leftover one.
+const XDG_CACHE = process.env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache");
+export const CACHE_DIR = path.resolve(
+ /* turbopackIgnore: true */
+ process.env.UMTOOL_CACHE_DIR || path.join(/* turbopackIgnore: true */ XDG_CACHE, "archilyzer", "umtool"),
+);
+export const OLD_CACHE_DIR = path.join(/* turbopackIgnore: true */ SONG_DATA, ".cache", "umtool");
/**
* A file in CACHE_DIR, by name. A route names its cache files through this
@@ -61,6 +77,53 @@ export const REPORTS_ROOT = path.resolve(
const dedupe = (list) => [...new Set(list.map((p) => path.resolve(p)))];
// ---------------------------------------------------------------------------
+// MEDIA_ROOT -- where a project's RENDER SCRATCH (`out/`) lives (release 17).
+//
+// A report project's manifest, revisions/, notes and sources are small text and
+// stay under REPORTS_ROOT. Its `out/` -- fetched windows, segments, the
+// deliverable, ~18 of the 20 GB in ~/reports -- is bulk that can be re-made,
+// and belongs on a media drive. With UMTOOL_MEDIA_DIR set, a project's `out` is
+// an absolute SYMLINK to the same project-relative path under it:
+//
+// <REPORTS_ROOT>/<folder>/<project>/out -> <MEDIA_ROOT>/<folder>/<project>/out
+//
+// made by the first writer (lib/report/storage.mjs ensureOutDir) or moved there
+// by `umtool storage move-out`. Every reader keeps opening `<project>/out/...`
+// by path; the link is the only place the media drive is named.
+//
+// UNSET, MEDIA_ROOT is REPORTS_ROOT, MEDIA_TIERED is false, and nothing changes:
+// `out/` is a plain directory in the project, as it always was.
+//
+// MEDIA_ROOT is READABLE -- so a client may name a media file by its real path
+// (one taken through a project's `out` link lands under it) to the mix bench's
+// /api/mix/{media,track} -- and never WRITABLE by a client-named path: what a
+// render may write to is still WRITE_ROOTS, judged lexically, so a write to
+// `<project>/out/...` is judged by the project's place, never the link's target.
+// ---------------------------------------------------------------------------
+export const MEDIA_ROOT = path.resolve(
+ /* turbopackIgnore: true */
+ process.env.UMTOOL_MEDIA_DIR || REPORTS_ROOT,
+);
+
+/** True when render scratch goes to a media root of its own. */
+export const MEDIA_TIERED = MEDIA_ROOT !== REPORTS_ROOT;
+
+/**
+ * Where `abs` (a path under REPORTS_ROOT) is mirrored under MEDIA_ROOT, or null
+ * when it is not under REPORTS_ROOT -- a project somewhere else is never tiered.
+ * Pure: it never touches the disk. The roots are parameters so a test can name
+ * its own; the defaults are the process's.
+ */
+export function mediaMirror(abs, { reportsRoot = REPORTS_ROOT, mediaRoot = MEDIA_ROOT } = {}) {
+ const rel = path.relative(
+ /* turbopackIgnore: true */ path.resolve(/* turbopackIgnore: true */ reportsRoot),
+ path.resolve(/* turbopackIgnore: true */ abs),
+ );
+ if (rel === "" || rel.startsWith("..") || path.isAbsolute(rel)) return null;
+ return path.join(/* turbopackIgnore: true */ path.resolve(/* turbopackIgnore: true */ mediaRoot), rel);
+}
+
+// ---------------------------------------------------------------------------
// READ vs WRITE, and why they are two lists.
//
// resolveInRoots() guards both what may be OPENED and what may be RENDERED TO.
@@ -148,7 +211,7 @@ export const CHANNELS_DIR = path.resolve(
export const READ_ROOTS = dedupe(
process.env.MIX_ROOTS
? process.env.MIX_ROOTS.split(":").filter(Boolean)
- : [SONG_REPORTS, REPORTS_ROOT, SONG_DATA, SONG_SCRATCH, CHANNELS_DIR],
+ : [SONG_REPORTS, REPORTS_ROOT, SONG_DATA, SONG_SCRATCH, CHANNELS_DIR, MEDIA_ROOT],
);
export const WRITE_ROOTS = dedupe(
diff --git a/umtool/lib/paths.ts b/umtool/lib/paths.ts
@@ -8,6 +8,7 @@ import path from "node:path";
// SONG_REPORTS the um-song deliverables -- quartering-*.mp4 and their .plan.json
// SONG_SCRATCH render scratch, ABOVE SONG_DATA
// REPORTS_ROOT the tree every PROJECT hangs off -- songs AND report videos
+// MEDIA_ROOT where a project's out/ is linked to (UMTOOL_MEDIA_DIR); = REPORTS_ROOT when unset
//
// Everything the bench reads or writes must resolve inside one of these. Not
// because this is exposed -- it is a local tool on a loopback port -- but
@@ -21,7 +22,9 @@ export {
CACHE_DIR,
cacheFile,
INDEX_DIR,
+ MEDIA_ROOT,
MEDIA_ROOTS,
+ MEDIA_TIERED,
MIX_CACHE,
READ_ROOTS,
REPORTS_ROOT,
@@ -31,6 +34,7 @@ export {
WRITE_ROOTS,
inside,
labelFor,
+ mediaMirror,
resolveInRoots,
} from "./paths.mjs";
diff --git a/umtool/lib/projects/kinds.mjs b/umtool/lib/projects/kinds.mjs
@@ -47,8 +47,31 @@ export const SKIP_DIRS = new Set([
// never reaches inside one -- but a snapshot directory left behind by a
// deleted manifest must not read as a project either.
"revisions",
+ // A report's cut clips. Like out/, it may be a link to the media root
+ // (release 17), and the walk follows a link to a directory with a stat --
+ // which, on a media drive that is unplugged or stalled, is a hang or a
+ // miss, never a project.
+ "clips",
]);
+/**
+ * Name PREFIXES the walk never descends into, for the same reason as `clips`:
+ * `share-<x>/` (a deliver's zip and its staging) may be a link to the media
+ * root, and none of them can contain a project.
+ */
+export const SKIP_PREFIXES = ["share-"];
+
+/**
+ * What a cut move leaves beside the directory it moved (lib/report/storage.mjs):
+ * `out.moved-<stamp>`, `out.incoming` -- only for the names that move, so a
+ * folder of projects that merely ends in `.incoming` is still walked.
+ */
+const MOVE_LEFTOVER = /^(out|clips|share-[^/]*)\.(moved-[^/]*|incoming)$/;
+
+/** Whether the walk skips a directory entry by its name. */
+export const skipsDir = (name) =>
+ SKIP_DIRS.has(name) || SKIP_PREFIXES.some((p) => name.startsWith(p)) || MOVE_LEFTOVER.test(name);
+
const has = (names, n) => names.has(n);
const someMatch = (names, re) => [...names].some((n) => re.test(n));
diff --git a/umtool/lib/projects/walk.mjs b/umtool/lib/projects/walk.mjs
@@ -15,7 +15,7 @@
// project. The 3.1 GB is never touched.
import { readdir, readFile, realpath, stat } from "node:fs/promises";
import path from "node:path";
-import { RESERVED_BROWSE, SKIP_DIRS, detectKind } from "./kinds.mjs";
+import { RESERVED_BROWSE, detectKind, skipsDir } from "./kinds.mjs";
/** A single safe path segment: no separators, no traversal, no dotfiles. */
export const isSegment = (v) => /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(v) && !v.includes("..");
@@ -103,7 +103,7 @@ export async function walkProjects(root, { maxDepth = MAX_DEPTH } = {}) {
for (const e of entries) {
if (e.name.startsWith(".")) continue;
- if (SKIP_DIRS.has(e.name)) continue;
+ if (skipsDir(e.name)) continue;
let isDir = e.isDirectory();
if (!isDir && e.isSymbolicLink()) {
isDir = await stat(path.join(abs, e.name)).then((s) => s.isDirectory(), () => false);
diff --git a/umtool/lib/report/export.mjs b/umtool/lib/report/export.mjs
@@ -19,6 +19,7 @@ import path from "node:path";
import { DEFAULT_VARIANT, selectVariant, segmentOffsets } from "umtool-report-to-video/build-video";
import { citeUrlFor, channelFor, readAvailability, readManifest } from "../projects/report.mjs";
import { teaserTitle } from "umtool-report-to-video/deck";
+import { outDirState } from "./storage.mjs";
export const EXPORT_FORMATS = ["toc-bbcode", "toc-markdown", "description", "chapters"];
@@ -72,6 +73,13 @@ const titleOf = (e, i) =>
export async function chapterOffsets(dir, manifest, variant) {
const entries = manifest.timeline ?? [];
const outDir = path.join(dir, "out");
+ // Read through, never made: an export writes nothing under out/. But a link
+ // to a media root that is not mounted would read below as "no build", which
+ // sends somebody off to rebuild a cut that is sitting on an unplugged drive.
+ const out = await outDirState(dir);
+ if (out.state === "dangling") {
+ return { error: `out/ is a link to ${out.target}, which is not there — is the media drive mounted?` };
+ }
const ffmetaCandidates = [path.join(outDir, variant, "chapters.ffmeta"), path.join(outDir, "chapters.ffmeta")];
for (const file of ffmetaCandidates) {
const text = await readFile(file, "utf8").catch(() => null);
diff --git a/umtool/lib/report/onscreen.mjs b/umtool/lib/report/onscreen.mjs
@@ -15,7 +15,7 @@
//
// The preview never renders and never touches the build's project or cache:
// compose-chrome's `preview: true` writes out/<variant>/chrome/deck-preview/.
-import { mkdir, readFile, rm } from "node:fs/promises";
+import { readFile, rm } from "node:fs/promises";
import path from "node:path";
import { composeChrome } from "umtool-report-to-video/compose-chrome";
import { selectVariant } from "umtool-report-to-video/build-video";
@@ -40,6 +40,7 @@ import {
} from "../projects/report.mjs";
import { normalizePostPatches } from "./manifest.mjs";
import { deckPreviewDir, postsPreviewDir } from "./serve.mjs";
+import { ensureWriteDir } from "./storage.mjs";
/** The schedule document deck.mjs defines, built or estimated. */
/** @typedef {ReturnType<typeof estimateSchedule>} DeckSchedule */
@@ -454,7 +455,7 @@ export async function deckStill(project, variant, schedule, t) {
const want = deckPreviewDir(project.dir, variant);
const stills = path.join(outDir, "chrome", "deck-stills");
return serialised(want, async () => {
- await mkdir(stills, { recursive: true });
+ await ensureWriteDir(stills); // through ensureOutDir: out/ may be a link to the media root
const png = path.join(stills, `still-${process.pid}-${Math.random().toString(36).slice(2, 8)}.png`);
try {
/** @type {Record<string, unknown>} */
diff --git a/umtool/lib/report/storage.mjs b/umtool/lib/report/storage.mjs
@@ -0,0 +1,569 @@
+// Where a project's bulk lives: its render scratch (`out/`) on a media root, the
+// manifest and everything small beside it (release 17).
+//
+// lib/paths.mjs names the roots: REPORTS_ROOT holds every project; MEDIA_ROOT
+// (UMTOOL_MEDIA_DIR) holds the bulk; MEDIA_TIERED says they differ. A tiered
+// project's `out` is an ABSOLUTE SYMLINK to the same project-relative path
+// under MEDIA_ROOT, so every reader keeps opening `<project>/out/...` by path
+// and none of them knows the media drive exists.
+//
+// ensureOutDir the first writer's call: makes `out/` -- a link when
+// tiered, a directory when not -- and refuses loudly on a
+// dangling link instead of building a new tree beside it
+// moveDirToMedia an existing directory of the project to MEDIA_ROOT:
+// copy, mirror, verify, park, link, delete the parked copy
+// moveDirToLocal the reverse: back to a real directory in the project
+//
+// The movers take a NAME, not "out": `umtool storage move-out` moves `out/`
+// with them, and a project's deliverables (`clips/`, `share-*/`) are moved by
+// the same two calls behind the deliverables switch.
+//
+// Modelled on the editor's common/controller/relocateDir.ts, not imported from
+// it (umtool's scripts are plain .mjs under node, and that is a TypeScript
+// controller): the source is never touched until the copy verifies; `--delete`
+// only ever points at the copy under construction; every step past the copy is
+// dispatched on what is ON DISK, so a run cut at any point is finished by
+// running it again.
+//
+// THE MEDIA ROOT IS NEVER CREATED HERE. A media drive that is not mounted
+// leaves its mountpoint as an empty directory or no directory at all, and a
+// recursive mkdir would build the tree on the root filesystem and fill it. So
+// the root is stat'd first and must already be a directory; everything BELOW
+// it may be made with a recursive mkdir.
+//
+// Every path and fs call carries `turbopackIgnore`: lib/report/onscreen.mjs
+// imports this module, and the app's routes import that (plans/FACTS.md, "A
+// path joined from `process.cwd()` …").
+import { spawn } from "node:child_process";
+import { lstat, mkdir, readdir, readlink, rename, rm, rmdir, stat, statfs, symlink, unlink } from "node:fs/promises";
+import path from "node:path";
+import { MEDIA_ROOT, REPORTS_ROOT, inside, mediaMirror } from "../paths.mjs";
+
+/** The free space a move keeps on the volume it copies to, beyond the copy. */
+export const MOVE_MARGIN_BYTES = 1024 ** 3;
+
+/** The project directories the movers may move. One path segment, never a dot name. */
+const assertName = (name) => {
+ if (typeof name !== "string" || !name || name.startsWith(".") || /[\/\\\0]/.test(name)) {
+ throw new Error(`not a project directory name: ${JSON.stringify(name)}`);
+ }
+};
+
+/** The roots a call works against: the process's, unless a test names its own. */
+function rootsOf(opts = {}) {
+ const reportsRoot = path.resolve(/* turbopackIgnore: true */ opts.reportsRoot ?? REPORTS_ROOT);
+ const mediaRoot = path.resolve(/* turbopackIgnore: true */ opts.mediaRoot ?? MEDIA_ROOT);
+ return { reportsRoot, mediaRoot, tiered: mediaRoot !== reportsRoot };
+}
+
+/**
+ * What a path is RIGHT NOW. lstat, never stat: a dangling link -- the state an
+ * unmounted media drive leaves behind -- reads as absent through stat.
+ * @returns {Promise<{ kind: "missing" } | { kind: "dir" } | { kind: "link", target: string, targetIsDir: boolean } | { kind: "other" }>}
+ */
+export async function pathState(p) {
+ let st;
+ try {
+ st = await lstat(/* turbopackIgnore: true */ p);
+ } catch {
+ return { kind: "missing" };
+ }
+ if (st.isSymbolicLink()) {
+ const raw = await readlink(/* turbopackIgnore: true */ p).catch(() => "");
+ const target = path.resolve(/* turbopackIgnore: true */ path.dirname(/* turbopackIgnore: true */ p), raw);
+ const targetIsDir = await stat(/* turbopackIgnore: true */ p).then((s) => s.isDirectory(), () => false);
+ return { kind: "link", target, targetIsDir };
+ }
+ if (st.isDirectory()) return { kind: "dir" };
+ return { kind: "other" };
+}
+
+/**
+ * Why render scratch cannot go to the media root now, or null when it can (or
+ * when nothing is tiered). The root must already exist as a directory -- it is
+ * never created -- and must neither sit inside REPORTS_ROOT nor contain it (a
+ * mirror inside the tree it mirrors would be walked as projects).
+ */
+export async function mediaRootProblem(opts = {}) {
+ const { reportsRoot, mediaRoot, tiered } = rootsOf(opts);
+ if (!tiered) return null;
+ if (inside(reportsRoot, mediaRoot) || inside(mediaRoot, reportsRoot)) {
+ return `the media root ${mediaRoot} (UMTOOL_MEDIA_DIR) must be outside the reports root ${reportsRoot}, and must not contain it`;
+ }
+ const st = await stat(/* turbopackIgnore: true */ mediaRoot).catch(() => null);
+ if (!st) {
+ return `the media root ${mediaRoot} (UMTOOL_MEDIA_DIR) is not there — is its drive mounted? Nothing was created.`;
+ }
+ if (!st.isDirectory()) return `the media root ${mediaRoot} (UMTOOL_MEDIA_DIR) is not a directory`;
+ return null;
+}
+
+/**
+ * The state of a project's `out`, for a reader that wants to say WHY a build's
+ * files are not there rather than "no build": `dangling` is a link whose target
+ * is gone (the media drive is not mounted).
+ * @returns {Promise<{ state: "absent" | "dir" | "link" | "dangling" | "other", target?: string }>}
+ */
+export async function outDirState(projectDir) {
+ const s = await pathState(path.join(/* turbopackIgnore: true */ projectDir, "out"));
+ if (s.kind === "missing") return { state: "absent" };
+ if (s.kind === "link") return { state: s.targetIsDir ? "link" : "dangling", target: s.target };
+ return { state: s.kind };
+}
+
+/**
+ * Make sure `<projectDir>/out` exists, and return its path.
+ *
+ * a directory -> it, as it is (an untiered project, or one not moved yet)
+ * a link to a directory -> it
+ * a DANGLING link -> throws: the media drive is not there, and writing
+ * through the link would fail anyway -- but a
+ * recursive mkdir under it would fail with a
+ * message about some subdirectory, so this says
+ * what is actually wrong. Nothing is created.
+ * absent, tiered -> `<MEDIA_ROOT>/<project-relative>/out`, then the link
+ * absent, not tiered -> a directory, as every writer always made it
+ *
+ * A project outside REPORTS_ROOT (a hand-run script's `--out` somewhere else)
+ * is never tiered.
+ */
+export async function ensureOutDir(projectDir, opts = {}) {
+ const out = path.join(/* turbopackIgnore: true */ projectDir, "out");
+ const s = await pathState(out);
+ if (s.kind === "dir") return out;
+ if (s.kind === "link") {
+ if (s.targetIsDir) return out;
+ throw new Error(
+ `${out} is a link to ${s.target}, which is not there — is the media drive mounted? ` +
+ `Nothing was written, and nothing was created in its place.`,
+ );
+ }
+ if (s.kind === "other") throw new Error(`${out} exists and is not a directory`);
+
+ // A cut move leaves `out.moved-<ts>` or `out.incoming` beside a missing
+ // `out`. Making a fresh, empty out/ there would let the next move-out mirror
+ // it over the complete media copy (review L5): refuse, and say how to finish.
+ await assertNoLeftovers(projectDir, "out", "nothing was created");
+
+ const roots = rootsOf(opts);
+ const mirror = roots.tiered ? mediaMirror(projectDir, roots) : null;
+ if (!mirror) {
+ await mkdir(/* turbopackIgnore: true */ out, { recursive: true });
+ return out;
+ }
+ const problem = await mediaRootProblem(roots);
+ if (problem) throw new Error(`cannot make ${out}: ${problem}`);
+ // The project must exist before its mirror is made: otherwise the symlink
+ // fails and leaves an empty mirror on the media root (review N5).
+ // stat, not lstat: a project directory may itself be a link (the walk follows them).
+ if (!(await stat(/* turbopackIgnore: true */ projectDir).then((st) => st.isDirectory(), () => false))) {
+ throw new Error(`cannot make ${out}: ${projectDir} is not a directory`);
+ }
+ const target = path.join(/* turbopackIgnore: true */ mirror, "out");
+ // Recursive is safe here: the root itself was just seen to exist.
+ await mkdir(/* turbopackIgnore: true */ target, { recursive: true });
+ try {
+ await symlink(/* turbopackIgnore: true */ target, out, "dir");
+ } catch (e) {
+ // Two writers starting at once: whichever linked first won, and a link (or
+ // directory) that now resolves is as good as ours.
+ if (e?.code !== "EEXIST") throw e;
+ const again = await pathState(out);
+ if (again.kind === "dir" || (again.kind === "link" && again.targetIsDir)) return out;
+ throw e;
+ }
+ return out;
+}
+
+/**
+ * Make a directory a pipeline step writes into, and return it.
+ *
+ * When it is a project's `out` or lies under one (`out/<variant>`,
+ * `out/<variant>/chrome/deck-stills`), that `out` is made by ensureOutDir FIRST
+ * -- a link when tiered -- and only then is the rest made under it. A plain
+ * recursive mkdir of `out/<variant>/segments` was how every script made `out/`,
+ * and it would make a real directory where the link belongs. Any other
+ * directory (a hand-run script's `--out` somewhere else) is made as it always
+ * was.
+ */
+export async function ensureWriteDir(dir) {
+ let d = path.resolve(/* turbopackIgnore: true */ dir);
+ for (let i = 0; i < 4; i++) {
+ if (path.basename(/* turbopackIgnore: true */ d) === "out") {
+ await ensureOutDir(path.dirname(/* turbopackIgnore: true */ d));
+ break;
+ }
+ const up = path.dirname(/* turbopackIgnore: true */ d);
+ if (up === d) break;
+ d = up;
+ }
+ await mkdir(/* turbopackIgnore: true */ dir, { recursive: true });
+ return dir;
+}
+
+// ---------------------------------------------------------------------------
+// Measuring
+// ---------------------------------------------------------------------------
+
+/** Files and bytes under a directory. Follows no symlink. */
+export async function measureTree(dir) {
+ let bytes = 0;
+ let files = 0;
+ const stack = [dir];
+ while (stack.length) {
+ const cur = stack.pop();
+ let entries;
+ try {
+ entries = await readdir(/* turbopackIgnore: true */ cur, { withFileTypes: true });
+ } catch {
+ continue;
+ }
+ for (const e of entries) {
+ const p = path.join(/* turbopackIgnore: true */ cur, e.name);
+ if (e.isDirectory()) stack.push(p);
+ else if (e.isFile()) {
+ const st = await stat(/* turbopackIgnore: true */ p).catch(() => null);
+ if (st) {
+ bytes += st.size;
+ files += 1;
+ }
+ }
+ }
+ }
+ return { bytes, files };
+}
+
+async function freeBytes(dir) {
+ const s = await statfs(/* turbopackIgnore: true */ dir).catch(() => null);
+ return s ? Number(s.bavail) * Number(s.bsize) : null;
+}
+
+async function assertRoom(dir, bytes) {
+ const free = await freeBytes(dir);
+ if (free !== null && free < bytes + MOVE_MARGIN_BYTES) {
+ throw new Error(
+ `not enough room on ${dir}: ${gb(free)} free, the copy needs ${gb(bytes)} plus ${gb(MOVE_MARGIN_BYTES)} to spare. Nothing moved.`,
+ );
+ }
+}
+
+const gb = (n) => `${(n / 1024 ** 3).toFixed(2)} GB`;
+
+// ---------------------------------------------------------------------------
+// rsync
+// ---------------------------------------------------------------------------
+
+/** `rsync <args> <src>/ <dest>/` -- the trailing slashes copy the CONTENTS. */
+function rsync(bin, args, src, dest, log) {
+ return new Promise((resolve, reject) => {
+ const argv = [...args, `${src}/`, `${dest}/`];
+ log(`$ ${bin} ${argv.join(" ")}`);
+ const child = spawn(bin, argv, { stdio: ["ignore", "pipe", "pipe"] });
+ let output = "";
+ child.stdout.on("data", (c) => (output += c));
+ child.stderr.on("data", (c) => (output += c));
+ child.on("error", reject);
+ child.on("close", (code) => resolve({ code: code ?? 1, output }));
+ });
+}
+
+// What a dry run itemizes that is not a difference: rsync's own chatter.
+const driftLines = (output) =>
+ output
+ .split("\n")
+ .map((l) => l.trim())
+ .filter((l) => l && !l.startsWith("sending incremental") && !/^(sent|total size)/.test(l));
+
+/**
+ * COPY, MIRROR, VERIFY -- the source is not touched by any of it.
+ *
+ * 1. `rsync -a --partial`: the bytes, resumable.
+ * 2. `rsync -a --delete --info=del` toward the COPY: what changed on the
+ * source meanwhile is re-sent, what it no longer has leaves the copy
+ * (each removal in the log). Never pointed the other way: the copy is
+ * checked to be neither the source nor inside it, nor around it.
+ * 3. `rsync -a --dry-run --itemize-changes --delete` must list nothing, and
+ * the two trees must measure the same. One more mirror pass if the source
+ * moved under the check; a second difference refuses.
+ */
+async function copyMirrorVerify(src, dest, { rsyncBin, log }) {
+ if (inside(src, dest) || inside(dest, src)) {
+ throw new Error(`refusing to copy ${src} into ${dest}: one is inside the other. Nothing moved.`);
+ }
+ const copied = await rsync(rsyncBin, ["-a", "--partial"], src, dest, log);
+ if (copied.code !== 0) throw new Error(`rsync failed (exit ${copied.code}): ${copied.output.trim().split("\n").pop() ?? ""}`);
+ for (let pass = 0; ; pass++) {
+ const mirrored = await rsync(rsyncBin, ["-a", "--delete", "--info=del"], src, dest, log);
+ if (mirrored.output.trim()) log(mirrored.output.trim());
+ if (mirrored.code !== 0) throw new Error(`the mirror pass failed (exit ${mirrored.code}). The source has NOT been touched.`);
+ const check = await rsync(rsyncBin, ["-a", "--dry-run", "--itemize-changes", "--delete"], src, dest, () => {});
+ if (check.code !== 0) throw new Error(`the verify failed (exit ${check.code}). The source has NOT been touched.`);
+ const drift = driftLines(check.output);
+ if (!drift.length) break;
+ if (pass >= 1) {
+ throw new Error(
+ `the copy still differs from the source after a second mirror pass (${drift.length} item(s), first: ${drift[0]}) — ` +
+ `something is still writing into ${src}. Stop it and run the move again. The source has NOT been touched.`,
+ );
+ }
+ log(`the source changed during the copy (${drift.length} item(s)) — one more mirror pass`);
+ }
+ const [a, b] = await Promise.all([measureTree(src), measureTree(dest)]);
+ if (a.files !== b.files || a.bytes !== b.bytes) {
+ throw new Error(
+ `the copy does not measure the same: ${a.files} file(s)/${a.bytes} B against ${b.files}/${b.bytes} B. The source has NOT been touched.`,
+ );
+ }
+ return a;
+}
+
+// ---------------------------------------------------------------------------
+// The movers
+// ---------------------------------------------------------------------------
+
+const stamp = () => new Date().toISOString().replace(/[-:]/g, "").replace(/\.\d+Z$/, "Z");
+
+/** `<name>.moved-<stamp>` siblings: a move-out parked them and was cut before deleting. */
+async function parkedOf(projectDir, name) {
+ const names = await readdir(/* turbopackIgnore: true */ projectDir).catch(() => []);
+ return names
+ .filter((n) => n.startsWith(`${name}.moved-`))
+ .sort()
+ .map((n) => path.join(/* turbopackIgnore: true */ projectDir, n));
+}
+
+/** `<name>.incoming`, when a move-back left it (a cut between its copy and its rename). */
+async function incomingOf(projectDir, name) {
+ const p = path.join(/* turbopackIgnore: true */ projectDir, `${name}.incoming`);
+ return (await pathState(p)).kind === "dir" ? p : null;
+}
+
+/** Every leftover of a cut move of `<name>`: parked copies and an incoming copy. */
+export async function leftoversOf(projectDir, name) {
+ const incoming = await incomingOf(projectDir, name);
+ return [...(await parkedOf(projectDir, name)), ...(incoming ? [incoming] : [])];
+}
+
+/**
+ * Refuse while a cut move of `<name>` has left something behind. A writer that
+ * made a fresh `<name>/` there, and a move that then mirrored it over the
+ * complete copy, would lose everything but the leftover nobody names.
+ */
+async function assertNoLeftovers(projectDir, name, what, { beside = false } = {}) {
+ const left = await leftoversOf(projectDir, name);
+ if (!left.length) return;
+ const names = left.map((p) => path.basename(/* turbopackIgnore: true */ p)).join(", ");
+ if (beside) {
+ // A real `<name>/` AND a leftover: running a move again would only refuse
+ // again (re-review R1). Only a person can say which one to keep.
+ throw new Error(
+ `${name}/ and ${names} both exist in ${projectDir}: a move of ${name}/ was cut, and something made a fresh ${name}/ since. ` +
+ `The leftover holds the moved data. Keep one and remove the other by hand, then run the move; ${what}`,
+ );
+ }
+ throw new Error(
+ `a move of ${name}/ in ${projectDir} was cut (left: ${left.map((p) => path.basename(/* turbopackIgnore: true */ p)).join(", ")}) — ` +
+ `run \`umtool storage ${left.some((p) => p.endsWith(".incoming")) ? "move-back" : "move-out"} <project>\` to finish it; ${what}`,
+ );
+}
+
+const defaults = (opts) => ({
+ rsyncBin: opts.rsyncBin ?? process.env.RSYNC_BIN ?? "rsync",
+ log: opts.log ?? (() => {}),
+ dryRun: !!opts.dryRun,
+});
+
+/**
+ * Move `<projectDir>/<name>` to the same project-relative path under the media
+ * root and leave an absolute link in its place.
+ *
+ * Dispatched on the disk, so running it again finishes a run that was cut:
+ *
+ * a link to the mirror -> "already" (and a parked copy left by a cut
+ * between the link and its deletion is removed)
+ * a link anywhere else -> refused; it is not this media root's
+ * absent, a parked copy -> the cut was between the park and the link
+ * (the copy had verified): link, delete the parked copy
+ * absent, nothing parked -> "absent", nothing to move
+ * a directory -> copy, mirror, verify, rename it to
+ * `<name>.moved-<stamp>`, link, delete the parked copy
+ * a directory + a leftover -> refused: a parked copy or `<name>.incoming`
+ * beside a real directory means a writer made a
+ * fresh one after a cut move (review L5)
+ * absent + `<name>.incoming` -> refused: a cut move-back is finished by move-back
+ *
+ * `dryRun` measures and changes nothing. Nothing in the project may be writing
+ * into `<name>` while it runs (the app's jobs are in its own memory, so a CLI
+ * caller cannot see them); the verify refuses if the tree keeps changing.
+ *
+ * @param {string} projectDir
+ * @param {string} name one path segment: "out", "clips", "share-<x>"
+ * @param {{ dryRun?: boolean, log?: (m: string) => void, rsyncBin?: string, reportsRoot?: string, mediaRoot?: string }} [opts]
+ * @returns {Promise<{ state: "moved" | "already" | "absent" | "would-move" | "would-finish", src: string, dest: string, bytes?: number, files?: number, resumed?: boolean }>}
+ */
+export async function moveDirToMedia(projectDir, name, opts = {}) {
+ assertName(name);
+ const { rsyncBin, log, dryRun } = defaults(opts);
+ const roots = rootsOf(opts);
+ if (!roots.tiered) {
+ throw new Error(`UMTOOL_MEDIA_DIR is not set: there is no media root to move ${name}/ to`);
+ }
+ const mirror = mediaMirror(projectDir, roots);
+ if (!mirror) throw new Error(`${projectDir} is not under the reports root ${roots.reportsRoot}`);
+ const src = path.join(/* turbopackIgnore: true */ projectDir, name);
+ const dest = path.join(/* turbopackIgnore: true */ mirror, name);
+ const s = await pathState(src);
+
+ if (s.kind === "link") {
+ if (s.target !== dest) {
+ throw new Error(`${src} is already a link, to ${s.target} — not to ${dest}. Nothing moved.`);
+ }
+ const parked = await parkedOf(projectDir, name);
+ if (!dryRun && s.targetIsDir) {
+ for (const p of parked) await rm(/* turbopackIgnore: true */ p, { recursive: true, force: true });
+ }
+ return { state: "already", src, dest };
+ }
+ if (s.kind === "other") throw new Error(`${src} is not a directory. Nothing moved.`);
+
+ if (s.kind === "missing") {
+ const parked = await parkedOf(projectDir, name);
+ if (!parked.length) {
+ // A cut move-BACK is finished by move-back, never overtaken by a move-out.
+ if (await incomingOf(projectDir, name)) await assertNoLeftovers(projectDir, name, "nothing moved");
+ return { state: "absent", src, dest };
+ }
+ if (parked.length > 1) {
+ throw new Error(`${src} is missing and there are ${parked.length} parked copies (${parked.join(", ")}) — settle them by hand`);
+ }
+ // A parked copy exists only after the copy verified, so the media side is
+ // complete -- provided it is reachable.
+ if ((await pathState(dest)).kind !== "dir") {
+ throw new Error(`${src} was parked at ${parked[0]}, but ${dest} is not there — is the media drive mounted? Nothing changed.`);
+ }
+ if (dryRun) return { state: "would-finish", src, dest };
+ log(`finishing a cut move: linking ${src} -> ${dest}`);
+ await symlink(/* turbopackIgnore: true */ dest, src, "dir");
+ await rm(/* turbopackIgnore: true */ parked[0], { recursive: true, force: true });
+ return { state: "moved", src, dest, resumed: true };
+ }
+
+ // A real directory: the move itself -- unless a cut move left something
+ // behind, in which case this directory is a writer's fresh one.
+ await assertNoLeftovers(projectDir, name, "nothing moved", { beside: true });
+ const problem = await mediaRootProblem(roots);
+ if (problem) throw new Error(`cannot move ${src}: ${problem}`);
+ const measured = await measureTree(src);
+ if (dryRun) return { state: "would-move", src, dest, ...measured };
+ await assertRoom(roots.mediaRoot, measured.bytes);
+ await mkdir(/* turbopackIgnore: true */ dest, { recursive: true });
+ const verified = await copyMirrorVerify(src, dest, { rsyncBin, log });
+ const parked = path.join(/* turbopackIgnore: true */ projectDir, `${name}.moved-${stamp()}`);
+ await rename(/* turbopackIgnore: true */ src, parked);
+ await symlink(/* turbopackIgnore: true */ dest, src, "dir");
+ await rm(/* turbopackIgnore: true */ parked, { recursive: true, force: true });
+ log(`moved ${src} -> ${dest} (${verified.files} file(s), ${gb(verified.bytes)})`);
+ return { state: "moved", src, dest, ...verified };
+}
+
+/**
+ * The reverse: `<projectDir>/<name>`, a link into the media root, becomes a
+ * real directory in the project again, and the media copy is deleted.
+ *
+ * a directory -> "already"
+ * absent, `<name>.incoming` -> the cut was between removing the link and
+ * renaming the verified copy: rename it, and
+ * report (never delete) the project's mirror
+ * absent, nothing incoming -> "absent"
+ * a link whose target is gone -> refused: the drive is not mounted
+ * a link -> copy the target into `<name>.incoming`,
+ * mirror, verify, remove the link, rename,
+ * then delete the copy only when it is this
+ * project's own mirror under a tiered media
+ * root (and the directories above it the move
+ * left empty, never the root); any other
+ * target is left and reported (mediaCopyLeft)
+ * a parked `<name>.moved-*` -> refused: a cut move-out is finished first
+ * a directory + a leftover -> refused (a writer's fresh directory)
+ *
+ * @param {string} projectDir
+ * @param {string} name
+ * @param {{ dryRun?: boolean, log?: (m: string) => void, rsyncBin?: string, reportsRoot?: string, mediaRoot?: string }} [opts]
+ * @returns {Promise<{ state: "moved" | "already" | "absent" | "would-move" | "would-finish", src: string, from?: string, bytes?: number, files?: number, resumed?: boolean }>}
+ */
+export async function moveDirToLocal(projectDir, name, opts = {}) {
+ assertName(name);
+ const { rsyncBin, log, dryRun } = defaults(opts);
+ const roots = rootsOf(opts);
+ const src = path.join(/* turbopackIgnore: true */ projectDir, name);
+ const incoming = path.join(/* turbopackIgnore: true */ projectDir, `${name}.incoming`);
+ const s = await pathState(src);
+
+ const mirror = roots.tiered ? mediaMirror(projectDir, roots) : null;
+ const ownCopy = mirror ? path.join(/* turbopackIgnore: true */ mirror, name) : null;
+ if (s.kind === "dir") {
+ await assertNoLeftovers(projectDir, name, "nothing moved", { beside: true });
+ // A move-back cut after its rename leaves the media copy behind (review
+ // L4): report it, never delete it blind.
+ const left = ownCopy && (await pathState(ownCopy)).kind === "dir" ? ownCopy : undefined;
+ return { state: "already", src, ...(left ? { mediaCopyLeft: left } : {}) };
+ }
+ if (s.kind === "other") throw new Error(`${src} is not a directory. Nothing moved.`);
+ // A cut move-OUT is finished by move-out first.
+ if ((await parkedOf(projectDir, name)).length) await assertNoLeftovers(projectDir, name, "nothing moved");
+
+ if (s.kind === "missing") {
+ if ((await pathState(incoming)).kind !== "dir") return { state: "absent", src };
+ if (dryRun) return { state: "would-finish", src };
+ // `.incoming` outlives the link only once it verified.
+ log(`finishing a cut move: ${incoming} -> ${src}`);
+ await rename(/* turbopackIgnore: true */ incoming, src);
+ // The link is gone, so what was copied cannot be told from the disk: the
+ // project's own mirror is reported, never deleted (review L3).
+ const left = ownCopy && (await pathState(ownCopy)).kind === "dir" ? ownCopy : undefined;
+ return { state: "moved", src, resumed: true, ...(left ? { mediaCopyLeft: left } : {}) };
+ }
+
+ // A link.
+ const from = s.target;
+ if (!s.targetIsDir) {
+ throw new Error(`${src} is a link to ${from}, which is not there — is the media drive mounted? Nothing moved.`);
+ }
+ const measured = await measureTree(from);
+ if (dryRun) return { state: "would-move", src, from, ...measured };
+ await assertRoom(projectDir, measured.bytes);
+ await mkdir(/* turbopackIgnore: true */ incoming, { recursive: true });
+ const verified = await copyMirrorVerify(from, incoming, { rsyncBin, log });
+ await unlink(/* turbopackIgnore: true */ src); // the link, not what it points at
+ await rename(/* turbopackIgnore: true */ incoming, src);
+ const left = await dropMediaCopy(from, ownCopy, roots.mediaRoot);
+ if (left) log(`left ${left} in place: it is not this project's own copy on the media root`);
+ log(`moved ${from} -> ${src} (${verified.files} file(s), ${gb(verified.bytes)})`);
+ return { state: "moved", src, from, ...verified, ...(left ? { mediaCopyLeft: left } : {}) };
+}
+
+/**
+ * Delete a media copy that has been brought home, then every directory above
+ * it the move left empty, stopping at -- never removing -- the media root.
+ *
+ * ONLY this project's own mirror (`ownCopy`, null when nothing is tiered), and
+ * only when the copy that came home IS that mirror (review L3). Without
+ * UMTOOL_MEDIA_DIR the "media root" would be the reports root itself, and a
+ * hand-made `out` link into another project's real out/ would be deleted after
+ * the copy. Anything else is left where it is, and its path returned so the
+ * caller can say so.
+ * @returns {Promise<string | undefined>} the path left in place, if any
+ */
+async function dropMediaCopy(from, ownCopy, mediaRoot) {
+ if (!ownCopy || from !== ownCopy || !inside(mediaRoot, from) || from === mediaRoot) return from;
+ if ((await pathState(from)).kind !== "dir") return undefined;
+ await rm(/* turbopackIgnore: true */ from, { recursive: true, force: true });
+ for (let d = path.dirname(/* turbopackIgnore: true */ from); d !== mediaRoot && inside(mediaRoot, d); d = path.dirname(/* turbopackIgnore: true */ d)) {
+ try {
+ await rmdir(/* turbopackIgnore: true */ d);
+ } catch {
+ break; // not empty: something else lives there
+ }
+ }
+ return undefined;
+}
diff --git a/umtool/lib/report/storage.test.mjs b/umtool/lib/report/storage.test.mjs
@@ -0,0 +1,469 @@
+// A project's render scratch on a media root (release 17): the roots in
+// lib/paths.mjs, ensureOutDir / ensureWriteDir, the two movers, and the walk's
+// skip rules for a media drive that is not there.
+//
+// Every call here names its own roots (`reportsRoot`, `mediaRoot`), so nothing
+// depends on, or touches, the process's REPORTS_ROOT. The env-derived roots are
+// tested in a child process, where the module is evaluated fresh.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { lstat, mkdir, mkdtemp, readFile, readdir, readlink, rename, rm, symlink, writeFile } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+import { mediaMirror } from "../paths.mjs";
+import { SKIP_DIRS, skipsDir } from "../projects/kinds.mjs";
+import { walkProjects } from "../projects/walk.mjs";
+import {
+ ensureOutDir,
+ ensureWriteDir,
+ mediaRootProblem,
+ moveDirToLocal,
+ moveDirToMedia,
+ outDirState,
+} from "./storage.mjs";
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+
+/** A reports root with one project, and a media root beside it (not inside). */
+async function world({ withOut = true } = {}) {
+ const base = await mkdtemp(path.join(tmpdir(), "umtool-storage-"));
+ const reportsRoot = path.join(base, "reports");
+ const mediaRoot = path.join(base, "media");
+ const projectDir = path.join(reportsRoot, "folder", "proj");
+ await mkdir(projectDir, { recursive: true });
+ await mkdir(mediaRoot);
+ await writeFile(path.join(projectDir, "video.manifest.json"), "{}\n");
+ if (withOut) {
+ await mkdir(path.join(projectDir, "out", "clips-raw"), { recursive: true });
+ await writeFile(path.join(projectDir, "out", "proj.mp4"), Buffer.alloc(4096, 1));
+ await writeFile(path.join(projectDir, "out", "clips-raw", "v_0-9.mp4"), Buffer.alloc(2048, 2));
+ }
+ const roots = { reportsRoot, mediaRoot };
+ const mirror = path.join(mediaRoot, "folder", "proj");
+ return { base, reportsRoot, mediaRoot, projectDir, roots, mirror, done: () => rm(base, { recursive: true, force: true }) };
+}
+
+const kind = async (p) => {
+ const st = await lstat(p).catch(() => null);
+ if (!st) return "missing";
+ return st.isSymbolicLink() ? "link" : st.isDirectory() ? "dir" : "file";
+};
+
+// ---------------------------------------------------------------------------
+// The roots
+// ---------------------------------------------------------------------------
+
+/** lib/paths.mjs's values under a given environment, evaluated fresh. */
+function rootsUnder(envPatch) {
+ const env = { ...process.env };
+ for (const k of ["UMTOOL_MEDIA_DIR", "UMTOOL_CACHE_DIR", "UMTOOL_INDEX_DIR", "XDG_CACHE_HOME", "MIX_ROOTS", "MIX_WRITE_ROOTS"]) delete env[k];
+ Object.assign(env, { REPORTS_DIR: "/r/reports", SONG_DIR: "/r/song-data-that-is-not-there" }, envPatch);
+ const code =
+ "const m = await import(process.argv[1]);" +
+ "console.log(JSON.stringify({ media: m.MEDIA_ROOT, tiered: m.MEDIA_TIERED, reports: m.REPORTS_ROOT, cache: m.CACHE_DIR," +
+ " old: m.OLD_CACHE_DIR, index: m.INDEX_DIR, read: m.READ_ROOTS, write: m.WRITE_ROOTS }));";
+ const out = execFileSync(process.execPath, ["--input-type=module", "-e", code, path.join(HERE, "..", "paths.mjs")], {
+ env,
+ encoding: "utf8",
+ });
+ return JSON.parse(out);
+}
+
+test("MEDIA_ROOT unset is REPORTS_ROOT: nothing tiered, no new root", () => {
+ const r = rootsUnder({});
+ assert.equal(r.media, "/r/reports");
+ assert.equal(r.tiered, false);
+ assert.equal(r.read.filter((p) => p === "/r/reports").length, 1);
+});
+
+test("UMTOOL_MEDIA_DIR is a READ root and never a write root", () => {
+ const r = rootsUnder({ UMTOOL_MEDIA_DIR: "/m/umtool" });
+ assert.equal(r.media, "/m/umtool");
+ assert.equal(r.tiered, true);
+ assert.ok(r.read.includes("/m/umtool"));
+ assert.ok(!r.write.includes("/m/umtool"));
+});
+
+test("CACHE_DIR: UMTOOL_CACHE_DIR, else XDG_CACHE_HOME/archilyzer/umtool, else ~/.cache — never SONG_DATA", () => {
+ assert.equal(rootsUnder({ UMTOOL_CACHE_DIR: "/c/u" }).cache, "/c/u");
+ assert.equal(rootsUnder({ XDG_CACHE_HOME: "/x" }).cache, "/x/archilyzer/umtool");
+ const plain = rootsUnder({ XDG_CACHE_HOME: "" });
+ assert.equal(plain.cache, path.join(homedir(), ".cache", "archilyzer", "umtool"));
+ assert.equal(plain.index, path.join(plain.cache, "index"));
+ assert.equal(plain.old, "/r/song-data-that-is-not-there/.cache/umtool");
+ assert.ok(!plain.cache.startsWith("/r/song-data"));
+});
+
+test("mediaMirror: the project-relative path under the media root, or null outside the reports root", () => {
+ const roots = { reportsRoot: "/r", mediaRoot: "/m" };
+ assert.equal(mediaMirror("/r/a/b", roots), "/m/a/b");
+ assert.equal(mediaMirror("/r", roots), null);
+ assert.equal(mediaMirror("/elsewhere/p", roots), null);
+ assert.equal(mediaMirror("/r-sibling/p", roots), null);
+});
+
+// ---------------------------------------------------------------------------
+// ensureOutDir / ensureWriteDir
+// ---------------------------------------------------------------------------
+
+test("ensureOutDir, not tiered: a plain directory, as every writer made it", async () => {
+ const w = await world({ withOut: false });
+ try {
+ const out = await ensureOutDir(w.projectDir, { reportsRoot: w.reportsRoot, mediaRoot: w.reportsRoot });
+ assert.equal(await kind(out), "dir");
+ assert.deepEqual(await readdir(w.mediaRoot), []);
+ } finally {
+ await w.done();
+ }
+});
+
+test("ensureOutDir, tiered and absent: the mirror is made under the media root and linked", async () => {
+ const w = await world({ withOut: false });
+ try {
+ const out = await ensureOutDir(w.projectDir, w.roots);
+ assert.equal(await kind(out), "link");
+ assert.equal(await readlink(out), path.join(w.mirror, "out"));
+ assert.equal(await kind(path.join(w.mirror, "out")), "dir");
+ assert.deepEqual(await outDirState(w.projectDir), { state: "link", target: path.join(w.mirror, "out") });
+ // Again: the link is kept as it is.
+ assert.equal(await ensureOutDir(w.projectDir, w.roots), out);
+ } finally {
+ await w.done();
+ }
+});
+
+test("ensureOutDir keeps an existing real out/ even when tiered (move-out moves it, not the writer)", async () => {
+ const w = await world();
+ try {
+ await ensureOutDir(w.projectDir, w.roots);
+ assert.equal(await kind(path.join(w.projectDir, "out")), "dir");
+ assert.deepEqual(await readdir(w.mediaRoot), []);
+ } finally {
+ await w.done();
+ }
+});
+
+test("ensureOutDir never creates the media root: an unplugged drive refuses and nothing is made", async () => {
+ const w = await world({ withOut: false });
+ try {
+ await rm(w.mediaRoot, { recursive: true });
+ await assert.rejects(ensureOutDir(w.projectDir, w.roots), /is not there — is its drive mounted\? Nothing was created/);
+ assert.equal(existsSync(w.mediaRoot), false);
+ assert.equal(await kind(path.join(w.projectDir, "out")), "missing");
+ } finally {
+ await w.done();
+ }
+});
+
+test("a dangling out link refuses loudly; nothing is materialised in its place or under it", async () => {
+ const w = await world({ withOut: false });
+ try {
+ await ensureOutDir(w.projectDir, w.roots);
+ await rename(w.mediaRoot, `${w.mediaRoot}.unplugged`);
+ assert.equal((await outDirState(w.projectDir)).state, "dangling");
+ await assert.rejects(ensureOutDir(w.projectDir, w.roots), /is the media drive mounted\?/);
+ await assert.rejects(ensureWriteDir(path.join(w.projectDir, "out", "sourced", "segments")), /is the media drive mounted\?/);
+ // And a plain recursive mkdir through the link fails too (ENOTDIR), making nothing.
+ await assert.rejects(mkdir(path.join(w.projectDir, "out", "clips-raw"), { recursive: true }));
+ assert.equal(await kind(path.join(w.projectDir, "out")), "link");
+ assert.equal(existsSync(w.mediaRoot), false);
+ } finally {
+ await w.done();
+ }
+});
+
+test("a media root inside the reports root (or around it) is refused", async () => {
+ const w = await world({ withOut: false });
+ try {
+ const inner = { reportsRoot: w.reportsRoot, mediaRoot: path.join(w.reportsRoot, "media") };
+ await mkdir(inner.mediaRoot);
+ assert.match(await mediaRootProblem(inner), /must be outside the reports root/);
+ assert.match(await mediaRootProblem({ reportsRoot: w.reportsRoot, mediaRoot: w.base }), /must be outside/);
+ assert.equal(await mediaRootProblem(w.roots), null);
+ assert.equal(await mediaRootProblem({ reportsRoot: w.reportsRoot, mediaRoot: w.reportsRoot }), null);
+ } finally {
+ await w.done();
+ }
+});
+
+test("ensureWriteDir makes a project's out/ through ensureOutDir before anything under it", async () => {
+ const w = await world({ withOut: false });
+ try {
+ // ensureWriteDir uses the process's roots; under the default (no
+ // UMTOOL_MEDIA_DIR in the test environment) it is a plain directory, the
+ // old behaviour. The tiered case is ensureOutDir's, tested above, and the
+ // storage e2e drives the pipeline scripts through it.
+ const deep = path.join(w.projectDir, "out", "sourced", "chrome", "deck-stills");
+ assert.equal(await ensureWriteDir(deep), deep);
+ assert.equal(await kind(deep), "dir");
+ // A directory that is not under any out/ is made as it always was.
+ const other = path.join(w.base, "elsewhere", "x");
+ await ensureWriteDir(other);
+ assert.equal(await kind(other), "dir");
+ } finally {
+ await w.done();
+ }
+});
+
+// ---------------------------------------------------------------------------
+// The movers
+// ---------------------------------------------------------------------------
+
+test("moveDirToMedia: copy, verify, link; the bytes are the same and nothing is left parked", async () => {
+ const w = await world();
+ try {
+ const logs = [];
+ const r = await moveDirToMedia(w.projectDir, "out", { ...w.roots, log: (m) => logs.push(m) });
+ assert.equal(r.state, "moved");
+ assert.equal(r.files, 2);
+ assert.equal(r.bytes, 4096 + 2048);
+ const out = path.join(w.projectDir, "out");
+ assert.equal(await kind(out), "link");
+ assert.equal(await readlink(out), path.join(w.mirror, "out"));
+ assert.deepEqual(await readFile(path.join(out, "proj.mp4")), Buffer.alloc(4096, 1));
+ assert.deepEqual((await readdir(w.projectDir)).sort(), ["out", "video.manifest.json"]);
+ assert.ok(logs.some((l) => l.includes("--delete")), "the mirror pass ran");
+ // Again: nothing to do.
+ assert.equal((await moveDirToMedia(w.projectDir, "out", w.roots)).state, "already");
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDirToMedia --dry-run measures and changes nothing", async () => {
+ const w = await world();
+ try {
+ const r = await moveDirToMedia(w.projectDir, "out", { ...w.roots, dryRun: true });
+ assert.equal(r.state, "would-move");
+ assert.equal(r.bytes, 6144);
+ assert.equal(await kind(path.join(w.projectDir, "out")), "dir");
+ assert.deepEqual(await readdir(w.mediaRoot), []);
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDirToMedia refuses without a media root, and on a link that is not its own", async () => {
+ const w = await world();
+ try {
+ await assert.rejects(
+ moveDirToMedia(w.projectDir, "out", { reportsRoot: w.reportsRoot, mediaRoot: w.reportsRoot }),
+ /UMTOOL_MEDIA_DIR is not set/,
+ );
+ await assert.rejects(moveDirToMedia(w.projectDir, "../x", w.roots), /not a project directory name/);
+ const other = path.join(w.base, "other");
+ await mkdir(other);
+ await symlink(other, path.join(w.projectDir, "clips"));
+ await assert.rejects(moveDirToMedia(w.projectDir, "clips", w.roots), /already a link, to .* not to/);
+ assert.equal((await moveDirToMedia(w.projectDir, "share-none", w.roots)).state, "absent");
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDirToMedia: a failed copy leaves the source untouched", async () => {
+ const w = await world();
+ try {
+ await assert.rejects(moveDirToMedia(w.projectDir, "out", { ...w.roots, rsyncBin: "false" }), /rsync failed/);
+ assert.equal(await kind(path.join(w.projectDir, "out")), "dir");
+ assert.deepEqual(await readFile(path.join(w.projectDir, "out", "proj.mp4")), Buffer.alloc(4096, 1));
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDirToMedia finishes a run cut between the park and the link", async () => {
+ const w = await world();
+ try {
+ // What a cut leaves: the verified copy on the media root, the source parked.
+ await mkdir(path.join(w.mirror), { recursive: true });
+ execFileSync("cp", ["-a", path.join(w.projectDir, "out"), path.join(w.mirror, "out")]);
+ await rename(path.join(w.projectDir, "out"), path.join(w.projectDir, "out.moved-20261001T000000Z"));
+ const r = await moveDirToMedia(w.projectDir, "out", w.roots);
+ assert.equal(r.state, "moved");
+ assert.equal(r.resumed, true);
+ assert.equal(await kind(path.join(w.projectDir, "out")), "link");
+ assert.deepEqual((await readdir(w.projectDir)).sort(), ["out", "video.manifest.json"]);
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDirToMedia removes a parked copy left by a cut after the link", async () => {
+ const w = await world();
+ try {
+ await moveDirToMedia(w.projectDir, "out", w.roots);
+ await mkdir(path.join(w.projectDir, "out.moved-20261001T000000Z"));
+ assert.equal((await moveDirToMedia(w.projectDir, "out", w.roots)).state, "already");
+ assert.deepEqual((await readdir(w.projectDir)).sort(), ["out", "video.manifest.json"]);
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDirToLocal: a real directory again, the media copy and its empty parents gone, the root kept", async () => {
+ const w = await world();
+ try {
+ await moveDirToMedia(w.projectDir, "out", w.roots);
+ const r = await moveDirToLocal(w.projectDir, "out", w.roots);
+ assert.equal(r.state, "moved");
+ assert.equal(r.bytes, 6144);
+ const out = path.join(w.projectDir, "out");
+ assert.equal(await kind(out), "dir");
+ assert.deepEqual(await readFile(path.join(out, "clips-raw", "v_0-9.mp4")), Buffer.alloc(2048, 2));
+ assert.deepEqual((await readdir(w.projectDir)).sort(), ["out", "video.manifest.json"]);
+ assert.equal(existsSync(path.join(w.mediaRoot, "folder")), false);
+ assert.equal(existsSync(w.mediaRoot), true);
+ assert.equal((await moveDirToLocal(w.projectDir, "out", w.roots)).state, "already");
+ } finally {
+ await w.done();
+ }
+});
+
+test("moveDirToLocal refuses a dangling link and finishes a cut rename", async () => {
+ const w = await world();
+ try {
+ await moveDirToMedia(w.projectDir, "out", w.roots);
+ await rename(w.mediaRoot, `${w.mediaRoot}.unplugged`);
+ await assert.rejects(moveDirToLocal(w.projectDir, "out", w.roots), /is the media drive mounted\? Nothing moved/);
+ assert.equal(await kind(path.join(w.projectDir, "out")), "link");
+ await rename(`${w.mediaRoot}.unplugged`, w.mediaRoot);
+
+ // A cut between removing the link and renaming the verified copy. The link
+ // is gone, so what was copied cannot be told: the mirror is reported, kept.
+ execFileSync("cp", ["-a", path.join(w.mirror, "out"), path.join(w.projectDir, "out.incoming")]);
+ await rm(path.join(w.projectDir, "out"));
+ const r = await moveDirToLocal(w.projectDir, "out", w.roots);
+ assert.equal(r.state, "moved");
+ assert.equal(r.resumed, true);
+ assert.equal(await kind(path.join(w.projectDir, "out")), "dir");
+ assert.equal(r.mediaCopyLeft, path.join(w.mirror, "out"));
+ assert.equal(existsSync(path.join(w.mirror, "out")), true);
+ // And from then on "already" keeps saying so (review L4).
+ const again = await moveDirToLocal(w.projectDir, "out", w.roots);
+ assert.equal(again.state, "already");
+ assert.equal(again.mediaCopyLeft, path.join(w.mirror, "out"));
+ } finally {
+ await w.done();
+ }
+});
+
+// ---------------------------------------------------------------------------
+// The review's fixes (L3, L5, N5)
+// ---------------------------------------------------------------------------
+
+test("move-back without a media root never deletes what the link pointed at (L3)", async () => {
+ const w = await world({ withOut: false });
+ try {
+ // Another project's REAL out/, and a hand-made link to it.
+ const other = path.join(w.reportsRoot, "other", "out");
+ await mkdir(other, { recursive: true });
+ await writeFile(path.join(other, "keep.mp4"), Buffer.alloc(1024, 3));
+ await symlink(other, path.join(w.projectDir, "out"));
+ const untiered = { reportsRoot: w.reportsRoot, mediaRoot: w.reportsRoot };
+ const r = await moveDirToLocal(w.projectDir, "out", untiered);
+ assert.equal(r.state, "moved");
+ assert.equal(r.mediaCopyLeft, other);
+ assert.equal(await kind(path.join(w.projectDir, "out")), "dir");
+ assert.deepEqual(await readFile(path.join(other, "keep.mp4")), Buffer.alloc(1024, 3));
+ } finally {
+ await w.done();
+ }
+});
+
+test("a tiered move-back of a link that is not the project's own mirror leaves the target (L3)", async () => {
+ const w = await world({ withOut: false });
+ try {
+ const elsewhere = path.join(w.mediaRoot, "someone-else", "out");
+ await mkdir(elsewhere, { recursive: true });
+ await writeFile(path.join(elsewhere, "x.mp4"), Buffer.alloc(512, 4));
+ await symlink(elsewhere, path.join(w.projectDir, "out"));
+ const r = await moveDirToLocal(w.projectDir, "out", w.roots);
+ assert.equal(r.mediaCopyLeft, elsewhere);
+ assert.equal(existsSync(path.join(elsewhere, "x.mp4")), true);
+ } finally {
+ await w.done();
+ }
+});
+
+test("a cut move's leftovers are a guard: no fresh out/, no move over them (L5)", async () => {
+ const w = await world({ withOut: false });
+ try {
+ // A move-out cut after the park: out/ is missing, the parked copy is the data.
+ const parked = path.join(w.projectDir, "out.moved-20261001T000000Z");
+ await mkdir(parked);
+ await writeFile(path.join(parked, "proj.mp4"), Buffer.alloc(4096, 1));
+ for (const roots of [w.roots, { reportsRoot: w.reportsRoot, mediaRoot: w.reportsRoot }]) {
+ await assert.rejects(ensureOutDir(w.projectDir, roots), /was cut \(left: out\.moved-20261001T000000Z\).*move-out <project>.*nothing was created/);
+ }
+ assert.equal(await kind(path.join(w.projectDir, "out")), "missing");
+ // A writer that made a fresh out/ anyway (an older binary): move-out refuses to mirror it over.
+ await mkdir(path.join(w.projectDir, "out"));
+ // Both exist: neither move sends the person to the other, which would only
+ // refuse again (re-review R1); the sentence says what to do by hand.
+ for (const move of [moveDirToMedia, moveDirToLocal]) {
+ await assert.rejects(
+ move(w.projectDir, "out", w.roots),
+ /out\/ and out\.moved-20261001T000000Z both exist .*The leftover holds the moved data\. Keep one and remove the other by hand, then run the move; nothing moved/,
+ );
+ }
+ assert.deepEqual(await readdir(w.mediaRoot), []);
+ await rm(path.join(w.projectDir, "out"), { recursive: true });
+ await rm(parked, { recursive: true });
+
+ // A move-back cut after the unlink: move-out refuses; move-back finishes it.
+ await mkdir(path.join(w.projectDir, "out.incoming"));
+ await assert.rejects(moveDirToMedia(w.projectDir, "out", w.roots), /move-back <project>.*nothing moved/);
+ await assert.rejects(ensureOutDir(w.projectDir, w.roots), /left: out\.incoming/);
+ assert.equal((await moveDirToLocal(w.projectDir, "out", w.roots)).state, "moved");
+ assert.equal(await kind(path.join(w.projectDir, "out")), "dir");
+ } finally {
+ await w.done();
+ }
+});
+
+test("ensureOutDir for a project that does not exist leaves no empty mirror (N5)", async () => {
+ const w = await world({ withOut: false });
+ try {
+ const ghost = path.join(w.reportsRoot, "ghost");
+ await assert.rejects(ensureOutDir(ghost, w.roots), /is not a directory/);
+ assert.deepEqual(await readdir(w.mediaRoot), []);
+ // A project directory that is itself a link is a directory (re-review R2).
+ const real = path.join(w.base, "elsewhere-proj");
+ await mkdir(real);
+ const linked = path.join(w.reportsRoot, "linked");
+ await symlink(real, linked);
+ assert.equal(await kind(await ensureOutDir(linked, w.roots)), "link");
+ } finally {
+ await w.done();
+ }
+});
+
+// ---------------------------------------------------------------------------
+// The walk never descends what may sit on the media drive
+// ---------------------------------------------------------------------------
+
+test("the project walk skips out, clips and share-* (a link into an unplugged drive is never stat'd)", async () => {
+ assert.ok(SKIP_DIRS.has("out") && SKIP_DIRS.has("clips"));
+ assert.ok(skipsDir("share-emancipation") && skipsDir("clips") && !skipsDir("shares") && !skipsDir("project"));
+ assert.ok(skipsDir("out.moved-20261001T000000Z") && skipsDir("out.incoming") && !skipsDir("incoming"));
+ assert.ok(skipsDir("share-x.incoming") && !skipsDir("drafts.incoming") && !skipsDir("old.moved-2026"));
+ const w = await world();
+ try {
+ for (const hidden of ["clips", "share-x"]) {
+ const d = path.join(w.reportsRoot, hidden, "inner");
+ await mkdir(d, { recursive: true });
+ await writeFile(path.join(d, "video.manifest.json"), "{}\n");
+ }
+ const ids = (await walkProjects(w.reportsRoot)).map((p) => p.id);
+ assert.deepEqual(ids, ["folder/proj"]);
+ } finally {
+ await w.done();
+ }
+});
diff --git a/umtool/next.config.ts b/umtool/next.config.ts
@@ -19,7 +19,8 @@ const nextConfig: NextConfig = {
// with "Can't resolve 'cbor-x'", which names a package nothing here uses.
serverExternalPackages: ["lmdb"],
// No route's trace may list the e2e fixture (.e2e-song, where
- // e2e/fixtures/make-fixture.mjs links the song data), the e2e server's own
+ // e2e/fixtures/make-fixture.mjs links the song data; .e2e-song-media, the
+ // storage spec's media root), the e2e server's own
// build directory (.next-e2e) or an env file: none is a run-time input. The
// clip-audio route's trace listed 1,704 such files (plans/release-15.md, slice
// UT). That was fixed at the call (lib/paths.mjs `cacheFile`); this is the
@@ -27,7 +28,7 @@ const nextConfig: NextConfig = {
// back to what the sibling routes list. scripts/next-build-trace.test.mjs
// reads the last build's traces back.
outputFileTracingExcludes: {
- "/*": ["./.e2e-song/**/*", "./.next-e2e/**/*", "./.env*"],
+ "/*": ["./.e2e-song/**/*", "./.e2e-song-media/**/*", "./.next-e2e/**/*", "./.env*"],
},
turbopack: {
// Same reasoning as editor/next.config.ts: Turbopack infers the workspace
diff --git a/umtool/playwright.config.ts b/umtool/playwright.config.ts
@@ -76,6 +76,15 @@ export default defineConfig({
// var. CHANNELS_DIR has to be said explicitly: it is where a report
// video's cue files live, and its default is the real 3 GB corpus.
`CHANNELS_DIR=${FIXTURE}/channels ` +
+ // The cache (the project index, posters, analyses) is no longer under
+ // SONG_DIR (release 17): its default is the user's ~/.cache, which a
+ // suite must never write. The fixture's own, rebuilt with it every run.
+ `UMTOOL_CACHE_DIR=${FIXTURE}/cache ` +
+ // And never the media root: Playwright hands the app this shell's whole
+ // environment, so a shell that exports UMTOOL_MEDIA_DIR would put every
+ // fixture build's out/ on the real media drive. Empty is unset
+ // (lib/paths.mjs reads it with ||); storage.spec.ts gives its CLI its own.
+ `UMTOOL_MEDIA_DIR= ` +
// Stub binaries, so a build spec is offline and deterministic. The
// pipeline already reads both as overrides; the fixture writes them.
`YTDLP_BIN=${FIXTURE}/bin/yt-dlp QRENCODE_BIN=${FIXTURE}/bin/qrencode ` +
diff --git a/umtool/report-to-video/build-video.mjs b/umtool/report-to-video/build-video.mjs
@@ -77,6 +77,7 @@ import {
cardWidth, contentWidth, reservedFooterHeight,
} from "./render-cards.mjs";
import { createCueSource, siteOriginFromManifest } from "./cues.mjs";
+import { ensureWriteDir } from "../lib/report/storage.mjs";
// The deck (`render.chrome`): its geometry, validation and schedule are pure
// and live in deck.mjs. This file only frames segments into its box and writes
// the schedule down -- it never has a copy of the arithmetic.
@@ -3143,6 +3144,11 @@ export async function buildVideo({ manifestPath, opts = {}, out, only, fetchOnly
const dirs = variantPaths(outRoot, manifest.slug, variant);
const outDir = dirs.dir;
+ // A project's out/ first, through ensureOutDir: with UMTOOL_MEDIA_DIR set it
+ // is a link to the media root, and the recursive mkdirs below would
+ // otherwise make it a real directory here. A dangling link refuses here,
+ // before a byte is fetched.
+ await ensureWriteDir(outRoot);
await mkdir(dirs.rawDir, { recursive: true });
for (const d of ["cards", "segments", "qr"]) {
await mkdir(path.join(outDir, d), { recursive: true });
diff --git a/umtool/report-to-video/check-availability.mjs b/umtool/report-to-video/check-availability.mjs
@@ -22,10 +22,11 @@
import { execFile } from "node:child_process";
import { promisify } from "node:util";
-import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { DEFAULT_CHANNELS_DIR } from "./cues.mjs";
+import { ensureWriteDir } from "../lib/report/storage.mjs";
// The per-platform yt-dlp args (Rumble's `--impersonate chrome`): the ONE table,
// in common, plain JS so bare `node` can load it.
import { platformArgsForUrl } from "yt-dlp-transcript-common/ytdlp/platformArgs.mjs";
@@ -149,7 +150,8 @@ export async function checkAvailability(manifestPath, { outDir, maxAgeDays = 0 }
}
const report = { manifest: path.resolve(manifestPath), checkedAt: new Date().toISOString(), sources };
- await mkdir(dir, { recursive: true });
+ // ensureWriteDir, not mkdir: a project's out/ may belong on the media root.
+ await ensureWriteDir(dir);
await writeFile(file, JSON.stringify(report, null, 2) + "\n", "utf8");
return { ...report, file };
}
diff --git a/umtool/report-to-video/compose-chrome.mjs b/umtool/report-to-video/compose-chrome.mjs
@@ -38,6 +38,7 @@ import path from "node:path";
import { ledgerTotals, dateKey } from "./ledger-totals.mjs";
import { selectVariant } from "./build-video.mjs";
+import { ensureWriteDir } from "../lib/report/storage.mjs";
import {
chromeCacheKey, deckLayout, frameCount, hyperframesCommand, postWindows, resolveDeck, sha256, validateTeaser,
} from "./deck.mjs";
@@ -737,6 +738,10 @@ export async function composeChrome({
const manifest = selectVariant(JSON.parse(await readFile(manifestPath, "utf8")), variant);
// Absolute: the still is a file:// URL, and a relative one is no page at all.
const base = path.resolve(outDir ?? path.join(path.dirname(path.resolve(manifestPath)), "out", variant));
+ // The project's out/ through ensureOutDir before anything lands under it: a
+ // link to the media root when UMTOOL_MEDIA_DIR is set, and a loud refusal
+ // when that link dangles.
+ await ensureWriteDir(base);
from = Number(from ?? 0);
// The regions keyed by the render cache: the two drawn from the deck's
diff --git a/umtool/report-to-video/render-cards.mjs b/umtool/report-to-video/render-cards.mjs
@@ -38,6 +38,7 @@ import { brandFaces, brandManifest, brandSvgFace, childOpts } from "./brand.mjs"
import { BRAND_CARD_STYLES, renderBrandCard } from "./brand-cards.mjs";
import { FIRA_SANS, textWidth } from "./svg-faces.mjs";
import { deckOn, resolveDeck } from "./deck.mjs";
+import { ensureWriteDir } from "../lib/report/storage.mjs";
const execFileP = promisify(execFile);
@@ -1607,6 +1608,7 @@ async function main() {
const outDir = flag("--out") ?? path.join(path.dirname(path.resolve(manifestPath)), "out");
const only = flag("--only");
+ await ensureWriteDir(outDir); // a project's out/ may be a link to the media root
await mkdir(path.join(outDir, "cards"), { recursive: true });
const cards = manifest.timeline.filter(
