commit 1245a42a7769b4196a0331b9d11b9da5086cb23f
parent eec2f36df6d2b02f6b17ff6ee36315efcbd8b1f1
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 29 Sep 2026 23:03:43 -0400
Merge r15/umtool-trace (release 15 slice UT) — umtool's build no longer traces its whole folder: one way to name a cache file, outputFileTracingExcludes for the dot-directories, a post-build .nft.json check that skips a stale build, the trace guard reads the modules a route imports and the open and write calls; under(first, ...rest); FACTS corrected; reviewed SHIP
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
11 files changed, 599 insertions(+), 57 deletions(-)
diff --git a/common/lib/paths.ts b/common/lib/paths.ts
@@ -178,9 +178,10 @@ let cached: Paths | null = null;
// reference — to every file under it when the path is a directory
// (plans/FACTS.md, "A path joined from `process.cwd()` …"). So every join on
// such a value goes through this one opted-out call. Nothing changes at run
-// time.
-function under(...parts: string[]): string {
- return path.join(/* turbopackIgnore: true */ ...parts);
+// time. The comment sits before a named first argument, not a spread: that is
+// the form Turbopack's own advice shows.
+function under(first: string, ...rest: string[]): string {
+ return path.join(/* turbopackIgnore: true */ first, ...rest);
}
export function getPaths(): Paths {
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -4,6 +4,7 @@
- **Transcripts that arrived after a video was first seen are counted.** The stats behind the homepage, the hub and every site's charts were cached per video and refreshed only when the video's metadata changed, so a transcript that came later — a Whisper run days after the download, or a video downloaded after the last index build — never reached them, and a video with YouTube captions alone had no transcription date. Counts and charts were low; the homepage could show a site with 0 transcripts, 0 channels and 0 hours while it served its videos. A stat is now also redone whenever the index re-reads the video, every transcript has a date, and a captioned video is dated by when its captions arrived rather than by a later Normalize run, so its place on "Transcribed over time" can move. **After updating, rebuild and restart the editor before anything else:** until then, **Build stats dataset** runs the old code and would undo the new stats, while a site, hub or homepage build already runs the new code — and the first stats build of any kind re-reads every video once (about 10–30 minutes on a large archive; it can be stopped and picks up where it stopped). Then build the index, the stats, the homepage, the hub, and the sites.
- **A stats build keeps the stats of a channel whose drive is not mounted, and will not undo a newer version's stats.** A channel whose media is on a drive that is not mounted (or is being moved) is left as it was instead of being read as a channel with no videos; a stats rebuild that has to start over refuses until the drive is back. A stats build refuses to clear stats written by a newer version of the editor; set `ARCHILYZER_STATS_ALLOW_DOWNGRADE=1` to roll back on purpose. Its log also says apart how many videos were downloaded since the last index build (they catch up after the next one) and how many the index skipped (no upload date, or it failed on them).
- **An index build keeps a channel whose drive is not mounted, instead of dropping it from the sites.** **Build index**, a site build's data phase and `archilyzer index` read a channel whose media is on a drive that is not mounted (or is being moved, or whose link and config disagree) as a channel with no videos: they removed its videos from the index, and the next site build published the channel as gone. Such a channel is now left as the last build had it — its videos stay in the index, its pages stay as they were, and the sites built next still list it — and the log names it, with its storage location: one line per channel, ` Held: N channel(s), K video(s) kept.` at the end of the `Diff:` line, and the channels again on the last line. A data folder that fails to read is held the same way, and a channel with no data folder at all is said in the log instead of passed over. An index rebuild that has to start over (after an update that changes the index's format, or with no index yet) refuses while any channel is held and says which; mount the drive first, or set `ARCHILYZER_INDEX_ALLOW_HELD=1` to rebuild without that channel until its drive is back and the index is built again — on the command for a command-line build (`ARCHILYZER_INDEX_ALLOW_HELD=1 pnpm archilyzer index`), or in the editor's own environment, with a restart, for **Build index** and the site builds started from the editor.
+- **umtool's build no longer lists its e2e test data, the e2e server's build folder or `.env.local` among a route's files.** The clip-audio route named its cache files in a way the bundler read as a pattern reaching into umtool's hidden folders, so its list of files took in the e2e fixture (where the tests link the song data), the e2e dev server's build folder and the env file: 1,704 of its 2,167 entries. It now lists what the other routes list (463). Those folders and env files are also excluded from every route's list, and `pnpm test:scripts` reads the last umtool build's lists back and fails on any such entry. A checkout whose umtool build predates its code (this change included) skips that check, saying so, until umtool is rebuilt (`pnpm --filter umtool exec next build`). Nothing changes when umtool runs.
- **Building the homepage now publishes the source: a read-only git mirror, its raw tree and a fresh tarball, behind a gate.** `archilyzer build homepage`, the `/sites` Homepage jobs and `pnpm ops build-homepage` run `archilyzer source publish` between compose and `next build`. It makes a fresh clone of the private `main` (the repository itself is never rewritten), rewrites that copy with git-filter-repo using your scrub rules (file contents and commit messages; your home directory becomes `/home/user` without a rule), and publishes it under `homepage/public` for `git clone https://archilyzer.pages.dev/source/archilyzer.git`, beside `/source/tree/` and the Downloads tarball. Before anything is written, every object of the rewritten history and every file about to be published is searched for every string you have denied; **one hit refuses the build**, and its log names the string only by where you wrote it (`denylist line 3 (len 5)`) and each hit by its object, field and byte offset — never a byte of the object. **A refusal withdraws the source**: the last publish is removed from `homepage/public` and the last build's copy from `homepage/out`, and **Deploy homepage refuses** a build whose source was not audited under today's rules and today's `main` ("run `archilyzer build homepage`, then deploy"). The rules live outside the repo, in `~/.config/archilyzer/source-scrub.txt` and `source-denylist.txt` (`ARCHILYZER_CONFIG_DIR`, `SOURCE_SCRUB_FILE`, `SOURCE_DENYLIST_FILE`); **without them the build refuses**, naming the missing file. **Put everything private in the denylist before any deploy, a preview included**: previews are public, and every deployment stays reachable at its own address until you delete it. Install git-filter-repo once (`pipx install git-filter-repo`; the editor's process needs `~/.local/bin` on its `PATH` to find it) — without it the build fetches it through `pipx run`, which needs the network — and gitleaks if you want its secret scan too. An unchanged `main` with unchanged rules is skipped, so a rebuild costs about 20 seconds only when something moved. A checkout with no git repository (the docker image, a tarball install) builds with the /source page's empty state. `archilyzer source publish --check` audits without writing, `archilyzer source audit <clone>/.git` checks any clone, `archilyzer build homepage --no-source` removes the published source instead, and `archilyzer doctor` reports the tools, the two files (rule counts and permissions, never their contents) and the last publish. `create-archives.sh` is gone. See PUBLISH.md, "The source mirror (homepage)".
- **umtool reads the corpus from its checkout (or `TRANSCRIPTS_DIR`), and the song project's data defaults to `~/.local/share/archilyzer/song`.** If yours is elsewhere, link it there before restarting umtool: `mkdir -p ~/.local/share/archilyzer && ln -s <where the data is> ~/.local/share/archilyzer/song` (the data stays where it is). With no `CHANNELS_DIR`, umtool reads the corpus at `$TRANSCRIPTS_DIR/channels`, else the checkout's own `transcripts/channels`; it used to fall back to an absolute path that existed on one machine only. The song project's videos default to `~/reports/quartering-uh-song/videos`; `SONG_DIR` and `VIDEO_ROOT` still win. The song project's tracked manifests record their paths relative to the song folders, and the twenty one-off `umtool/song/*.sh` run logs, which only ever ran on the machine that wrote them, are gone.
- **umtool's production build no longer reads the corpus folder.** Since umtool began finding the corpus from its checkout (the bullet above), `next build` treated the checkout's whole `transcripts/channels` as files to bundle. On a real archive it ran out of memory and was killed, so umtool could not be rebuilt. The build now ignores that folder and finishes in about 25 s at under 1 GB, the same as a checkout with no corpus. Nothing changes when umtool runs.
diff --git a/plans/FACTS.md b/plans/FACTS.md
@@ -7339,7 +7339,8 @@ Slices Q (`4855f70b`) and R (`ffdeb2cd`): [`release-12.md`](release-12.md), the
`path.resolve` / fs call on the result becomes an asset reference: to a file, or, when the joined
path is a directory, to every file under it (`DirAssetReference`).
- **A worktree build will not show it.** A worktree has no `transcripts/`, so the reference is
- empty. **Test with the corpus visible.**
+ empty. **Test with the corpus visible.** Nor does it have umtool's e2e fixture (`.e2e-song`,
+ `.next-e2e`), which is what slice UT's pattern reached (below).
- **What happened:** slice Q wrote `path.join(REPO_ROOT, "transcripts", "channels")` with
`REPO_ROOT = findRepoRoot(process.cwd())` in `umtool/lib/paths.mjs`. The walk's fallback,
`path.resolve(start, "..")`, evaluates to the project root.
@@ -7350,18 +7351,51 @@ Slices Q (`4855f70b`) and R (`ffdeb2cd`): [`release-12.md`](release-12.md), the
ModuleReference>::resolve_reference failed … Symlink [project]/transcripts/channels/<slug>/archive
is invalid, it points out of the filesystem root`.
- A worktree built it in 30 s at 0.8 GB, which is how the gate passed.
-- **The opt-out is per expression and documented:** `path.join(/* turbopackIgnore: true */
- process.cwd(), bar)`. That exact text is Turbopack's own advice in its "whole project was traced"
- message; the table is in the Next docs, `03-api-reference/08-turbopack.md`, "Magic comments". It
- goes before the FIRST argument of each path or fs call on such a value, and it changes nothing
- at run time. Per call, not per value:
+- **The opt-out is per expression, and it is not documented for path or fs calls** (corrected by
+ release 15, slice UT). `path.join(/* turbopackIgnore: true */ process.cwd(), bar)` is Turbopack's
+ own advice, in the text of its "Encountered unexpected file in NFT list" issue (the "whole project
+ was traced" warning; the 16.2.3 native binary carries it), and Next's own server uses it, on
+ the join and on the fs call around it: `next/dist/server/next-server.js:620` (16.2.3) reads
+ `existsSync(/* turbopackIgnore: true */ (0, _path.join)(/* turbopackIgnore: true */ this.dir,
+ 'static'))`. The Next docs (`08-turbopack.md` and
+ `02-guides/lazy-loading.md`, "Magic Comments") list the comment only for `import()`, `require()`,
+ `require.resolve()` and `new Worker()`. It goes before the FIRST argument of each call, and it
+ changes nothing at run time. Measured in slice UT, it works on a `path.join` and on an fs call.
+ Per call, not per value:
- a nested call needs its own marker (`path.dirname(/* turbopackIgnore: true */
fileURLToPath(import.meta.url))`);
- - an outer fs call on an opted-out `path.join` is covered.
-- **Not followed by the tracer:** `os.homedir()` and `process.env.*`. A build with `HOME` pointed at
- a synthetic home inside the project, full of out-of-root symlinks under `reports/`,
- `.local/share/archilyzer/song` and `.cache/`, succeeded. So `~/reports` and the XDG song path are
- safe as `path.join(os.homedir(), …)`.
+ - **an fs call on an opted-out `path.join` is NOT covered**, nested or through a variable: it
+ traces the join's value. Slice UT, on umtool's clip-audio route. With 4 probe files in its dot
+ directories: the join opted out and the fs calls on its value kept, 4 traced; those calls
+ stubbed, 0. With the primary's fixture: one `existsSync(path.join(/* opt-out */ CACHE_DIR, …))`,
+ 1,704; the same with the `existsSync` opted out as well, 0; the join held in a variable and
+ only `existsSync(/* opt-out */ cached)` reading it, 0. On a cwd-derived value the join is a
+ known path, so the outer call traces the one file it names; the guard lets that through (its
+ comment says so; the release 15 review ruled its four sites safe). Next's own
+ `next-server.js:620` (above) opts out both calls.
+- **A value the tracer cannot know is a dynamic part, not ignored** (corrected by release 15, slice
+ UT). `process.env.*`, `os.homedir()`, a parameter and an imported binding all make patterns over
+ the app's own directory (`umtool/`):
+ - 66 of umtool's 68 routes trace its whole tree outside dot-directories (361 files,
+ `next.config.ts` among them). Opting out every path op in `song/paths.mjs`, `lib/paths.mjs`
+ and `lib/paths.ts` (`path.resolve(process.env.X ?? path.join(os.homedir(), …))` and the like)
+ left 31 of the 68 routes clean; every path op in all 53 modules that have one (319 calls),
+ 49 (`_global-error` and `_not-found` were clean before). The rest come through fs calls
+ (`lib/report/snapshots.mjs` is the first the warning names).
+ - The clip-audio route's `path.join(CACHE_DIR, `${stamp}.${asMp3 ? "mp3" : "wav"}`)` (CACHE_DIR
+ imported, built on the env or home directory) also took in the dot-directories: 1,704 of its
+ 2,167 entries were the e2e fixture (`.e2e-song`), the e2e server's build directory
+ (`.next-e2e`) and `.env.local`. Hoisting the ternary into a `const` changed nothing. Without
+ the ternary (`${stamp}.wav`), as a ternary of two joins, or with the name handed to a function
+ from another module (the fix, `cacheFile` in `umtool/lib/paths.mjs`), it did not. The video
+ route's `.mp4` join did not reach the `.mp4` inside `.e2e-song`.
+ - **A pattern walk does not enter a symlinked directory**, in the root or out of it: a link at
+ `umtool/.e2e-song/data/planted` to `<primary>/transcripts/channels` gave 0 entries before and
+ after the fix, while the old route's walk listed the real files beside it (`data/mkvocals`);
+ one to the worktree's `common/` (675 files) gave 0 on the old route. That is what the
+ synthetic-HOME build showed, not that the env and home directory are unfollowed. A KNOWN
+ directory (slice Q's `<root>/transcripts/channels`) is different: it is walked through its
+ symlinks.
- **The guard is `scripts/next-build-trace.test.mjs`** (in `test:scripts`; it was
`umtool-build-trace.test.mjs` until release 14 widened it). It scans umtool's app, components,
lib, `report-to-video/*.mjs` and `song/paths.mjs`, and, since release 14 (F8), `homepage/app`,
@@ -7370,14 +7404,37 @@ Slices Q (`4855f70b`) and R (`ffdeb2cd`): [`release-12.md`](release-12.md), the
`process.cwd()`, `import.meta.url|dirname|filename`, `__dirname`, or a call to a function
declared in the file whose body carries one, must open with the opt-out.
- It is static and per module, as Turbopack's value analysis is: an imported binding is opaque to
- it.
+ it (to Turbopack it is an unknown, which is a dynamic part: see above).
- With the slice Q `paths.mjs` it fails on the defect's line.
+ - **Since release 15 (slice UT):**
+ - The scan set also follows every relative import out of those folders, which adds
+ `common/bin/_publicFile.ts`, `homepage/content/docs.ts` and seven `umtool/song` modules
+ (umtool 211 → 218 modules, the rest 895 → 897); a test pins them.
+ - The checked calls include `open`, `writeFile`, `appendFile`, `createWriteStream` (and their
+ Sync forms) and the `fs.promises.` / `fsPromises.` prefixes. No new finding.
+ - **It reads umtool's last build back.** Every `.nft.json` under `umtool/.next` but the
+ build's own `cache/` and `dev/` fails on an entry outside the repo, under `transcripts/`, or
+ through any name starting with a dot other than the build's own directory and
+ `node_modules/.pnpm`.
+ - **It skips, saying so,** when there is no build (`umtool/.next/server` or `BUILD_ID`
+ missing), and when `BUILD_ID` is older than `umtool/next.config.ts` or any umtool module it
+ scans (review M1). A merge or checkout gives the changed files new mtimes, and umtool runs
+ under `next dev`, which does not refresh `.next`, so a stale build skips until umtool is
+ rebuilt instead of failing on a call already fixed.
+ - **What it can see** (review L2). Without the excludes, the old clip-audio route failed it with
+ 1,704 entries (the primary's pre-fix build: 1,705, `test-results/.last-run.json` too). With
+ the excludes umtool's config now carries, such a pattern shows only through a name they
+ miss: `test-results/.last-run.json` after an e2e run, `.next-shots`, the corpus, a path
+ outside the repo. A checkout with no e2e run behind it is blind to it; the fix at the call
+ is what keeps the route clean.
- **The build gate** is run with the corpus visible and under a memory cap (the command is in
`plans/tools/implementer-rules.md`). Linking `<primary>/transcripts` into a worktree is for a
BUILD only. Remove the link afterwards: never run an app, an index or a fixture builder through it.
- **The other apps were safe by accident; since release 14 (F8) they are by rule:**
- `common/lib/paths.ts` builds every path on the repo root through one opted-out `under()`, and
- `findMonorepoRoot()`'s walk and its `process.cwd()` fallback are opted out.
+ `findMonorepoRoot()`'s walk and its `process.cwd()` fallback are opted out. Since release 15
+ `under(first, ...rest)` puts the marker before a named first argument, not a spread (the
+ release 14 review's L3); `getPaths()` is unchanged.
- `homepage/app/lib/source.ts`' directory join on `public` (the source mirror) and the homepage's
and export's other cwd joins carry the opt-out.
- The guard covers them. A homepage build with the published source measured the same with and
diff --git a/plans/release-15.md b/plans/release-15.md
@@ -216,4 +216,251 @@ checkout's code, so they hold from the moment `main` has this branch. The editor
**Build index** button runs its built bundle, so it holds only after the editor is rebuilt and
restarted.
+### Slice UT, as shipped — umtool's build stops tracing its dot-directories (2026-09-29)
+
+Branch `r15/umtool-trace` off `main` `ccf90892`, worktree `~/Projects/r12-source-mirror` (block
+#13: editor 4301, test 4311, export 4310), one Opus implementer. Scratch files `ut-*` in the job's
+`tmp`. The ruling: find the one expression that widens the clip-audio route's trace and fix it
+there; exclude the fixture, the e2e build and env files as a second line; narrow or drop the
+`ignoreIssue`; make the trace guard read a build back and close the release 14 review's L1 and L3.
+
+**What was wrong.** With the primary's e2e fixture in place, umtool's
+`app/api/clip/[key]/audio/route.js.nft.json` listed 2,167 files: the 463 its sibling routes list,
+178 under `.e2e-song/` (the fixture, where `make-fixture.mjs` links the song data), 1,525 under
+`.next-e2e/` (the e2e dev server's build directory, 1.1 GB) and `.env.local`. Turbopack's warning
+for it ("Encountered unexpected file in NFT list", the "whole project was traced" text) was silenced
+by the config's `ignoreIssue`. The traces are not consumed while `output: "standalone"` stays off,
+so nothing broke; a fixture with more in it, or a standalone build, would have carried it.
+
+**The bisect.** One change per build, in this worktree with four probe files planted in
+`.e2e-song/probe/` and `.next-e2e/probe/`; the audio route's trace, total / under dot-directories.
+The route as on `main`: **467 / 4**.
+
+| Change (line on `main`) | Entries |
+|---|---|
+| `existsSync(file)` :51 stubbed | 467 / 4 |
+| **the join `path.join(CACHE_DIR, `${stamp}.${asMp3 ? "mp3" : "wav"}`)` :58 written as a string concatenation** | **463 / 0** |
+| `existsSync(cached)` :61, `readFile(cached)` :62, `mkdir(CACHE_DIR)` :78, `readFile(tmpMp3)` :89, `rename(tmpMp3, cached)` :90, `writeFile(tmp)` :93 or `rename(tmp, cached)` :94 stubbed, each alone | 467 / 4 each |
+| the `tmpWav` / `tmpMp3` joins :82-83 as concatenations; `writeFile(tmpWav)` :84 stubbed; both `writeFile`s stubbed; either `writeFile` opted out | 467 / 4 each |
+| the opt-out on the :58 join | 467 / 4 |
+| the :58 ternary hoisted into a `const ext` | 467 / 4 |
+| **:58 without the ternary** (`${stamp}.wav`) | **463 / 0** |
+| :58 as `path.join(CACHE_DIR, asMp3 ? `${stamp}.mp3` : `${stamp}.wav`)` | 463 / 0 |
+| :58 as a ternary of two joins | 463 / 0 |
+| opt-outs on `existsSync(cached)` and `readFile(cached)`, or either alone; both stubbed | 467 / 4 each |
+| all five readers and writers of `cached` stubbed | 467 / 4 |
+| **all five stubbed, and the opt-out on the :58 join** | **463 / 0** |
+| all five stubbed, and the join as a concatenation | 463 / 0 |
+
+With the primary's fixture (the table's 4 are 1,704 there), on top of the last-but-one row:
+
+| Change | Entries |
+|---|---|
+| one `existsSync(path.join(/* opt-out */ CACHE_DIR, …))` | 2,167 / 1,704 |
+| the same `existsSync` opted out as well | 463 / 0 |
+| `existsSync(/* opt-out */ cached)` as the only reader of the opted-out join | 463 / 0 |
+
+**The expression** is the :58 join, and in it the ternary inside the template literal. The join and
+the fs calls on its value each trace the pattern (the join alone with every reader stubbed; the
+readers alone with the join opted out; `existsSync` is one such reader), so no single opt-out or
+stub cleared it. Without the ternary, the pattern stays out of the dot-directories.
+
+**The fix** (`a305b956`). `umtool/lib/paths.mjs` gains `cacheFile(name)`, `path.join(/* opt-out */
+CACHE_DIR, name)`, re-exported by `lib/paths.ts`. The audio route names all four of its cache files
+through it (`cached`, `tmpWav`, `tmpMp3`, and `tmp`, now `cacheFile(`${stamp}.wav.tmp`)`, the same
+path as `${cached}.tmp` in that branch). A value returned by a function from another module is
+opaque to the tracer, so the call site traces nothing: the route lists 463, as its siblings do. The
+video and face-frame routes join `CACHE_DIR` with a fixed extension; they were measured clean (the
+video route's `.mp4` join did not reach the `.mp4` in `.e2e-song`) and name their cache files the
+same way, so no route joins `CACHE_DIR` itself. The run-time paths are unchanged.
+
+**The second line** (`f18fa034`). `outputFileTracingExcludes: { "/*": ["./.e2e-song/**/*",
+"./.next-e2e/**/*", "./.env*"] }` (Next 16.2.3's `05-config/01-next-config-js/output.md`: route
+globs to globs from the project root; Turbopack reads the key natively, `collect-build-traces.js`
+is the webpack path). Measured alone, with `main`'s route: 2,167 → 463.
+
+**The warning stays silenced, narrow as it was (path + title), with its measured reason.** Dropping
+it shows one warning on every build, and after the fix it is still true:
+- 66 of the 68 routes trace umtool's whole tree outside dot-directories: its 361 files,
+ `next.config.ts` (the file the warning names) among them. Only `_global-error` and `_not-found`
+ do not.
+- Path and fs calls on env, home-directory and parameter values do it. Opting out every path op in
+ `song/paths.mjs`, `lib/paths.mjs` and `lib/paths.ts` left 31 of the 68 routes clean (the audio
+ route 463 → 102). Opting out all 319 path ops in the 53 modules that have one left 49 clean. The
+ two clean before are among them. The rest come through fs calls; the next import trace the
+ warning names is `lib/report/snapshots.mjs`.
+- That walk skips dot-directories and does not enter symlinks. The warning names the same file for
+ it as for the audio route's, so it cannot tell the two apart; the second line and the guard below
+ cover the dot-directories instead. The config's comment says all of this.
+
+**A symlinked directory is not entered by these patterns.** The planted link
+`umtool/.e2e-song/data/planted` → `<primary>/transcripts/channels` gave 0 entries before and after
+the fix, and an in-root link to `common/` (675 files) gave 0 on `main`'s route. What the pattern
+reached was the fixture's real files.
+
+**The guard** (`scripts/next-build-trace.test.mjs`, `9a375ecd`, and after the review `a4d100b4`,
+`c3e8a2c7`), 6 → 10 tests:
+- **(a) It reads umtool's last build back.** Every `.nft.json` under `umtool/.next` but the
+ build's own `cache/` and `dev/` fails on an entry outside the repo, under `transcripts/`, or
+ through any name starting with a dot but the build's own directory and `node_modules/.pnpm`.
+ Unit tests pin `forbiddenTrace` and which directories are read.
+ - **It skips, saying so,** with no build, and (review M1) when `umtool/.next/BUILD_ID` is older
+ than `umtool/next.config.ts` or any umtool module the guard scans. The message gives the
+ build's time, the first newer file and how to rebuild. A merge or checkout gives the changed
+ files new mtimes, and umtool runs under `next dev`, which does not refresh `.next`, so a stale
+ build skips instead of failing on a call already fixed.
+ - **What it can see** (review L2). Without the excludes, `main`'s route failed it with 1,704
+ entries (the first 20 listed); the primary's pre-fix build fails it with 1,705 (the review:
+ `test-results/.last-run.json` too). With the excludes in place, a pattern like that one shows
+ only through a name they miss: `test-results/.last-run.json` after an e2e run, `.next-shots`,
+ the corpus, a path outside the repo. A checkout with no e2e run behind it is blind to it; the
+ fix at the call is what keeps the route clean.
+- **(b) The scan set follows relative imports** out of the listed folders, to any depth. It adds
+ the review's L1 modules and no others: `common/bin/_publicFile.ts`, `homepage/content/docs.ts`
+ (895 → 897) and seven `umtool/song` modules, `reasons`, `archive-url`, `pitch`, `flatness`,
+ `clipwindow`, `deplosive`, `orderfeat` (211 → 218; `song/paths.mjs` was listed by hand before and
+ is now reached). A test pins them, and that a song CLI and a common CLI stay out. The comments
+ that called them CLI-only are gone.
+- **(c) The checked calls** add `open`, `writeFile`, `appendFile`, `createWriteStream` and their
+ Sync forms, and the `fs.promises.` / `fsPromises.` prefixes. No new finding.
+- **(d)** `common/lib/paths.ts` `under(first, ...rest)` (`8b3409c2`): the opt-out sits before a
+ named first argument. `getPaths()` hashed identical, old module against new, from the repo root,
+ `editor/` and a directory outside the repo (56 keys). The guard passes; every caller type-checks.
+- The header no longer says the env and home directory are unfollowed or that the opt-out is
+ documented. The nested-call exemption's comment says it is a simplification (below).
+
+**FACTS**, "A path joined from `process.cwd()` …", corrected:
+- "documented": the Next docs list `turbopackIgnore` only for `import()`, `require()`,
+ `require.resolve()` and `new Worker()`. The path form is Turbopack's own advice, in the warning's
+ text in the 16.2.3 binary, and Next's own server uses it on the join and on the fs call around
+ it (`next/dist/server/next-server.js:620`; review L4).
+- "an outer fs call on an opted-out `path.join` is covered": it is not (the second bisect table).
+- "Not followed by the tracer: `os.homedir()` and `process.env.*`": such values are dynamic parts,
+ and make patterns over the app directory. The synthetic-HOME build showed that a pattern walk
+ does not enter symlinks.
+- The guard's entry, the worktree caveat (no fixture either) and `under()`'s shape.
+
+**Commits**
+
+| Commit | What |
+|---|---|
+| `a305b956` | `umtool:` `cacheFile`; the audio, video and face-frame routes name their cache files through it |
+| `f18fa034` | `umtool:` `outputFileTracingExcludes`; the `ignoreIssue` kept, its comment the measured reason |
+| `8b3409c2` | `common:` `under(first, ...rest)` |
+| `9a375ecd` | `scripts:` the post-build check, the relative-import scan set, the opens and writes, the header |
+| `750fc850` | `plans:` this section; FACTS; the editor changelog |
+| `a4d100b4` | `scripts:` review M1: the post-build check skips a build older than the code it judges |
+| `c3e8a2c7` | `scripts:` review L1 (only the build's own `cache/` and `dev/` unread, pinned), L2 (the check's comment says what it can see), L3 |
+| `b4d6607d` | `umtool:` review L3 in the `ignoreIssue` comment |
+| `31bb0f8c` | `plans:` the review's findings to their commits; FACTS (M1, L2, L3, L4); the changelog (M1); the rollout note |
+| `7c6d4969` | merge of `main` (`721ed0eb`, release 14 HS and S1); clean, the changelog bullet still under `[Unreleased]` |
+| this commit | `plans:` the gates after the review and the merge |
+
+#### Gates (logs `$T/ut-*.log`)
+
+- **tsc** clean over the combined tree before the first commit, 241 s (load average ~26). The
+ commits are independent pieces of that tree; since then only a comment changed in a
+ type-checked file (`cacheFile`'s, in `umtool/lib/paths.mjs`).
+- **`test:scripts`:** 194 passed, 1 skipped (195): `main`'s 191 + 1 and the guard's three new
+ tests. Before the final build it failed exactly the post-build test, on a build of `main`'s route.
+- **common:** 2,220/2,220, 107 s.
+- **umtool's build, capped at 5 GB with no swap, with the primary's `transcripts/` linked in, the
+ primary's `.e2e-song` and `.next-e2e` hard-linked in, an empty `.env.local`, and the planted
+ link** (all removed afterwards; none committed):
+
+ | Tree | Wall | User | Max RSS | Audio route | `.e2e-song` | `.next-e2e` | `.env*` | `transcripts` / planted |
+ |---|---|---|---|---|---|---|---|---|
+ | `main`'s route and config | 23 s | 57 s | 809 MB | 2,167 | 178 | 1,525 | 1 | 0 / 0 |
+ | the branch | 34 s | 64 s | 765 MB | 463 | 0 | 0 | 0 | 0 / 0 |
+
+ The wall times swing with the machine's load (another slice's e2e and builds ran alongside);
+ compile was 7.5 s and 10.7 s, TypeScript 12 s and 19 s.
+- **The editor's build**, capped, with the corpus linked in (`paths.ts` changed): 97 s wall, 159 s
+ user, max RSS 1,576 MB (IG's: 64 s / 1,642 MB, under less load); 0 of its 81 traces' entries
+ under `transcripts/`, and none that `forbiddenTrace` refuses.
+- **e2e** (umtool's own filter, `SONG_DIR=~/reports/quartering-uh-song/data`, queued):
+ `find.spec.ts` and `triage.spec.ts` fetch the audio route, `faces.spec.ts` the face-frame route,
+ and `find.spec.ts` names the video route: **6 passed, 29 skipped, 0 failed, 27 s** (6.3 min with
+ the queue). The skips are the fixture's: this machine has no `wav48/`, `asr/` or `media/`, so
+ every spec that fetches one of the three routes skipped, and they are not exercised at run time
+ here. What stands for that: calling `cacheFile` gives the same path as the old expression for
+ all six names (the four audio names, and the video and frame temporaries).
+- **After the e2e run** (which built this worktree its own fixture and `.next-e2e`), a last capped
+ build with the corpus linked: the audio route 463, none under a dot-directory; `test:scripts`
+ 194 passed, 1 skipped. The first run of that `test:scripts` failed `queue-lock.test.mjs`'s FIFO
+ case once (`S1E1S3E3S2E2`) under a load average of about 26; it passed on the rerun, and this
+ slice does not touch the queue lock.
+- **Numbers tool:** none.
+- **After the review and the merge of `main`** (at `7c6d4969`):
+ - tsc clean, 98 s;
+ - common **2,229/2,229** (`main`'s 2,229), 108 s;
+ - `test:scripts` **195 passed, 1 skipped (196)**: `main`'s 191 + 1 and the guard's four new
+ tests. The two runs before it each failed `queue-lock.test.mjs`'s "prints a banner naming the
+ holder while waiting" under a load average of about 26, the known flake (alone, 11/11 twice);
+ this slice does not touch the queue lock;
+ - umtool's build, capped, with the corpus linked and this worktree's own e2e fixture present:
+ 48 s wall, max RSS 796 MB, the audio route 463, none under a dot-directory; the post-build
+ check passes on it;
+ - the same build with its `BUILD_ID` set back to 2026-09-01 (in place, then put back; nothing
+ committed): the check skips, `umtool/.next was built 2026-09-01T04:00:00.000Z, before
+ umtool/next.config.ts (219 changed since); rebuild umtool (…) to check its traces`. With the
+ mtime put back it passes again.
+
+#### Found and left
+
+- **The whole-folder trace in 66 routes** (above). Bounded to umtool's own files; cleaning it means
+ opt-outs on hundreds of path and fs calls on unknown values, which no static check can find.
+- **The guard's nested-call exemption.** Without it, four calls would need an outer opt-out:
+ `common/lib/paths.ts:311` (`existsSync` of one file), `export/app/changelog/page.tsx:9` and
+ `homepage/app/changelog/page.tsx:24` (one file each; other slices own them), and
+ `umtool/report-to-video/brand.mjs:82` (the brand kits, which a standalone build needs). Each
+ traces the file or files it reads. Left, and the comment says so.
+- **Which fs calls Turbopack traces is not established per call.** `existsSync` does (the second
+ table); the writes were added to the guard without a measurement, since an extra opt-out costs
+ nothing.
+- **The post-build check reads umtool only.** In this worktree the editor's 81 traces pass the same
+ rule. The review's Info saw `editor/.env` in the primary's; a worktree has none, so it was not
+ re-measured.
+- **The gate command in `implementer-rules.md`, in a worktree that has a `transcripts/` directory,**
+ makes `transcripts/transcripts` and builds without the corpus where the paths point. This worktree
+ had one (an `index.mdb` from 2026-09-28), and my first two corpus-linked builds ran like that. The
+ numbers above are from builds that set it aside and put it back. `ln -sT` would refuse instead.
+- **A checkout whose umtool build predates its code skips the post-build check** until umtool is
+ rebuilt (review M1). The primary's `umtool/.next` is from before this slice, so the check skips
+ there until the rollout rebuilds it.
+
+#### Decisions the operator could overturn
+
+| What I did | The alternative |
+|---|---|
+| A `cacheFile` helper in `lib/paths.mjs`, used by all three routes that join `CACHE_DIR`. **Ruled at review: keep; one way to name a cache file.** | Opt-outs on the audio route's join and on every fs call on its value, in that route only |
+| The `ignoreIssue` stays, narrow, with the measured reason in its comment | Drop it: one warning on every build, naming one route's import trace |
+| The post-build check refuses any dot-named path but the build's own and `node_modules/.pnpm` | Refuse only `.e2e-song`, `.next-e2e`, `.env*` and `.git` |
+| The post-build check covers umtool only. **Ruled at review: umtool only.** In the primary the editor's traces would fail it (75 of 81, `editor/.env` among them) and the export's `.export-index` entries would be misjudged. | Also read the editor's, the export's and the homepage's builds |
+| The static check keeps its nested-call exemption, documented as a simplification. **Ruled at review: it stays; the four sites are safe.** | Require the outer opt-out: four new findings, two in files other slices own |
+| A stale build skips the post-build check (review M1) | Fail on it, as first shipped |
+
+#### Review
+
+**Verdict: SHIP AFTER FIXES** (`ut-review.md` in the job's scratch). No High. The reviewer found
+the fix's nine call-site paths byte-identical, the bisect logs in agreement with the tables, the
+excludes' key and shape right, and 0 dot entries across all 70 traces of a fresh build.
+
+| Finding | Where |
+|---|---|
+| M1: a stale umtool build turns `test:scripts` red, pointing at a call already fixed | `a4d100b4`: the check skips a build older than `next.config.ts` or any module it scans; the changelog, FACTS and "Found and left" say so; the rollout note below |
+| L1: `cache`/`dev` skipped at any depth | `c3e8a2c7`: only directly under `.next`; a test with a route directory named each |
+| L2: with the excludes in place the check cannot see the original defect | `c3e8a2c7` (the test's comment), guard (a) above and FACTS: it sees only a name the excludes miss |
+| L3: "cleaned 31 / 49" | `c3e8a2c7`, `b4d6607d`, this commit: "left 31 / 49 of the 68 clean" |
+| L4: FACTS cited the native binary's string as Next's runtime | this commit: `next-server.js:620` |
+| L5: the build-gate command's `ln -s` | The parent's (`implementer-rules.md`) |
+| Info: the editor's and export's primary builds carry the same class of widening | Recorded in the decisions table; a later slice's |
+
+**For the rollout.** Rebuild umtool in the primary first, under the cap
+(`timeout -s KILL 240 systemd-run --user --scope -q -p MemoryMax=5G -p MemorySwapMax=0 pnpm
+--filter umtool exec next build`), then run `pnpm run test:scripts` there. That is the only proof
+on the real fixture, which carries `.env.local`, `.next-shots` and `test-results/.last-run.json`,
+two of them names the excludes do not cover. Until that rebuild the post-build check skips in the
+primary, saying why. umtool's code changes nothing at run time.
+
## Rollout
diff --git a/scripts/next-build-trace.test.mjs b/scripts/next-build-trace.test.mjs
@@ -15,17 +15,28 @@
// per module; an imported binding is opaque to it): every path or fs call whose
// arguments carry a value derived IN THAT FILE from `process.cwd()`,
// `import.meta.url`, `import.meta.dirname|filename` or `__dirname` must open its
-// argument list with the documented opt-out, `/* turbopackIgnore: true */`.
+// argument list with Turbopack's opt-out, `/* turbopackIgnore: true */` (the
+// form its own "whole project was traced" warning advises; the Next docs list
+// the comment only for import(), require(), require.resolve() and new Worker()).
// The comment changes nothing at run time. A function declared in the file whose
// body carries a source is a source too (`const ROOT = findMonorepoRoot()`).
-// `os.homedir()` is NOT a source: the
-// tracer does not follow it (a build with HOME pointed at a synthetic home full
-// of out-of-root symlinks inside the project succeeds), and neither is
-// `process.env.*`.
+//
+// A value Turbopack cannot know -- `process.env.*`, `os.homedir()`, a
+// parameter, an imported binding -- is not one of these sources, and it is not
+// ignored either: it is a dynamic part, and a path or fs call on it becomes a
+// PATTERN over the app's own directory. Measured in umtool (release 15, slice
+// UT): the path ops on env and home-directory values in its path modules took
+// in the app's whole tree outside dot-directories (opting them out left 31 of
+// the 68 routes clean), and the clip-audio route's join with a dynamic extension took
+// in the dot-directories too, the e2e fixture and `.env.local` among them.
+// Neither walk entered a symlinked directory. No static check here can tell
+// such a pattern from a harmless one, so the last test reads a build's traces
+// back instead.
//
// Run with: pnpm test:scripts
import assert from "node:assert/strict";
-import { readdirSync, readFileSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
import path from "node:path";
import test from "node:test";
import { fileURLToPath } from "node:url";
@@ -38,10 +49,13 @@ const MARK = "__TURBOPACK_IGNORE__";
const IGNORE_COMMENT = /\/\*\s*turbopackIgnore\s*:\s*true\s*\*\//g;
// path ops, and fs calls either bare (`existsSync(`) or on a namespace
-// (`fs.readdir(`, `fsp.stat(`). A method on anything else (`obj.stat(`) is not
-// one.
+// (`fs.readdir(`, `fsp.stat(`, `fs.promises.readFile(`). A method on anything
+// else (`obj.stat(`) is not one. The opens and writes are checked too
+// (`open`, `writeFile`, `appendFile`, `createWriteStream`): which fs calls
+// Turbopack traces is not documented, and an opt-out on one it does not trace
+// costs nothing.
const SINK =
- /(?:\bpath\.(?:join|resolve|dirname|relative)|(?<![\w$.])(?:fs\.|fsp\.|promises\.)?(?:existsSync|readFileSync|readdirSync|statSync|lstatSync|realpathSync|opendirSync|readFile|readdir|stat|lstat|opendir|createReadStream))\s*\(/g;
+ /(?:\bpath\.(?:join|resolve|dirname|relative)|(?<![\w$.])(?:fs\.promises\.|fsPromises\.|fs\.|fsp\.|promises\.)?(?:existsSync|readFileSync|readdirSync|statSync|lstatSync|realpathSync|opendirSync|openSync|writeFileSync|appendFileSync|readFile|readdir|stat|lstat|opendir|open|writeFile|appendFile|createReadStream|createWriteStream))\s*\(/g;
/** Comments out, except the opt-out, which becomes a marker. Strings stay. */
function prepare(text) {
@@ -154,9 +168,15 @@ export function untracedCalls(text) {
const carries = (args) =>
SOURCE.test(args) ||
[...names].some((n) => new RegExp(`(?<![\\w$.])${n.replace(/\$/g, "\\$")}\\b(?!\\s*:)`).test(args));
- // A call nested in these arguments that opts out is opaque to this one too:
+ // A call nested in these arguments that opts out is let through:
// `readFileSync(path.join(/* turbopackIgnore: true */ HERE, "a.json"))`. Its
// own arguments are cut out before asking whether this call carries a source.
+ // That is a simplification, not how Turbopack reads it: the outer call traces
+ // the join's value all the same (release 15, slice UT, measured). On a
+ // cwd-derived value that value is a known path, so the outer call traces the
+ // one file it names (a changelog), or the files of one known directory (the
+ // brand kits). Where the join names a directory, or a dynamic part could
+ // reach past the files the call reads, give the outer call its own opt-out.
const withoutOptedOut = (args) => {
let out = args;
for (let i = out.search(new RegExp(`\\(\\s*${MARK}`)); i !== -1; i = out.search(new RegExp(`\\(\\s*${MARK}`))) {
@@ -187,24 +207,75 @@ function modulesUnder(dir, out = []) {
return out;
}
-/** umtool's modules that a Next build can reach: the app, its libs, the pipeline. */
+const MODULE_EXT = [".ts", ".tsx", ".mjs", ".js", ".cjs"];
+const isModule = (p) => MODULE_EXT.some((x) => p.endsWith(x)) && !/\.test\./.test(p) && !p.endsWith(".d.ts");
+const isFile = (p) => {
+ try {
+ return statSync(p).isFile();
+ } catch {
+ return false;
+ }
+};
+
+/** The module a relative specifier names, the way the bundler resolves it, or null. */
+function resolveRelative(from, spec) {
+ const base = path.resolve(path.dirname(from), spec);
+ const candidates = [base, ...MODULE_EXT.map((x) => base + x), ...MODULE_EXT.map((x) => path.join(base, "index" + x))];
+ return candidates.find((c) => isModule(c) && isFile(c)) ?? null;
+}
+
+// `import … from "./x"`, `export … from "../x"`, `import "./x"`, `import("./x")`,
+// `require("./x")`: the relative specifiers only. A package import (`next`,
+// `yt-dlp-transcript-common/…`) is covered by the package's own directory in
+// the set, or is not this repo's code.
+const RELATIVE_IMPORT = /(?:\bfrom\s*|\bimport\s*\(?\s*|\brequire\s*\(\s*)(["'])(\.{1,2}\/[^"'\n]+)\1/g;
+
+/**
+ * `files` plus every module of this repo they reach by relative imports, to
+ * any depth. A directory list alone misses a module one app imports from a
+ * folder the list treats as CLI-only (`common/bin/_publicFile.ts`, imported by
+ * `common/publish/source.ts`; umtool's `song/pitch.mjs`, imported by
+ * `lib/verdict.ts`) or keeps outside `app/` (`homepage/content/docs.ts`).
+ */
+export function withRelativeImports(files) {
+ const seen = new Set(files);
+ const queue = [...files];
+ while (queue.length) {
+ const file = queue.pop();
+ for (const m of readFileSync(file, "utf8").matchAll(RELATIVE_IMPORT)) {
+ const target = resolveRelative(file, m[2]);
+ if (!target || seen.has(target) || target.includes(`${path.sep}node_modules${path.sep}`)) continue;
+ if (path.relative(REPO, target).startsWith("..")) continue;
+ seen.add(target);
+ queue.push(target);
+ }
+ }
+ return [...seen];
+}
+
+/**
+ * umtool's modules that a Next build can reach: the app, its components and
+ * libs, the report pipeline, and whatever of `song/` they import (`paths.mjs`
+ * through `lib/paths.mjs`, `pitch.mjs`, `reasons.mjs` and the rest through the
+ * libs; the other song scripts are CLIs nothing in the app imports).
+ */
function umtoolModules() {
const out = [];
for (const d of ["app", "components", "lib"]) modulesUnder(path.join(UMTOOL, d), out);
for (const e of readdirSync(path.join(UMTOOL, "report-to-video"))) {
if (e.endsWith(".mjs") && !e.includes(".test.")) out.push(path.join(UMTOOL, "report-to-video", e));
}
- // song/paths.mjs is imported by lib/paths.mjs; the other song scripts are CLIs.
- out.push(path.join(UMTOOL, "song", "paths.mjs"));
- return out;
+ return withRelativeImports(out);
}
/**
* The editor's, the export's and the homepage's modules a Next build can
* reach: each app's `app/` (the editor's `lib/` and `instrumentation.ts` too),
- * and every module of common/ but its CLIs (`bin/`, which no app imports). The
- * common set is wider than what the apps import today, on purpose: a module
- * that starts being imported is already covered.
+ * every module of common/ but its CLIs in `bin/`, and every module those reach
+ * by a relative import — which brings in the few `bin/` modules an app does
+ * import (`_publicFile.ts`) and the homepage's `content/docs.ts`. The common set
+ * is wider than what the apps import today, on purpose: a module that starts
+ * being imported is already covered.
*/
function nextAppModules() {
const out = [];
@@ -214,7 +285,7 @@ function nextAppModules() {
if (!e.isDirectory() || e.name === "bin" || e.name === "node_modules" || e.name.startsWith(".")) continue;
modulesUnder(path.join(REPO, "common", e.name), out);
}
- return out;
+ return withRelativeImports(out);
}
function untracedIn(files) {
@@ -298,6 +369,135 @@ test("no umtool module the app can import joins a cwd-derived path without optin
);
});
+test("the scan set follows relative imports out of the listed folders", () => {
+ const rel = (files) => new Set(files.map((f) => path.relative(REPO, f)));
+ const um = rel(umtoolModules());
+ for (const m of ["paths", "reasons", "archive-url", "pitch", "flatness", "clipwindow", "deplosive", "orderfeat"]) {
+ assert.ok(um.has(`umtool/song/${m}.mjs`), `umtool/song/${m}.mjs is not scanned`);
+ }
+ // A song CLI nothing in the app imports stays out.
+ assert.ok(!um.has("umtool/song/build-um.mjs"));
+ const apps = rel(nextAppModules());
+ assert.ok(apps.has("common/bin/_publicFile.ts"), "common/bin/_publicFile.ts is not scanned");
+ assert.ok(apps.has("homepage/content/docs.ts"), "homepage/content/docs.ts is not scanned");
+ assert.ok(!apps.has("common/bin/compose-site.ts"), "a common CLI no app imports is scanned");
+});
+
+/**
+ * Every `.nft.json` a build wrote under its directory `dist`, its cache and
+ * dev-server output (`dist/cache`, `dist/dev`) aside. Only those two: a route
+ * directory named `cache` or `dev` deeper down is read like any other.
+ */
+function traceFilesUnder(dist, dir = dist, out = []) {
+ for (const e of readdirSync(dir, { withFileTypes: true })) {
+ const p = path.join(dir, e.name);
+ if (e.isDirectory()) {
+ if (dir !== dist || (e.name !== "cache" && e.name !== "dev")) traceFilesUnder(dist, p, out);
+ } else if (e.name.endsWith(".nft.json")) out.push(p);
+ }
+ return out;
+}
+
+/**
+ * Why `abs`, a file a build traced, must not be in the trace, or null.
+ *
+ * The build's own directory (`dist`) holds the chunks every trace lists, and
+ * `node_modules/.pnpm` is where pnpm keeps the packages; any other path through
+ * a directory or file whose name starts with a dot is something no server
+ * needs at run time — a fixture (`.e2e-song`, where the e2e fixture links the
+ * song data), another build (`.next-e2e`), a secret (`.env.local`), `.git` —
+ * and is the mark of a pattern Turbopack could not bound. So are the corpus
+ * and anything outside the repo.
+ */
+export function forbiddenTrace(abs, dist) {
+ const rel = path.relative(REPO, abs);
+ if (rel === "" || rel.startsWith("..") || path.isAbsolute(rel)) return "outside the repo";
+ if (abs.startsWith(dist + path.sep)) return null;
+ const parts = rel.split(path.sep);
+ if (parts[0] === "transcripts") return "the corpus";
+ for (let i = 0; i < parts.length; i += 1) {
+ if (!parts[i].startsWith(".")) continue;
+ if (parts[i] === ".pnpm" && parts[i - 1] === "node_modules") continue;
+ return `under ${parts.slice(0, i + 1).join("/")}`;
+ }
+ return null;
+}
+
+test("traceFilesUnder: only the build's own cache/ and dev/ are left out", () => {
+ const dist = mkdtempSync(path.join(tmpdir(), "next-build-trace-"));
+ try {
+ for (const f of ["cache/a.nft.json", "dev/b.nft.json", "server/app/api/cache/route.js.nft.json", "server/app/dev/page.js.nft.json"]) {
+ mkdirSync(path.dirname(path.join(dist, f)), { recursive: true });
+ writeFileSync(path.join(dist, f), '{"files":[]}');
+ }
+ const found = traceFilesUnder(dist).map((f) => path.relative(dist, f)).sort();
+ assert.deepEqual(found, ["server/app/api/cache/route.js.nft.json", "server/app/dev/page.js.nft.json"]);
+ } finally {
+ rmSync(dist, { recursive: true, force: true });
+ }
+});
+
+test("forbiddenTrace: a fixture, another build, a secret, the corpus, outside the repo", () => {
+ const dist = path.join(UMTOOL, ".next");
+ const at = (p) => forbiddenTrace(path.join(REPO, p), dist);
+ assert.equal(at("umtool/.next/server/chunks/ssr/a.js"), null);
+ assert.equal(at("node_modules/.pnpm/next@16.2.3/node_modules/next/dist/server/next.js"), null);
+ assert.equal(at("umtool/lib/paths.mjs"), null);
+ assert.equal(at("umtool/.e2e-song/data/planted/x/config.json"), "under umtool/.e2e-song");
+ assert.equal(at("umtool/.next-e2e/dev/server/a.js"), "under umtool/.next-e2e");
+ assert.equal(at("umtool/.env.local"), "under umtool/.env.local");
+ assert.equal(at(".git/config"), "under .git");
+ assert.equal(at("transcripts/channels/x/config.json"), "the corpus");
+ assert.equal(forbiddenTrace(path.resolve(REPO, "..", "elsewhere", "a.json"), dist), "outside the repo");
+});
+
+// The static checks above cannot see a value Turbopack reads through an
+// import: `path.join(CACHE_DIR, `${stamp}.${asMp3 ? "mp3" : "wav"}`)` in
+// umtool's clip-audio route took umtool's dot-directories into that route's
+// trace, the fixture and the e2e build's directory included (plans/release-15.md,
+// slice UT). So the last build's traces are read back, when there is one.
+//
+// What this can see: umtool/next.config.ts now excludes `.e2e-song`,
+// `.next-e2e` and `.env*` from every trace, so a pattern like that one shows
+// here only through a name the excludes miss -- `test-results/.last-run.json`
+// after an e2e run, `.next-shots`, the corpus, a path outside the repo. A
+// checkout with no e2e run behind it is blind to it; the fix at the call is
+// what keeps the route clean.
+test("umtool's last build traced no dot-directory, no corpus file and nothing outside the repo", (t) => {
+ const dist = path.join(UMTOOL, ".next");
+ const id = path.join(dist, "BUILD_ID");
+ if (!existsSync(path.join(dist, "server")) || !existsSync(id)) {
+ t.skip("no umtool build to read (umtool/.next/server); `pnpm --filter umtool exec next build` makes one");
+ return;
+ }
+ // A build older than the code that decides its traces judges code that is
+ // gone: after a merge or a checkout it would fail on a call already fixed.
+ // umtool runs under `next dev` day to day, so nothing else refreshes it.
+ const built = statSync(id).mtimeMs;
+ const newer = [path.join(UMTOOL, "next.config.ts"), ...umtoolModules()].filter((f) => statSync(f).mtimeMs > built);
+ if (newer.length) {
+ t.skip(
+ `umtool/.next was built ${new Date(built).toISOString()}, before ${path.relative(REPO, newer[0])}` +
+ ` (${newer.length} changed since); rebuild umtool (\`pnpm --filter umtool exec next build\`) to check its traces`,
+ );
+ return;
+ }
+ const bad = [];
+ for (const nft of traceFilesUnder(dist)) {
+ const { files } = JSON.parse(readFileSync(nft, "utf8"));
+ for (const f of files) {
+ const why = forbiddenTrace(path.resolve(path.dirname(nft), f), dist);
+ if (why) bad.push(`${path.relative(dist, nft)}: ${f} (${why})`);
+ }
+ }
+ assert.deepEqual(
+ bad.slice(0, 20),
+ [],
+ `${bad.length} traced file(s) no server needs; find the fs or path call whose value Turbopack could not bound (this file's header):\n` +
+ bad.slice(0, 20).join("\n"),
+ );
+});
+
test("no module the editor, the export or the homepage can bundle joins a cwd-derived path without opting out", () => {
const files = nextAppModules();
assert.ok(files.length > 500, `only ${files.length} modules found`);
diff --git a/umtool/app/api/clip/[key]/audio/route.ts b/umtool/app/api/clip/[key]/audio/route.ts
@@ -1,10 +1,9 @@
import { createHash } from "node:crypto";
import { existsSync } from "node:fs";
import { mkdir, readFile, writeFile, rename } from "node:fs/promises";
-import path from "node:path";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
-import { CACHE_DIR } from "@/lib/paths";
+import { CACHE_DIR, cacheFile } from "@/lib/paths";
import { sourceWav } from "@/lib/clips";
import { readWavWindow, encodeWav, peakOver } from "@/lib/wav";
@@ -55,7 +54,8 @@ export async function GET(request: Request, ctx: { params: Promise<{ key: string
.update(`${video}|${from.toFixed(3)}|${to.toFixed(3)}|${asMp3 ? "mp3" : "wav"}`)
.digest("hex")
.slice(0, 16);
- const cached = path.join(CACHE_DIR, `${stamp}.${asMp3 ? "mp3" : "wav"}`);
+ // Through cacheFile, never a join here: see its comment in lib/paths.mjs.
+ const cached = cacheFile(`${stamp}.${asMp3 ? "mp3" : "wav"}`);
const type = asMp3 ? "audio/mpeg" : "audio/wav";
if (existsSync(cached)) {
@@ -79,8 +79,8 @@ export async function GET(request: Request, ctx: { params: Promise<{ key: string
const wav = encodeWav(x, region.sampleRate);
let body: Buffer = wav;
if (asMp3) {
- const tmpWav = path.join(CACHE_DIR, `${stamp}.in.wav`);
- const tmpMp3 = path.join(CACHE_DIR, `${stamp}.out.mp3`);
+ const tmpWav = cacheFile(`${stamp}.in.wav`);
+ const tmpMp3 = cacheFile(`${stamp}.out.mp3`);
await writeFile(tmpWav, wav);
await run("ffmpeg", [
"-nostdin", "-v", "error", "-y", "-i", tmpWav,
@@ -89,7 +89,7 @@ export async function GET(request: Request, ctx: { params: Promise<{ key: string
body = await readFile(tmpMp3);
await rename(tmpMp3, cached).catch(() => {});
} else {
- const tmp = `${cached}.tmp`;
+ const tmp = cacheFile(`${stamp}.wav.tmp`);
await writeFile(tmp, wav);
await rename(tmp, cached).catch(() => {});
}
diff --git a/umtool/app/api/clip/[key]/video/route.ts b/umtool/app/api/clip/[key]/video/route.ts
@@ -1,10 +1,9 @@
import { createHash } from "node:crypto";
import { existsSync } from "node:fs";
import { mkdir, readFile, rename } from "node:fs/promises";
-import path from "node:path";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
-import { CACHE_DIR } from "@/lib/paths";
+import { CACHE_DIR, cacheFile } from "@/lib/paths";
import { sourceVideo } from "@/lib/clips";
const run = promisify(execFile);
@@ -44,7 +43,8 @@ export async function GET(request: Request, ctx: { params: Promise<{ key: string
.update(`v1|${video}|${from.toFixed(3)}|${to.toFixed(3)}`)
.digest("hex")
.slice(0, 16);
- const cached = path.join(CACHE_DIR, `${stamp}.mp4`);
+ // Through cacheFile, never a join here: see its comment in lib/paths.mjs.
+ const cached = cacheFile(`${stamp}.mp4`);
if (existsSync(cached)) {
return new Response(new Uint8Array(await readFile(cached)), {
headers: { "content-type": "video/mp4", "cache-control": "no-store" },
@@ -52,7 +52,7 @@ export async function GET(request: Request, ctx: { params: Promise<{ key: string
}
await mkdir(CACHE_DIR, { recursive: true });
- const tmp = `${cached}.tmp.mp4`;
+ const tmp = cacheFile(`${stamp}.mp4.tmp.mp4`);
// -ss BEFORE -i for the fast seek, then -t for the length. Re-encoded rather
// than copied because a stream copy starts at the previous keyframe, which
// would slide the picture against the audio by up to several seconds.
diff --git a/umtool/app/api/face/frame/route.ts b/umtool/app/api/face/frame/route.ts
@@ -1,11 +1,10 @@
import { createHash } from "node:crypto";
import { existsSync } from "node:fs";
import { mkdir, readFile, rename } from "node:fs/promises";
-import path from "node:path";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { sourceVideo } from "@/lib/clips";
-import { CACHE_DIR } from "@/lib/paths";
+import { CACHE_DIR, cacheFile } from "@/lib/paths";
const run = promisify(execFile);
@@ -55,11 +54,12 @@ export async function GET(request: Request) {
.update(`face1|${video}|${at.toFixed(3)}|${w ?? "native"}`)
.digest("hex")
.slice(0, 16);
- const cached = path.join(CACHE_DIR, `${stamp}.jpg`);
+ // Through cacheFile, never a join here: see its comment in lib/paths.mjs.
+ const cached = cacheFile(`${stamp}.jpg`);
if (!existsSync(cached)) {
await mkdir(CACHE_DIR, { recursive: true });
- const tmp = `${cached}.tmp.jpg`;
+ const tmp = cacheFile(`${stamp}.jpg.tmp.jpg`);
// -ss BEFORE -i for the fast seek. When a width is asked for it is scaled to
// an even one with the aspect preserved; when it is not, the frame comes out
// at the source's own size and no mapping is needed at all.
diff --git a/umtool/lib/paths.mjs b/umtool/lib/paths.mjs
@@ -18,6 +18,21 @@ export { SONG_DATA, SONG_REPORTS };
// the data, not in the repo, and is safe to delete at any time.
export const CACHE_DIR = path.join(SONG_DATA, ".cache", "umtool");
+/**
+ * A file in CACHE_DIR, by name. A route names its cache files through this
+ * rather than joining CACHE_DIR itself. Turbopack reads a path it can see as a
+ * pattern of files to trace, in the join and in every fs call its value
+ * reaches, and CACHE_DIR is unknown to it (an env var or the home directory).
+ * So `path.join(CACHE_DIR, `${stamp}.${asMp3 ? "mp3" : "wav"}`)` in the
+ * clip-audio route was a pattern that reached into umtool's dot-directories:
+ * that route's trace listed the e2e fixture, the e2e server's build directory
+ * and `.env.local` (plans/release-15.md, slice UT). A value returned by a
+ * function from another module is opaque to it, so a call site traces nothing.
+ */
+export function cacheFile(name) {
+ return path.join(/* turbopackIgnore: true */ CACHE_DIR, name);
+}
+
// Render scratch: the body render and the cut background sit in the job temp
// dir ABOVE SONG_DATA, not inside it, because render-poly.mjs writes them next
// to its logs.
@@ -105,8 +120,10 @@ const dedupe = (list) => [...new Set(list.map((p) => path.resolve(p)))];
* REFERENCE: `next build` walked the whole corpus (hundreds of GB, `data/`
* symlinked to another drive) and was OOM-killed, or died on the first symlink
* out of the root. A worktree with no transcripts/ builds fine, which is how it
- * shipped. The comment is the documented per-expression opt-out; the values at
- * run time are unchanged. scripts/next-build-trace.test.mjs holds the line.
+ * shipped. The comment is Turbopack's per-expression opt-out (the form its own
+ * "whole project was traced" warning advises; the Next docs list the comment
+ * for import(), require(), require.resolve() and new Worker() only); the values
+ * at run time are unchanged. scripts/next-build-trace.test.mjs holds the line.
*/
export function findRepoRoot(start) {
let dir = path.resolve(/* turbopackIgnore: true */ start);
diff --git a/umtool/lib/paths.ts b/umtool/lib/paths.ts
@@ -19,6 +19,7 @@ import path from "node:path";
// ---------------------------------------------------------------------------
export {
CACHE_DIR,
+ cacheFile,
INDEX_DIR,
MEDIA_ROOTS,
MIX_CACHE,
diff --git a/umtool/next.config.ts b/umtool/next.config.ts
@@ -18,19 +18,37 @@ const nextConfig: NextConfig = {
// fail the whole module graph. Every page importing lib/projects then 500s
// with "Can't resolve 'cbor-x'", which names a package nothing here uses.
serverExternalPackages: ["lmdb"],
+ // No route's trace may list the e2e fixture (.e2e-song, where
+ // e2e/fixtures/make-fixture.mjs links the song data), the e2e server's own
+ // build directory (.next-e2e) or an env file: none is a run-time input. The
+ // clip-audio route's trace listed 1,704 such files (plans/release-15.md, slice
+ // UT). That was fixed at the call (lib/paths.mjs `cacheFile`); this is the
+ // second line, measured on its own: with the old route it takes the trace
+ // back to what the sibling routes list. scripts/next-build-trace.test.mjs
+ // reads the last build's traces back.
+ outputFileTracingExcludes: {
+ "/*": ["./.e2e-song/**/*", "./.next-e2e/**/*", "./.env*"],
+ },
turbopack: {
// Same reasoning as editor/next.config.ts: Turbopack infers the workspace
// root by walking up for the outermost lockfile, and a stray pnpm-lock.yaml
// above the checkout silently relocates it. Nothing here lives above the
// monorepo root, so pinning it costs nothing.
root: path.join(__dirname, ".."),
- // Suppress the harmless "whole project was traced unintentionally" NFT
- // warning, exactly as editor/next.config.ts does. It fires because the
- // server genuinely does runtime-dynamic fs reads it cannot statically bound
- // -- lib/paths.ts resolves SONG_DATA from an env var and the routes read
- // wav48/<video>.wav by name. We don't use `output: 'standalone'`, so the
- // .nft.json traces are never consumed and the over-tracing is cosmetic.
- // Scoped to this exact issue (path + title) so other warnings still surface.
+ // Silences the "whole project was traced unintentionally" warning, and only
+ // it (path + title), for one measured reason: 66 of the 68 routes trace
+ // umtool's own tree, its 361 files outside dot-directories, next.config.ts
+ // (the file the warning names) among them. A path or fs call on a value
+ // Turbopack cannot know (an env var, the home directory, a parameter) is a
+ // pattern over the project, and umtool has hundreds. Opting out every path
+ // op in the three path modules left 31 of the 68 routes clean; opting out
+ // all 319 in the 53 modules that have one left 49 clean, and the rest come
+ // through fs calls (lib/report/snapshots.mjs, among others). That walk skips
+ // dot-directories and does not enter symlinks, and the traces are not
+ // consumed while `output: "standalone"` stays off. The warning cannot tell
+ // that walk from one that does reach a dot-directory (both name
+ // next.config.ts), so that case is excluded above and checked after the
+ // build by scripts/next-build-trace.test.mjs instead.
ignoreIssue: [
{
path: "**/next.config.ts",
