commit 2a4598de02cde54d3096e31493117892ea5f7595
parent 72438bd04609528a6c7a2043171ea573be90ed8b
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 28 Sep 2026 20:13:11 -0400
Merge main (88db32b9, slice Q's umtool build trace) into homepage/social-visible
editor/CHANGELOG.md: both sides kept; main's umtool bullet stays under the
bullet it refers to, this branch's two stay at the end of [Unreleased].
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
13 files changed, 372 insertions(+), 24 deletions(-)
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -5,6 +5,7 @@
- **A stats build keeps the stats of a channel whose drive is not mounted, and will not undo a newer version's stats.** A channel whose media is on a drive that is not mounted (or is being moved) is left as it was instead of being read as a channel with no videos; a stats rebuild that has to start over refuses until the drive is back. A stats build refuses to clear stats written by a newer version of the editor; set `ARCHILYZER_STATS_ALLOW_DOWNGRADE=1` to roll back on purpose. Its log also says apart how many videos were downloaded since the last index build (they catch up after the next one) and how many the index skipped (no upload date, or it failed on them).
- **Building the homepage now publishes the source: a read-only git mirror, its raw tree and a fresh tarball, behind a gate.** `archilyzer build homepage`, the `/sites` Homepage jobs and `pnpm ops build-homepage` run `archilyzer source publish` between compose and `next build`. It makes a fresh clone of the private `main` (the repository itself is never rewritten), rewrites that copy with git-filter-repo using your scrub rules (file contents and commit messages; your home directory becomes `/home/user` without a rule), and publishes it under `homepage/public` for `git clone https://archilyzer.pages.dev/source/archilyzer.git`, beside `/source/tree/` and the Downloads tarball. Before anything is written, every object of the rewritten history and every file about to be published is searched for every string you have denied; **one hit refuses the build**, and its log names the string only by where you wrote it (`denylist line 3 (len 5)`) and each hit by its object, field and byte offset — never a byte of the object. **A refusal withdraws the source**: the last publish is removed from `homepage/public` and the last build's copy from `homepage/out`, and **Deploy homepage refuses** a build whose source was not audited under today's rules and today's `main` ("run `archilyzer build homepage`, then deploy"). The rules live outside the repo, in `~/.config/archilyzer/source-scrub.txt` and `source-denylist.txt` (`ARCHILYZER_CONFIG_DIR`, `SOURCE_SCRUB_FILE`, `SOURCE_DENYLIST_FILE`); **without them the build refuses**, naming the missing file. **Put everything private in the denylist before any deploy, a preview included**: previews are public, and every deployment stays reachable at its own address until you delete it. Install git-filter-repo once (`pipx install git-filter-repo`; the editor's process needs `~/.local/bin` on its `PATH` to find it) — without it the build fetches it through `pipx run`, which needs the network — and gitleaks if you want its secret scan too. An unchanged `main` with unchanged rules is skipped, so a rebuild costs about 20 seconds only when something moved. A checkout with no git repository (the docker image, a tarball install) builds with the /source page's empty state. `archilyzer source publish --check` audits without writing, `archilyzer source audit <clone>/.git` checks any clone, `archilyzer build homepage --no-source` removes the published source instead, and `archilyzer doctor` reports the tools, the two files (rule counts and permissions, never their contents) and the last publish. `create-archives.sh` is gone. See PUBLISH.md, "The source mirror (homepage)".
- **umtool reads the corpus from its checkout (or `TRANSCRIPTS_DIR`), and the song project's data defaults to `~/.local/share/archilyzer/song`.** If yours is elsewhere, link it there before restarting umtool: `mkdir -p ~/.local/share/archilyzer && ln -s <where the data is> ~/.local/share/archilyzer/song` (the data stays where it is). With no `CHANNELS_DIR`, umtool reads the corpus at `$TRANSCRIPTS_DIR/channels`, else the checkout's own `transcripts/channels`; it used to fall back to an absolute path that existed on one machine only. The song project's videos default to `~/reports/quartering-uh-song/videos`; `SONG_DIR` and `VIDEO_ROOT` still win. The song project's tracked manifests record their paths relative to the song folders, and the twenty one-off `umtool/song/*.sh` run logs, which only ever ran on the machine that wrote them, are gone.
+- **umtool's production build no longer reads the corpus folder.** Since umtool began finding the corpus from its checkout (the bullet above), `next build` treated the checkout's whole `transcripts/channels` as files to bundle. On a real archive it ran out of memory and was killed, so umtool could not be rebuilt. The build now ignores that folder and finishes in about 25 s at under 1 GB, the same as a checkout with no corpus. Nothing changes when umtool runs.
- **A social icon pasted with only a width and height is accepted, and each social link can be shown in a header.** The social-link editors (Settings, a site's form) refused an SVG with no `viewBox`, so a vendor's logo file as downloaded, which often carries only its size, was refused. On save, a root with a numeric width and height (unitless or px) and no viewBox is now given `viewBox="0 0 W H"`; a percentage, `em`, or a missing or zero side is still refused. Each link has a **Show in header** checkbox, stored as `featured: true` only when checked, with the hint "With none checked, the header shows the last four.": a header shows at most four links, the checked ones when any is checked, else the last four (the homepage's header reads it). A file with neither is read and rendered as before. `SETTINGS.md` and `SITE.md` list `featured`.
- **A social icon is checked by what it may contain, on save and every time it is shown, and a refused one says why.** An icon must be one well-formed `<svg>` of shapes, groups, gradients, clips, masks, filters, text and simple animation, with SVG presentation attributes: no script, `style` block, `foreignObject`, link, embedded image or HTML element; no event handler, however it is written; a reference only to something inside the icon, even when it is spelled with character references. Comments, a leading XML declaration and a plain DOCTYPE are removed. A refused save ends with the reason ("… has an invalid SVG: it has an event handler attribute.", "… it links to something outside the icon.", "… it has an element an icon has no use for (style).") and never repeats the markup. A stored icon that fails the check — a file edited by hand, or saved by an older build — is not shown: the homepage and every site's footer show the link's label instead. A file saved from Illustrator or Inkscape may need its `<style>` block, `<metadata>` or `inkscape:` attributes removed first.
diff --git a/plans/FACTS.md b/plans/FACTS.md
@@ -7320,6 +7320,57 @@ Slices Q (`4855f70b`) and R (`ffdeb2cd`): [`release-12.md`](release-12.md), the
- **The live :3001 editor runs its BUILT bundle.** Until it is rebuilt on a tree with release 12,
its `/sites` Homepage jobs have no source step, no withdrawal and no deploy check.
+### A path joined from `process.cwd()` is a directory of assets to Turbopack (slice Q, after rollout)
+
+**The hazard.** In a module a Next app imports, Turbopack traces a path joined from
+`process.cwd()` as a directory of assets. The same goes for a module's own `import.meta.url` or
+`__dirname`.
+- **How:** Turbopack evaluates these statically as paths in the project. A `path.join` /
+ `path.resolve` / fs call on the result becomes an asset reference: to a file, or, when the joined
+ path is a directory, to every file under it (`DirAssetReference`).
+- **A worktree build will not show it.** A worktree has no `transcripts/`, so the reference is
+ empty. **Test with the corpus visible.**
+- **What happened:** slice Q wrote `path.join(REPO_ROOT, "transcripts", "channels")` with
+ `REPO_ROOT = findRepoRoot(process.cwd())` in `umtool/lib/paths.mjs`. The walk's fallback,
+ `path.resolve(start, "..")`, evaluates to the project root.
+ - In `<primary>`, `pnpm --filter umtool exec next build` walked the corpus (hundreds of GB, with
+ channel `data/` symlinked to another drive). It grew until the kernel OOM-killed it: twice, at
+ about 3.7 GB RSS. The live umtool was down until the fix.
+ - Under a memory cap it dies at once instead: `<DirAssetReference as
+ ModuleReference>::resolve_reference failed … Symlink [project]/transcripts/channels/<slug>/archive
+ is invalid, it points out of the filesystem root`.
+ - A worktree built it in 30 s at 0.8 GB, which is how the gate passed.
+- **The opt-out is per expression and documented:** `path.join(/* turbopackIgnore: true */
+ process.cwd(), bar)`. That exact text is Turbopack's own advice in its "whole project was traced"
+ message; the table is in the Next docs, `03-api-reference/08-turbopack.md`, "Magic comments". It
+ goes before the FIRST argument of each path or fs call on such a value, and it changes nothing
+ at run time. Per call, not per value:
+ - a nested call needs its own marker (`path.dirname(/* turbopackIgnore: true */
+ fileURLToPath(import.meta.url))`);
+ - an outer fs call on an opted-out `path.join` is covered.
+- **Not followed by the tracer:** `os.homedir()` and `process.env.*`. A build with `HOME` pointed at
+ a synthetic home inside the project, full of out-of-root symlinks under `reports/`,
+ `.local/share/archilyzer/song` and `.cache/`, succeeded. So `~/reports` and the XDG song path are
+ safe as `path.join(os.homedir(), …)`.
+- **The guard is `scripts/umtool-build-trace.test.mjs`** (in `test:scripts`). It scans umtool's
+ app, components, lib, `report-to-video/*.mjs` and `song/paths.mjs`, per module. A path or fs call
+ carrying a value derived in that file from `process.cwd()`, `import.meta.url|dirname|filename` or
+ `__dirname` must open with the opt-out.
+ - It is static and per module, as Turbopack's value analysis is: an imported binding is opaque to
+ it.
+ - With the slice Q `paths.mjs` it fails on the defect's line.
+- **The build gate** is run with the corpus visible and under a memory cap (the command is in
+ `plans/tools/implementer-rules.md`). Linking `<primary>/transcripts` into a worktree is for a
+ BUILD only. Remove the link afterwards: never run an app, an index or a fixture builder through it.
+- **The other apps are safe by accident, not by rule:**
+ - `common/lib/paths.ts`' `findMonorepoRoot()` falls back to `process.cwd()` (the app's own
+ directory, which has no `transcripts/`). An editor build with the corpus present is 39 s today.
+ A fallback that evaluated to the repo root would make `path.join(monorepoRoot, "transcripts")`
+ this same bug.
+ - `homepage/app/lib/source.ts` joins `process.cwd()` + `public`, which holds the source mirror.
+ The homepage builds in about 20 s today.
+ - Neither has a guard.
+
## The stats cache key (verified 2026-09-28, branch `fix/stats-cache-key`)
The record is [`stats-cache-key.md`](stats-cache-key.md). Anchors are at the branch tip. Two notes
diff --git a/plans/release-12.md b/plans/release-12.md
@@ -281,6 +281,87 @@ No High or Medium findings. The coordinator asked for two of the Lows to be fixe
- **After the fixes:** tsc is clean (41 s, all seven packages), and the grep gate is empty at the
new tip. Per the coordinator, e2e was not re-run for a type comment and a changelog line.
+### Slice Q, found after rollout — the umtool build walked the corpus (2026-09-28)
+
+**What.** Slice Q's `umtool/lib/paths.mjs` set `CHANNELS_DIR` to
+`path.join(REPO_ROOT, "transcripts", "channels")`, with
+`REPO_ROOT = findRepoRoot(process.cwd())`.
+- **Turbopack evaluated that statically** as `[project]/transcripts/channels` and made it a
+ directory asset reference. The walk's fallback, `path.resolve(start, "..")`, is the project
+ root.
+- **In `<primary>`,** `pnpm --filter umtool exec next build` walked the real corpus (hundreds of GB,
+ channel `data/` symlinked to another drive). It was OOM-killed twice, at about 3.7 GB RSS, so the
+ live umtool on :3050 stayed down until this fix.
+- **Slice Q's gate built in a worktree,** which has no `transcripts/`, so the reference was empty
+ and the build took 30 s. The hazard and the rule are in FACTS: "A path joined from
+ `process.cwd()` is a directory of assets to Turbopack".
+
+**The repro.** Link the corpus into a worktree for the BUILD only, and cap memory:
+```
+ln -s <primary>/transcripts <worktree>/transcripts
+timeout -s KILL 240 systemd-run --user --scope -q -p MemoryMax=5G -p MemorySwapMax=0 pnpm --filter umtool exec next build
+rm <worktree>/transcripts
+```
+On `main` `10cefd15` it fails in 6.4 s (0.97 GB): `TurbopackInternalError: Failed to write app
+endpoint /page … [project]/umtool/lib/paths.mjs … <DirAssetReference as
+ModuleReference>::resolve_reference failed … Symlink [project]/transcripts/channels/<slug>/archive
+is invalid, it points out of the filesystem root`.
+
+**The fix** (branch `fix/umtool-build-trace` off `main` `10cefd15`). Every path or fs call on a
+value derived from `process.cwd()` or `import.meta.url`, in a umtool module the app imports, now
+opens with `/* turbopackIgnore: true */`. That is the per-expression opt-out Turbopack's own
+message documents. The calls are in:
+- `lib/paths.mjs` (`findRepoRoot`, `CHANNELS_DIR`);
+- `lib/paths.ts` (`SONG_CODE`, `stateFile`);
+- `lib/tools.mjs`, `lib/trim.ts`, `lib/report/driver.mjs`;
+- `report-to-video/brand.mjs` and `cues.mjs`.
+
+The values at run time are unchanged. From `umtool/`, `REPO_ROOT`, `CHANNELS_DIR` and `SONG_DATA`
+print the same as on `main`, and `CHANNELS_DIR` / `TRANSCRIPTS_DIR` / `SONG_DIR` still win.
+`os.homedir()` joins are left as they are. A build with `HOME` pointed at a synthetic home inside
+the project, holding out-of-root symlinks at every home-derived root, succeeded, so the tracer does
+not follow `os.homedir()`.
+
+| build (5 GB cap, `/usr/bin/time -v`) | result | wall | max RSS | `.next` |
+|---|---|---|---|---|
+| `main` `10cefd15`, corpus linked | **fails** (the error above) | 6.4 s | 0.97 GB | — |
+| fix, no corpus | ok | 42.2 s (a busy machine; 22.2 s on an earlier run) | 0.80 GB | 1,009 files, 25,417,403 B |
+| fix, corpus linked | ok | 24.2 s | 0.80 GB | 1,009 files, 25,417,285 B |
+
+- **The two outputs are the same file set,** differing only in the build-id directory.
+- **Nothing from the corpus is traced.** 0 `.nft.json` entries are under `transcripts/`. The 14
+ files that contain the string `transcripts/channels` are all `.js.map` source maps of the code;
+ none is a chunk or an asset.
+
+**The guard: `scripts/umtool-build-trace.test.mjs`** (in `test:scripts`, 3 tests). It is a static,
+per-module check of umtool's app, components, lib, `report-to-video/*.mjs` and `song/paths.mjs`. A
+path or fs call carrying a value derived in that file from `process.cwd()`,
+`import.meta.url|dirname|filename` or `__dirname` must open with the opt-out. It costs about 0.1 s.
+With `main`'s `lib/paths.mjs` swapped in, it fails and names the defect:
+`umtool/lib/paths.mjs:118 : path.join(REPO_ROOT, "transcripts", "channels")`.
+
+| sha | what |
+|---|---|
+| `8346f824` | `umtool:` cwd-derived paths opt out of Turbopack's asset tracing |
+| `55699a20` | `scripts:` the guard |
+| _this_ | `plans:` this note, FACTS, the implementer rules' umtool build gate; the `[Unreleased]` bullet |
+
+**Gates:**
+- tsc is clean (all seven packages).
+- `node --check` passes on the five changed `.mjs`.
+- `test:scripts` **188 + 1 skipped** (`main` 185 + 1, plus the guard's 3).
+- The capped builds are in the table above.
+- umtool e2e (`faces`, `deck`, `clip-bench`, with the corpus link removed): **67 passed, 8
+ skipped, 0 failed** (1.9 min). The skips are the song-data specs, as in slice Q.
+
+**The other apps are safe by accident, not by rule.**
+- `common/lib/paths.ts`' `findMonorepoRoot()` falls back to `process.cwd()` (the app's own
+ directory).
+- `homepage/app/lib/source.ts` joins `process.cwd()` + `public`.
+- Both build today, so there is no finding to fix, only a note in FACTS.
+
+**For the parent:** merge, then rebuild and restart :3050. That is the parent's step.
+
### Slice R, as shipped — `archilyzer source publish` + `/source` (2026-09-28)
Branch `r12/source-mirror` off `main` `e6c5d2e3` (slice Q merged), worktree
diff --git a/plans/tools/implementer-rules.md b/plans/tools/implementer-rules.md
@@ -77,6 +77,16 @@ the Next.js reference for this version.
test:scripts` (156 + 1 skip); mcp `pnpm --filter yt-dlp-transcript-mcp test` (219) — check
`package.json` for the exact script names before running.
- `pnpm --filter editor exec next build` and `pnpm --filter export exec next build`.
+- **umtool's build runs with the corpus visible, under a memory cap.** A worktree has no
+ `transcripts/`, so a path Turbopack traces as a directory is empty there. It was hundreds of GB in
+ the primary, and the build was OOM-killed (FACTS, "A path joined from `process.cwd()` …"). From
+ the worktree root:
+ ```
+ ln -s <primary>/transcripts transcripts && timeout -s KILL 240 systemd-run --user --scope -q -p MemoryMax=5G -p MemorySwapMax=0 pnpm --filter umtool exec next build; rm transcripts
+ ```
+ It is for the BUILD only. Always remove the link, never commit it, and never run an app, an index
+ or a fixture builder through it. It should take about 25 s at under 1 GB, the same as without
+ the corpus.
- The slice's e2e spec list (in the prompt), detached and waited on as above.
- The numbers tool the prompt names, diff-empty (or "none", stated).
diff --git a/plans/tools/rollout/smoke.sh b/plans/tools/rollout/smoke.sh
@@ -25,7 +25,7 @@ done
echo "BUILD_ID: $(cat $R/editor/.next/BUILD_ID)"
echo "ZodError in start log: $(grep -c ZodError $T/editor-start-r8.log 2>/dev/null)"
# release 7 additions
-d=$(curl -s --max-time 300 "$B/api/view/autoQueueStatus" | jq -c '.download.deferred | if type=="array" and all(.[]; type=="object" and has("videoId") and has("until")) then "ok \(length)" else "bad" end' 2>/dev/null); case "$d" in '"ok '*) echo "DEFERRED_OK download.deferred well-formed ($d)";; *) echo "DEFERRED_UNEXPECTED $d"; fail=1;; esac
+d=$(curl -s --max-time 300 "$B/api/view/autoQueueStatus" | jq -c '.download.deferred | if type=="array" and all(.[]; type=="object" and has("videoId") and has("untilMs")) then "ok \(length)" else "bad" end' 2>/dev/null); case "$d" in '"ok '*) echo "DEFERRED_OK download.deferred well-formed ($d)";; *) echo "DEFERRED_UNEXPECTED $d"; fail=1;; esac
f="$T/$REL-page_operations_download.html"; c=$(curl -s -o "$f" --max-time 300 -w %{http_code} "$B/operations/download"); n=$(grep -c 'aria-label="Rate-limit cooldown"' "$f")
[ "$c" = 200 ] && echo "PAGE_OK /operations/download rate-limit-regions=$n" || { echo "PAGE_FAIL /operations/download $c"; fail=1; }
f="$T/$REL-page_sites.html"; c=$(curl -s -o "$f" --max-time 300 -w %{http_code} "$B/sites"); echo "PAGE $c /sites Build hub=$(grep -c 'Build hub' "$f") Deploy hub=$(grep -c 'Deploy hub' "$f")"
diff --git a/scripts/umtool-build-trace.test.mjs b/scripts/umtool-build-trace.test.mjs
@@ -0,0 +1,195 @@
+// umtool's modules must not hand Turbopack a directory to bundle.
+//
+// Turbopack evaluates `process.cwd()` (and a module's own `import.meta.url` /
+// `__dirname`) statically, as a path in the project, and a `path.join` /
+// `path.resolve` / fs call on such a value becomes an ASSET REFERENCE: to a
+// file, or, when the joined path is a directory, to EVERY file under it. Release
+// 12 slice Q wrote `path.join(REPO_ROOT, "transcripts", "channels")` with
+// `REPO_ROOT = findRepoRoot(process.cwd())`, and `next build` in the primary
+// checkout walked the whole corpus (hundreds of GB, `data/` symlinked to another
+// drive) until the kernel killed it -- while a worktree, which has no
+// `transcripts/`, built in 30 s. So no build-in-a-worktree gate can see this.
+//
+// The rule, checked statically and per module (Turbopack's value analysis is
+// per module; an imported binding is opaque to it): every path or fs call whose
+// arguments carry a value derived IN THAT FILE from `process.cwd()`,
+// `import.meta.url`, `import.meta.dirname|filename` or `__dirname` must open its
+// argument list with the documented opt-out, `/* turbopackIgnore: true */`.
+// The comment changes nothing at run time. `os.homedir()` is NOT a source: the
+// tracer does not follow it (a build with HOME pointed at a synthetic home full
+// of out-of-root symlinks inside the project succeeds), and neither is
+// `process.env.*`.
+//
+// Run with: pnpm test:scripts
+import assert from "node:assert/strict";
+import { readdirSync, readFileSync } from "node:fs";
+import path from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const UMTOOL = path.join(REPO, "umtool");
+
+const SOURCE = /process\.cwd\(\)|import\.meta\.(?:url|dirname|filename)|\b__dirname\b/;
+const MARK = "__TURBOPACK_IGNORE__";
+const IGNORE_COMMENT = /\/\*\s*turbopackIgnore\s*:\s*true\s*\*\//g;
+
+// path ops, and fs calls either bare (`existsSync(`) or on a namespace
+// (`fs.readdir(`, `fsp.stat(`). A method on anything else (`obj.stat(`) is not
+// one.
+const SINK =
+ /(?:\bpath\.(?:join|resolve|dirname|relative)|(?<![\w$.])(?:fs\.|fsp\.|promises\.)?(?:existsSync|readFileSync|readdirSync|statSync|lstatSync|realpathSync|opendirSync|readFile|readdir|stat|lstat|opendir|createReadStream))\s*\(/g;
+
+/** Comments out, except the opt-out, which becomes a marker. Strings stay. */
+function prepare(text) {
+ let s = text.replace(IGNORE_COMMENT, ` ${MARK} `);
+ s = s.replace(/\/\*[\s\S]*?\*\//g, (m) => m.replace(/[^\n]/g, " "));
+ // A line comment: `//` not preceded by `:` (URLs, `file://` templates).
+ s = s.replace(/(^|[^:\\])\/\/[^\n]*/g, (m, pre) => pre + " ".repeat(m.length - pre.length));
+ return s;
+}
+
+/** Index just past the bracket that closes the one at `open`. */
+function closeOf(s, open) {
+ let depth = 0;
+ let quote = null;
+ for (let i = open; i < s.length; i += 1) {
+ const c = s[i];
+ if (quote) {
+ if (c === "\\") i += 1;
+ else if (c === quote) quote = null;
+ continue;
+ }
+ if (c === '"' || c === "'" || c === "`") quote = c;
+ else if (c === "(" || c === "[" || c === "{") depth += 1;
+ else if (c === ")" || c === "]" || c === "}") {
+ depth -= 1;
+ if (depth === 0) return i + 1;
+ }
+ }
+ return s.length;
+}
+
+/** Names assigned, in this file, from a source or from another such name. */
+function taintedNames(s) {
+ const decls = [];
+ const re = /\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=/g;
+ for (let m; (m = re.exec(s)); ) {
+ // The right-hand side runs to the first `;` or newline at depth 0 -- good
+ // enough for this repo's formatter, which ends statements with `;`.
+ let depth = 0;
+ let end = s.length;
+ for (let i = re.lastIndex; i < s.length; i += 1) {
+ const c = s[i];
+ if (c === "(" || c === "[" || c === "{") depth += 1;
+ else if (c === ")" || c === "]" || c === "}") depth -= 1;
+ else if (c === ";" && depth <= 0) {
+ end = i;
+ break;
+ }
+ }
+ decls.push({ name: m[1], rhs: s.slice(re.lastIndex, end) });
+ }
+ const names = new Set();
+ for (let grew = true; grew; ) {
+ grew = false;
+ for (const { name, rhs } of decls) {
+ if (names.has(name)) continue;
+ if (SOURCE.test(rhs) || [...names].some((n) => new RegExp(`(?<![\\w$.])${n.replace(/\$/g, "\\$")}\\b`).test(rhs))) {
+ names.add(name);
+ grew = true;
+ }
+ }
+ }
+ return names;
+}
+
+/** Every path/fs call on a cwd-derived value that does not opt out. */
+export function untracedCalls(text) {
+ const s = prepare(text);
+ const names = taintedNames(s);
+ const carries = (args) =>
+ SOURCE.test(args) ||
+ [...names].some((n) => new RegExp(`(?<![\\w$.])${n.replace(/\$/g, "\\$")}\\b(?!\\s*:)`).test(args));
+ // A call nested in these arguments that opts out is opaque to this one too:
+ // `readFileSync(path.join(/* turbopackIgnore: true */ HERE, "a.json"))`. Its
+ // own arguments are cut out before asking whether this call carries a source.
+ const withoutOptedOut = (args) => {
+ let out = args;
+ for (let i = out.search(new RegExp(`\\(\\s*${MARK}`)); i !== -1; i = out.search(new RegExp(`\\(\\s*${MARK}`))) {
+ out = out.slice(0, i) + out.slice(closeOf(out, i));
+ }
+ return out;
+ };
+ const out = [];
+ for (let m; (m = SINK.exec(s)); ) {
+ const open = SINK.lastIndex - 1;
+ const args = s.slice(open + 1, closeOf(s, open) - 1);
+ if (args.trimStart().startsWith(MARK)) continue;
+ if (!carries(withoutOptedOut(args))) continue;
+ const line = s.slice(0, m.index).split("\n").length;
+ out.push({ line, call: text.split("\n")[line - 1].trim() });
+ }
+ return out;
+}
+
+/** umtool's modules that a Next build can reach: the app, its libs, the pipeline. */
+function appModules() {
+ const out = [];
+ const walk = (dir) => {
+ for (const e of readdirSync(dir, { withFileTypes: true })) {
+ if (e.name === "node_modules" || e.name.startsWith(".")) continue;
+ const p = path.join(dir, e.name);
+ if (e.isDirectory()) walk(p);
+ else if (/\.(?:mjs|js|ts|tsx)$/.test(e.name) && !/\.test\./.test(e.name) && !e.name.endsWith(".d.ts")) out.push(p);
+ }
+ };
+ for (const d of ["app", "components", "lib"]) walk(path.join(UMTOOL, d));
+ for (const e of readdirSync(path.join(UMTOOL, "report-to-video"))) {
+ if (e.endsWith(".mjs") && !e.includes(".test.")) out.push(path.join(UMTOOL, "report-to-video", e));
+ }
+ // song/paths.mjs is imported by lib/paths.mjs; the other song scripts are CLIs.
+ out.push(path.join(UMTOOL, "song", "paths.mjs"));
+ return out;
+}
+
+test("the check flags slice Q's join and passes the opted-out form", () => {
+ const bad = [
+ 'const REPO_ROOT = findRepoRoot(process.cwd());',
+ 'export const CHANNELS_DIR = path.resolve(',
+ ' process.env.CHANNELS_DIR ?? path.join(REPO_ROOT, "transcripts", "channels"),',
+ ');',
+ ].join("\n");
+ const found = untracedCalls(bad);
+ assert.ok(found.some((f) => f.call.includes('path.join(REPO_ROOT, "transcripts"')), JSON.stringify(found));
+
+ const good = bad
+ .replace("path.resolve(", "path.resolve(/* turbopackIgnore: true */")
+ .replace("path.join(REPO_ROOT", "path.join(/* turbopackIgnore: true */ REPO_ROOT");
+ assert.deepEqual(untracedCalls(good), []);
+});
+
+test("sources: cwd, import.meta, __dirname; not homedir, env, or a comment", () => {
+ assert.equal(untracedCalls('const X = path.join(process.cwd(), "song");').length, 1);
+ assert.equal(untracedCalls("const H = path.dirname(fileURLToPath(import.meta.url));").length, 1);
+ assert.equal(untracedCalls('readFileSync(path.join(__dirname, "a.json"));').length, 2);
+ assert.equal(untracedCalls('const R = path.join(os.homedir(), "reports");').length, 0);
+ assert.equal(untracedCalls('const R = path.join(process.env.X, "channels");').length, 0);
+ assert.equal(untracedCalls('// path.join(process.cwd(), "x")\nconst y = 1;').length, 0);
+ // An object key named like a tainted value is not a use of it.
+ assert.equal(untracedCalls('const cwd = path.join(/* turbopackIgnore: true */ process.cwd(), "s");\nf(path.join(a, { cwd: 1 }));').length, 0);
+});
+
+test("no umtool module the app can import joins a cwd-derived path without opting out", () => {
+ const bad = [];
+ for (const file of appModules()) {
+ for (const f of untracedCalls(readFileSync(file, "utf8"))) {
+ bad.push(`${path.relative(REPO, file)}:${f.line} ${f.call}`);
+ }
+ }
+ assert.deepEqual(
+ bad,
+ [],
+ "add /* turbopackIgnore: true */ as the first argument (see this file's header):\n" + bad.join("\n"),
+ );
+});
diff --git a/umtool/lib/paths.mjs b/umtool/lib/paths.mjs
@@ -98,13 +98,22 @@ const dedupe = (list) => [...new Set(list.map((p) => path.resolve(p)))];
* all run with cwd inside the checkout (the app at umtool/, which is what the
* fallback assumes). report-to-video/cues.mjs keeps its import.meta.url walk:
* only its CLI entry points (build-video, resolve-windows) read that default.
+ *
+ * EVERY PATH OP ON A cwd-DERIVED VALUE CARRIES `turbopackIgnore`. Turbopack
+ * evaluates `process.cwd()` statically as the project, so an un-annotated
+ * `path.join(REPO_ROOT, "transcripts", "channels")` became a DIRECTORY ASSET
+ * REFERENCE: `next build` walked the whole corpus (hundreds of GB, `data/`
+ * symlinked to another drive) and was OOM-killed, or died on the first symlink
+ * out of the root. A worktree with no transcripts/ builds fine, which is how it
+ * shipped. The comment is the documented per-expression opt-out; the values at
+ * run time are unchanged. scripts/umtool-build-trace.test.mjs holds the line.
*/
export function findRepoRoot(start) {
- let dir = path.resolve(start);
+ let dir = path.resolve(/* turbopackIgnore: true */ start);
for (;;) {
- if (existsSync(path.join(dir, "pnpm-workspace.yaml"))) return dir;
- const up = path.dirname(dir);
- if (up === dir) return path.resolve(start, "..");
+ if (existsSync(path.join(/* turbopackIgnore: true */ dir, "pnpm-workspace.yaml"))) return dir;
+ const up = path.dirname(/* turbopackIgnore: true */ dir);
+ if (up === dir) return path.resolve(/* turbopackIgnore: true */ start, "..");
dir = up;
}
}
@@ -112,10 +121,11 @@ export function findRepoRoot(start) {
export const REPO_ROOT = findRepoRoot(process.cwd());
export const CHANNELS_DIR = path.resolve(
+ /* turbopackIgnore: true */
process.env.CHANNELS_DIR ??
(process.env.TRANSCRIPTS_DIR
- ? path.join(process.env.TRANSCRIPTS_DIR, "channels")
- : path.join(REPO_ROOT, "transcripts", "channels")),
+ ? path.join(/* turbopackIgnore: true */ process.env.TRANSCRIPTS_DIR, "channels")
+ : path.join(/* turbopackIgnore: true */ REPO_ROOT, "transcripts", "channels")),
);
export const READ_ROOTS = dedupe(
diff --git a/umtool/lib/paths.ts b/umtool/lib/paths.ts
@@ -39,9 +39,9 @@ export {
// cwd at the package root.
export const SONG_CODE = process.env.SONG_CODE_DIR
? path.resolve(process.env.SONG_CODE_DIR)
- : path.join(process.cwd(), "song");
+ : path.join(/* turbopackIgnore: true */ process.cwd(), "song");
-export const stateFile = (name: string) => path.join(SONG_CODE, name);
+export const stateFile = (name: string) => path.join(/* turbopackIgnore: true */ SONG_CODE, name);
// dataFile needs SONG_DATA at module scope, which the re-export above does not
// bind locally -- so it is imported again rather than duplicated.
diff --git a/umtool/lib/report/driver.mjs b/umtool/lib/report/driver.mjs
@@ -11,9 +11,9 @@
import path from "node:path";
/** Where the pipeline lives. One place, so a move is one edit. */
-export const PIPELINE_DIR = path.resolve(process.cwd(), "report-to-video");
+export const PIPELINE_DIR = path.resolve(/* turbopackIgnore: true */ process.cwd(), "report-to-video");
-const script = (name) => path.join(PIPELINE_DIR, name);
+const script = (name) => path.join(/* turbopackIgnore: true */ PIPELINE_DIR, name);
/**
* Presets, in the order somebody actually works.
@@ -322,9 +322,9 @@ export function checkSourcesSteps(projects, env = {}) {
// ---------------------------------------------------------------------------
/** This package's own root. bin/ lives here, and so does report-to-video/. */
-export const UMTOOL_DIR = path.resolve(process.cwd());
+export const UMTOOL_DIR = path.resolve(/* turbopackIgnore: true */ process.cwd());
-const tool = (name) => path.join(UMTOOL_DIR, "bin", name);
+const tool = (name) => path.join(/* turbopackIgnore: true */ UMTOOL_DIR, "bin", name);
/** The interpreter a project's own scripts are run with. */
export const PYTHON = process.env.PYTHON_BIN ?? "python3";
diff --git a/umtool/lib/tools.mjs b/umtool/lib/tools.mjs
@@ -19,8 +19,8 @@ import { SONG_SCRATCH } from "./paths.mjs";
export const facedetPython = () =>
process.env.FACEDET_PYTHON ?? path.join(SONG_SCRATCH, "facedet", "bin", "python");
export const songCode = () =>
- process.env.SONG_CODE_DIR ? path.resolve(process.env.SONG_CODE_DIR) : path.join(process.cwd(), "song");
-export const facecropPy = () => path.join(songCode(), "facecrop.py");
+ process.env.SONG_CODE_DIR ? path.resolve(process.env.SONG_CODE_DIR) : path.join(/* turbopackIgnore: true */ process.cwd(), "song");
+export const facecropPy = () => path.join(/* turbopackIgnore: true */ songCode(), "facecrop.py");
/**
* The tools, with the env override each pipeline script honours. `required`
diff --git a/umtool/lib/trim.ts b/umtool/lib/trim.ts
@@ -222,7 +222,7 @@ export function hookRecipe(
const trimDir = `mkvocals/hooks-trim-${stamp}`;
const takeDir = `mkvocals/hooks-best-${stamp}`;
const outOverlays = dataFile(pair.to.replace(/\.json$/, `-${stamp}.json`));
- const cwd = path.join(process.cwd(), "song");
+ const cwd = path.join(/* turbopackIgnore: true */ process.cwd(), "song");
const base = { SONG_DIR: SONG_DATA };
return [
diff --git a/umtool/report-to-video/brand.mjs b/umtool/report-to-video/brand.mjs
@@ -37,17 +37,17 @@ import { readFileSync } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
-const HERE = path.dirname(fileURLToPath(import.meta.url));
+const HERE = path.dirname(/* turbopackIgnore: true */ fileURLToPath(import.meta.url));
import { BRAND_CHOICES, BRAND_IDS } from "./brand-ids.mjs";
import { IBM_PLEX_SANS } from "./svg-faces.mjs";
export { BRAND_CHOICES, BRAND_IDS };
-export const FONTS_DIR = path.join(HERE, "fonts");
-export const FONTCONFIG_FILE = path.join(FONTS_DIR, "fonts.conf");
-export const MONO_FONT_FILE = path.join(FONTS_DIR, "IBMPlexMono-Regular.ttf");
+export const FONTS_DIR = path.join(/* turbopackIgnore: true */ HERE, "fonts");
+export const FONTCONFIG_FILE = path.join(/* turbopackIgnore: true */ FONTS_DIR, "fonts.conf");
+export const MONO_FONT_FILE = path.join(/* turbopackIgnore: true */ FONTS_DIR, "IBMPlexMono-Regular.ttf");
/** Plex Mono's static Bold: `render.fontBold`, which compose-chrome's HyperFrames band sets its bold in. */
-export const MONO_BOLD_FONT_FILE = path.join(FONTS_DIR, "IBMPlexMono-Bold.ttf");
+export const MONO_BOLD_FONT_FILE = path.join(/* turbopackIgnore: true */ FONTS_DIR, "IBMPlexMono-Bold.ttf");
/** The end card's default length: YouTube's end screen runs in the last 5–20 s. */
export const END_CARD_DEFAULT_SECONDS = 20;
@@ -79,7 +79,7 @@ export function brandKit(id) {
throw new Error(`render.brand "${id}" is not a preset — one of ${BRAND_IDS.join(", ")}`);
}
if (!kits.has(id)) {
- kits.set(id, JSON.parse(readFileSync(path.join(HERE, "brands", `${id}.json`), "utf8")));
+ kits.set(id, JSON.parse(readFileSync(path.join(/* turbopackIgnore: true */ HERE, "brands", `${id}.json`), "utf8")));
}
return kits.get(id);
}
diff --git a/umtool/report-to-video/cues.mjs b/umtool/report-to-video/cues.mjs
@@ -58,10 +58,10 @@ import { fileURLToPath } from "node:url";
// checkout would have is two levels up. Previously this defaulted to an absolute
// path inside the original author's home directory, which meant every other
// clone silently looked in a directory that does not exist.
-const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
+const REPO_ROOT = path.resolve(/* turbopackIgnore: true */ path.dirname(/* turbopackIgnore: true */ fileURLToPath(import.meta.url)), "..", "..");
export const DEFAULT_CHANNELS_DIR =
- process.env.CHANNELS_DIR ?? path.join(REPO_ROOT, "transcripts", "channels");
+ process.env.CHANNELS_DIR ?? path.join(/* turbopackIgnore: true */ REPO_ROOT, "transcripts", "channels");
const DEFAULT_CACHE_DIR =
process.env.REPORT_CACHE_DIR ??
