// No module a Next app bundles may hand Turbopack a directory to trace: umtool, // the editor, the export and the homepage, and the common/ modules they import. // // Turbopack evaluates `process.cwd()` (and a module's own `import.meta.url` / // `__dirname`) statically, as a path in the project, and a `path.join` / // `path.resolve` / fs call on such a value becomes an ASSET REFERENCE: to a // file, or, when the joined path is a directory, to EVERY file under it. Release // 12 slice Q wrote `path.join(REPO_ROOT, "transcripts", "channels")` with // `REPO_ROOT = findRepoRoot(process.cwd())`, and `next build` in the primary // checkout walked the whole corpus (hundreds of GB, `data/` symlinked to another // drive) until the kernel killed it -- while a worktree, which has no // `transcripts/`, built in 30 s. So no build-in-a-worktree gate can see this. // // The rule, checked statically and per module (Turbopack's value analysis is // per module; an imported binding is opaque to it): every path or fs call whose // arguments carry a value derived IN THAT FILE from `process.cwd()`, // `import.meta.url`, `import.meta.dirname|filename` or `__dirname` must open its // argument list with Turbopack's opt-out, `/* turbopackIgnore: true */` (the // form its own "whole project was traced" warning advises; the Next docs list // the comment only for import(), require(), require.resolve() and new Worker()). // The comment changes nothing at run time. A function declared in the file whose // body carries a source is a source too (`const ROOT = findMonorepoRoot()`). // // A value Turbopack cannot know -- `process.env.*`, `os.homedir()`, a // parameter, an imported binding -- is not one of these sources, and it is not // ignored either: it is a dynamic part, and a path or fs call on it becomes a // PATTERN over the app's own directory. Measured in umtool (release 15, slice // UT): the path ops on env and home-directory values in its path modules took // in the app's whole tree outside dot-directories (opting them out left 31 of // the 68 routes clean), and the clip-audio route's join with a dynamic extension took // in the dot-directories too, the e2e fixture and `.env.local` among them. // Neither walk entered a symlinked directory. No static check here can tell // such a pattern from a harmless one, so the last test reads a build's traces // back instead. // // Run with: pnpm test:scripts import assert from "node:assert/strict"; import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const UMTOOL = path.join(REPO, "umtool"); const SOURCE = /process\.cwd\(\)|import\.meta\.(?:url|dirname|filename)|\b__dirname\b/; const MARK = "__TURBOPACK_IGNORE__"; const IGNORE_COMMENT = /\/\*\s*turbopackIgnore\s*:\s*true\s*\*\//g; // path ops, and fs calls either bare (`existsSync(`) or on a namespace // (`fs.readdir(`, `fsp.stat(`, `fs.promises.readFile(`). A method on anything // else (`obj.stat(`) is not one. The opens and writes are checked too // (`open`, `writeFile`, `appendFile`, `createWriteStream`): which fs calls // Turbopack traces is not documented, and an opt-out on one it does not trace // costs nothing. const SINK = /(?:\bpath\.(?:join|resolve|dirname|relative)|(? m.replace(/[^\n]/g, " ")); // A line comment: `//` not preceded by `:` (URLs, `file://` templates). s = s.replace(/(^|[^:\\])\/\/[^\n]*/g, (m, pre) => pre + " ".repeat(m.length - pre.length)); return s; } /** Index just past the regex literal whose opening `/` is at `i`, or -1 when * the `/` there is a division. A `/` opens a regex after an operator, an * opening bracket, a separator or `return` — good enough for this repo. */ function regexEnd(s, i) { let j = i - 1; while (j >= 0 && /\s/.test(s[j])) j -= 1; const before = j < 0 ? "" : s[j]; const word = s.slice(0, j + 1).match(/[A-Za-z_$][\w$]*$/)?.[0]; if (!(before === "" || "(,=:[!&|?{};+-*%<>~^".includes(before) || word === "return" || word === "typeof")) return -1; if (s[i + 1] === "/" || s[i + 1] === "*") return -1; let inClass = false; for (let k = i + 1; k < s.length; k += 1) { const c = s[k]; if (c === "\n") return -1; if (c === "\\") k += 1; else if (c === "[") inClass = true; else if (c === "]") inClass = false; else if (c === "/" && !inClass) return k + 1; } return -1; } /** Index just past the bracket that closes the one at `open`. Strings and * regex literals are skipped whole. */ function closeOf(s, open) { let depth = 0; let quote = null; for (let i = open; i < s.length; i += 1) { const c = s[i]; if (quote) { if (c === "\\") i += 1; else if (c === quote) quote = null; continue; } if (c === "/") { const end = regexEnd(s, i); if (end !== -1) { i = end - 1; continue; } } if (c === '"' || c === "'" || c === "`") quote = c; else if (c === "(" || c === "[" || c === "{") depth += 1; else if (c === ")" || c === "]" || c === "}") { depth -= 1; if (depth === 0) return i + 1; } } return s.length; } /** Names assigned, in this file, from a source or from another such name. */ function taintedNames(s) { const decls = []; const re = /\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=/g; for (let m; (m = re.exec(s)); ) { // The right-hand side runs to the first `;` or newline at depth 0 -- good // enough for this repo's formatter, which ends statements with `;`. let depth = 0; let end = s.length; for (let i = re.lastIndex; i < s.length; i += 1) { const c = s[i]; if (c === "(" || c === "[" || c === "{") depth += 1; else if (c === ")" || c === "]" || c === "}") depth -= 1; else if (c === ";" && depth <= 0) { end = i; break; } } decls.push({ name: m[1], rhs: s.slice(re.lastIndex, end) }); } // A function declared in this file whose body carries a source: a call to it // is a source too (`const ROOT = findMonorepoRoot()`, where the function // walks up from `process.cwd()` and falls back to it). const fnRe = /\bfunction\s*\*?\s*([A-Za-z_$][\w$]*)\s*\(/g; for (let m; (m = fnRe.exec(s)); ) { const params = fnRe.lastIndex - 1; const bodyOpen = s.indexOf("{", closeOf(s, params)); if (bodyOpen === -1) continue; decls.push({ name: m[1], rhs: s.slice(params, closeOf(s, bodyOpen)) }); } const names = new Set(); for (let grew = true; grew; ) { grew = false; for (const { name, rhs } of decls) { if (names.has(name)) continue; if (SOURCE.test(rhs) || [...names].some((n) => new RegExp(`(? SOURCE.test(args) || [...names].some((n) => new RegExp(`(? { let out = args; for (let i = out.search(new RegExp(`\\(\\s*${MARK}`)); i !== -1; i = out.search(new RegExp(`\\(\\s*${MARK}`))) { out = out.slice(0, i) + out.slice(closeOf(out, i)); } return out; }; const out = []; for (let m; (m = SINK.exec(s)); ) { const open = SINK.lastIndex - 1; const args = s.slice(open + 1, closeOf(s, open) - 1); if (args.trimStart().startsWith(MARK)) continue; if (!carries(withoutOptedOut(args))) continue; const line = s.slice(0, m.index).split("\n").length; out.push({ line, call: text.split("\n")[line - 1].trim() }); } return out; } /** Every module under `dir`, tests and declarations aside. */ function modulesUnder(dir, out = []) { for (const e of readdirSync(dir, { withFileTypes: true })) { if (e.name === "node_modules" || e.name.startsWith(".")) continue; const p = path.join(dir, e.name); if (e.isDirectory()) modulesUnder(p, out); else if (/\.(?:mjs|cjs|js|ts|tsx)$/.test(e.name) && !/\.test\./.test(e.name) && !e.name.endsWith(".d.ts")) out.push(p); } return out; } const MODULE_EXT = [".ts", ".tsx", ".mjs", ".js", ".cjs"]; const isModule = (p) => MODULE_EXT.some((x) => p.endsWith(x)) && !/\.test\./.test(p) && !p.endsWith(".d.ts"); const isFile = (p) => { try { return statSync(p).isFile(); } catch { return false; } }; /** The module a relative specifier names, the way the bundler resolves it, or null. */ function resolveRelative(from, spec) { const base = path.resolve(path.dirname(from), spec); const candidates = [base, ...MODULE_EXT.map((x) => base + x), ...MODULE_EXT.map((x) => path.join(base, "index" + x))]; return candidates.find((c) => isModule(c) && isFile(c)) ?? null; } // `import … from "./x"`, `export … from "../x"`, `import "./x"`, `import("./x")`, // `require("./x")`: the relative specifiers only. A package import (`next`, // `yt-dlp-transcript-common/…`) is covered by the package's own directory in // the set, or is not this repo's code. const RELATIVE_IMPORT = /(?:\bfrom\s*|\bimport\s*\(?\s*|\brequire\s*\(\s*)(["'])(\.{1,2}\/[^"'\n]+)\1/g; /** * `files` plus every module of this repo they reach by relative imports, to * any depth. A directory list alone misses a module one app imports from a * folder the list treats as CLI-only (`common/bin/_publicFile.ts`, imported by * `common/publish/source.ts`; umtool's `song/pitch.mjs`, imported by * `lib/verdict.ts`) or keeps outside `app/` (`homepage/content/docs.ts`). */ export function withRelativeImports(files) { const seen = new Set(files); const queue = [...files]; while (queue.length) { const file = queue.pop(); for (const m of readFileSync(file, "utf8").matchAll(RELATIVE_IMPORT)) { const target = resolveRelative(file, m[2]); if (!target || seen.has(target) || target.includes(`${path.sep}node_modules${path.sep}`)) continue; if (path.relative(REPO, target).startsWith("..")) continue; seen.add(target); queue.push(target); } } return [...seen]; } /** * umtool's modules that a Next build can reach: the app, its components and * libs, the report pipeline, and whatever of `song/` they import (`paths.mjs` * through `lib/paths.mjs`, `pitch.mjs`, `reasons.mjs` and the rest through the * libs; the other song scripts are CLIs nothing in the app imports). */ function umtoolModules() { const out = []; for (const d of ["app", "components", "lib"]) modulesUnder(path.join(UMTOOL, d), out); for (const e of readdirSync(path.join(UMTOOL, "report-to-video"))) { if (e.endsWith(".mjs") && !e.includes(".test.")) out.push(path.join(UMTOOL, "report-to-video", e)); } return withRelativeImports(out); } /** * The editor's, the export's and the homepage's modules a Next build can * reach: each app's `app/` (the editor's `lib/` and `instrumentation.ts` too), * every module of common/ but its CLIs in `bin/`, and every module those reach * by a relative import — which brings in the few `bin/` modules an app does * import (`_publicFile.ts`) and the homepage's `content/docs.ts`. The common set * is wider than what the apps import today, on purpose: a module that starts * being imported is already covered. */ function nextAppModules() { const out = []; for (const d of ["homepage/app", "export/app", "editor/app", "editor/lib"]) modulesUnder(path.join(REPO, d), out); out.push(path.join(REPO, "editor", "instrumentation.ts")); for (const e of readdirSync(path.join(REPO, "common"), { withFileTypes: true })) { if (!e.isDirectory() || e.name === "bin" || e.name === "node_modules" || e.name.startsWith(".")) continue; modulesUnder(path.join(REPO, "common", e.name), out); } return withRelativeImports(out); } function untracedIn(files) { const bad = []; for (const file of files) { for (const f of untracedCalls(readFileSync(file, "utf8"))) { bad.push(`${path.relative(REPO, file)}:${f.line} ${f.call}`); } } return bad; } test("the check flags slice Q's join and passes the opted-out form", () => { const bad = [ 'const REPO_ROOT = findRepoRoot(process.cwd());', 'export const CHANNELS_DIR = path.resolve(', ' process.env.CHANNELS_DIR ?? path.join(REPO_ROOT, "transcripts", "channels"),', ');', ].join("\n"); const found = untracedCalls(bad); assert.ok(found.some((f) => f.call.includes('path.join(REPO_ROOT, "transcripts"')), JSON.stringify(found)); const good = bad .replace("path.resolve(", "path.resolve(/* turbopackIgnore: true */") .replace("path.join(REPO_ROOT", "path.join(/* turbopackIgnore: true */ REPO_ROOT"); assert.deepEqual(untracedCalls(good), []); }); test("sources: cwd, import.meta, __dirname; not homedir, env, or a comment", () => { assert.equal(untracedCalls('const X = path.join(process.cwd(), "song");').length, 1); assert.equal(untracedCalls("const H = path.dirname(fileURLToPath(import.meta.url));").length, 1); assert.equal(untracedCalls('readFileSync(path.join(__dirname, "a.json"));').length, 2); assert.equal(untracedCalls('const R = path.join(os.homedir(), "reports");').length, 0); assert.equal(untracedCalls('const R = path.join(process.env.X, "channels");').length, 0); assert.equal(untracedCalls('// path.join(process.cwd(), "x")\nconst y = 1;').length, 0); // An object key named like a tainted value is not a use of it. assert.equal(untracedCalls('const cwd = path.join(/* turbopackIgnore: true */ process.cwd(), "s");\nf(path.join(a, { cwd: 1 }));').length, 0); }); test("the check flags the homepage's source.ts and paths.ts' walk as main had them", () => { // homepage/app/lib/source.ts before the opt-out: a DIRECTORY join on the cwd. const source = [ "export function loadSourceManifest(", ' pub: string = path.join(process.cwd(), "public"),', "): SourceManifest | null {", ' return fs.readFileSync(path.join(pub, "source", "manifest.json"), "utf8");', "}", ].join("\n"); assert.ok(untracedCalls(source).some((f) => f.call.includes('path.join(process.cwd(), "public")'))); // common/lib/paths.ts before: the walk, and a join on the root it returns. const walk = [ "function findMonorepoRoot(): string {", " let dir = process.cwd();", ' if (fs.existsSync(path.join(dir, "pnpm-workspace.yaml"))) return dir;', " return process.cwd();", "}", "const monorepoRoot = findMonorepoRoot();", 'const transcriptsDir = process.env.TRANSCRIPTS_DIR ?? path.join(monorepoRoot, "transcripts");', ].join("\n"); const found = untracedCalls(walk).map((f) => f.call); assert.ok(found.some((c) => c.includes('path.join(dir, "pnpm-workspace.yaml")')), JSON.stringify(found)); assert.ok(found.some((c) => c.includes('path.join(monorepoRoot, "transcripts")')), JSON.stringify(found)); }); test("brackets inside a regex literal or a string do not end a call early", () => { // A regex holding a quote used to swallow the rest of the file as one body. const text = [ "function esc(s) { return s.replace(/['\"&]/g, \"\"); }", "const out = path.join(dir, esc(x));", "if (import.meta.url === `file://${process.argv[1]}`) main();", ].join("\n"); assert.deepEqual(untracedCalls(text), []); }); test("no umtool module the app can import joins a cwd-derived path without opting out", () => { const bad = untracedIn(umtoolModules()); assert.deepEqual( bad, [], "add /* turbopackIgnore: true */ as the first argument (see this file's header):\n" + bad.join("\n"), ); }); test("the scan set follows relative imports out of the listed folders", () => { const rel = (files) => new Set(files.map((f) => path.relative(REPO, f))); const um = rel(umtoolModules()); for (const m of ["paths", "reasons", "archive-url", "pitch", "flatness", "clipwindow", "deplosive", "orderfeat"]) { assert.ok(um.has(`umtool/song/${m}.mjs`), `umtool/song/${m}.mjs is not scanned`); } // A song CLI nothing in the app imports stays out. assert.ok(!um.has("umtool/song/build-um.mjs")); const apps = rel(nextAppModules()); assert.ok(apps.has("common/bin/_publicFile.ts"), "common/bin/_publicFile.ts is not scanned"); assert.ok(apps.has("homepage/content/docs.ts"), "homepage/content/docs.ts is not scanned"); assert.ok(!apps.has("common/bin/compose-site.ts"), "a common CLI no app imports is scanned"); }); /** * Every `.nft.json` a build wrote under its directory `dist`, its cache and * dev-server output (`dist/cache`, `dist/dev`) aside. Only those two: a route * directory named `cache` or `dev` deeper down is read like any other. */ function traceFilesUnder(dist, dir = dist, out = []) { for (const e of readdirSync(dir, { withFileTypes: true })) { const p = path.join(dir, e.name); if (e.isDirectory()) { if (dir !== dist || (e.name !== "cache" && e.name !== "dev")) traceFilesUnder(dist, p, out); } else if (e.name.endsWith(".nft.json")) out.push(p); } return out; } /** * Why `abs`, a file a build traced, must not be in the trace, or null. * * The build's own directory (`dist`) holds the chunks every trace lists, and * `node_modules/.pnpm` is where pnpm keeps the packages; any other path through * a directory or file whose name starts with a dot is something no server * needs at run time — a fixture (`.e2e-song`, where the e2e fixture links the * song data), another build (`.next-e2e`), a secret (`.env.local`), `.git` — * and is the mark of a pattern Turbopack could not bound. So are the corpus * and anything outside the repo. */ export function forbiddenTrace(abs, dist) { const rel = path.relative(REPO, abs); if (rel === "" || rel.startsWith("..") || path.isAbsolute(rel)) return "outside the repo"; if (abs.startsWith(dist + path.sep)) return null; const parts = rel.split(path.sep); if (parts[0] === "transcripts") return "the corpus"; for (let i = 0; i < parts.length; i += 1) { if (!parts[i].startsWith(".")) continue; if (parts[i] === ".pnpm" && parts[i - 1] === "node_modules") continue; return `under ${parts.slice(0, i + 1).join("/")}`; } return null; } test("traceFilesUnder: only the build's own cache/ and dev/ are left out", () => { const dist = mkdtempSync(path.join(tmpdir(), "next-build-trace-")); try { for (const f of ["cache/a.nft.json", "dev/b.nft.json", "server/app/api/cache/route.js.nft.json", "server/app/dev/page.js.nft.json"]) { mkdirSync(path.dirname(path.join(dist, f)), { recursive: true }); writeFileSync(path.join(dist, f), '{"files":[]}'); } const found = traceFilesUnder(dist).map((f) => path.relative(dist, f)).sort(); assert.deepEqual(found, ["server/app/api/cache/route.js.nft.json", "server/app/dev/page.js.nft.json"]); } finally { rmSync(dist, { recursive: true, force: true }); } }); test("forbiddenTrace: a fixture, another build, a secret, the corpus, outside the repo", () => { const dist = path.join(UMTOOL, ".next"); const at = (p) => forbiddenTrace(path.join(REPO, p), dist); assert.equal(at("umtool/.next/server/chunks/ssr/a.js"), null); assert.equal(at("node_modules/.pnpm/next@16.2.3/node_modules/next/dist/server/next.js"), null); assert.equal(at("umtool/lib/paths.mjs"), null); assert.equal(at("umtool/.e2e-song/data/planted/x/config.json"), "under umtool/.e2e-song"); assert.equal(at("umtool/.next-e2e/dev/server/a.js"), "under umtool/.next-e2e"); assert.equal(at("umtool/.env.local"), "under umtool/.env.local"); assert.equal(at(".git/config"), "under .git"); assert.equal(at("transcripts/channels/x/config.json"), "the corpus"); assert.equal(forbiddenTrace(path.resolve(REPO, "..", "elsewhere", "a.json"), dist), "outside the repo"); }); // The static checks above cannot see a value Turbopack reads through an // import: `path.join(CACHE_DIR, `${stamp}.${asMp3 ? "mp3" : "wav"}`)` in // umtool's clip-audio route took umtool's dot-directories into that route's // trace, the fixture and the e2e build's directory included (plans/release-15.md, // slice UT). So the last build's traces are read back, when there is one. // // What this can see: umtool/next.config.ts now excludes `.e2e-song`, // `.next-e2e` and `.env*` from every trace, so a pattern like that one shows // here only through a name the excludes miss -- `test-results/.last-run.json` // after an e2e run, `.next-shots`, the corpus, a path outside the repo. A // checkout with no e2e run behind it is blind to it; the fix at the call is // what keeps the route clean. test("umtool's last build traced no dot-directory, no corpus file and nothing outside the repo", (t) => { const dist = path.join(UMTOOL, ".next"); const id = path.join(dist, "BUILD_ID"); if (!existsSync(path.join(dist, "server")) || !existsSync(id)) { t.skip("no umtool build to read (umtool/.next/server); `pnpm --filter umtool exec next build` makes one"); return; } // A build older than the code that decides its traces judges code that is // gone: after a merge or a checkout it would fail on a call already fixed. // umtool runs under `next dev` day to day, so nothing else refreshes it. const built = statSync(id).mtimeMs; const newer = [path.join(UMTOOL, "next.config.ts"), ...umtoolModules()].filter((f) => statSync(f).mtimeMs > built); if (newer.length) { t.skip( `umtool/.next was built ${new Date(built).toISOString()}, before ${path.relative(REPO, newer[0])}` + ` (${newer.length} changed since); rebuild umtool (\`pnpm --filter umtool exec next build\`) to check its traces`, ); return; } const bad = []; for (const nft of traceFilesUnder(dist)) { const { files } = JSON.parse(readFileSync(nft, "utf8")); for (const f of files) { const why = forbiddenTrace(path.resolve(path.dirname(nft), f), dist); if (why) bad.push(`${path.relative(dist, nft)}: ${f} (${why})`); } } assert.deepEqual( bad.slice(0, 20), [], `${bad.length} traced file(s) no server needs; find the fs or path call whose value Turbopack could not bound (this file's header):\n` + bad.slice(0, 20).join("\n"), ); }); test("no module the editor, the export or the homepage can bundle joins a cwd-derived path without opting out", () => { const files = nextAppModules(); assert.ok(files.length > 500, `only ${files.length} modules found`); const bad = untracedIn(files); assert.deepEqual( bad, [], "add /* turbopackIgnore: true */ as the first argument (see this file's header):\n" + bad.join("\n"), ); });