import { test } from "node:test"; import assert from "node:assert/strict"; import { readdir, readFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; // Run with: // pnpm --filter yt-dlp-transcript-common exec tsx --test common/controller/noCorpusWalkInRenderPaths.test.ts // // THIS IS THE TEST THAT STOPS THE WHOLE PROBLEM RECURRING. // // listChannelStatsFromDisk walks every video directory in the corpus (~474,559 // file touches, ~4 seconds). One call to it from the root layout — for a badge // describing 98 videos — put a 4.4 second floor under every page in the editor // and re-ran on a 5 second timer forever. It is ground truth and the batch jobs // need it, so it can't just be deleted; what it can't do is appear in anything // that renders. // // It lives in common's suite rather than editor's because this suite is the one // that runs in a second without a browser, and a guard nobody runs guards // nothing. // // If this test fails: you want listChannelBriefs (slug + config + snapshot), // listChannelConfigs (slug + config), or listChannelStatsFromSnapshots (the // same ChannelStat shape, projected from snapshots) — all in ./channels. const HERE = path.dirname(fileURLToPath(import.meta.url)); const REPO = path.resolve(HERE, "..", ".."); const EDITOR_APP = path.resolve(REPO, "editor", "app"); // The view layer renders too. `common/views/` holds the payload builders the // editor's pages fold their live state through (one-core phase 3 slice 1) — // which is to say it is a render path that happens to live in this package, so // the same ban applies to it. Walking only `editor/app` would let a builder // dodge this guard simply by moving down a layer. const VIEWS = path.resolve(REPO, "common", "views"); // EVERY identifier that reaches the corpus walk, not just the walk itself. // // The guard grepped for one name and that was not enough: the retired // `buildBackfillSweepPlan` CALLED listChannelStatsFromDisk, so importing it into // a page would have walked 474,559 files on a 3-second poll and passed this test // with room to spare. `buildDigestSweepPlan` is the same hazard and is still // here — an LMDB scan plus a per-video freshness check, priced for the // `bin/digest-plan` ETA rather than for a render. // // If you are here because you want it on a screen: what a lane's console draws // is `computeLeafPending` (controller/autoRunner.ts), which folds the channel // SNAPSHOTS the payload has already read. // // THE MATCH IS TEXTUAL AND CONTEXT-BLIND, so a mere MENTION in a comment fails // it too. That is deliberate and not worth softening: a guard that skipped // comments and strings is a guard an offending call can hide behind, and the // cost of the false positive is one reworded comment. // // It IS word-bounded, which is a different thing from context: `measureTree` // must not match `measureTreeCached`, the bounded wrapper that exists so the // raw walk has one caller. A substring match would make the safe name // indistinguishable from the banned one and force every mention of the wrapper // into the allow map — which is how an allow map stops meaning anything. // // `measureTree` is here for a different reason from the other two: it is not a // corpus walk, it is a walk of ONE directory — but the directory is the // saved-video store, which grows on every pin, every `keepSourceVideo` and // every full-source fetch, and /storage is `force-dynamic` with the global // AutoRefresh re-rendering it on a timer. An uncapped walk there gets slower // exactly as the thing it is measuring gets bigger. const BANNED = [ "listChannelStatsFromDisk", "buildDigestSweepPlan", "measureTree", ]; // THE ONE FILE ALLOWED TO NAME EACH BANNED IDENTIFIER, and why it is a map // rather than a blanket exception: the exemption is a claim about a specific // file's specific mitigation, and the next call site added elsewhere must // still fail. `lib/measureStore.ts` wraps `measureTree` in a 60 s, // bounded-size cache; that wrapper is what `buildStorage.ts` calls. const ALLOWED: Record = { "editor/app/storage/lib/measureStore.ts": ["measureTree"], }; async function walk(dir: string): Promise { const out: string[] = []; const entries = await readdir(dir, { withFileTypes: true }); for (const e of entries) { const full = path.join(dir, e.name); if (e.isDirectory()) { if (e.name === "node_modules" || e.name === ".next") continue; out.push(...(await walk(full))); } else if (/\.tsx?$/.test(e.name)) { out.push(full); } } return out; } test("the corpus walk never reaches a render path", async () => { const editorFiles = await walk(EDITOR_APP); const viewFiles = await walk(VIEWS); // Guard the guard: if either traversal finds nothing, the assertion below // would pass while checking zero files. The roots are explicit, so a // directory nobody added is a directory nobody scans. assert.ok( editorFiles.length > 100, `expected to scan the editor app tree, found ${editorFiles.length} files under ${EDITOR_APP}`, ); assert.ok( viewFiles.length > 0, `expected to scan the view layer, found ${viewFiles.length} files under ${VIEWS}`, ); const files = [...editorFiles, ...viewFiles]; const offenders: string[] = []; await Promise.all( files.map(async (file) => { const source = await readFile(file, "utf8"); const rel = path.relative(REPO, file); for (const banned of BANNED) { if (!new RegExp(`\\b${banned}\\b`).test(source)) continue; if (ALLOWED[rel]?.includes(banned)) continue; offenders.push(`${banned} in ${rel}`); } }), ); assert.deepEqual( offenders.sort(), [], `these walk the whole corpus and must not be reachable from a page, ` + `layout, API route or server action. Found: ${offenders.join(", ")}`, ); }); test("the cheap channel readers are the ones the editor actually uses", async () => { // The converse check: if someone "fixes" the test above by inlining a readdir // loop instead, the named readers would quietly stop being used. This asserts // the intended replacements are still wired in. // // Editor-only on purpose: these are READERS, and a view-model is forbidden to // call one (common/architecture.test.ts). They are called by the shells, // which stay in editor/app. const files = await walk(EDITOR_APP); const sources = await Promise.all(files.map((f) => readFile(f, "utf8"))); const joined = sources.join("\n"); for (const wanted of [ "listChannelBriefs", "listChannelConfigs", "listChannelStatsFromSnapshots", ]) { assert.ok( joined.includes(wanted), `${wanted} is no longer referenced anywhere in editor/app — did a render path go back to walking the corpus?`, ); } });