commit 597669b68bd1ae676c384c4c88f91e59d2c09fbb
parent d2efce8e88fec486342f663c2508eb93db0604b7
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 8 Sep 2026 01:02:31 -0400
plans: the bucket lanes' new work lists are checked against the live corpus
phase1-worklist-check.ts folds the two entries in memory from each of the 68
snapshots already on disk — regenerating nothing — and compares them against a
union re-derived by hand from the literal bucket names, so the check is not the
fold testing itself. It also asserts no live snapshot carries an entry yet, that
ids.length is the reachable count, and that presentOperationWork round-trips
totals.downloaded / totals.transcribed.
Result: OK, 68 channels, 0 already regenerated; download 9, transcription 882.
The 882 is the DEDUPED union: failedListed is 25 and 24 of them are already in
downloadedNoTranscript (881).
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
1 file changed, 186 insertions(+), 0 deletions(-)
diff --git a/plans/tools/phase1-worklist-check.ts b/plans/tools/phase1-worklist-check.ts
@@ -0,0 +1,186 @@
+#!/usr/bin/env tsx
+// Slice 1.5's proof over the LIVE corpus: the work list the two bucket lanes
+// now carry in a snapshot is the list they have always been handed.
+//
+// WHY THIS IS A SCRIPT AND NOT A TEST. `generateChannelSnapshot` writes
+// `backfill.download` and `backfill.transcription` from
+// `bucketLaneWorkIds(lane, snapshot)`; `autoRunner.buildChannelWork` reads the
+// entry back and falls back to that same fold for a snapshot that predates it.
+// Because both sides are one function, a unit test of it can only be
+// tautological. What is NOT tautological is that the fold, over all 68 real
+// snapshots, equals the union the runner used to walk bucket-by-bucket — so
+// this re-derives that union INDEPENDENTLY here (the literal bucket names, in
+// the literal priority order, deduped by hand) and compares.
+//
+// STRICTLY READ-ONLY, and the one thing it must never do is regenerate a
+// snapshot. That is a WRITE into production data, and it would replace the
+// evidence with something this process just produced. It reads
+// `channels/<slug>/snapshot.json` and nothing else; it starts no server and
+// arms no runner.
+//
+// Usage, from anywhere in the repo:
+// pnpm --filter yt-dlp-transcript-common exec tsx ../plans/tools/phase1-worklist-check.ts
+
+import { getPaths } from "../../common/lib/paths";
+import {
+ listChannelConfigs,
+ readChannelSnapshot,
+} from "../../common/controller/channels";
+import type { ChannelSnapshot } from "../../common/controller/channelSnapshot";
+import { foldBucketLaneEntry } from "../../common/controller/channelSnapshot";
+import {
+ bucketLaneWorkIds,
+ bucketsForKind,
+} from "../../common/jobs/autoQueuePolicy";
+import { bucketLaneOperationId } from "../../common/lib/operations";
+import {
+ presentOperationWork,
+ reachableOperationWork,
+} from "../../common/lib/operations";
+
+// The lanes with a bucket-derived work list, and the buckets each draws, SPELLED
+// OUT rather than imported. This is the independent half of the comparison: if
+// `bucketsForKind` ever changes, this list does not follow it and the check
+// fails, which is the point.
+const LANE_BUCKETS = {
+ download: ["partialDownloads", "undownloadedIds"],
+ transcription: ["downloadedNoTranscript", "failedListed"],
+} as const;
+
+type Lane = keyof typeof LANE_BUCKETS;
+const LANES: readonly Lane[] = ["download", "transcription"];
+
+// Element-wise, because ORDER is half of what is being checked and a
+// join-and-compare needs a separator no video id can contain.
+function sameList(a: readonly string[], b: readonly string[]): boolean {
+ return a.length === b.length && a.every((id, i) => id === b[i]);
+}
+
+// The union, re-derived by hand: walk the names in priority order, take each
+// list as it lies, drop a repeat. No sort — `undownloadedIds` is playlist order.
+function unionByHand(snap: ChannelSnapshot, lane: Lane): string[] {
+ const out: string[] = [];
+ const seen = new Set<string>();
+ for (const name of LANE_BUCKETS[lane]) {
+ const ids: string[] =
+ name === "undownloadedIds"
+ ? (snap.undownloadedIds ?? [])
+ : ((snap.buckets as unknown as Record<string, string[] | undefined>)?.[
+ name
+ ] ?? []);
+ for (const id of ids) {
+ if (seen.has(id)) continue;
+ seen.add(id);
+ out.push(id);
+ }
+ }
+ return out;
+}
+
+async function main(): Promise<void> {
+ const paths = getPaths();
+ const configs = await listChannelConfigs(paths);
+ const slugs = configs.map((c) => c.slug).sort();
+
+ console.log("# one-core phase 1.5 — bucket-lane work lists over the live corpus");
+ console.log(`# channelsDir: ${paths.channelsDir}`);
+ console.log(`# channels: ${slugs.length}`);
+ console.log("");
+
+ const mismatches: string[] = [];
+ const totals: Record<Lane, number> = { download: 0, transcription: 0 };
+ const present: Record<Lane, number> = { download: 0, transcription: 0 };
+ let alreadyRegenerated = 0;
+ let missingSnapshots = 0;
+
+ for (const slug of slugs) {
+ const snap = await readChannelSnapshot(paths, slug);
+ if (!snap) {
+ missingSnapshots += 1;
+ mismatches.push(`${slug}: NO SNAPSHOT`);
+ continue;
+ }
+ for (const lane of LANES) {
+ const opId = bucketLaneOperationId(lane);
+ if (opId !== lane) {
+ mismatches.push(`${lane}: bucketLaneOperationId said ${opId}`);
+ continue;
+ }
+ // THE ENTRY MUST NOT BE THERE YET. This slice regenerates nothing, so a
+ // live snapshot carrying one would mean something wrote into
+ // `transcripts/` — which is the failure this script exists to make loud.
+ if (snap.backfill?.[opId]) alreadyRegenerated += 1;
+
+ const byHand = unionByHand(snap, lane);
+ const folded = bucketLaneWorkIds(lane, snap);
+ if (!sameList(folded, byHand)) {
+ mismatches.push(
+ `${slug} ${lane}: fold (${folded.length}) != union (${byHand.length})`,
+ );
+ continue;
+ }
+
+ // The entry the next regen will write, folded here in memory from the
+ // snapshot already on disk.
+ const entry = foldBucketLaneEntry(lane, {
+ buckets: snap.buckets as unknown as Record<string, string[] | undefined>,
+ undownloadedIds: snap.undownloadedIds,
+ present: lane === "transcription" ? snap.totals.transcribed : snap.totals.downloaded,
+ });
+ if (entry.ids.length !== reachableOperationWork(entry)) {
+ mismatches.push(
+ `${slug} ${lane}: ids.length ${entry.ids.length} != reachable ${reachableOperationWork(entry)}`,
+ );
+ }
+ if (!sameList(entry.ids, byHand)) {
+ mismatches.push(`${slug} ${lane}: entry.ids != union`);
+ }
+ const back = presentOperationWork(entry);
+ const expected =
+ lane === "transcription" ? snap.totals.transcribed : snap.totals.downloaded;
+ if (back !== expected) {
+ mismatches.push(
+ `${slug} ${lane}: presentOperationWork ${back} != totals ${expected}`,
+ );
+ }
+ totals[lane] += entry.ids.length;
+ present[lane] += expected;
+ console.log(`${slug} ${lane} ids = ${entry.ids.length}`);
+ }
+ }
+
+ console.log("");
+ console.log("## totals");
+ console.log(`channels = ${slugs.length}`);
+ console.log(`missingSnapshots = ${missingSnapshots}`);
+ console.log(`snapshotsAlreadyCarryingAnEntry = ${alreadyRegenerated}`);
+ for (const lane of LANES) {
+ console.log(`${lane} work list = ${totals[lane]}`);
+ console.log(`${lane} present = ${present[lane]}`);
+ }
+ console.log("");
+
+ // The independent cross-check on the shape of the whole thing: the two lane
+ // bucket lists this script spells out must still be the ones the code draws.
+ for (const lane of LANES) {
+ const declared = [...bucketsForKind(lane)];
+ if (declared.join(",") !== LANE_BUCKETS[lane].join(",")) {
+ mismatches.push(
+ `${lane}: bucketsForKind is [${declared}], this script checks [${LANE_BUCKETS[lane]}]`,
+ );
+ }
+ }
+
+ if (mismatches.length === 0) {
+ console.log("OK");
+ return;
+ }
+ console.log(`MISMATCHES (${mismatches.length}):`);
+ for (const m of mismatches) console.log(` ${m}`);
+ process.exitCode = 1;
+}
+
+main().catch((err) => {
+ console.error(err);
+ process.exit(1);
+});
