commit 30a5ac502d0f68b127873fae6f23c4b59c7ca8bd
parent 1724aca9ef6b4607aa6d750ec27c145170f2e4d4
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 7 Sep 2026 18:50:59 -0400
plans/tools: the phase-1 numbers script
Every live number a Phase 1 slice could move, printed deterministically so two
runs diff. Read-only over `settings.json` and the 68 snapshots: per channel and
summed, every `backfill[op]` count field plus `ids.length`,
`undownloadedIds.length`, the four buckets the lanes draw from, the per-lane
`computeLeafPending` totals, and the four pause gates with both sweep flags.
The lane sections come from the keys present in `settings.json`, not from the
code's lane list — a slice that only ADDS a lane to the in-memory default must
not print two new zero sections and call the diff noise.
Baseline on the live corpus agrees with the plan's "Facts that shape the
slices" to the digit: digest missing 55,956 / stale 267 / deferred 15 /
blocked 1,722; diarization missing 424 / missingInput 75,591; attribution-text
missing 77,888; attribution-diarized missing 1,881 / blocked 76,015;
undownloadedIds 9; downloadedNoTranscript 881.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
1 file changed, 205 insertions(+), 0 deletions(-)
diff --git a/plans/tools/phase1-numbers.ts b/plans/tools/phase1-numbers.ts
@@ -0,0 +1,205 @@
+#!/usr/bin/env tsx
+// The one-core Phase 1 measurement: every live number a lane could move,
+// printed deterministically so two runs can be diffed.
+//
+// WHY THIS IS A SCRIPT AND NOT A TEST. Phase 1 rewires dispatch over a corpus
+// of 78,000 videos that no fixture reproduces. The claim each slice makes is
+// "no live number moved", and the only way to check it is to read the real
+// `transcripts/` before and after and diff two files. A test would have to
+// carry the expected numbers, and they change every time a sweep runs.
+//
+// STRICTLY READ-ONLY. It opens `settings.json` and the 68
+// `channels/<slug>/snapshot.json` and writes nothing anywhere. It must never be
+// made to regenerate a snapshot: that is a WRITE into production data, and the
+// numbers it would then "verify" would be the ones it just produced.
+//
+// NEVER BOOT AN EDITOR FOR THIS. computeLeafPending is called in-process,
+// offline; instrumentation.ts is not loaded, so no runner, sweep or scheduler
+// is armed against the live corpus.
+//
+// LANE COVERAGE IS DRIVEN BY THE FILE, NOT BY THE CODE. The lane sections come
+// from the keys actually present in settings.json's `autoQueue` object, not
+// from LANES — otherwise slice 1.1, which only ADDS two lanes to the in-memory
+// default, would print two new zero sections and the before/after diff would
+// be noise instead of evidence. The pause flags are the four PauseLane values,
+// which have all existed since pauseGates.ts landed.
+//
+// Usage, from anywhere in the repo:
+// pnpm --filter yt-dlp-transcript-common exec tsx ../plans/tools/phase1-numbers.ts
+
+import { readFile } from "node:fs/promises";
+import { getPaths } from "../../common/lib/paths";
+import { getSettings } from "../../common/lib/settings";
+import { isGateHeld, type PauseLane } from "../../common/lib/pauseGates";
+import {
+ listChannelConfigs,
+ readChannelSnapshot,
+} from "../../common/controller/channels";
+import { computeLeafPending } from "../../common/controller/autoRunner";
+import type { AutoQueueKind } from "../../common/lib/autoQueueTypes";
+
+// Every count field on OperationSnapshotEntry, in a fixed order. `partial` is
+// here alongside the four the plan names because it is part of OperationCounts
+// and IS summed into the reachable set — a slice that moved it would otherwise
+// move `ids.length` with no visible cause.
+const COUNT_FIELDS = [
+ "missing",
+ "stale",
+ "partial",
+ "deferred",
+ "blocked",
+ "missingInput",
+ "eligible",
+] as const;
+
+// The buckets the two existing lanes draw from, plus the opt-in one. Not
+// selectableBucketsForKind(): this list must not change when that function does,
+// or the diff would report a code change as a data change.
+const BUCKETS = [
+ "partialDownloads",
+ "downloadedNoTranscript",
+ "failedListed",
+ "downloadedAutoSubsOnly",
+] as const;
+
+const PAUSE_LANES: ReadonlyArray<PauseLane> = [
+ "backfill",
+ "digest",
+ "download",
+ "transcription",
+];
+
+type Row = Record<string, number>;
+
+function add(into: Row, from: Row): void {
+ for (const [k, v] of Object.entries(from)) into[k] = (into[k] ?? 0) + v;
+}
+
+function printRow(prefix: string, row: Row): void {
+ for (const key of Object.keys(row).sort()) {
+ console.log(`${prefix} ${key} = ${row[key]}`);
+ }
+}
+
+async function main(): Promise<void> {
+ const paths = getPaths();
+ const settings = getSettings();
+
+ console.log("# one-core phase 1 — live numbers");
+ console.log(`# settingsFile: ${paths.settingsFile}`);
+ console.log(`# channelsDir: ${paths.channelsDir}`);
+ console.log("");
+
+ // --- flags ---------------------------------------------------------------
+ console.log("## flags");
+ for (const lane of PAUSE_LANES) {
+ console.log(`held ${lane} = ${isGateHeld(settings, lane)}`);
+ }
+ console.log(`digest.digestsPaused = ${settings.digest.digestsPaused}`);
+ console.log(`digest.sweepEnabled = ${settings.digest.sweepEnabled}`);
+ console.log(`digest.remoteEnabled = ${settings.digest.remoteEnabled}`);
+ console.log(`backfill.enabled = ${settings.backfill.enabled}`);
+ console.log(`backfill.sweepEnabled = ${settings.backfill.sweepEnabled}`);
+ console.log(`transcriptionsPaused = ${settings.transcriptionsPaused}`);
+ console.log(`downloadsPaused = ${settings.downloadsPaused}`);
+ console.log("");
+
+ // --- lanes, as the file spells them --------------------------------------
+ const raw = JSON.parse(await readFile(paths.settingsFile, "utf8")) as {
+ autoQueue?: Record<string, unknown>;
+ };
+ const fileLanes = Object.keys(raw.autoQueue ?? {}).sort();
+
+ console.log("## lanes (from settings.json autoQueue)");
+ for (const lane of fileLanes) {
+ const policy = (settings.autoQueue as Record<string, unknown>)[lane] as
+ | { enabled: boolean; order?: string; maxWorkers: number | null; replaceAutoSubs?: boolean }
+ | undefined;
+ if (!policy) {
+ console.log(`${lane} MISSING FROM SANITIZED SETTINGS`);
+ continue;
+ }
+ console.log(`${lane} enabled = ${policy.enabled}`);
+ console.log(`${lane} order = ${policy.order ?? "listed"}`);
+ console.log(`${lane} maxWorkers = ${policy.maxWorkers}`);
+ console.log(`${lane} replaceAutoSubs = ${policy.replaceAutoSubs === true}`);
+ }
+ console.log("");
+
+ console.log("## computeLeafPending (per lane, per leaf, sorted by leaf id)");
+ for (const lane of fileLanes) {
+ const pending = await computeLeafPending(lane as AutoQueueKind, paths);
+ let total = 0;
+ for (const leafId of Object.keys(pending.counts).sort()) {
+ console.log(`${lane} leaf ${leafId} = ${pending.counts[leafId]}`);
+ total += pending.counts[leafId];
+ }
+ console.log(`${lane} TOTAL = ${total}`);
+ console.log(
+ `${lane} nextUp = ${pending.nextUp ? `${pending.nextUp.leafId}/${pending.nextUp.videoId}` : "none"}`,
+ );
+ }
+ console.log("");
+
+ // --- snapshots -----------------------------------------------------------
+ const configs = await listChannelConfigs(paths);
+ const slugs = configs.map((c) => c.slug).sort();
+ console.log(`## snapshots (${slugs.length} channels)`);
+
+ const opTotals: Record<string, Row> = {};
+ const bucketTotals: Row = {};
+ let undownloadedTotal = 0;
+ let missingSnapshots = 0;
+
+ for (const slug of slugs) {
+ const snap = await readChannelSnapshot(paths, slug);
+ if (!snap) {
+ console.log(`${slug} NO SNAPSHOT`);
+ missingSnapshots += 1;
+ continue;
+ }
+ const undownloaded = snap.undownloadedIds?.length ?? 0;
+ undownloadedTotal += undownloaded;
+ console.log(`${slug} undownloadedIds.length = ${undownloaded}`);
+
+ const buckets = (snap.buckets ?? {}) as Record<string, string[] | undefined>;
+ for (const name of BUCKETS) {
+ const n = buckets[name]?.length ?? 0;
+ bucketTotals[name] = (bucketTotals[name] ?? 0) + n;
+ console.log(`${slug} bucket ${name} = ${n}`);
+ }
+
+ const backfill = snap.backfill ?? {};
+ for (const op of Object.keys(backfill).sort()) {
+ const entry = backfill[op] as unknown as Record<string, unknown>;
+ const row: Row = {};
+ for (const field of COUNT_FIELDS) {
+ row[field] = typeof entry[field] === "number" ? (entry[field] as number) : 0;
+ }
+ row["ids.length"] = Array.isArray(entry.ids) ? entry.ids.length : 0;
+ printRow(`${slug} op ${op}`, row);
+ opTotals[op] ??= {};
+ add(opTotals[op], row);
+ }
+ }
+ console.log("");
+
+ console.log("## totals");
+ console.log(`channels = ${slugs.length}`);
+ console.log(`missingSnapshots = ${missingSnapshots}`);
+ console.log(`undownloadedIds.length = ${undownloadedTotal}`);
+ for (const name of [...BUCKETS].sort()) {
+ console.log(`bucket ${name} = ${bucketTotals[name] ?? 0}`);
+ }
+ for (const op of Object.keys(opTotals).sort()) {
+ printRow(`op ${op}`, opTotals[op]);
+ }
+}
+
+main().then(
+ () => process.exit(0),
+ (err) => {
+ console.error(err);
+ process.exit(1);
+ },
+);
