commit 2164c33e1b3f6f69253f91390316e344ecb9ecef parent b0cca40e63435b4b7447dc1a7c7c9f2ff2b913ed Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st> Date: Mon, 14 Sep 2026 16:51:19 -0400 views/pipeline: the five that already were view-models One-core phase 3 slice 1, sub-slice A2. `band`, `buildBands`, `stageStatus`, `channelFlow` and `tone` fold a channel snapshot into what the pipeline draws. They were already pure functions of their arguments — no getter, no reader, no clock — so this is a rename, not a refactor: `git mv` plus import rewrites, and not one line of logic changed. They move together because they import each other. `views/pipeline/` is the only nest in the layer for exactly that reason. Every import of common inside them is rewritten from the package name to a relative specifier. That is not cosmetic: the layer guard only reads RELATIVE specifiers, so `yt-dlp-transcript-common/controller/channelSnapshot` left in place would resolve fine, compile fine, and hide the edge from the very guard this slice turned on. `BARE_FORBIDDEN` now fails that mistake loudly. What the guard sees as a result, and is meant to: `channelFlow → digestCountOf` and `buildBands → excludedDownloadIdSet` are views/ → controller/, which is downward and allowed. `stageStatus`'s `channelMedia` import stays `import type` — it reaches `node:fs`, it is imported by client components, and only `next build` can see that violation. `buildBands.test.ts`'s two directive guards come along. The `band.ts` read is still `./band.ts`; the `StateBand.tsx` read now reaches back into `editor/app`, because the client half stays where it renders. It is a textual read, not an import — nothing in common/ depends on editor/ at build time — and it is worth keeping across the boundary: it is the guard that caught a `"use client"` module exporting a callable helper that every channel page then 500'd on at request time, which `next build` does not catch. Twenty-two editor files repointed at `yt-dlp-transcript-common/views/pipeline/*`. No component changed, no payload type renamed, nothing rendered differently. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Diffstat:
37 files changed, 2370 insertions(+), 2363 deletions(-)
diff --git a/editor/app/components/pipelines/band.ts b/common/views/pipeline/band.ts diff --git a/common/views/pipeline/buildBands.test.ts b/common/views/pipeline/buildBands.test.ts @@ -0,0 +1,435 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import type { ChannelSnapshot } from "../../controller/channelSnapshot"; +import type { OperationSnapshotEntry } from "../../lib/operations"; +import { + bandCoverage, + buildChannelBands, + buildOperationBands, + sumOrNull, + type OperationBand, +} from "./buildBands"; + +// Run from this directory: +// cd editor/app/components/pipelines && ../../../../node_modules/.bin/tsx --test buildBands.test.ts + +function snapshotOf(patch: Partial<ChannelSnapshot> = {}): ChannelSnapshot { + return { + generatedAt: "2026-08-21T00:00:00.000Z", + totals: { videos: 100, transcribed: 40, downloaded: 60 }, + buckets: {} as ChannelSnapshot["buckets"], + ...patch, + } as ChannelSnapshot; +} + +function entryOf(patch: Partial<OperationSnapshotEntry>): OperationSnapshotEntry { + return { + missing: 0, + stale: 0, + partial: 0, + missingInput: 0, + deferred: 0, + blocked: 0, + ids: [], + ...patch, + } as OperationSnapshotEntry; +} + +const bandOf = (bands: OperationBand[], id: string): OperationBand => { + const found = bands.find((b) => b.id === id); + assert.ok(found, `no band for ${id}`); + return found; +}; + +test("the four work states are kept apart and never summed", () => { + // The measured shape of this corpus in miniature: diarization is dominated by + // missing media and attribution-diarized by blocked work. Any code that added + // them would report both lanes as busy. + const bands = buildOperationBands({ + snapshots: [ + snapshotOf({ + backfill: { + diarization: entryOf({ + missing: 647, + missingInput: 77_276, + eligible: 78_019, + }), + "attribution-diarized": entryOf({ + missing: 94, + blocked: 77_923, + eligible: 78_019, + }), + }, + }), + ], + operationIds: ["diarization", "attribution-diarized"], + }); + const dia = bandOf(bands, "diarization"); + assert.equal(dia.reachable, 647); + assert.equal(dia.missingInput, 77_276); + assert.equal(dia.blocked, 0); + const attr = bandOf(bands, "attribution-diarized"); + assert.equal(attr.reachable, 94); + assert.equal(attr.blocked, 77_923); + assert.equal(attr.missingInput, 0); +}); + +test("one channel that cannot report `eligible` voids the whole denominator", () => { + // The partial-sum trap. A snapshot predating `eligible` contributes videos to + // the corpus but nothing to the denominator, so summing what IS known gives a + // denominator smaller than its own numerator. + const bands = buildOperationBands({ + snapshots: [ + snapshotOf({ + backfill: { diarization: entryOf({ missing: 1, eligible: 500 }) }, + }), + snapshotOf({ + // No `eligible` — an older snapshot. + backfill: { diarization: entryOf({ missing: 2 }) }, + }), + ], + operationIds: ["diarization"], + }); + const dia = bandOf(bands, "diarization"); + assert.equal(dia.reachable, 3, "work counts still sum"); + assert.equal(dia.eligible, null, "the denominator does not"); + assert.equal(dia.present, null); + assert.equal(bandCoverage(dia), null, "and coverage renders as unknown"); +}); + +test("coverage is null, never 0, when the denominator is unknown", () => { + // A 0 here would read as "nothing digested" on a fully digested channel. + assert.equal( + bandCoverage({ present: null, eligible: 10 } as OperationBand), + null, + ); + assert.equal( + bandCoverage({ present: 5, eligible: null } as OperationBand), + null, + ); + assert.equal(bandCoverage({ present: 5, eligible: 0 } as OperationBand), null); + assert.equal(bandCoverage({ present: 5, eligible: 10 } as OperationBand), 0.5); +}); + +test("digest with no registry entry is an unfilled outline, never 0 %", () => { + // A channel with no `backfill.digest` must not read "all digested" — nor + // "none digested". Digest is a plain registry entry here, exactly like every + // other operation: no entry means no work KNOWN and coverage UNKNOWN. + const bands = buildOperationBands({ + snapshots: [snapshotOf({ backfill: {} })], + operationIds: ["digest"], + }); + const digest = bandOf(bands, "digest"); + assert.equal(digest.reachable, 0); + // Nothing to have an opinion about, and bandCoverage refuses to divide by it: + // the band draws as an empty outline rather than a filled 0 %. + assert.equal(digest.eligible, 0); + assert.equal(bandCoverage(digest), null); +}); + +test("the external pipelines get bands from totals and buckets", () => { + const bands = buildOperationBands({ + snapshots: [ + snapshotOf({ + totals: { videos: 100, transcribed: 40, downloaded: 60 }, + undownloadedIds: ["u1", "u2", "u3"], + buckets: { + downloadedNoTranscript: ["d1", "d2"], + noTranscript: Array.from({ length: 60 }, (_, i) => `n${i}`), + untranscribable: ["x1", "x2"], + partialDownloads: ["p1"], + } as unknown as ChannelSnapshot["buckets"], + }), + ], + operationIds: [], + }); + const download = bandOf(bands, "download"); + // Every video the playlist knows about, not just the dirs that exist. + assert.equal(download.eligible, 103); + assert.equal(download.present, 60); + assert.equal(download.reachable, 4, "3 never fetched + 1 partial"); + assert.equal(download.dispatched, false); + + const transcription = bandOf(bands, "transcription"); + assert.equal(transcription.eligible, 98, "100 videos less 2 untranscribable"); + assert.equal(transcription.present, 40); + assert.equal(transcription.reachable, 2, "audio in hand"); + // The rest of noTranscript is waiting on the DOWNLOAD lane — blocked on an + // operation this system produces, not reachable and not missing media. + assert.equal(transcription.blocked, 56); + assert.equal(transcription.missingInput, 0); +}); + +test("ids excluded from download are deferred, not reachable", () => { + // A channel deliberately not fetching members-only videos is not a lane with + // work to do, and the two must stay separable rather than one being netted + // off the other. + const bands = buildOperationBands({ + snapshots: [ + snapshotOf({ + totals: { videos: 0, transcribed: 0, downloaded: 0 }, + undownloadedIds: ["ok", "gone"], + excludedFromDownload: { deleted: ["gone"] }, + } as Partial<ChannelSnapshot>), + ], + operationIds: [], + }); + const download = bandOf(bands, "download"); + assert.equal(download.reachable, 1); + assert.equal(download.deferred, 1); +}); + +test("a switched-off operation gets no band at all", () => { + // Absent, not zero: an empty work list because nobody enabled the feature is + // not the same as being finished, and a full green bar would claim it was. + const bands = buildOperationBands({ + snapshots: [snapshotOf({ backfill: { diarization: entryOf({ missing: 5 }) } })], + operationIds: [], + }); + assert.equal( + bands.find((b) => b.id === "diarization"), + undefined, + ); +}); + +test("sumOrNull latches null and never returns a partial total", () => { + assert.equal(sumOrNull([1, 2, 3]), 6); + assert.equal(sumOrNull([1, null, 3]), null); + assert.equal(sumOrNull([]), 0); +}); + +// ── THE PER-CHANNEL PROJECTION ────────────────────────────────────────────── +// +// The /channels strip and the channel page's station foot are the same fold as +// the corpus rail, over one snapshot. These tests pin the three cases the strip +// actually meets on the live corpus: every operation present, one operation the +// snapshot has no entry for, and one that predates `eligible`. + +const CHANNEL_OPS = [ + "diarization", + "attribution-diarized", + "attribution-text", + "digest", +]; + +test("a channel's bands carry every operation, and the two external ones", () => { + const bands = buildChannelBands( + snapshotOf({ + totals: { videos: 11_344, transcribed: 11_339, downloaded: 11_340 }, + buckets: { + downloadedNoTranscript: ["a", "b"], + noTranscript: ["a", "b", "c"], + untranscribable: ["c"], + } as ChannelSnapshot["buckets"], + undownloadedIds: ["x", "y", "z", "w"], + backfill: { + // The measured shape of the-quartering, in miniature: digest is all + // reachable with nothing done, diarization is almost all media-gone, + // and attribution-diarized is almost all blocked behind it. + diarization: entryOf({ missing: 1, missingInput: 11_333, eligible: 11_338 }), + "attribution-diarized": entryOf({ + missing: 4, + blocked: 11_334, + eligible: 11_338, + }), + "attribution-text": entryOf({ missing: 11_337, eligible: 11_338 }), + digest: entryOf({ missing: 11_329, blocked: 2, eligible: 11_340 }), + }, + }), + CHANNEL_OPS, + ); + + assert.deepEqual( + bands.map((b) => b.id), + ["download", "transcription", ...CHANNEL_OPS], + ); + + // Digest: ALL accent, nothing done. This is the row that makes a percent bar + // useless and the state band useful — "0% complete" is true of every large + // channel and says nothing; "11,329 can run now" is the whole story. + const digest = bandOf(bands, "digest"); + assert.equal(digest.reachable, 11_329); + assert.equal(digest.blocked, 2); + assert.equal(digest.present, 9); + assert.equal(bandCoverage(digest), 9 / 11_340); + + // Diarization: all hollow. 11,333 with no media left is not work, and must + // never be added to the 1 video that is. + const diarize = bandOf(bands, "diarization"); + assert.equal(diarize.reachable, 1); + assert.equal(diarize.missingInput, 11_333); + + // Attribution-diarized: all hatched, waiting on the lane above it. + const named = bandOf(bands, "attribution-diarized"); + assert.equal(named.reachable, 4); + assert.equal(named.blocked, 11_334); + + // The external pipelines use the SAME definitions the transit line does, so a + // channel figure and a corpus figure cannot disagree about "downloaded". + const download = bandOf(bands, "download"); + assert.equal(download.eligible, 11_344 + 4); + assert.equal(download.present, 11_340); + assert.equal(download.reachable, 4); +}); + +test("an operation the snapshot has no entry for is an EMPTY band, not a missing column", () => { + // A channel whose report predates a kind still gets a cell — drawn empty, + // with a known denominator of 0, which bandCoverage reports as unknown rather + // than as 0% done. Dropping the column instead would make the table ragged + // and hide the fact that nothing has been measured yet. + const bands = buildChannelBands( + snapshotOf({ + backfill: { diarization: entryOf({ missing: 5, eligible: 10 }) }, + }), + CHANNEL_OPS, + ); + const text = bandOf(bands, "attribution-text"); + assert.equal(text.reachable, 0); + assert.equal(text.blocked, 0); + assert.equal(text.missingInput, 0); + assert.equal(text.eligible, 0); + assert.equal(bandCoverage(text), null); +}); + +test("an entry with no `eligible` draws an outline, never 0%", () => { + // UNKNOWN IS NOT ZERO, at channel scale. A snapshot written before the field + // existed — or one that has lapsed — has work counts but no denominator, and + // the band must say "we cannot tell you the coverage" rather than "none of it + // is done", which on a fully-diarized channel would be a lie. + const bands = buildChannelBands( + snapshotOf({ + backfill: { + diarization: entryOf({ missing: 3, missingInput: 90 }), + }, + }), + ["diarization"], + ); + const diarize = bandOf(bands, "diarization"); + assert.equal(diarize.eligible, null); + assert.equal(bandCoverage(diarize), null); + // The work counts survive the unknown denominator — they are separately + // known, and the cell still says how much can run now. + assert.equal(diarize.reachable, 3); + assert.equal(diarize.missingInput, 90); +}); + +test("a channel with no snapshot at all is every band empty", () => { + // A brand-new channel, before its first report. Every column present, every + // one empty, coverage unknown — the freshness note under the table is what + // explains why. + const bands = buildChannelBands(null, CHANNEL_OPS); + assert.equal(bands.length, 2 + CHANNEL_OPS.length); + for (const band of bands) { + assert.equal(band.reachable, 0); + assert.equal(bandCoverage(band), null); + } +}); + +// ── THE CLIENT/SERVER SPLIT, GUARDED ──────────────────────────────────────── + +test("band.ts stays directive-free, so a server component can call it", () => { + // band.ts carries the TYPE and every pure reading of a band. The channel + // page's station foot is a SERVER component and calls bandSentence() and + // bandHeadline() directly; adding "use client" here would break it at request + // time with "attempted to call bandSentence() from the server". + const src = readFileSync(new URL("./band.ts", import.meta.url), "utf8"); + // A DIRECTIVE, not the string — this file discusses "use client" in prose. + // A directive is a bare expression statement before any other code. + assert.ok( + !/^\s*(?:"use client"|'use client');?\s*$/m.test(src), + "band.ts must not be a client module", + ); + // And it must import nothing but types — buildBands.ts pulls in + // channelSnapshot → execa, which would put node:child_process in the browser + // bundle and fail `next build`. + const valueImports = [...src.matchAll(/^import\s+(?!type\b)/gm)]; + assert.equal( + valueImports.length, + 0, + "band.ts must not take a value import — see its header", + ); +}); + +test("StateBand.tsx exports only components, never callable helpers", () => { + // THE BUG THIS CAUGHT, ONCE. A plain function exported from a `"use client"` + // module cannot be CALLED by a server component — only rendered. bandSentence + // lived here, the server-rendered station foot called it, and every channel + // page 500'd at request time. + // + // `pnpm build` does NOT catch this: the route is force-dynamic, so nothing + // prerenders it and the error only appears on a request. e2e found it and the + // build did not, which is why this guard is a unit test and not a build step. + // StateBand.tsx is the client half and stays in the editor — this file moved + // down to common/views/ in phase 3 slice 1 and the guard came with it, so the + // read reaches back across the package boundary. A textual check, not an + // import: nothing in common/ may depend on editor/ at build time. + const src = readFileSync( + new URL("../../../editor/app/components/pipelines/StateBand.tsx", import.meta.url), + "utf8", + ); + assert.ok( + /^\s*(?:"use client"|'use client');?\s*$/m.test(src), + "StateBand.tsx is the client half", + ); + const exported = [...src.matchAll(/^export\s+(?:function|const)\s+(\w+)/gm)].map( + (m) => m[1], + ); + assert.ok(exported.length > 0, "found no exports to check — regex drifted"); + for (const name of exported) { + assert.ok( + /^[A-Z]/.test(name), + `${name} is exported from a client module but is not a component — a server component that calls it throws at request time. Move it to band.ts.`, + ); + } +}); + +test("a download/transcription entry in the snapshot is NOT folded twice", () => { + // Slice 1.5 gave the two bucket lanes a snapshot entry, and `/channels` passes + // `[...EXTERNAL_BAND_IDS, ...allOperations]` as operationIds — so without the + // filter in buildOperationBands the entry lands on top of addExternalBands and + // a three-video channel reads "6 done of 6". backfill.spec.ts caught it in the + // browser; this is the unit that pins it. + // + // The entries here are the ones generateChannelSnapshot writes: ids = the + // lane's default bucket union, eligible = present + the work counts. + const snapshot = snapshotOf({ + totals: { videos: 3, transcribed: 0, downloaded: 3 }, + buckets: { + downloadedNoTranscript: ["vidA", "vidB", "vidC"], + failedListed: [], + partialDownloads: [], + noTranscript: [], + untranscribable: [], + } as unknown as ChannelSnapshot["buckets"], + undownloadedIds: [], + backfill: { + download: entryOf({ missing: 0, ids: [], eligible: 3 }), + transcription: entryOf({ + missing: 3, + ids: ["vidA", "vidB", "vidC"], + eligible: 3, + }), + }, + }); + const bands = buildOperationBands({ + snapshots: [snapshot], + operationIds: ["download", "transcription", "diarization"], + }); + const dl = bandOf(bands, "download"); + assert.equal(dl.eligible, 3, "eligible must be the playlist, counted once"); + assert.equal(dl.present, 3); + assert.equal(dl.reachable, 0); + const tr = bandOf(bands, "transcription"); + assert.equal(tr.eligible, 3); + assert.equal(tr.present, 0); + assert.equal(tr.reachable, 3); + // And the band is still the BUCKET definition, not the entry's: the same + // numbers come back with the entries absent, which is every live snapshot. + const stripped = buildOperationBands({ + snapshots: [snapshotOf({ ...snapshot, backfill: {} })], + operationIds: ["download", "transcription", "diarization"], + }); + assert.deepEqual(bandOf(stripped, "download"), dl); + assert.deepEqual(bandOf(stripped, "transcription"), tr); +}); diff --git a/common/views/pipeline/buildBands.ts b/common/views/pipeline/buildBands.ts @@ -0,0 +1,248 @@ +import type { ChannelSnapshot } from "../../controller/channelSnapshot"; +import { excludedDownloadIdSet } from "../../controller/channelSnapshot"; +import { + EXTERNAL_OPERATIONS, + operationCostBasis, + operationLabel, + presentOperationWork, + reachableOperationWork, +} from "../../lib/operations"; +import { sumOrNull, type OperationBand } from "./band"; + +export type { OperationBand } from "./band"; +export { bandCoverage, sumOrNull } from "./band"; + +// THE COMPARISON RAIL'S MODEL: one band per pipeline, summed across the corpus. +// +// This is the corpus-wide twin of channelFlow's transit line, and it holds the +// same two invariants for the same reasons — they are the two ways every earlier +// version of this number was wrong: +// +// 1. WORK THE LANE CAN DO IS NEVER SUMMED WITH WORK IT CANNOT. `reachable`, +// `blocked`, `missingInput` and `deferred` are four separate fields on four +// different axes, and nothing here adds them. On the live corpus that is not +// pedantry: attribution-diarized is 94 reachable against 77,923 blocked, and +// diarization is 647 against 77,276 with no media. A single "remaining" +// figure would say the same thing about a lane that is finished and a lane +// that cannot start. +// 2. UNKNOWN IS NOT ZERO. `eligible` and `present` are `number | null`, and one +// null poisons the whole sum deliberately — a third of the snapshots on disk +// predate `eligible`, and "three channels are done and the fourth is +// unknown" is not a number. The band renders a null denominator as an +// unfilled outline, never as 0% progress. +// +// WHY THE RATIO IS THE STORY, AND WHY EACH BAND KEEPS ITS OWN DENOMINATOR. +// Three of the four pipelines are dominated by a non-actionable state, so a +// count renders them as "94" and "647" and tells you nothing. And digest's +// eligible population is genuinely a different set from diarization's — sharing +// one denominator across the rail to make the bars comparable would be a lie +// about what is being compared. Each band states its own, in its own header. +// +// Pure and snapshot-only: common/controller/noCorpusWalkInRenderPaths.test.ts +// bans a corpus walk from a render path, and this feeds a 3-second poll. + +function emptyBand(id: string, dispatched: boolean): OperationBand { + return { + id, + label: operationLabel(id), + costBasis: operationCostBasis(id), + eligible: 0, + present: 0, + reachable: 0, + blocked: 0, + missingInput: 0, + deferred: 0, + dispatched, + }; +} + +// Fold one snapshot's entry for a registry operation into a band. `null` for +// either coverage half latches for the whole corpus. +function addRegistryEntry(band: OperationBand, snapshot: ChannelSnapshot): void { + const entry = snapshot.backfill?.[band.id]; + if (!entry) return; + band.reachable += reachableOperationWork(entry); + band.missingInput += entry.missingInput; + // `?? 0` at every read: snapshots written before these fields existed lack + // them, and undefined poisons the sum to NaN. + band.blocked += entry.blocked ?? 0; + band.deferred += entry.deferred ?? 0; + band.eligible = sumOrNull([band.eligible, entry.eligible ?? null]); + band.present = sumOrNull([band.present, presentOperationWork(entry)]); +} + +export type BuildOperationBandsInput = { + snapshots: ReadonlyArray<ChannelSnapshot | null>; + // Registry operations to build a band for, in rail order. Comes from + // allOperations(), so a switched-off feature is simply absent — which is + // the honest rendering: an empty work list because nobody enabled it is not + // the same as being finished. + operationIds: ReadonlyArray<string>; +}; + +// The two pipelines this system counts but does not dispatch through the +// operation registry. They are on the rail anyway, and deliberately: +// +// The rail exists so you never have to switch lanes to learn that THIS lane is +// idle because ANOTHER one is — and on this corpus that is the normal case, not +// the exception (diarization is 99.2% media-gone; attribution-diarized is 99.9% +// blocked behind diarization). Leaving transcription and download off it would +// remove exactly the two lanes whose state explains the other four. +// +// Their numbers do NOT come from Operation.state() — they have no registry +// entry, because EXTERNAL_OPERATIONS registers them for the dependency graph +// and not for dispatch. They come from `totals` and the buckets, using the SAME +// definitions the channel transit line already uses for its Download and +// Transcribe stations, so a corpus figure and a channel figure cannot disagree +// about what "downloaded" means. +// +// SLICE 1.5 GAVE THEM A SNAPSHOT ENTRY, AND THIS STILL DOES NOT READ IT. +// `snapshot.backfill.download` and `.transcription` are the DISPATCH work list — +// what the runner would hand out — and that is a different set from what these +// five numbers mean, in two places that both matter: +// +// * transcription's `reachable` here is downloadedNoTranscript ALONE. The +// lane's work list also carries `failedListed`, the retry bucket, which is +// 1,873 videos corpus-wide against 881 — reading the entry would nearly +// quadruple a rendered figure. +// * this band's `blocked` is "no audio yet", which the entry calls +// `missingInput`, and its `eligible`/`present` are the playlist and +// `totals.downloaded`/`totals.transcribed` — coverage measures the entry +// states rather than counts. +// +// So the entry and the band are two honest answers to two different questions, +// and the rail keeps its own. What DID stop being a special case is the id list +// below: it is the catalog's own, not a hand-written pair. +// +// Sync is catalogued beside them and still gets NO band, here or anywhere: its +// populations are channels, not videos (`scope: "channel"`), so every one of a +// band's five numbers would be a category error and the rail would draw +// "coverage unknown" over a hollow outline. Its rail row is composed instead — +// see SyncRailRow in OperationRail.tsx. +function addExternalBands( + bands: Map<string, OperationBand>, + snapshot: ChannelSnapshot, +): void { + const totals = snapshot.totals ?? { videos: 0, transcribed: 0, downloaded: 0 }; + const buckets = snapshot.buckets; + const undownloaded = snapshot.undownloadedIds ?? []; + const excluded = excludedDownloadIdSet(snapshot); + + const download = bands.get("download"); + if (download) { + // Eligible is every video the playlist knows about: the dirs that exist + // plus the ids that have never been fetched. `totals.videos` alone would be + // a denominator that grows only as work completes. + download.eligible = sumOrNull([ + download.eligible, + totals.videos + undownloaded.length, + ]); + download.present = sumOrNull([download.present, totals.downloaded]); + // Partial downloads are reachable work like any other — the same rule + // the dispatch decision applies to `partial`. + download.reachable += + undownloaded.filter((id) => !excluded.has(id)).length + + (buckets?.partialDownloads?.length ?? 0); + // Excluded ids have LEFT the line: a channel deliberately not fetching + // them is not a lane with work to do. They are deferred, not reachable — + // and never subtracted from anything, so the two stay separable. + download.deferred += undownloaded.filter((id) => excluded.has(id)).length; + } + + const transcription = bands.get("transcription"); + if (transcription) { + // A video marked untranscribable is not eligible — it is not work anyone is + // waiting on, and counting it would put a permanent ceiling under 100%. + const untranscribable = buckets?.untranscribable?.length ?? 0; + transcription.eligible = sumOrNull([ + transcription.eligible, + Math.max(0, totals.videos - untranscribable), + ]); + transcription.present = sumOrNull([ + transcription.present, + totals.transcribed, + ]); + // Reachable = the audio is in hand. Everything else without a transcript is + // waiting on the DOWNLOAD lane, which is exactly what `blocked` means here + // — an operation this system produces, one station upstream. + const downloadedNoTranscript = ( + buckets?.downloadedNoTranscript ?? [] + ).filter((id) => !excluded.has(id)).length; + const noTranscript = buckets?.noTranscript?.length ?? 0; + transcription.reachable += downloadedNoTranscript; + transcription.blocked += Math.max( + 0, + noTranscript - untranscribable - downloadedNoTranscript, + ); + } +} + +// The rail, left to right. Download and transcription lead because everything +// else depends on them; digest and the backfill kinds follow in registry order. +// +// OFF THE CATALOG, not a literal pair. EXTERNAL_OPERATIONS is the declaration of +// exactly this set — the media-derived pipelines this system counts and does not +// dispatch through the registry — and sync is deliberately not in it (its +// populations are channels, not videos). A third external pipeline would join +// the rail by being declared, the way pauseLaneFor and bucketLaneOperationId +// already read that same field. +export const EXTERNAL_BAND_IDS: readonly string[] = EXTERNAL_OPERATIONS.map( + (op) => op.id, +); + +export function buildOperationBands({ + snapshots, + operationIds, +}: BuildOperationBandsInput): OperationBand[] { + const bands = new Map<string, OperationBand>(); + for (const id of EXTERNAL_BAND_IDS) bands.set(id, emptyBand(id, false)); + for (const id of operationIds) { + if (!bands.has(id)) bands.set(id, emptyBand(id, true)); + } + // EXACTLY ONE FOLD PER BAND, and the two external ids are addExternalBands'. + // + // `/channels` passes `[...EXTERNAL_BAND_IDS, ...allOperations]` in — it wants + // a column for every band — and until slice 1.5 that was harmless here + // because `snapshot.backfill.download` did not exist and addRegistryEntry + // returned early. Now it does exist, and folding it on top of + // addExternalBands DOUBLES a channel's Download and Transcribe coverage. + // (Caught by backfill.spec.ts, which read the row as "6 done of 6" on a + // three-video channel.) + // + // This is the same trap backfillLaneOperationEntriesOf was written for, at + // the one surface that does not go through it: the moment the snapshot map + // stopped being one lane, "every id in this map is mine" stopped being true. + const registryIds = operationIds.filter( + (id) => !EXTERNAL_BAND_IDS.includes(id), + ); + + for (const snapshot of snapshots) { + if (!snapshot) continue; + addExternalBands(bands, snapshot); + for (const id of registryIds) { + const band = bands.get(id); + if (!band) continue; + // Digest is a plain registry entry here, like every other operation: a + // snapshot with no entry contributes no work and no coverage, and the + // band stays an unfilled outline rather than reading 0 %. + addRegistryEntry(band, snapshot); + } + } + + return [...bands.values()]; +} + +// ONE CHANNEL'S BANDS — the strip on /channels and the station foot on a +// channel page. +// +// A thin wrapper and deliberately not a second implementation: a channel figure +// and the corpus figure it contributes to MUST agree about what "downloaded" +// or "reachable" means, and the only way to guarantee that is for both to be +// the same fold over the same snapshot. buildOperationBands already takes an +// array; a channel is an array of one. +export function buildChannelBands( + snapshot: ChannelSnapshot | null, + operationIds: ReadonlyArray<string>, +): OperationBand[] { + return buildOperationBands({ snapshots: [snapshot], operationIds }); +} diff --git a/common/views/pipeline/channelFlow.test.ts b/common/views/pipeline/channelFlow.test.ts @@ -0,0 +1,339 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import type { ChannelSnapshot } from "../../controller/channelSnapshot"; +import type { ChannelConfig } from "../../lib/channelConfig"; +import type { + Operation, + OperationSnapshotEntry, +} from "../../lib/operations"; +import { computeChannelFlow, type FlowStationId } from "./channelFlow"; +import { computeStageStatuses, normalizeBuckets } from "./stageStatus"; + +// Run from this directory (the [slug] segment is a glob to node's test runner): +// cd "editor/app/channels/[slug]/lib" && ../../../../../node_modules/.bin/tsx --test channelFlow.test.ts + +const CONFIG: ChannelConfig = { handling: "transcribe", url: "https://x/y" }; + +function snapshotOf(patch: Partial<ChannelSnapshot> = {}): ChannelSnapshot { + return { + generatedAt: "2026-08-01T00:00:00.000Z", + totals: { videos: 100, transcribed: 40, downloaded: 60 }, + buckets: normalizeBuckets(undefined), + undownloadedIds: [], + ...patch, + }; +} + +// A registry entry is a big object with three async methods on it; none of them +// are reachable from computeChannelFlow, which only ever reads id/label/hints. +function kind(patch: Partial<Operation> & { id: string }): Operation { + return { label: patch.id, hint: "", ...patch } as Operation; +} + +// A snapshot entry AS WRITTEN TO DISK. The cast is the point of the helper: +// OperationCounts declares deferred/blocked/partial required, but every snapshot +// currently on disk predates them, which is why every read site carries `?? 0`. +// Omitting them here is how these tests exercise the real files. +function entry(patch: Partial<OperationSnapshotEntry>): OperationSnapshotEntry { + return { ids: [], missing: 0, stale: 0, missingInput: 0, ...patch } as OperationSnapshotEntry; +} + +function flowOf( + snapshot: ChannelSnapshot, + opts: { + laneOperations?: Operation[]; + playlistCount?: number | null; + failedVideoIds?: string[]; + } = {}, +) { + const failedVideoIds = opts.failedVideoIds ?? []; + return computeChannelFlow({ + snapshot, + stages: computeStageStatuses({ + snapshot, + failedVideoIds, + config: CONFIG, + runningJobs: [], + }), + config: CONFIG, + failedVideoIds, + laneOperations: opts.laneOperations ?? [kind({ id: "diarization" })], + playlistCount: opts.playlistCount ?? null, + }); +} + +function station( + flow: ReturnType<typeof flowOf>, + id: FlowStationId, +) { + const s = flow.stations.find((st) => st.id === id); + assert.ok(s, `expected a ${id} station`); + return s; +} + +test("a pre-`eligible` snapshot reports unknown coverage, not zero", () => { + // The shape a third of the snapshots on disk are still in: per-kind counts + // written before `eligible` existed. A 0 here renders as "nothing digested" + // on a channel that may be fully digested. + const flow = flowOf( + snapshotOf({ + backfill: { + digest: entry({ missing: 5 }), + diarization: entry({ missing: 3 }), + }, + }), + ); + + assert.equal(station(flow, "digest").through, null); + assert.equal(station(flow, "digest").denominator, null); + assert.equal(station(flow, "digest").coverage, null); + assert.equal(station(flow, "speakers").through, null); + assert.equal(station(flow, "speakers").coverage, null); +}); + +test("the lane station does NOT sum its operations — it reads the lead one", () => { + // THE BUG THIS STATION USED TO BE. `through` and `denominator` were the sum + // across every kind on the lane, which on the live corpus added diarization's + // coverage (one audio pass per video, 4 done of 11,338) to attribution-text's + // (~1 model call per transcript CHUNK, 1 done of 11,338) and printed the + // result under a label that named the queue. Two different populations in two + // different units, added, and no screen said so. + // + // The numeral now belongs to exactly ONE operation — the first in dependency + // order — and the rest state themselves separately in the station foot. + const flow = flowOf( + snapshotOf({ + backfill: { + diarization: entry({ eligible: 10 }), + // Same lane, a different population. Nothing may fold it in. + "attribution-diarized": entry({ eligible: 1000, missing: 400 }), + }, + }), + { + laneOperations: [ + kind({ id: "diarization" }), + kind({ id: "attribution-diarized" }), + ], + }, + ); + + const lane = station(flow, "speakers"); + assert.equal(lane.through, 10); + assert.equal(lane.denominator, 10); + // The sum would be 1,010. Asserting the negative is the point. + assert.notEqual(lane.denominator, 1010); + // Both operations are carried, each with its own band and its own + // denominator, so nothing is hidden by not being summed. + assert.deepEqual( + lane.operations.map((o) => o.id), + ["diarization", "attribution-diarized"], + ); + assert.equal(lane.operations[1].band.eligible, 1000); + assert.equal(lane.operations[1].band.reachable, 400); +}); + +test("the lane station is named after its operations, not its queue key", () => { + // "Backfill" is a scheduler key. An operator cannot arm, pause or run "a + // backfill" — they can run speaker work. The label is DERIVED from the group + // its kinds declare, so a lane holding a mix degrades to the generic name + // rather than advertising one member's. + const speakers = flowOf(snapshotOf(), { + laneOperations: [ + kind({ id: "diarization" }), + kind({ id: "attribution-text" }), + ], + }); + assert.equal(station(speakers, "speakers").label, "Speakers"); + + const mixed = flowOf(snapshotOf(), { + laneOperations: [kind({ id: "diarization" }), kind({ id: "digest" })], + }); + assert.equal(station(mixed, "speakers").label, "Derived data"); + + // Nothing enabled: the generic name, and no operations to state. + const off = flowOf(snapshotOf(), { laneOperations: [] }); + assert.equal(station(off, "speakers").label, "Derived data"); + assert.deepEqual(station(off, "speakers").operations, []); +}); + +test("an unknown `eligible` on the lead operation still renders unknown, not zero", () => { + // Invariant 2, at the one station whose denominator moved. A snapshot written + // before `eligible` existed has work counts and no denominator, and the + // station must say "—" rather than 0%. + const flow = flowOf( + snapshotOf({ backfill: { diarization: entry({ missing: 3 }) } }), + { laneOperations: [kind({ id: "diarization" })] }, + ); + const lane = station(flow, "speakers"); + assert.equal(lane.through, null); + assert.equal(lane.denominator, null); + assert.equal(lane.coverage, null); +}); + +test("coverage is a real ratio once the snapshot can say", () => { + const flow = flowOf(snapshotOf(), { playlistCount: 125 }); + assert.equal(station(flow, "playlist").through, 100); + assert.equal(station(flow, "playlist").denominator, 125); + assert.equal(station(flow, "playlist").coverage, 0.8); + assert.equal(station(flow, "download").coverage, 0.6); + // transcribed / downloaded, not / videos: the denominator is the eligible + // population, and an undownloaded video is not eligible for transcription. + assert.equal( + station(flow, "transcribe").coverage, + 40 / 60, + ); +}); + +test("a lane that is switched off reads neutral, never ok and never amber", () => { + const snapshot = snapshotOf({ + backfill: { + diarization: entry({ missing: 7, eligible: 50, ids: ["a", "b"] }), + }, + }); + + const off = flowOf(snapshot, { laneOperations: [] }); + assert.equal(station(off, "speakers").tone, "neutral"); + // …and the work it recorded is not offered as something to press, because + // nothing would run it. + assert.equal(off.gaps.find((g) => g.to === "speakers")?.reachable, 0); + // NOT the pre-rename literal. This assertion passed vacuously the moment the + // stage id changed — a notEqual against a value the union can no longer hold + // is always true — so it is spelled with the live id and would fail if the + // disabled lane were ever offered as the next action again. + assert.notEqual(off.next?.stage, "speakers"); + + const on = flowOf(snapshot, { laneOperations: [kind({ id: "diarization" })] }); + assert.equal(on.gaps.find((g) => g.to === "speakers")?.reachable, 7); +}); + +test("deferred, blocked and missing-input never enter a gap's reachable count", () => { + const flow = flowOf( + snapshotOf({ + backfill: { + digest: entry({ + missing: 2, + stale: 1, + partial: 1, + missingInput: 900, + deferred: 40, + blocked: 1631, + eligible: 3000, + ids: ["a", "b", "c", "d"], + }), + diarization: entry({ + missing: 3, + missingInput: 500, + deferred: 11, + blocked: 70, + eligible: 1000, + ids: ["x", "y", "z"], + }), + }, + }), + ); + + const toDigest = flow.gaps.find((g) => g.to === "digest"); + const toBackfill = flow.gaps.find((g) => g.to === "speakers"); + // missing + stale + partial, and nothing else. + assert.equal(toDigest?.reachable, 4); + assert.equal(toBackfill?.reachable, 3); + + // The excluded populations are present — on the other axis. + const sidingCount = ( + gapTo: FlowStationId, + label: string, + ): number | undefined => + flow.gaps + .find((g) => g.to === gapTo) + ?.sidings.find((s) => s.label === label)?.count; + + assert.equal(sidingCount("digest", "waiting on a transcript"), 1631); + assert.equal(sidingCount("digest", "deferred"), 40); + assert.equal(sidingCount("speakers", "needs media re-acquired"), 500); + assert.equal(sidingCount("speakers", "deferred"), 11); + assert.equal(sidingCount("speakers", "waiting on an earlier backfill"), 70); + + // The invariant stated as the sum nobody should be able to write: a gap's + // reachable count is not the total of everything hanging under it. + for (const gap of flow.gaps) { + const sidingTotal = gap.sidings.reduce((n, s) => n + s.count, 0); + if (sidingTotal > 0) { + assert.notEqual(gap.reachable, gap.reachable + sidingTotal); + } + } +}); + +test("the digest station reads the registry entry", () => { + // The operation registry's entry is the one definition of the digest work + // list. `eligible` is absent here, so the station reports the work and still + // refuses to say how many are done. + const flow = flowOf( + snapshotOf({ + backfill: { digest: entry({ missing: 3, ids: ["a", "b", "c"] }) }, + }), + ); + assert.equal(flow.gaps.find((g) => g.to === "digest")?.reachable, 3); + // …but it still cannot say how many are done. + assert.equal(station(flow, "digest").through, null); +}); + +test("the bottleneck is the biggest gap; the next action is the furthest upstream one", () => { + const flow = flowOf( + snapshotOf({ + undownloadedIds: ["a", "b"], + backfill: { + digest: entry({ missing: 1675, eligible: 1797 }), + }, + }), + ); + + // The eye goes to the 1,675-video digest shortfall… + assert.equal(flow.bottleneck, "transcribe"); + // …but the button offers the two downloads, because a line clears from the + // front and the digest gap shrinks on its own as the upstream one does. + assert.equal(flow.next?.stage, "download"); + assert.equal(flow.next?.count, 2); +}); + +test("an idle, clean, reported channel offers no action at all", () => { + const flow = flowOf( + snapshotOf({ totals: { videos: 10, transcribed: 10, downloaded: 10 } }), + ); + assert.equal(flow.next, null); + assert.equal(flow.bottleneck, null); +}); + +test("a channel with no report offers nothing here — NoReportYet owns that", () => { + // Every count is zero because nothing has looked yet, not because the work is + // done. The page says so in the NoReportYet banner, which carries the only + // "Refresh report" button; putting a second one here duplicates the affordance + // and makes the name ambiguous. + const flow = flowOf( + snapshotOf({ + generatedAt: "", + totals: { videos: 0, transcribed: 0, downloaded: 0 }, + }), + ); + assert.equal(flow.next, null); +}); + +test("every optional snapshot field survives being absent", () => { + // The render-path crash this guards: `.toLocaleString()` on an undefined + // count. Nothing here is defaulted defensively — it is defaulted because + // snapshots on disk genuinely predate each field. + const bare = { + generatedAt: "", + totals: { videos: 0, transcribed: 0, downloaded: 0 }, + buckets: {}, + undownloadedIds: [], + } as unknown as ChannelSnapshot; + const flow = flowOf(bare); + for (const s of flow.stations) { + assert.ok(s.through === null || Number.isFinite(s.through)); + assert.ok(s.coverage === null || Number.isFinite(s.coverage)); + } + for (const g of flow.gaps) { + assert.ok(Number.isFinite(g.reachable)); + for (const sd of g.sidings) assert.ok(Number.isFinite(sd.count)); + } +}); diff --git a/common/views/pipeline/channelFlow.ts b/common/views/pipeline/channelFlow.ts @@ -0,0 +1,582 @@ +import type { ChannelConfig } from "../../lib/channelConfig"; +import { + digestWorkOf, + excludedDownloadIdSet, + type ChannelSnapshot, +} from "../../controller/channelSnapshot"; +import { digestCountOf } from "../../controller/channels"; +import { + backfillLaneEntriesOf, + operationCatalog, + operationLabel, + operationsActionLabel, + operationsGroupLabel, + presentOperationWork, + reachableOperationWork, + type Operation, +} from "../../lib/operations"; +import type { OperationBand } from "./band"; +import { buildChannelBands } from "./buildBands"; +import { + normalizeBuckets, + type StageId, + type StageStatus, + type StageTone, +} from "./stageStatus"; + +// THE CHANNEL LINE. +// +// A channel's lifecycle is not ten sibling cards — it is a conserved quantity +// moving through stations. Every video enters at the playlist and either +// advances or leaves the line. This module is the model behind that picture: +// stations are stages, and the GAP between two stations carries the shortfall, +// because the gap is the work. +// +// SNAPSHOT-ONLY, and deliberately pure. common/controller/noCorpusWalkInRender- +// Paths.test.ts bans the corpus walk from render paths, and page.tsx documents +// the multi-minute regression from generating a report inside a GET. The one +// value this cannot derive from the snapshot — how many videos the playlist +// file names — is passed IN by the caller (one small readFile), never read here. +// +// Two invariants this file exists to hold: +// +// 1. WORK ON THE LINE IS NEVER SUMMED WITH WORK OFF IT. `reachable` is what the +// lane can do today (missing + stale + partial). `missingInput`, `deferred`, +// `blocked`, `excludedFromDownload` and `untranscribable` have LEFT the line +// and live in `sidings`. Four surfaces once summed Object.values(snapshot +// .backfill) and put every channel permanently at the top of every list. +// Different fields on different axes is what makes the mistake unspellable. +// 2. UNKNOWN IS NOT ZERO. A third of the snapshots on disk predate `eligible`, +// so `present`/`coverage` are `number | null` and a reader must render "—". +// A 0 there reads as "nothing digested" on a fully digested channel. +// +// AND THE ONE THIS FILE USED TO BREAK. The `backfill` station set `through` and +// `denominator` by SUMMING THREE OPERATIONS — the one thing invariant 1 forbids +// everywhere else, hidden behind a station label that named the queue rather +// than the work. On the live corpus it was adding audio passes (diarization: 4 +// done of 11,338) to per-chunk model calls (attribution-text: 1 done of 11,338, +// and its cost basis is the transcript CHUNK, not the video), and calling the result +// "Backfill". A station now carries its group's OPERATIONS, each with its own +// band and its own denominator, and the numeral above them belongs to exactly +// one of them — see leadOf. + +export type FlowStationId = + | "playlist" + | "download" + | "transcribe" + | "digest" + // Renamed with StageId — see stageStatus.ts. The station and the stage card it + // links to must carry the same id or `?stage=${station.stage}` opens the wrong + // panel. + | "speakers"; + +// One pipeline drawn under a station. The band is the same instrument the +// /channels strip and the /operations rail draw, at station scale — which is +// what makes a figure here and a figure there impossible to disagree. +export type StationOperation = { + id: string; + label: string; + shortLabel: string; + // What one video costs, in words — the cost basis. Printed wherever the operation is armed, so + // an 11,337-video backlog of per-chunk model calls cannot read as a quiet row. + costBasis: string; + band: OperationBand; +}; + +export type FlowStation = { + id: FlowStationId; + // DERIVED for a station that holds several operations, never hardcoded: a + // lane holding a mix of groups falls back to "Derived data" rather than + // advertising one member's name. See operationsGroupLabel. + label: string; + // Videos that have cleared this station. NULL when the snapshot cannot say — + // see invariant 2 above. Only the digest and backfill stations can be null; + // the rest come from `totals`, which every snapshot carries. + through: number | null; + // The eligible population. null = unknown. + denominator: number | null; + // through / denominator, 0..1. Null whenever either side is unknown — render + // "—", never 0. + coverage: number | null; + running: boolean; + tone: StageTone; + // Which stage panel this station opens (?stage=). + stage: StageId; + // The pipelines that run at this station, in dependency order. Empty for + // playlist, which is not an operation the registry dispatches or counts — it + // keeps the plain coverage meter. + operations: StationOperation[]; +}; + +// A population that has LEFT the line: it is not work the lane can pick up, and +// it must never be added to a gap's `reachable`. +export type Siding = { + label: string; + count: number; + stage: StageId; + hint?: string; +}; + +export type FlowGap = { + from: FlowStationId; + to: FlowStationId; + // Work the lane can do today. NEVER summed with `sidings`. + reachable: number; + // "to download", "to transcribe", … + label: string; + // Where ?stage= sends you — the stage that OWNS this gap's work, i.e. the + // destination station's stage. + stage: StageId; + // The destination station's off-line populations. Sidings hang below the gap + // in the rendered line, on a different axis from `reachable`, so the layout + // itself cannot sum them. + sidings: Siding[]; +}; + +export type ChannelFlow = { + stations: FlowStation[]; + gaps: FlowGap[]; + // The `from` id of the largest reachable gap — the one the renderer promotes + // typographically. Null when nothing is reachable anywhere. + bottleneck: FlowStationId | null; + // The single primary action offered on the page. See pickNext below for why + // this is the FURTHEST UPSTREAM gap rather than the biggest one. + next: { stage: StageId; label: string; count: number } | null; +}; + +export type ComputeChannelFlowInput = { + snapshot: ChannelSnapshot; + // Reused, never recomputed: tone and running are stageStatus's job and a + // second opinion about them is a second thing to keep in sync. + stages: Record<StageId, StageStatus>; + config: ChannelConfig; + failedVideoIds: string[]; + // Enabled lane kinds. EMPTY MEANS THE LANE IS OFF, which is not the same as + // finished — see the tone rule at the bottom of this file. + laneOperations: Operation[]; + // How many videos the channel's `playlist` file names, or null when there is + // no playlist file. Read by the caller (countPlaylist) so this stays pure. + playlistCount: number | null; +}; + +function ratio(through: number | null, denominator: number | null): number | null { + if (through == null || denominator == null || denominator <= 0) return null; + return Math.min(1, through / denominator); +} + +function siding( + label: string, + count: number, + stage: StageId, + hint?: string, +): Siding[] { + return count > 0 ? [{ label, count, stage, ...(hint ? { hint } : {}) }] : []; +} + +// THE PIPELINES DRAWN UNDER EACH STATION, off the registry. +// +// The band for a channel is the same fold over the same snapshot the corpus +// rail uses, so a channel figure and a corpus figure cannot disagree about what +// "downloaded" or "reachable" means. Everything else here — the label, the +// column-width label, the cost basis — is read from the operation catalog +// rather than restated, which is what stops this file drifting from the two +// other surfaces that group the same operations. +function stationOperations( + snapshot: ChannelSnapshot, + laneKindIds: ReadonlyArray<string>, +): Map<string, StationOperation> { + const catalog = new Map(operationCatalog().map((o) => [o.id, o])); + const bands = buildChannelBands(snapshot, laneKindIds); + const out = new Map<string, StationOperation>(); + for (const band of bands) { + const op = catalog.get(band.id); + if (!op) continue; + out.set(band.id, { + id: band.id, + label: op.label, + shortLabel: op.shortLabel, + costBasis: op.costBasis, + band, + }); + } + return out; +} + +// The operation whose coverage the station's big numeral belongs to: the FIRST +// in dependency order, which is the one every other member of the group either +// consumes or runs beside. +// +// Explicitly NOT a sum, and not an average either. The three speaker operations +// are three different populations with two different cost bases — one audio +// pass per video against ~1 model call per transcript chunk — and any single +// figure over all three is the mistake this station used to make. One member +// owns the numeral; the rest state themselves, separately, in the foot. +function leadOf( + ops: ReadonlyArray<StationOperation>, +): StationOperation | null { + return ops[0] ?? null; +} + +export function computeChannelFlow( + input: ComputeChannelFlowInput, +): ChannelFlow { + const { + snapshot, + stages, + failedVideoIds, + laneOperations, + playlistCount, + } = input; + + const buckets = normalizeBuckets(snapshot.buckets); + const digestWarnings = snapshot.buckets?.digestWarnings ?? []; + const totals = snapshot.totals ?? { videos: 0, transcribed: 0, downloaded: 0 }; + const undownloadedIds = snapshot.undownloadedIds ?? []; + const excluded = snapshot.excludedFromDownload; + const excludedIds = excludedDownloadIdSet(snapshot); + const actionableDownloadedNoTranscript = buckets.downloadedNoTranscript.filter( + (id) => !excludedIds.has(id), + ); + + // Read the digest operation through digestWorkOf — the operation registry's + // entry. The old `noDigest` bucket had no cues-staleness gate and no + // transcript gate, so it called deferred and blocked videos done. + const digestWork = digestWorkOf(snapshot); + + // backfillLaneEntriesOf, never Object.values: the per-kind map now carries EVERY + // catalog operation including digest (~75k videos on the live corpus), and + // digest has its own station one step upstream. + const laneEntries = backfillLaneEntriesOf(snapshot.backfill); + // An empty kind list means the operator switched the feature off. That is not + // an empty work list in the "finished" sense, and the tone rule below says so. + const laneOff = laneOperations.length === 0; + + const laneKindIds = laneOperations.map((k) => k.id); + const operationsById = stationOperations(snapshot, laneKindIds); + const opsFor = (...ids: string[]): StationOperation[] => + ids + .map((id) => operationsById.get(id)) + .filter((o): o is StationOperation => o != null); + // The lane's own operations, in dependency order — the group the station is + // named after, and the members its foot states one by one. + const laneOps = laneOff ? [] : opsFor(...laneKindIds); + const laneLead = leadOf(laneOps); + + const laneReachable = laneOff + ? 0 + : laneEntries.reduce((n, e) => n + reachableOperationWork(e), 0); + const laneMissingInput = laneEntries.reduce((n, e) => n + e.missingInput, 0); + // `?? 0` is load-bearing, not defensive: snapshots written before these fields + // existed have neither, and .toLocaleString() on undefined throws in a render. + const laneDeferred = laneEntries.reduce((n, e) => n + (e.deferred ?? 0), 0); + const laneBlocked = laneEntries.reduce((n, e) => n + (e.blocked ?? 0), 0); + + // NO CROSS-OPERATION `present` OR `eligible` SUM LIVES HERE ANY MORE, and the + // helper that made one is gone with it. It used to add diarization's coverage + // to attribution-text's, which is an audio pass plus a per-chunk model call + // over two different populations. The station reads its LEAD operation and + // the foot states each member on its own — see leadOf. + // + // The four WORK counts above are still summed, and legitimately: a siding is + // "how many videos have left the line for this reason", and that reason is + // the same reason whichever operation reported it. + + // digestCountOf sums `digestEngines`, which 11 of the 65 live snapshots lack + // entirely — it returns 0 for those, which would read as "nothing digested". + // So it is only consulted when the map is actually present. + const digestEnginesTotal = + snapshot.digestEngines != null ? digestCountOf(snapshot) : null; + + const stationById: Record<FlowStationId, FlowStation> = { + playlist: { + id: "playlist", + label: "Playlist", + through: totals.videos, + denominator: playlistCount, + coverage: ratio(totals.videos, playlistCount), + running: stages.playlist.running, + tone: stages.playlist.tone, + stage: "playlist", + operations: [], + }, + download: { + id: "download", + label: "Download", + through: totals.downloaded, + denominator: totals.videos, + coverage: ratio(totals.downloaded, totals.videos), + running: stages.download.running, + tone: stages.download.tone, + stage: "download", + operations: opsFor("download"), + }, + transcribe: { + id: "transcribe", + label: "Transcribe", + through: totals.transcribed, + denominator: totals.downloaded, + coverage: ratio(totals.transcribed, totals.downloaded), + running: stages.transcribe.running, + tone: stages.transcribe.tone, + stage: "transcribe", + operations: opsFor("transcription"), + }, + digest: { + id: "digest", + label: "Digest", + through: digestWork.present ?? digestEnginesTotal, + denominator: digestWork.eligible, + coverage: ratio(digestWork.present ?? digestEnginesTotal, digestWork.eligible), + running: stages.digest.running, + tone: stages.digest.tone, + stage: "digest", + operations: opsFor("digest"), + }, + speakers: { + id: "speakers", + // NAMES THE WORK, NOT THE QUEUE. "Backfill" is a scheduler key — three + // operations happen to share it — and an operator cannot control, arm or + // pause "a backfill". They can pause speaker work. The name is derived + // from the group its members declare, so a lane that gains a kind from + // another group degrades to "Derived data" instead of lying. + label: operationsGroupLabel(laneKindIds), + // THE NUMERAL BELONGS TO ONE OPERATION, not to a sum of three. See leadOf. + // Read off the band rather than recomputed: the band IS presentBackfill- + // Work over this snapshot, and a second derivation is a second thing that + // can disagree with the strip on /channels. + through: laneLead?.band.present ?? null, + denominator: laneLead?.band.eligible ?? null, + coverage: laneLead + ? ratio(laneLead.band.present, laneLead.band.eligible) + : null, + running: stages.speakers.running, + // A station whose lane is DISABLED is neutral — never "ok" and never + // amber. An empty work list because a feature is off is not the same as + // being finished, and colouring it green claims a thing nobody checked. + tone: laneOff ? "neutral" : stages.speakers.tone, + stage: "speakers", + operations: laneOps, + }, + }; + + const order: FlowStationId[] = [ + "playlist", + "download", + "transcribe", + "digest", + "speakers", + ]; + const stations = order.map((id) => stationById[id]); + + // Sidings belong to the DESTINATION station's stage: "4 need cookies" hangs + // under "to download", "1,631 blocked" under "to digest". Same rule for every + // gap, so nothing is homeless and nothing is counted twice. + const sidingsOf: Record<FlowStationId, Siding[]> = { + playlist: [], + download: [ + ...siding( + "never fetched", + snapshot.missingNeverFetched?.length ?? 0, + "diagnostics", + "Known to the roster, never downloaded, and gone from the current listing.", + ), + ...siding("members-only", excluded?.membersOnly?.length ?? 0, "diagnostics"), + ...siding("deleted", excluded?.deleted?.length ?? 0, "diagnostics"), + ...siding("private", excluded?.private?.length ?? 0, "diagnostics"), + ...siding( + "skipped by filter", + buckets.skippedByFilter.length, + "diagnostics", + "Declined as currently live or upcoming; retried on a later sync.", + ), + ...siding( + "need cookies", + buckets.needsCookies.length, + "download", + "Browser cookies could recover these.", + ), + ...siding("partial downloads", buckets.partialDownloads.length, "download"), + ...siding( + "corrupt source", + buckets.corruptSource.length, + "download", + "Malformed source; needs re-downloading.", + ), + ...siding( + "corrupt full source", + buckets.corruptFullSource.length, + "download", + "Download completed but the audio stayed malformed. File kept for inspection; re-downloading is futile.", + ), + ], + transcribe: [ + ...siding("failed", failedVideoIds.length, "transcribe"), + ...siding( + "untranscribable", + buckets.untranscribable.length, + "diagnostics", + "Marked untranscribable by hand — an intentional decision, not an anomaly.", + ), + ...siding( + "incomplete transcript", + buckets.incompleteTranscript.length, + "transcribe", + "The transcript covers a fraction of the runtime — the audio download truncated silently.", + ), + ...siding( + "short audio", + buckets.shortAudio.length, + "transcribe", + "The source served a truncated stream; the short file is kept so it is not re-downloaded into a loop.", + ), + ], + digest: [ + ...siding( + "waiting on a transcript", + digestWork.blocked, + "digest", + "Nothing the digest lane can do about these — the number falls on its own as transcription runs.", + ), + ...siding( + "deferred", + digestWork.deferred, + "digest", + deferredHintFor(laneOperations, "digest"), + ), + ...siding( + "digest warnings", + digestWarnings.length, + "digest", + "The digest pass recorded something a human should look at.", + ), + ], + speakers: laneOff + ? [] + : [ + ...siding( + "needs media re-acquired", + laneMissingInput, + "speakers", + "The source audio is gone; re-acquiring it is an opt-in re-download.", + ), + ...siding( + "deferred", + laneDeferred, + "speakers", + deferredHintFor(laneOperations), + ), + ...siding( + "waiting on an earlier backfill", + laneBlocked, + "speakers", + dependsOnHint(laneOperations), + ), + ], + }; + + // Work owned by each DESTINATION station — the shortfall carried by the gap + // that leads into it. + const reachableInto: Record<FlowStationId, number> = { + playlist: 0, + download: undownloadedIds.length, + transcribe: actionableDownloadedNoTranscript.length, + digest: digestWork.reachable, + speakers: laneReachable, + }; + + const GAP_LABEL: Record<FlowStationId, string> = { + playlist: "", + download: "to download", + transcribe: "to transcribe", + digest: "to digest", + // A VALUE tsc keys but does not spell, so it is hand-checked: + // channel-line.spec.ts asserts this string. + speakers: "to speakers", + }; + + const gaps: FlowGap[] = []; + for (let i = 0; i < order.length - 1; i++) { + const from = order[i]; + const to = order[i + 1]; + gaps.push({ + from, + to, + reachable: reachableInto[to], + label: GAP_LABEL[to], + stage: stationById[to].stage, + sidings: sidingsOf[to], + }); + } + + const biggest = gaps.reduce<FlowGap | null>( + (best, g) => (g.reachable > (best?.reachable ?? 0) ? g : best), + null, + ); + + return { + stations, + gaps, + bottleneck: biggest ? biggest.from : null, + next: pickNext(gaps, laneKindIds), + }; +} + +// The ONE primary action. Deliberately the FURTHEST UPSTREAM gap with work +// rather than the biggest one: the pipeline is a line, so 1,600 videos waiting +// to be digested behind 113 that were never downloaded is not 1,600 jobs you can +// start — clearing the upstream gap is what makes the downstream one shrink. +// The bottleneck is still reported separately; it is the thing to LOOK at, not +// necessarily the thing to press. +function pickNext( + gaps: FlowGap[], + laneKindIds: ReadonlyArray<string>, +): ChannelFlow["next"] { + const ACTIONABLE: Partial<Record<StageId, string>> = { + download: "Download missing", + transcribe: "Transcribe pending", + digest: "Digest channel", + // A verb and its object, derived from the group — "Run speaker work". The + // button used to read "Backfill", which is a queue key with no object and + // nothing an operator recognises as a thing they wanted done. + speakers: `Run ${operationsActionLabel(laneKindIds)}`, + }; + for (const gap of gaps) { + if (gap.reachable <= 0) continue; + const label = ACTIONABLE[gap.stage]; + if (!label) continue; + return { stage: gap.stage, label, count: gap.reachable }; + } + // Nothing reachable — including on a channel that has never been reported, + // where the counts are all zero because nothing has looked yet. That case is + // NOT offered here: NoReportYet already sits at the top of the page saying so + // and carrying the button, and a second control with the same accessible name + // is both a duplicate affordance and, as it turns out, a locator that matches + // two elements. + return null; +} + +// What a deferred video of this kind is waiting for, from the registry rather +// than hardcoded here. SpeakersStage used to say "too long to diarize", which +// was true only while diarization was the sole kind that could defer. +function deferredHintFor( + kinds: Operation[], + onlyId?: string, +): string | undefined { + const hints = kinds + .filter((k) => (onlyId ? k.id === onlyId : true)) + .map((k) => k.deferredHint) + .filter((h): h is string => Boolean(h)); + const unique = [...new Set(hints)]; + if (unique.length === 0) return undefined; + return unique.map((h) => `Waiting because they are ${h}`).join(" · "); +} + +// Which operations' output the lane's kinds are blocked on, by label — so a +// blocked count says what it is waiting FOR rather than merely that it is stuck. +function dependsOnHint(kinds: Operation[]): string | undefined { + const labels = [ + ...new Set(kinds.flatMap((k) => (k.dependsOn ?? []).map(operationLabel))), + ]; + if (labels.length === 0) return undefined; + return `Waiting on ${labels.join(", ")}.`; +} diff --git a/common/views/pipeline/stageOrder.test.ts b/common/views/pipeline/stageOrder.test.ts @@ -0,0 +1,69 @@ +// Run with: +// node_modules/.bin/tsx --test "editor/app/channels/[slug]/lib/stageOrder.test.ts" +// +// page.tsx builds its stage list from the registry rather than from a literal: +// +// ["configure", "playlist", +// ...OPERATION_GROUP_ORDER.flatMap((g) => GROUP_STAGES[g]), +// "cleanup", "diagnostics", "danger"] +// +// That array is what `?stage=` is resolved against and what the switcher renders +// in order, so a change to it is a change to every stage link on the page. tsc +// checks MEMBERSHIP (GROUP_STAGES is an exhaustive Record<OperationGroup, …>) +// but not ORDER and not CARDINALITY — a group that gained a second stage, or an +// OPERATION_GROUP_ORDER someone resorted, would compile and silently reorder the +// page. This pins both against the literal list that shipped before the +// derivation. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { OPERATION_GROUP_ORDER } from "../../lib/operations"; +import { GROUP_STAGES, type StageId } from "./stageStatus"; + +// The expression from page.tsx, verbatim. Duplicated rather than exported and +// imported because page.tsx is a server component that reads the filesystem at +// module scope; what is worth pinning is the SHAPE, and a copy that drifted from +// the page would fail this test by construction on the next edit to either. +function stageOrder(): StageId[] { + return [ + "configure", + "playlist", + ...OPERATION_GROUP_ORDER.flatMap((g) => GROUP_STAGES[g]), + "cleanup", + "diagnostics", + "danger", + ]; +} + +test("the derived stage order is the list that shipped", () => { + assert.deepEqual(stageOrder(), [ + "configure", + "playlist", + "download", + "transcribe", + "digest", + "speakers", + "cleanup", + "diagnostics", + "danger", + ]); +}); + +test("every operation group names at least one stage, and none is orphaned", () => { + // The exhaustiveness tsc gives is on the KEYS. This is the other half: a group + // mapped to an empty array would compile and would mean an operation group + // with nowhere to appear on the channel page. + for (const group of OPERATION_GROUP_ORDER) { + assert.ok( + GROUP_STAGES[group].length > 0, + `${group} names no stage, so its operations have no card`, + ); + } + // And no stage is claimed by two groups, which would render it twice. + const claimed = OPERATION_GROUP_ORDER.flatMap((g) => GROUP_STAGES[g]); + assert.equal( + claimed.length, + new Set(claimed).size, + `a stage is claimed by more than one group: ${claimed.join(",")}`, + ); +}); diff --git a/common/views/pipeline/stageStatus.ts b/common/views/pipeline/stageStatus.ts @@ -0,0 +1,628 @@ +import type { ChannelConfig } from "../../lib/channelConfig"; +import { + digestWorkOf, + excludedDownloadIdSet, + type ChannelSnapshot, +} from "../../controller/channelSnapshot"; +import { + backfillLaneEntriesOf, + operationsGroupLabel, + reachableOperationWork, + type OperationGroup, +} from "../../lib/operations"; +// TYPE ONLY. channelMedia.ts imports node:fs, and this module is imported by +// three client components (AttentionStrip, NextAction, OverviewPanel) for its +// types. A type import is erased, a value import would not be. +import type { ChannelMediaLocation } from "../../lib/channelMedia"; + +export type SnapshotBuckets = ChannelSnapshot["buckets"]; + +// Older snapshots on disk may pre-date some bucket fields. Normalize to +// always-present arrays so callers can read .length without guards. +export function normalizeBuckets( + raw: Partial<SnapshotBuckets> | undefined, +): SnapshotBuckets { + return { + noTranscript: raw?.noTranscript ?? [], + downloadedNoTranscript: raw?.downloadedNoTranscript ?? [], + wrongFormatAudio: raw?.wrongFormatAudio ?? [], + multipleAudioFormats: raw?.multipleAudioFormats ?? [], + transcribedWithAudio: raw?.transcribedWithAudio ?? [], + untranscribable: raw?.untranscribable ?? [], + noMetadata: raw?.noMetadata ?? [], + failedListed: raw?.failedListed ?? [], + missingFromArchive: raw?.missingFromArchive ?? [], + duplicateDirs: raw?.duplicateDirs ?? [], + partialDownloads: raw?.partialDownloads ?? [], + corruptSource: raw?.corruptSource ?? [], + corruptFullSource: raw?.corruptFullSource ?? [], + nonStandardVtt: raw?.nonStandardVtt ?? [], + skippedByFilter: raw?.skippedByFilter ?? [], + incompleteTranscript: raw?.incompleteTranscript ?? [], + shortAudio: raw?.shortAudio ?? [], + autoSubsOnly: raw?.autoSubsOnly ?? [], + downloadedAutoSubsOnly: raw?.downloadedAutoSubsOnly ?? [], + supersededAutoSubs: raw?.supersededAutoSubs ?? [], + needsCookies: raw?.needsCookies ?? [], + }; +} + +export type StageId = + | "configure" + | "playlist" + | "download" + | "transcribe" + | "digest" + // THE OPERATIONS, NOT THE QUEUE. This card was called "backfill" — a queue key + // wearing a stage's name. Nobody can arm, pause or run "a backfill"; what the + // card actually holds is the speaker work (diarization and the two attribution + // kinds), which is a thing an operator recognises. The queue key BACKFILL_QUEUE + // is untouched: it is a scheduler key and correctly named as one. + // + // ?stage=backfill still resolves here — see STAGE_ALIASES in page.tsx. + | "speakers" + | "cleanup" + | "diagnostics" + // WHERE THE MEDIA PHYSICALLY IS (plans/relocate-channel-media.md). A channel + // CHORE like cleanup and diagnostics, not an operation — nothing registers it + // and no group owns it — so it is hand-listed at the call site alongside them + // and is deliberately absent from GROUP_STAGES below. + | "storage" + | "danger"; + +// THE STAGES EACH OPERATION GROUP OWNS, in the group's own order. +// +// Record<> is EXHAUSTIVE, which is the whole point: a new OperationGroup does +// not compile until it names its stage(s). Before this, the channel page's stage +// list was a hand-written literal, so a registered operation in a new group got +// a card only if someone remembered to add one — and the failure mode was a +// silent absence, not an error. +// +// NOT 1:1 with the groups, and no honest derivation makes it so: `sync` owns +// NO stage (below), and a group may own more than one — `media` did while the +// transcode stage existed (retired 2026-08-30). So this is a Record of ARRAYS, +// spread in OPERATION_GROUP_ORDER — which yields exactly today's order and +// today's cardinality. It is a compile-time membership check, not a re-shaping +// of the page. +// +// DELIBERATELY ONLY THE MIDDLE. configure/playlist and cleanup/diagnostics/ +// danger are channel CHORES, not operations — nothing registers them and no +// group owns them — so they stay hand-listed at the call site. Do not "finish" +// this derivation by inventing groups for them. +export const GROUP_STAGES: Record<OperationGroup, readonly StageId[]> = { + media: ["download"], + transcript: ["transcribe"], + digest: ["digest"], + speakers: ["speakers"], + // Empty on purpose, and this is the paragraph above in practice: sync's + // channel surface is the hand-listed `playlist` bookend (JOB_KIND_TO_STAGE + // maps the sync job kind to it), which is a channel chore, not a stage this + // Record owns. Do not "finish" the derivation by giving sync a stage here — + // the channel page would grow a second, duplicate playlist card. + sync: [], +}; + +export type StageTone = "neutral" | "attention" | "danger" | "running" | "ok"; + +export type StageStatus = { + id: StageId; + title: string; + pending: number; + failed: number; + running: boolean; + defaultOpen: boolean; + summary: string; + tone: StageTone; +}; + +const JOB_KIND_TO_STAGE: Record<string, StageId> = { + "store-playlist": "playlist", + sync: "playlist", + "download-from-playlist": "download", + "download-missing": "download", + "whisper-all": "transcribe", + "whisper-retry": "transcribe", + "transcribe-one": "transcribe", + "whisper-video": "transcribe", + "whisper-bucket-auto-subs": "transcribe", + "digest-channel-local": "digest", + "digest-channel-remote": "digest", + "digest-share-cluster": "digest", + "backfill-channel": "speakers", + // The pre-registry per-channel diarization button lands on the channel queue + // but is the same work this card is about, so it lights this card too. + "diarize-channel": "speakers", + "clean-audio-transcribed": "cleanup", + "purge-superseded-auto-subs": "cleanup", + "clean-extra-audio-formats": "cleanup", + "remove-wrong-format-audio": "cleanup", + "check-availability": "diagnostics", +}; + +function pluralize(n: number, singular: string, plural?: string): string { + return `${n} ${n === 1 ? singular : plural ?? `${singular}s`}`; +} + +function pickTone(args: { + running: boolean; + pending: number; + failed: number; + fallback?: StageTone; +}): StageTone { + if (args.running) return "running"; + if (args.failed > 0) return "danger"; + if (args.pending > 0) return "attention"; + return args.fallback ?? "neutral"; +} + +export type ComputeStageStatusesInput = { + snapshot: ChannelSnapshot; + failedVideoIds: string[]; + config: ChannelConfig; + // Only `status` and `kind` are read (which stage has work in flight), so this + // takes the SHAPE rather than the record: the channel page now gets its rows + // from the one job-row builder (liveJobRows) and no longer holds JobRecords. + runningJobs: ReadonlyArray<{ status: string; kind: string }>; + // Whether ANY backfill kind is switched on. + // + // The snapshot's per-kind counts are a record of what was true when it was + // written, and they survive the operator turning the feature off. Without + // this flag the card reports "7 videos need derived data" — amber, with a + // count — directly above its own body copy saying "No backfill is enabled", + // and nothing would ever run the work it is advertising. Defaults to true so + // an omitted flag behaves as it always did. + backfillEnabled?: boolean; + // The ids of the lane's enabled kinds, so the stage can be titled after what + // it HOLDS rather than after its queue key. Optional and defaulting to the + // generic name, because a caller that only needs tone and counts should not + // have to resolve the registry. + backfillKindIds?: ReadonlyArray<string>; + // Where this channel's media actually is, from inspectChannelMedia. Optional + // because the two stats it costs belong to the caller that already has the + // config in hand, and a caller that only wants tone and counts should not + // have to do I/O to get them — an omitted location reads as "in place", which + // is what every channel was before relocation existed. + media?: ChannelMediaLocation | null; +}; + +export function computeStageStatuses( + input: ComputeStageStatusesInput, +): Record<StageId, StageStatus> { + const { + snapshot, + failedVideoIds, + config, + runningJobs, + backfillEnabled = true, + backfillKindIds, + media, + } = input; + + const buckets = normalizeBuckets(snapshot.buckets); + const undownloadedIds = snapshot.undownloadedIds ?? []; + const excludedDownloadIds = excludedDownloadIdSet(snapshot); + const actionableNoTranscript = buckets.noTranscript.filter( + (id) => !excludedDownloadIds.has(id), + ); + const actionableDownloadedNoTranscript = buckets.downloadedNoTranscript.filter( + (id) => !excludedDownloadIds.has(id), + ); + + const runningByStage = new Set<StageId>(); + for (const job of runningJobs) { + if (job.status !== "running" && job.status !== "queued") continue; + const stage = JOB_KIND_TO_STAGE[job.kind]; + if (stage) runningByStage.add(stage); + } + + const downloadPending = + undownloadedIds.length + + actionableNoTranscript.length + + buckets.partialDownloads.length; + const transcribePending = actionableDownloadedNoTranscript.length; + const transcribeFailed = failedVideoIds.length; + const cleanupPending = buckets.multipleAudioFormats.length; + // `untranscribable` is deliberately excluded: those videos are an intentional + // user decision ("mark untranscribable"), not an anomaly with an action. + const diagnosticsPending = + buckets.noMetadata.length + + buckets.missingFromArchive.length + + buckets.duplicateDirs.length; + + // Cards that carry primary actions stay open by default so the user can + // always reach the buttons; the summary line communicates idle/busy state + // instead of collapsing the controls out of sight. Configure collapses once + // the channel has a URL; Danger zone stays collapsed unless opened. + const configure: StageStatus = { + id: "configure", + title: "Configure", + pending: 0, + failed: 0, + running: false, + defaultOpen: true, + summary: config.url + ? `${config.handling}${config.platform ? ` · ${config.platform}` : ""}` + : "Channel has no URL — open to configure.", + tone: config.url ? "neutral" : "attention", + }; + + const playlistRunning = runningByStage.has("playlist"); + const playlist: StageStatus = { + id: "playlist", + title: "Playlist", + pending: 0, + failed: 0, + running: playlistRunning, + defaultOpen: true, + summary: playlistRunning + ? "Running…" + : config.lastSyncedAt + ? `Last sync ${new Date(config.lastSyncedAt).toLocaleString()}` + : "Never synced.", + tone: pickTone({ running: playlistRunning, pending: 0, failed: 0 }), + }; + + const downloadRunning = runningByStage.has("download"); + const downloadParts: string[] = []; + if (undownloadedIds.length > 0) { + downloadParts.push(pluralize(undownloadedIds.length, "undownloaded")); + } + if (actionableNoTranscript.length > 0) { + downloadParts.push( + pluralize( + actionableNoTranscript.length, + "dir missing transcript & audio", + "dirs missing transcript & audio", + ), + ); + } + if (buckets.partialDownloads.length > 0) { + downloadParts.push( + pluralize( + buckets.partialDownloads.length, + "partial download", + "partial downloads", + ), + ); + } + if (buckets.corruptSource.length > 0) { + downloadParts.push( + pluralize( + buckets.corruptSource.length, + "corrupt source (needs re-download)", + "corrupt sources (need re-download)", + ), + ); + } + if (buckets.corruptFullSource.length > 0) { + downloadParts.push( + pluralize( + buckets.corruptFullSource.length, + "corrupt full source (file kept)", + "corrupt full sources (files kept)", + ), + ); + } + const download: StageStatus = { + id: "download", + title: "Download", + pending: downloadPending, + failed: 0, + running: downloadRunning, + defaultOpen: true, + summary: downloadRunning + ? "Running…" + : downloadParts.length > 0 + ? downloadParts.join(" · ") + : "Nothing to download.", + tone: pickTone({ + running: downloadRunning, + pending: downloadPending, + failed: 0, + }), + }; + + const transcribeRunning = runningByStage.has("transcribe"); + const transcribeParts: string[] = []; + if (actionableDownloadedNoTranscript.length > 0) { + transcribeParts.push( + pluralize( + actionableDownloadedNoTranscript.length, + "video awaiting whisper", + "videos awaiting whisper", + ), + ); + } + if (failedVideoIds.length > 0) { + transcribeParts.push(pluralize(failedVideoIds.length, "failed")); + } + // Informational only — the replace-auto-captions lane is opt-in, so these are + // NOT counted as pending work (that would light every YouTube channel up + // amber forever). + const autoSubsCandidates = + buckets.autoSubsOnly.length + buckets.downloadedAutoSubsOnly.length; + if (autoSubsCandidates > 0) { + transcribeParts.push( + pluralize( + autoSubsCandidates, + "video with only auto-captions", + "videos with only auto-captions", + ), + ); + } + const transcribe: StageStatus = { + id: "transcribe", + title: "Transcribe", + pending: transcribePending, + failed: transcribeFailed, + running: transcribeRunning, + defaultOpen: true, + summary: transcribeRunning + ? "Running…" + : transcribeParts.length > 0 + ? transcribeParts.join(" · ") + : "All transcribed.", + tone: pickTone({ + running: transcribeRunning, + pending: transcribePending, + failed: transcribeFailed, + }), + }; + + // Videos whose digest is missing, stale or part-done against the local lane's + // current identity. Read from the operation registry via digestWorkOf, which + // is the one definition: a channel with no entry reports unknown coverage + // rather than reading as fully digested. + // + // Counted as pending work rather than merely informational: unlike the + // auto-captions lane, every transcribed video is eventually meant to have one. + // Videos BLOCKED on transcription are deliberately not in this number — there + // is nothing the digest lane can do about them — and the stage card names them + // separately. + const digestRunning = runningByStage.has("digest"); + const digestWork = digestWorkOf(snapshot); + const digestPending = digestWork.reachable; + const digest: StageStatus = { + id: "digest", + title: "Digest", + pending: digestPending, + failed: 0, + running: digestRunning, + defaultOpen: true, + summary: digestRunning + ? "Running…" + : digestPending > 0 + ? pluralize( + digestPending, + "transcript needs a digest", + "transcripts need a digest", + ) + : // "All digested" MUST NOT be said over a channel that simply has + // nothing to digest yet. Before the registry classified them, videos + // with no transcript were absent from every digest bucket, so a + // channel of untranscribed videos read as finished — the exact failure + // declaring the transcription dependency exists to end. + digestWork.blocked > 0 + ? pluralize( + digestWork.blocked, + "video is waiting on a transcript", + "videos are waiting on transcripts", + ) + : "All digested at the current settings.", + tone: pickTone({ + running: digestRunning, + pending: digestPending, + failed: 0, + fallback: "ok", + }), + }; + + // The backfill lane's work list, summed across every registered kind. + // + // `pending` counts ONLY the reachable half. The needs-re-acquiring population + // is reported in the summary line and never folded in: it is 91x larger on the + // measured corpus, so counting it would hold every channel permanently amber + // for work that cannot be done without an opt-in re-download — precisely the + // trap /api/widget/actionable documents for the digest work count. + const backfillRunning = runningByStage.has("speakers"); + // backfillLaneEntriesOf, not Object.values. The snapshot's per-kind map carries every + // operation in the catalog now, including digest — which runs on its own queue + // key, has its own stage card directly above, and would otherwise add ~75,000 + // videos to this instrument on the measured corpus. The filter is by the kind's + // declared lane rather than by its id, so the next operation registered on a + // lane of its own does not re-arm the same trap. + // A disabled lane has NO work, whatever the snapshot recorded before it was + // switched off — see `backfillEnabled` above. + const backfillEntries = backfillEnabled + ? backfillLaneEntriesOf(snapshot.backfill) + : []; + const backfillPending = backfillEntries.reduce( + (n, e) => n + reachableOperationWork(e), + 0, + ); + const backfillMissingInput = backfillEntries.reduce( + (n, e) => n + e.missingInput, + 0, + ); + // Same treatment as missingInput: reported in the summary line, never folded + // into `pending`. `?? 0` because snapshots written before the cap existed have + // no such field. + const backfillDeferred = backfillEntries.reduce( + (n, e) => n + (e.deferred ?? 0), + 0, + ); + // Same treatment again, and the wording matters: this number falls on its own + // as the prerequisite lane runs, so it must not read as something to fix. + const backfillBlocked = backfillEntries.reduce( + (n, e) => n + (e.blocked ?? 0), + 0, + ); + const backfillParts: string[] = []; + if (backfillPending > 0) { + backfillParts.push( + pluralize( + backfillPending, + "video needs derived data", + "videos need derived data", + ), + ); + } + if (backfillMissingInput > 0) { + backfillParts.push( + `${backfillMissingInput.toLocaleString()} needing media re-acquired`, + ); + } + if (backfillDeferred > 0) { + backfillParts.push( + `${backfillDeferred.toLocaleString()} deferred (too long to diarize)`, + ); + } + if (backfillBlocked > 0) { + backfillParts.push( + `${backfillBlocked.toLocaleString()} waiting on an earlier backfill`, + ); + } + const speakers: StageStatus = { + id: "speakers", + // NAMED AFTER THE OPERATIONS, NOT THE QUEUE. "Backfill" is a scheduler key + // that on this install stands for three different operations; nobody can + // arm, pause or run "a backfill". Derived, so a lane that gains a kind from + // another group degrades to "Derived data" rather than going stale. + title: operationsGroupLabel(backfillKindIds ?? []), + pending: backfillPending, + failed: 0, + running: backfillRunning, + defaultOpen: true, + summary: backfillRunning + ? "Running…" + : backfillEntries.length === 0 + ? "Nothing here is enabled." + : backfillParts.length > 0 + ? backfillParts.join(" · ") + : "Everything reachable is current.", + tone: pickTone({ + running: backfillRunning, + pending: backfillPending, + failed: 0, + // Neutral rather than "ok" when nothing is enabled: an empty work list + // because a feature is off is not the same as being finished. + fallback: backfillEntries.length === 0 ? "neutral" : "ok", + }), + }; + + const cleanupRunning = runningByStage.has("cleanup"); + const cleanupParts: string[] = []; + if (cleanupPending > 0) { + cleanupParts.push( + pluralize( + cleanupPending, + "dir has extra audio formats", + "dirs have extra audio formats", + ), + ); + } + // Kept auto-caption backups. Informational (not folded into `pending`): they + // are deliberately retained until purged by hand, so they are inventory, not + // a chore. + if (buckets.supersededAutoSubs.length > 0) { + cleanupParts.push( + pluralize( + buckets.supersededAutoSubs.length, + "superseded auto-caption backup", + "superseded auto-caption backups", + ), + ); + } + const cleanup: StageStatus = { + id: "cleanup", + title: "Cleanup", + pending: cleanupPending, + failed: 0, + running: cleanupRunning, + defaultOpen: true, + summary: cleanupRunning + ? "Running…" + : cleanupParts.length > 0 + ? cleanupParts.join(" · ") + : "Nothing to clean.", + tone: pickTone({ + running: cleanupRunning, + pending: cleanupPending, + failed: 0, + }), + }; + + const diagnosticsRunning = runningByStage.has("diagnostics"); + const diagnostics: StageStatus = { + id: "diagnostics", + title: "Diagnostics", + pending: diagnosticsPending, + failed: 0, + running: diagnosticsRunning, + defaultOpen: true, + summary: diagnosticsRunning + ? "Running…" + : diagnosticsPending > 0 + ? pluralize(diagnosticsPending, "anomaly", "anomalies") + : "All clear.", + tone: pickTone({ + running: diagnosticsRunning, + pending: diagnosticsPending, + failed: 0, + fallback: "ok", + }), + }; + + // WHERE THE MEDIA IS. The only stage whose tone comes from a filesystem fact + // rather than from a count: an unreachable channel is a channel whose numbers + // everywhere else on this page are about to be wrong (an unmounted drive reads + // as "nothing downloaded"), so this card is red the moment inspect() says so + // and neutral the rest of the time. "in-place" is not an achievement, so it is + // never "ok" — the fallback tone for a healthy relocation is neutral too. + const mediaStatus = media?.status ?? "in-place"; + const storage: StageStatus = { + id: "storage", + title: "Storage", + pending: 0, + failed: 0, + running: mediaStatus === "in-transition", + defaultOpen: true, + summary: + mediaStatus === "in-place" + ? "Media is in the channel directory." + : mediaStatus === "ok" + ? `Media relocated to ${media?.target ?? "another drive"}.` + : (media?.detail ?? mediaStatus), + tone: + mediaStatus === "in-transition" + ? "running" + : mediaStatus === "unreachable" || + mediaStatus === "inconsistent" + ? "danger" + : "neutral", + }; + + const danger: StageStatus = { + id: "danger", + title: "Danger zone", + pending: 0, + failed: 0, + running: false, + defaultOpen: true, + summary: "Delete this channel.", + tone: "neutral", + }; + + return { + configure, + playlist, + download, + transcribe, + digest, + speakers, + cleanup, + diagnostics, + storage, + danger, + }; +} diff --git a/common/views/pipeline/tone.ts b/common/views/pipeline/tone.ts @@ -0,0 +1,41 @@ +import type { StageTone } from "./stageStatus"; + +// The line invents no colours. Every value below is one of the semantic tokens +// the repo already carries across four theme families × light/dark +// (common/styles/tokens.css); a bespoke hue here would be wrong in eight +// palettes at once. + +export const STATION_DOT: Record<StageTone, string> = { + // A station with nothing to say is an outline, not a filled dot: "○" in the + // sketch. Notably this is also where a DISABLED lane lands — see the tone rule + // in channelFlow.ts. + neutral: "border border-border-strong bg-transparent", + ok: "bg-success", + attention: "bg-warning", + danger: "bg-destructive", + // The travelling pulse. The only animated thing on the page. + running: "bg-info animate-pulse", +}; + +export const STATION_TEXT: Record<StageTone, string> = { + neutral: "text-muted-foreground", + ok: "text-foreground", + attention: "text-warning", + danger: "text-destructive", + running: "text-info", +}; + +export function formatCount(n: number | null): string { + // "—", never "0". A zero here would claim a measurement nobody took. + return n == null ? "—" : n.toLocaleString(); +} + +export function formatCoverage(coverage: number | null): string { + if (coverage == null) return "—"; + const pct = coverage * 100; + // Whole percents once you are past 10% — the third significant figure on + // "94.37%" is noise at this size. Below that, one decimal, because the + // difference between 6.8% and 6% is the difference between a sweep that is + // moving and one that is not. + return `${pct >= 10 ? Math.round(pct) : Math.round(pct * 10) / 10}%`; +} diff --git a/editor/app/channels/[slug]/components/PipelineStageCard.tsx b/editor/app/channels/[slug]/components/PipelineStageCard.tsx @@ -1,7 +1,7 @@ "use client"; import type { ReactNode } from "react"; -import type { StageTone } from "../lib/stageStatus"; +import type { StageTone } from "yt-dlp-transcript-common/views/pipeline/stageStatus"; type Props = { id: string; diff --git a/editor/app/channels/[slug]/components/flow/AttentionStrip.tsx b/editor/app/channels/[slug]/components/flow/AttentionStrip.tsx @@ -1,6 +1,6 @@ import Link from "next/link"; import { Badge } from "yt-dlp-transcript-common/components/ui/badge"; -import type { SnapshotBuckets, StageId } from "../../lib/stageStatus"; +import type { SnapshotBuckets, StageId } from "yt-dlp-transcript-common/views/pipeline/stageStatus"; // ANOMALIES, one chip each, every one a link to the stage that owns it. // diff --git a/editor/app/channels/[slug]/components/flow/ChannelLine.tsx b/editor/app/channels/[slug]/components/flow/ChannelLine.tsx @@ -1,4 +1,4 @@ -import type { ChannelFlow } from "../../lib/channelFlow"; +import type { ChannelFlow } from "yt-dlp-transcript-common/views/pipeline/channelFlow"; import { GapFoot, GapHead, GapRail } from "./FlowGap"; import { StationFoot, StationHead, StationRail } from "./FlowStation"; diff --git a/editor/app/channels/[slug]/components/flow/FlowGap.tsx b/editor/app/channels/[slug]/components/flow/FlowGap.tsx @@ -1,5 +1,5 @@ import Link from "next/link"; -import type { FlowGap as Gap } from "../../lib/channelFlow"; +import type { FlowGap as Gap } from "yt-dlp-transcript-common/views/pipeline/channelFlow"; import { SidingList } from "./SidingList"; // THE GAP IS THE WORK. diff --git a/editor/app/channels/[slug]/components/flow/FlowStation.tsx b/editor/app/channels/[slug]/components/flow/FlowStation.tsx @@ -3,7 +3,7 @@ import { Progress } from "yt-dlp-transcript-common/components/ui/progress"; import type { FlowStation as Station, StationOperation, -} from "../../lib/channelFlow"; +} from "yt-dlp-transcript-common/views/pipeline/channelFlow"; // THE SPLIT MATTERS HERE. This is a SERVER component, and StateBand.tsx is // `"use client"` — a plain function exported from a client module cannot be // CALLED from the server ("attempted to call bandSentence() from the server"), @@ -12,9 +12,9 @@ import type { import { bandHeadline, bandSentence, -} from "../../../../components/pipelines/band"; +} from "yt-dlp-transcript-common/views/pipeline/band"; import { StateBand } from "../../../../components/pipelines/StateBand"; -import { formatCount, formatCoverage, STATION_DOT } from "./tone"; +import { formatCount, formatCoverage, STATION_DOT } from "yt-dlp-transcript-common/views/pipeline/tone"; // One station on the line, emitted as THREE siblings so the parent grid can put // every dot on the same horizontal rule regardless of how tall the labels above diff --git a/editor/app/channels/[slug]/components/flow/NextAction.tsx b/editor/app/channels/[slug]/components/flow/NextAction.tsx @@ -1,6 +1,6 @@ -import type { ChannelFlow } from "../../lib/channelFlow"; +import type { ChannelFlow } from "yt-dlp-transcript-common/views/pipeline/channelFlow"; import { InlineActionButton } from "../../../../components/actions/InlineActionButton"; -import type { StageId } from "../../lib/stageStatus"; +import type { StageId } from "yt-dlp-transcript-common/views/pipeline/stageStatus"; // THE ONE PRIMARY ACTION, and the only brand-coloured thing on the page. // diff --git a/editor/app/channels/[slug]/components/flow/OverviewPanel.tsx b/editor/app/channels/[slug]/components/flow/OverviewPanel.tsx @@ -6,7 +6,7 @@ import { CardTitle, } from "yt-dlp-transcript-common/components/ui/card"; import { Separator } from "yt-dlp-transcript-common/components/ui/separator"; -import type { StageId, StageStatus } from "../../lib/stageStatus"; +import type { StageId, StageStatus } from "yt-dlp-transcript-common/views/pipeline/stageStatus"; // THE LANDING PANEL. // diff --git a/editor/app/channels/[slug]/components/flow/SidingList.tsx b/editor/app/channels/[slug]/components/flow/SidingList.tsx @@ -11,7 +11,7 @@ import { HoverCardContent, HoverCardTrigger, } from "yt-dlp-transcript-common/components/ui/hover-card"; -import type { Siding } from "../../lib/channelFlow"; +import type { Siding } from "yt-dlp-transcript-common/views/pipeline/channelFlow"; // How many sidings stay visible before the rest collapse. The tail of the // pipeline can legitimately carry six at once (digest blocked + deferred + diff --git a/editor/app/channels/[slug]/components/flow/StageSwitcher.tsx b/editor/app/channels/[slug]/components/flow/StageSwitcher.tsx @@ -1,5 +1,5 @@ import Link from "next/link"; -import type { StageId, StageStatus } from "../../lib/stageStatus"; +import type { StageId, StageStatus } from "yt-dlp-transcript-common/views/pipeline/stageStatus"; // The stage picker. ONE PANEL IS OPEN AT A TIME and the selection lives in // `?stage=`, not a hash — so it is server-rendered, shareable, and survives the diff --git a/editor/app/channels/[slug]/components/flow/tone.ts b/editor/app/channels/[slug]/components/flow/tone.ts @@ -1,41 +0,0 @@ -import type { StageTone } from "../../lib/stageStatus"; - -// The line invents no colours. Every value below is one of the semantic tokens -// the repo already carries across four theme families × light/dark -// (common/styles/tokens.css); a bespoke hue here would be wrong in eight -// palettes at once. - -export const STATION_DOT: Record<StageTone, string> = { - // A station with nothing to say is an outline, not a filled dot: "○" in the - // sketch. Notably this is also where a DISABLED lane lands — see the tone rule - // in channelFlow.ts. - neutral: "border border-border-strong bg-transparent", - ok: "bg-success", - attention: "bg-warning", - danger: "bg-destructive", - // The travelling pulse. The only animated thing on the page. - running: "bg-info animate-pulse", -}; - -export const STATION_TEXT: Record<StageTone, string> = { - neutral: "text-muted-foreground", - ok: "text-foreground", - attention: "text-warning", - danger: "text-destructive", - running: "text-info", -}; - -export function formatCount(n: number | null): string { - // "—", never "0". A zero here would claim a measurement nobody took. - return n == null ? "—" : n.toLocaleString(); -} - -export function formatCoverage(coverage: number | null): string { - if (coverage == null) return "—"; - const pct = coverage * 100; - // Whole percents once you are past 10% — the third significant figure on - // "94.37%" is noise at this size. Below that, one decimal, because the - // difference between 6.8% and 6% is the difference between a sweep that is - // moving and one that is not. - return `${pct >= 10 ? Math.round(pct) : Math.round(pct * 10) / 10}%`; -} diff --git a/editor/app/channels/[slug]/lib/channelFlow.test.ts b/editor/app/channels/[slug]/lib/channelFlow.test.ts @@ -1,339 +0,0 @@ -import { test } from "node:test"; -import assert from "node:assert/strict"; -import type { ChannelSnapshot } from "yt-dlp-transcript-common/controller/channelSnapshot"; -import type { ChannelConfig } from "yt-dlp-transcript-common/lib/channelConfig"; -import type { - Operation, - OperationSnapshotEntry, -} from "yt-dlp-transcript-common/lib/operations"; -import { computeChannelFlow, type FlowStationId } from "./channelFlow"; -import { computeStageStatuses, normalizeBuckets } from "./stageStatus"; - -// Run from this directory (the [slug] segment is a glob to node's test runner): -// cd "editor/app/channels/[slug]/lib" && ../../../../../node_modules/.bin/tsx --test channelFlow.test.ts - -const CONFIG: ChannelConfig = { handling: "transcribe", url: "https://x/y" }; - -function snapshotOf(patch: Partial<ChannelSnapshot> = {}): ChannelSnapshot { - return { - generatedAt: "2026-08-01T00:00:00.000Z", - totals: { videos: 100, transcribed: 40, downloaded: 60 }, - buckets: normalizeBuckets(undefined), - undownloadedIds: [], - ...patch, - }; -} - -// A registry entry is a big object with three async methods on it; none of them -// are reachable from computeChannelFlow, which only ever reads id/label/hints. -function kind(patch: Partial<Operation> & { id: string }): Operation { - return { label: patch.id, hint: "", ...patch } as Operation; -} - -// A snapshot entry AS WRITTEN TO DISK. The cast is the point of the helper: -// OperationCounts declares deferred/blocked/partial required, but every snapshot -// currently on disk predates them, which is why every read site carries `?? 0`. -// Omitting them here is how these tests exercise the real files. -function entry(patch: Partial<OperationSnapshotEntry>): OperationSnapshotEntry { - return { ids: [], missing: 0, stale: 0, missingInput: 0, ...patch } as OperationSnapshotEntry; -} - -function flowOf( - snapshot: ChannelSnapshot, - opts: { - laneOperations?: Operation[]; - playlistCount?: number | null; - failedVideoIds?: string[]; - } = {}, -) { - const failedVideoIds = opts.failedVideoIds ?? []; - return computeChannelFlow({ - snapshot, - stages: computeStageStatuses({ - snapshot, - failedVideoIds, - config: CONFIG, - runningJobs: [], - }), - config: CONFIG, - failedVideoIds, - laneOperations: opts.laneOperations ?? [kind({ id: "diarization" })], - playlistCount: opts.playlistCount ?? null, - }); -} - -function station( - flow: ReturnType<typeof flowOf>, - id: FlowStationId, -) { - const s = flow.stations.find((st) => st.id === id); - assert.ok(s, `expected a ${id} station`); - return s; -} - -test("a pre-`eligible` snapshot reports unknown coverage, not zero", () => { - // The shape a third of the snapshots on disk are still in: per-kind counts - // written before `eligible` existed. A 0 here renders as "nothing digested" - // on a channel that may be fully digested. - const flow = flowOf( - snapshotOf({ - backfill: { - digest: entry({ missing: 5 }), - diarization: entry({ missing: 3 }), - }, - }), - ); - - assert.equal(station(flow, "digest").through, null); - assert.equal(station(flow, "digest").denominator, null); - assert.equal(station(flow, "digest").coverage, null); - assert.equal(station(flow, "speakers").through, null); - assert.equal(station(flow, "speakers").coverage, null); -}); - -test("the lane station does NOT sum its operations — it reads the lead one", () => { - // THE BUG THIS STATION USED TO BE. `through` and `denominator` were the sum - // across every kind on the lane, which on the live corpus added diarization's - // coverage (one audio pass per video, 4 done of 11,338) to attribution-text's - // (~1 model call per transcript CHUNK, 1 done of 11,338) and printed the - // result under a label that named the queue. Two different populations in two - // different units, added, and no screen said so. - // - // The numeral now belongs to exactly ONE operation — the first in dependency - // order — and the rest state themselves separately in the station foot. - const flow = flowOf( - snapshotOf({ - backfill: { - diarization: entry({ eligible: 10 }), - // Same lane, a different population. Nothing may fold it in. - "attribution-diarized": entry({ eligible: 1000, missing: 400 }), - }, - }), - { - laneOperations: [ - kind({ id: "diarization" }), - kind({ id: "attribution-diarized" }), - ], - }, - ); - - const lane = station(flow, "speakers"); - assert.equal(lane.through, 10); - assert.equal(lane.denominator, 10); - // The sum would be 1,010. Asserting the negative is the point. - assert.notEqual(lane.denominator, 1010); - // Both operations are carried, each with its own band and its own - // denominator, so nothing is hidden by not being summed. - assert.deepEqual( - lane.operations.map((o) => o.id), - ["diarization", "attribution-diarized"], - ); - assert.equal(lane.operations[1].band.eligible, 1000); - assert.equal(lane.operations[1].band.reachable, 400); -}); - -test("the lane station is named after its operations, not its queue key", () => { - // "Backfill" is a scheduler key. An operator cannot arm, pause or run "a - // backfill" — they can run speaker work. The label is DERIVED from the group - // its kinds declare, so a lane holding a mix degrades to the generic name - // rather than advertising one member's. - const speakers = flowOf(snapshotOf(), { - laneOperations: [ - kind({ id: "diarization" }), - kind({ id: "attribution-text" }), - ], - }); - assert.equal(station(speakers, "speakers").label, "Speakers"); - - const mixed = flowOf(snapshotOf(), { - laneOperations: [kind({ id: "diarization" }), kind({ id: "digest" })], - }); - assert.equal(station(mixed, "speakers").label, "Derived data"); - - // Nothing enabled: the generic name, and no operations to state. - const off = flowOf(snapshotOf(), { laneOperations: [] }); - assert.equal(station(off, "speakers").label, "Derived data"); - assert.deepEqual(station(off, "speakers").operations, []); -}); - -test("an unknown `eligible` on the lead operation still renders unknown, not zero", () => { - // Invariant 2, at the one station whose denominator moved. A snapshot written - // before `eligible` existed has work counts and no denominator, and the - // station must say "—" rather than 0%. - const flow = flowOf( - snapshotOf({ backfill: { diarization: entry({ missing: 3 }) } }), - { laneOperations: [kind({ id: "diarization" })] }, - ); - const lane = station(flow, "speakers"); - assert.equal(lane.through, null); - assert.equal(lane.denominator, null); - assert.equal(lane.coverage, null); -}); - -test("coverage is a real ratio once the snapshot can say", () => { - const flow = flowOf(snapshotOf(), { playlistCount: 125 }); - assert.equal(station(flow, "playlist").through, 100); - assert.equal(station(flow, "playlist").denominator, 125); - assert.equal(station(flow, "playlist").coverage, 0.8); - assert.equal(station(flow, "download").coverage, 0.6); - // transcribed / downloaded, not / videos: the denominator is the eligible - // population, and an undownloaded video is not eligible for transcription. - assert.equal( - station(flow, "transcribe").coverage, - 40 / 60, - ); -}); - -test("a lane that is switched off reads neutral, never ok and never amber", () => { - const snapshot = snapshotOf({ - backfill: { - diarization: entry({ missing: 7, eligible: 50, ids: ["a", "b"] }), - }, - }); - - const off = flowOf(snapshot, { laneOperations: [] }); - assert.equal(station(off, "speakers").tone, "neutral"); - // …and the work it recorded is not offered as something to press, because - // nothing would run it. - assert.equal(off.gaps.find((g) => g.to === "speakers")?.reachable, 0); - // NOT the pre-rename literal. This assertion passed vacuously the moment the - // stage id changed — a notEqual against a value the union can no longer hold - // is always true — so it is spelled with the live id and would fail if the - // disabled lane were ever offered as the next action again. - assert.notEqual(off.next?.stage, "speakers"); - - const on = flowOf(snapshot, { laneOperations: [kind({ id: "diarization" })] }); - assert.equal(on.gaps.find((g) => g.to === "speakers")?.reachable, 7); -}); - -test("deferred, blocked and missing-input never enter a gap's reachable count", () => { - const flow = flowOf( - snapshotOf({ - backfill: { - digest: entry({ - missing: 2, - stale: 1, - partial: 1, - missingInput: 900, - deferred: 40, - blocked: 1631, - eligible: 3000, - ids: ["a", "b", "c", "d"], - }), - diarization: entry({ - missing: 3, - missingInput: 500, - deferred: 11, - blocked: 70, - eligible: 1000, - ids: ["x", "y", "z"], - }), - }, - }), - ); - - const toDigest = flow.gaps.find((g) => g.to === "digest"); - const toBackfill = flow.gaps.find((g) => g.to === "speakers"); - // missing + stale + partial, and nothing else. - assert.equal(toDigest?.reachable, 4); - assert.equal(toBackfill?.reachable, 3); - - // The excluded populations are present — on the other axis. - const sidingCount = ( - gapTo: FlowStationId, - label: string, - ): number | undefined => - flow.gaps - .find((g) => g.to === gapTo) - ?.sidings.find((s) => s.label === label)?.count; - - assert.equal(sidingCount("digest", "waiting on a transcript"), 1631); - assert.equal(sidingCount("digest", "deferred"), 40); - assert.equal(sidingCount("speakers", "needs media re-acquired"), 500); - assert.equal(sidingCount("speakers", "deferred"), 11); - assert.equal(sidingCount("speakers", "waiting on an earlier backfill"), 70); - - // The invariant stated as the sum nobody should be able to write: a gap's - // reachable count is not the total of everything hanging under it. - for (const gap of flow.gaps) { - const sidingTotal = gap.sidings.reduce((n, s) => n + s.count, 0); - if (sidingTotal > 0) { - assert.notEqual(gap.reachable, gap.reachable + sidingTotal); - } - } -}); - -test("the digest station reads the registry entry", () => { - // The operation registry's entry is the one definition of the digest work - // list. `eligible` is absent here, so the station reports the work and still - // refuses to say how many are done. - const flow = flowOf( - snapshotOf({ - backfill: { digest: entry({ missing: 3, ids: ["a", "b", "c"] }) }, - }), - ); - assert.equal(flow.gaps.find((g) => g.to === "digest")?.reachable, 3); - // …but it still cannot say how many are done. - assert.equal(station(flow, "digest").through, null); -}); - -test("the bottleneck is the biggest gap; the next action is the furthest upstream one", () => { - const flow = flowOf( - snapshotOf({ - undownloadedIds: ["a", "b"], - backfill: { - digest: entry({ missing: 1675, eligible: 1797 }), - }, - }), - ); - - // The eye goes to the 1,675-video digest shortfall… - assert.equal(flow.bottleneck, "transcribe"); - // …but the button offers the two downloads, because a line clears from the - // front and the digest gap shrinks on its own as the upstream one does. - assert.equal(flow.next?.stage, "download"); - assert.equal(flow.next?.count, 2); -}); - -test("an idle, clean, reported channel offers no action at all", () => { - const flow = flowOf( - snapshotOf({ totals: { videos: 10, transcribed: 10, downloaded: 10 } }), - ); - assert.equal(flow.next, null); - assert.equal(flow.bottleneck, null); -}); - -test("a channel with no report offers nothing here — NoReportYet owns that", () => { - // Every count is zero because nothing has looked yet, not because the work is - // done. The page says so in the NoReportYet banner, which carries the only - // "Refresh report" button; putting a second one here duplicates the affordance - // and makes the name ambiguous. - const flow = flowOf( - snapshotOf({ - generatedAt: "", - totals: { videos: 0, transcribed: 0, downloaded: 0 }, - }), - ); - assert.equal(flow.next, null); -}); - -test("every optional snapshot field survives being absent", () => { - // The render-path crash this guards: `.toLocaleString()` on an undefined - // count. Nothing here is defaulted defensively — it is defaulted because - // snapshots on disk genuinely predate each field. - const bare = { - generatedAt: "", - totals: { videos: 0, transcribed: 0, downloaded: 0 }, - buckets: {}, - undownloadedIds: [], - } as unknown as ChannelSnapshot; - const flow = flowOf(bare); - for (const s of flow.stations) { - assert.ok(s.through === null || Number.isFinite(s.through)); - assert.ok(s.coverage === null || Number.isFinite(s.coverage)); - } - for (const g of flow.gaps) { - assert.ok(Number.isFinite(g.reachable)); - for (const sd of g.sidings) assert.ok(Number.isFinite(sd.count)); - } -}); diff --git a/editor/app/channels/[slug]/lib/channelFlow.ts b/editor/app/channels/[slug]/lib/channelFlow.ts @@ -1,582 +0,0 @@ -import type { ChannelConfig } from "yt-dlp-transcript-common/lib/channelConfig"; -import { - digestWorkOf, - excludedDownloadIdSet, - type ChannelSnapshot, -} from "yt-dlp-transcript-common/controller/channelSnapshot"; -import { digestCountOf } from "yt-dlp-transcript-common/controller/channels"; -import { - backfillLaneEntriesOf, - operationCatalog, - operationLabel, - operationsActionLabel, - operationsGroupLabel, - presentOperationWork, - reachableOperationWork, - type Operation, -} from "yt-dlp-transcript-common/lib/operations"; -import type { OperationBand } from "../../../components/pipelines/band"; -import { buildChannelBands } from "../../../components/pipelines/buildBands"; -import { - normalizeBuckets, - type StageId, - type StageStatus, - type StageTone, -} from "./stageStatus"; - -// THE CHANNEL LINE. -// -// A channel's lifecycle is not ten sibling cards — it is a conserved quantity -// moving through stations. Every video enters at the playlist and either -// advances or leaves the line. This module is the model behind that picture: -// stations are stages, and the GAP between two stations carries the shortfall, -// because the gap is the work. -// -// SNAPSHOT-ONLY, and deliberately pure. common/controller/noCorpusWalkInRender- -// Paths.test.ts bans the corpus walk from render paths, and page.tsx documents -// the multi-minute regression from generating a report inside a GET. The one -// value this cannot derive from the snapshot — how many videos the playlist -// file names — is passed IN by the caller (one small readFile), never read here. -// -// Two invariants this file exists to hold: -// -// 1. WORK ON THE LINE IS NEVER SUMMED WITH WORK OFF IT. `reachable` is what the -// lane can do today (missing + stale + partial). `missingInput`, `deferred`, -// `blocked`, `excludedFromDownload` and `untranscribable` have LEFT the line -// and live in `sidings`. Four surfaces once summed Object.values(snapshot -// .backfill) and put every channel permanently at the top of every list. -// Different fields on different axes is what makes the mistake unspellable. -// 2. UNKNOWN IS NOT ZERO. A third of the snapshots on disk predate `eligible`, -// so `present`/`coverage` are `number | null` and a reader must render "—". -// A 0 there reads as "nothing digested" on a fully digested channel. -// -// AND THE ONE THIS FILE USED TO BREAK. The `backfill` station set `through` and -// `denominator` by SUMMING THREE OPERATIONS — the one thing invariant 1 forbids -// everywhere else, hidden behind a station label that named the queue rather -// than the work. On the live corpus it was adding audio passes (diarization: 4 -// done of 11,338) to per-chunk model calls (attribution-text: 1 done of 11,338, -// and its cost basis is the transcript CHUNK, not the video), and calling the result -// "Backfill". A station now carries its group's OPERATIONS, each with its own -// band and its own denominator, and the numeral above them belongs to exactly -// one of them — see leadOf. - -export type FlowStationId = - | "playlist" - | "download" - | "transcribe" - | "digest" - // Renamed with StageId — see stageStatus.ts. The station and the stage card it - // links to must carry the same id or `?stage=${station.stage}` opens the wrong - // panel. - | "speakers"; - -// One pipeline drawn under a station. The band is the same instrument the -// /channels strip and the /operations rail draw, at station scale — which is -// what makes a figure here and a figure there impossible to disagree. -export type StationOperation = { - id: string; - label: string; - shortLabel: string; - // What one video costs, in words — the cost basis. Printed wherever the operation is armed, so - // an 11,337-video backlog of per-chunk model calls cannot read as a quiet row. - costBasis: string; - band: OperationBand; -}; - -export type FlowStation = { - id: FlowStationId; - // DERIVED for a station that holds several operations, never hardcoded: a - // lane holding a mix of groups falls back to "Derived data" rather than - // advertising one member's name. See operationsGroupLabel. - label: string; - // Videos that have cleared this station. NULL when the snapshot cannot say — - // see invariant 2 above. Only the digest and backfill stations can be null; - // the rest come from `totals`, which every snapshot carries. - through: number | null; - // The eligible population. null = unknown. - denominator: number | null; - // through / denominator, 0..1. Null whenever either side is unknown — render - // "—", never 0. - coverage: number | null; - running: boolean; - tone: StageTone; - // Which stage panel this station opens (?stage=). - stage: StageId; - // The pipelines that run at this station, in dependency order. Empty for - // playlist, which is not an operation the registry dispatches or counts — it - // keeps the plain coverage meter. - operations: StationOperation[]; -}; - -// A population that has LEFT the line: it is not work the lane can pick up, and -// it must never be added to a gap's `reachable`. -export type Siding = { - label: string; - count: number; - stage: StageId; - hint?: string; -}; - -export type FlowGap = { - from: FlowStationId; - to: FlowStationId; - // Work the lane can do today. NEVER summed with `sidings`. - reachable: number; - // "to download", "to transcribe", … - label: string; - // Where ?stage= sends you — the stage that OWNS this gap's work, i.e. the - // destination station's stage. - stage: StageId; - // The destination station's off-line populations. Sidings hang below the gap - // in the rendered line, on a different axis from `reachable`, so the layout - // itself cannot sum them. - sidings: Siding[]; -}; - -export type ChannelFlow = { - stations: FlowStation[]; - gaps: FlowGap[]; - // The `from` id of the largest reachable gap — the one the renderer promotes - // typographically. Null when nothing is reachable anywhere. - bottleneck: FlowStationId | null; - // The single primary action offered on the page. See pickNext below for why - // this is the FURTHEST UPSTREAM gap rather than the biggest one. - next: { stage: StageId; label: string; count: number } | null; -}; - -export type ComputeChannelFlowInput = { - snapshot: ChannelSnapshot; - // Reused, never recomputed: tone and running are stageStatus's job and a - // second opinion about them is a second thing to keep in sync. - stages: Record<StageId, StageStatus>; - config: ChannelConfig; - failedVideoIds: string[]; - // Enabled lane kinds. EMPTY MEANS THE LANE IS OFF, which is not the same as - // finished — see the tone rule at the bottom of this file. - laneOperations: Operation[]; - // How many videos the channel's `playlist` file names, or null when there is - // no playlist file. Read by the caller (countPlaylist) so this stays pure. - playlistCount: number | null; -}; - -function ratio(through: number | null, denominator: number | null): number | null { - if (through == null || denominator == null || denominator <= 0) return null; - return Math.min(1, through / denominator); -} - -function siding( - label: string, - count: number, - stage: StageId, - hint?: string, -): Siding[] { - return count > 0 ? [{ label, count, stage, ...(hint ? { hint } : {}) }] : []; -} - -// THE PIPELINES DRAWN UNDER EACH STATION, off the registry. -// -// The band for a channel is the same fold over the same snapshot the corpus -// rail uses, so a channel figure and a corpus figure cannot disagree about what -// "downloaded" or "reachable" means. Everything else here — the label, the -// column-width label, the cost basis — is read from the operation catalog -// rather than restated, which is what stops this file drifting from the two -// other surfaces that group the same operations. -function stationOperations( - snapshot: ChannelSnapshot, - laneKindIds: ReadonlyArray<string>, -): Map<string, StationOperation> { - const catalog = new Map(operationCatalog().map((o) => [o.id, o])); - const bands = buildChannelBands(snapshot, laneKindIds); - const out = new Map<string, StationOperation>(); - for (const band of bands) { - const op = catalog.get(band.id); - if (!op) continue; - out.set(band.id, { - id: band.id, - label: op.label, - shortLabel: op.shortLabel, - costBasis: op.costBasis, - band, - }); - } - return out; -} - -// The operation whose coverage the station's big numeral belongs to: the FIRST -// in dependency order, which is the one every other member of the group either -// consumes or runs beside. -// -// Explicitly NOT a sum, and not an average either. The three speaker operations -// are three different populations with two different cost bases — one audio -// pass per video against ~1 model call per transcript chunk — and any single -// figure over all three is the mistake this station used to make. One member -// owns the numeral; the rest state themselves, separately, in the foot. -function leadOf( - ops: ReadonlyArray<StationOperation>, -): StationOperation | null { - return ops[0] ?? null; -} - -export function computeChannelFlow( - input: ComputeChannelFlowInput, -): ChannelFlow { - const { - snapshot, - stages, - failedVideoIds, - laneOperations, - playlistCount, - } = input; - - const buckets = normalizeBuckets(snapshot.buckets); - const digestWarnings = snapshot.buckets?.digestWarnings ?? []; - const totals = snapshot.totals ?? { videos: 0, transcribed: 0, downloaded: 0 }; - const undownloadedIds = snapshot.undownloadedIds ?? []; - const excluded = snapshot.excludedFromDownload; - const excludedIds = excludedDownloadIdSet(snapshot); - const actionableDownloadedNoTranscript = buckets.downloadedNoTranscript.filter( - (id) => !excludedIds.has(id), - ); - - // Read the digest operation through digestWorkOf — the operation registry's - // entry. The old `noDigest` bucket had no cues-staleness gate and no - // transcript gate, so it called deferred and blocked videos done. - const digestWork = digestWorkOf(snapshot); - - // backfillLaneEntriesOf, never Object.values: the per-kind map now carries EVERY - // catalog operation including digest (~75k videos on the live corpus), and - // digest has its own station one step upstream. - const laneEntries = backfillLaneEntriesOf(snapshot.backfill); - // An empty kind list means the operator switched the feature off. That is not - // an empty work list in the "finished" sense, and the tone rule below says so. - const laneOff = laneOperations.length === 0; - - const laneKindIds = laneOperations.map((k) => k.id); - const operationsById = stationOperations(snapshot, laneKindIds); - const opsFor = (...ids: string[]): StationOperation[] => - ids - .map((id) => operationsById.get(id)) - .filter((o): o is StationOperation => o != null); - // The lane's own operations, in dependency order — the group the station is - // named after, and the members its foot states one by one. - const laneOps = laneOff ? [] : opsFor(...laneKindIds); - const laneLead = leadOf(laneOps); - - const laneReachable = laneOff - ? 0 - : laneEntries.reduce((n, e) => n + reachableOperationWork(e), 0); - const laneMissingInput = laneEntries.reduce((n, e) => n + e.missingInput, 0); - // `?? 0` is load-bearing, not defensive: snapshots written before these fields - // existed have neither, and .toLocaleString() on undefined throws in a render. - const laneDeferred = laneEntries.reduce((n, e) => n + (e.deferred ?? 0), 0); - const laneBlocked = laneEntries.reduce((n, e) => n + (e.blocked ?? 0), 0); - - // NO CROSS-OPERATION `present` OR `eligible` SUM LIVES HERE ANY MORE, and the - // helper that made one is gone with it. It used to add diarization's coverage - // to attribution-text's, which is an audio pass plus a per-chunk model call - // over two different populations. The station reads its LEAD operation and - // the foot states each member on its own — see leadOf. - // - // The four WORK counts above are still summed, and legitimately: a siding is - // "how many videos have left the line for this reason", and that reason is - // the same reason whichever operation reported it. - - // digestCountOf sums `digestEngines`, which 11 of the 65 live snapshots lack - // entirely — it returns 0 for those, which would read as "nothing digested". - // So it is only consulted when the map is actually present. - const digestEnginesTotal = - snapshot.digestEngines != null ? digestCountOf(snapshot) : null; - - const stationById: Record<FlowStationId, FlowStation> = { - playlist: { - id: "playlist", - label: "Playlist", - through: totals.videos, - denominator: playlistCount, - coverage: ratio(totals.videos, playlistCount), - running: stages.playlist.running, - tone: stages.playlist.tone, - stage: "playlist", - operations: [], - }, - download: { - id: "download", - label: "Download", - through: totals.downloaded, - denominator: totals.videos, - coverage: ratio(totals.downloaded, totals.videos), - running: stages.download.running, - tone: stages.download.tone, - stage: "download", - operations: opsFor("download"), - }, - transcribe: { - id: "transcribe", - label: "Transcribe", - through: totals.transcribed, - denominator: totals.downloaded, - coverage: ratio(totals.transcribed, totals.downloaded), - running: stages.transcribe.running, - tone: stages.transcribe.tone, - stage: "transcribe", - operations: opsFor("transcription"), - }, - digest: { - id: "digest", - label: "Digest", - through: digestWork.present ?? digestEnginesTotal, - denominator: digestWork.eligible, - coverage: ratio(digestWork.present ?? digestEnginesTotal, digestWork.eligible), - running: stages.digest.running, - tone: stages.digest.tone, - stage: "digest", - operations: opsFor("digest"), - }, - speakers: { - id: "speakers", - // NAMES THE WORK, NOT THE QUEUE. "Backfill" is a scheduler key — three - // operations happen to share it — and an operator cannot control, arm or - // pause "a backfill". They can pause speaker work. The name is derived - // from the group its members declare, so a lane that gains a kind from - // another group degrades to "Derived data" instead of lying. - label: operationsGroupLabel(laneKindIds), - // THE NUMERAL BELONGS TO ONE OPERATION, not to a sum of three. See leadOf. - // Read off the band rather than recomputed: the band IS presentBackfill- - // Work over this snapshot, and a second derivation is a second thing that - // can disagree with the strip on /channels. - through: laneLead?.band.present ?? null, - denominator: laneLead?.band.eligible ?? null, - coverage: laneLead - ? ratio(laneLead.band.present, laneLead.band.eligible) - : null, - running: stages.speakers.running, - // A station whose lane is DISABLED is neutral — never "ok" and never - // amber. An empty work list because a feature is off is not the same as - // being finished, and colouring it green claims a thing nobody checked. - tone: laneOff ? "neutral" : stages.speakers.tone, - stage: "speakers", - operations: laneOps, - }, - }; - - const order: FlowStationId[] = [ - "playlist", - "download", - "transcribe", - "digest", - "speakers", - ]; - const stations = order.map((id) => stationById[id]); - - // Sidings belong to the DESTINATION station's stage: "4 need cookies" hangs - // under "to download", "1,631 blocked" under "to digest". Same rule for every - // gap, so nothing is homeless and nothing is counted twice. - const sidingsOf: Record<FlowStationId, Siding[]> = { - playlist: [], - download: [ - ...siding( - "never fetched", - snapshot.missingNeverFetched?.length ?? 0, - "diagnostics", - "Known to the roster, never downloaded, and gone from the current listing.", - ), - ...siding("members-only", excluded?.membersOnly?.length ?? 0, "diagnostics"), - ...siding("deleted", excluded?.deleted?.length ?? 0, "diagnostics"), - ...siding("private", excluded?.private?.length ?? 0, "diagnostics"), - ...siding( - "skipped by filter", - buckets.skippedByFilter.length, - "diagnostics", - "Declined as currently live or upcoming; retried on a later sync.", - ), - ...siding( - "need cookies", - buckets.needsCookies.length, - "download", - "Browser cookies could recover these.", - ), - ...siding("partial downloads", buckets.partialDownloads.length, "download"), - ...siding( - "corrupt source", - buckets.corruptSource.length, - "download", - "Malformed source; needs re-downloading.", - ), - ...siding( - "corrupt full source", - buckets.corruptFullSource.length, - "download", - "Download completed but the audio stayed malformed. File kept for inspection; re-downloading is futile.", - ), - ], - transcribe: [ - ...siding("failed", failedVideoIds.length, "transcribe"), - ...siding( - "untranscribable", - buckets.untranscribable.length, - "diagnostics", - "Marked untranscribable by hand — an intentional decision, not an anomaly.", - ), - ...siding( - "incomplete transcript", - buckets.incompleteTranscript.length, - "transcribe", - "The transcript covers a fraction of the runtime — the audio download truncated silently.", - ), - ...siding( - "short audio", - buckets.shortAudio.length, - "transcribe", - "The source served a truncated stream; the short file is kept so it is not re-downloaded into a loop.", - ), - ], - digest: [ - ...siding( - "waiting on a transcript", - digestWork.blocked, - "digest", - "Nothing the digest lane can do about these — the number falls on its own as transcription runs.", - ), - ...siding( - "deferred", - digestWork.deferred, - "digest", - deferredHintFor(laneOperations, "digest"), - ), - ...siding( - "digest warnings", - digestWarnings.length, - "digest", - "The digest pass recorded something a human should look at.", - ), - ], - speakers: laneOff - ? [] - : [ - ...siding( - "needs media re-acquired", - laneMissingInput, - "speakers", - "The source audio is gone; re-acquiring it is an opt-in re-download.", - ), - ...siding( - "deferred", - laneDeferred, - "speakers", - deferredHintFor(laneOperations), - ), - ...siding( - "waiting on an earlier backfill", - laneBlocked, - "speakers", - dependsOnHint(laneOperations), - ), - ], - }; - - // Work owned by each DESTINATION station — the shortfall carried by the gap - // that leads into it. - const reachableInto: Record<FlowStationId, number> = { - playlist: 0, - download: undownloadedIds.length, - transcribe: actionableDownloadedNoTranscript.length, - digest: digestWork.reachable, - speakers: laneReachable, - }; - - const GAP_LABEL: Record<FlowStationId, string> = { - playlist: "", - download: "to download", - transcribe: "to transcribe", - digest: "to digest", - // A VALUE tsc keys but does not spell, so it is hand-checked: - // channel-line.spec.ts asserts this string. - speakers: "to speakers", - }; - - const gaps: FlowGap[] = []; - for (let i = 0; i < order.length - 1; i++) { - const from = order[i]; - const to = order[i + 1]; - gaps.push({ - from, - to, - reachable: reachableInto[to], - label: GAP_LABEL[to], - stage: stationById[to].stage, - sidings: sidingsOf[to], - }); - } - - const biggest = gaps.reduce<FlowGap | null>( - (best, g) => (g.reachable > (best?.reachable ?? 0) ? g : best), - null, - ); - - return { - stations, - gaps, - bottleneck: biggest ? biggest.from : null, - next: pickNext(gaps, laneKindIds), - }; -} - -// The ONE primary action. Deliberately the FURTHEST UPSTREAM gap with work -// rather than the biggest one: the pipeline is a line, so 1,600 videos waiting -// to be digested behind 113 that were never downloaded is not 1,600 jobs you can -// start — clearing the upstream gap is what makes the downstream one shrink. -// The bottleneck is still reported separately; it is the thing to LOOK at, not -// necessarily the thing to press. -function pickNext( - gaps: FlowGap[], - laneKindIds: ReadonlyArray<string>, -): ChannelFlow["next"] { - const ACTIONABLE: Partial<Record<StageId, string>> = { - download: "Download missing", - transcribe: "Transcribe pending", - digest: "Digest channel", - // A verb and its object, derived from the group — "Run speaker work". The - // button used to read "Backfill", which is a queue key with no object and - // nothing an operator recognises as a thing they wanted done. - speakers: `Run ${operationsActionLabel(laneKindIds)}`, - }; - for (const gap of gaps) { - if (gap.reachable <= 0) continue; - const label = ACTIONABLE[gap.stage]; - if (!label) continue; - return { stage: gap.stage, label, count: gap.reachable }; - } - // Nothing reachable — including on a channel that has never been reported, - // where the counts are all zero because nothing has looked yet. That case is - // NOT offered here: NoReportYet already sits at the top of the page saying so - // and carrying the button, and a second control with the same accessible name - // is both a duplicate affordance and, as it turns out, a locator that matches - // two elements. - return null; -} - -// What a deferred video of this kind is waiting for, from the registry rather -// than hardcoded here. SpeakersStage used to say "too long to diarize", which -// was true only while diarization was the sole kind that could defer. -function deferredHintFor( - kinds: Operation[], - onlyId?: string, -): string | undefined { - const hints = kinds - .filter((k) => (onlyId ? k.id === onlyId : true)) - .map((k) => k.deferredHint) - .filter((h): h is string => Boolean(h)); - const unique = [...new Set(hints)]; - if (unique.length === 0) return undefined; - return unique.map((h) => `Waiting because they are ${h}`).join(" · "); -} - -// Which operations' output the lane's kinds are blocked on, by label — so a -// blocked count says what it is waiting FOR rather than merely that it is stuck. -function dependsOnHint(kinds: Operation[]): string | undefined { - const labels = [ - ...new Set(kinds.flatMap((k) => (k.dependsOn ?? []).map(operationLabel))), - ]; - if (labels.length === 0) return undefined; - return `Waiting on ${labels.join(", ")}.`; -} diff --git a/editor/app/channels/[slug]/lib/stageOrder.test.ts b/editor/app/channels/[slug]/lib/stageOrder.test.ts @@ -1,69 +0,0 @@ -// Run with: -// node_modules/.bin/tsx --test "editor/app/channels/[slug]/lib/stageOrder.test.ts" -// -// page.tsx builds its stage list from the registry rather than from a literal: -// -// ["configure", "playlist", -// ...OPERATION_GROUP_ORDER.flatMap((g) => GROUP_STAGES[g]), -// "cleanup", "diagnostics", "danger"] -// -// That array is what `?stage=` is resolved against and what the switcher renders -// in order, so a change to it is a change to every stage link on the page. tsc -// checks MEMBERSHIP (GROUP_STAGES is an exhaustive Record<OperationGroup, …>) -// but not ORDER and not CARDINALITY — a group that gained a second stage, or an -// OPERATION_GROUP_ORDER someone resorted, would compile and silently reorder the -// page. This pins both against the literal list that shipped before the -// derivation. - -import { test } from "node:test"; -import assert from "node:assert/strict"; -import { OPERATION_GROUP_ORDER } from "yt-dlp-transcript-common/lib/operations"; -import { GROUP_STAGES, type StageId } from "./stageStatus"; - -// The expression from page.tsx, verbatim. Duplicated rather than exported and -// imported because page.tsx is a server component that reads the filesystem at -// module scope; what is worth pinning is the SHAPE, and a copy that drifted from -// the page would fail this test by construction on the next edit to either. -function stageOrder(): StageId[] { - return [ - "configure", - "playlist", - ...OPERATION_GROUP_ORDER.flatMap((g) => GROUP_STAGES[g]), - "cleanup", - "diagnostics", - "danger", - ]; -} - -test("the derived stage order is the list that shipped", () => { - assert.deepEqual(stageOrder(), [ - "configure", - "playlist", - "download", - "transcribe", - "digest", - "speakers", - "cleanup", - "diagnostics", - "danger", - ]); -}); - -test("every operation group names at least one stage, and none is orphaned", () => { - // The exhaustiveness tsc gives is on the KEYS. This is the other half: a group - // mapped to an empty array would compile and would mean an operation group - // with nowhere to appear on the channel page. - for (const group of OPERATION_GROUP_ORDER) { - assert.ok( - GROUP_STAGES[group].length > 0, - `${group} names no stage, so its operations have no card`, - ); - } - // And no stage is claimed by two groups, which would render it twice. - const claimed = OPERATION_GROUP_ORDER.flatMap((g) => GROUP_STAGES[g]); - assert.equal( - claimed.length, - new Set(claimed).size, - `a stage is claimed by more than one group: ${claimed.join(",")}`, - ); -}); diff --git a/editor/app/channels/[slug]/lib/stageStatus.ts b/editor/app/channels/[slug]/lib/stageStatus.ts @@ -1,628 +0,0 @@ -import type { ChannelConfig } from "yt-dlp-transcript-common/lib/channelConfig"; -import { - digestWorkOf, - excludedDownloadIdSet, - type ChannelSnapshot, -} from "yt-dlp-transcript-common/controller/channelSnapshot"; -import { - backfillLaneEntriesOf, - operationsGroupLabel, - reachableOperationWork, - type OperationGroup, -} from "yt-dlp-transcript-common/lib/operations"; -// TYPE ONLY. channelMedia.ts imports node:fs, and this module is imported by -// three client components (AttentionStrip, NextAction, OverviewPanel) for its -// types. A type import is erased, a value import would not be. -import type { ChannelMediaLocation } from "yt-dlp-transcript-common/lib/channelMedia"; - -export type SnapshotBuckets = ChannelSnapshot["buckets"]; - -// Older snapshots on disk may pre-date some bucket fields. Normalize to -// always-present arrays so callers can read .length without guards. -export function normalizeBuckets( - raw: Partial<SnapshotBuckets> | undefined, -): SnapshotBuckets { - return { - noTranscript: raw?.noTranscript ?? [], - downloadedNoTranscript: raw?.downloadedNoTranscript ?? [], - wrongFormatAudio: raw?.wrongFormatAudio ?? [], - multipleAudioFormats: raw?.multipleAudioFormats ?? [], - transcribedWithAudio: raw?.transcribedWithAudio ?? [], - untranscribable: raw?.untranscribable ?? [], - noMetadata: raw?.noMetadata ?? [], - failedListed: raw?.failedListed ?? [], - missingFromArchive: raw?.missingFromArchive ?? [], - duplicateDirs: raw?.duplicateDirs ?? [], - partialDownloads: raw?.partialDownloads ?? [], - corruptSource: raw?.corruptSource ?? [], - corruptFullSource: raw?.corruptFullSource ?? [], - nonStandardVtt: raw?.nonStandardVtt ?? [], - skippedByFilter: raw?.skippedByFilter ?? [], - incompleteTranscript: raw?.incompleteTranscript ?? [], - shortAudio: raw?.shortAudio ?? [], - autoSubsOnly: raw?.autoSubsOnly ?? [], - downloadedAutoSubsOnly: raw?.downloadedAutoSubsOnly ?? [], - supersededAutoSubs: raw?.supersededAutoSubs ?? [], - needsCookies: raw?.needsCookies ?? [], - }; -} - -export type StageId = - | "configure" - | "playlist" - | "download" - | "transcribe" - | "digest" - // THE OPERATIONS, NOT THE QUEUE. This card was called "backfill" — a queue key - // wearing a stage's name. Nobody can arm, pause or run "a backfill"; what the - // card actually holds is the speaker work (diarization and the two attribution - // kinds), which is a thing an operator recognises. The queue key BACKFILL_QUEUE - // is untouched: it is a scheduler key and correctly named as one. - // - // ?stage=backfill still resolves here — see STAGE_ALIASES in page.tsx. - | "speakers" - | "cleanup" - | "diagnostics" - // WHERE THE MEDIA PHYSICALLY IS (plans/relocate-channel-media.md). A channel - // CHORE like cleanup and diagnostics, not an operation — nothing registers it - // and no group owns it — so it is hand-listed at the call site alongside them - // and is deliberately absent from GROUP_STAGES below. - | "storage" - | "danger"; - -// THE STAGES EACH OPERATION GROUP OWNS, in the group's own order. -// -// Record<> is EXHAUSTIVE, which is the whole point: a new OperationGroup does -// not compile until it names its stage(s). Before this, the channel page's stage -// list was a hand-written literal, so a registered operation in a new group got -// a card only if someone remembered to add one — and the failure mode was a -// silent absence, not an error. -// -// NOT 1:1 with the groups, and no honest derivation makes it so: `sync` owns -// NO stage (below), and a group may own more than one — `media` did while the -// transcode stage existed (retired 2026-08-30). So this is a Record of ARRAYS, -// spread in OPERATION_GROUP_ORDER — which yields exactly today's order and -// today's cardinality. It is a compile-time membership check, not a re-shaping -// of the page. -// -// DELIBERATELY ONLY THE MIDDLE. configure/playlist and cleanup/diagnostics/ -// danger are channel CHORES, not operations — nothing registers them and no -// group owns them — so they stay hand-listed at the call site. Do not "finish" -// this derivation by inventing groups for them. -export const GROUP_STAGES: Record<OperationGroup, readonly StageId[]> = { - media: ["download"], - transcript: ["transcribe"], - digest: ["digest"], - speakers: ["speakers"], - // Empty on purpose, and this is the paragraph above in practice: sync's - // channel surface is the hand-listed `playlist` bookend (JOB_KIND_TO_STAGE - // maps the sync job kind to it), which is a channel chore, not a stage this - // Record owns. Do not "finish" the derivation by giving sync a stage here — - // the channel page would grow a second, duplicate playlist card. - sync: [], -}; - -export type StageTone = "neutral" | "attention" | "danger" | "running" | "ok"; - -export type StageStatus = { - id: StageId; - title: string; - pending: number; - failed: number; - running: boolean; - defaultOpen: boolean; - summary: string; - tone: StageTone; -}; - -const JOB_KIND_TO_STAGE: Record<string, StageId> = { - "store-playlist": "playlist", - sync: "playlist", - "download-from-playlist": "download", - "download-missing": "download", - "whisper-all": "transcribe", - "whisper-retry": "transcribe", - "transcribe-one": "transcribe", - "whisper-video": "transcribe", - "whisper-bucket-auto-subs": "transcribe", - "digest-channel-local": "digest", - "digest-channel-remote": "digest", - "digest-share-cluster": "digest", - "backfill-channel": "speakers", - // The pre-registry per-channel diarization button lands on the channel queue - // but is the same work this card is about, so it lights this card too. - "diarize-channel": "speakers", - "clean-audio-transcribed": "cleanup", - "purge-superseded-auto-subs": "cleanup", - "clean-extra-audio-formats": "cleanup", - "remove-wrong-format-audio": "cleanup", - "check-availability": "diagnostics", -}; - -function pluralize(n: number, singular: string, plural?: string): string { - return `${n} ${n === 1 ? singular : plural ?? `${singular}s`}`; -} - -function pickTone(args: { - running: boolean; - pending: number; - failed: number; - fallback?: StageTone; -}): StageTone { - if (args.running) return "running"; - if (args.failed > 0) return "danger"; - if (args.pending > 0) return "attention"; - return args.fallback ?? "neutral"; -} - -export type ComputeStageStatusesInput = { - snapshot: ChannelSnapshot; - failedVideoIds: string[]; - config: ChannelConfig; - // Only `status` and `kind` are read (which stage has work in flight), so this - // takes the SHAPE rather than the record: the channel page now gets its rows - // from the one job-row builder (liveJobRows) and no longer holds JobRecords. - runningJobs: ReadonlyArray<{ status: string; kind: string }>; - // Whether ANY backfill kind is switched on. - // - // The snapshot's per-kind counts are a record of what was true when it was - // written, and they survive the operator turning the feature off. Without - // this flag the card reports "7 videos need derived data" — amber, with a - // count — directly above its own body copy saying "No backfill is enabled", - // and nothing would ever run the work it is advertising. Defaults to true so - // an omitted flag behaves as it always did. - backfillEnabled?: boolean; - // The ids of the lane's enabled kinds, so the stage can be titled after what - // it HOLDS rather than after its queue key. Optional and defaulting to the - // generic name, because a caller that only needs tone and counts should not - // have to resolve the registry. - backfillKindIds?: ReadonlyArray<string>; - // Where this channel's media actually is, from inspectChannelMedia. Optional - // because the two stats it costs belong to the caller that already has the - // config in hand, and a caller that only wants tone and counts should not - // have to do I/O to get them — an omitted location reads as "in place", which - // is what every channel was before relocation existed. - media?: ChannelMediaLocation | null; -}; - -export function computeStageStatuses( - input: ComputeStageStatusesInput, -): Record<StageId, StageStatus> { - const { - snapshot, - failedVideoIds, - config, - runningJobs, - backfillEnabled = true, - backfillKindIds, - media, - } = input; - - const buckets = normalizeBuckets(snapshot.buckets); - const undownloadedIds = snapshot.undownloadedIds ?? []; - const excludedDownloadIds = excludedDownloadIdSet(snapshot); - const actionableNoTranscript = buckets.noTranscript.filter( - (id) => !excludedDownloadIds.has(id), - ); - const actionableDownloadedNoTranscript = buckets.downloadedNoTranscript.filter( - (id) => !excludedDownloadIds.has(id), - ); - - const runningByStage = new Set<StageId>(); - for (const job of runningJobs) { - if (job.status !== "running" && job.status !== "queued") continue; - const stage = JOB_KIND_TO_STAGE[job.kind]; - if (stage) runningByStage.add(stage); - } - - const downloadPending = - undownloadedIds.length + - actionableNoTranscript.length + - buckets.partialDownloads.length; - const transcribePending = actionableDownloadedNoTranscript.length; - const transcribeFailed = failedVideoIds.length; - const cleanupPending = buckets.multipleAudioFormats.length; - // `untranscribable` is deliberately excluded: those videos are an intentional - // user decision ("mark untranscribable"), not an anomaly with an action. - const diagnosticsPending = - buckets.noMetadata.length + - buckets.missingFromArchive.length + - buckets.duplicateDirs.length; - - // Cards that carry primary actions stay open by default so the user can - // always reach the buttons; the summary line communicates idle/busy state - // instead of collapsing the controls out of sight. Configure collapses once - // the channel has a URL; Danger zone stays collapsed unless opened. - const configure: StageStatus = { - id: "configure", - title: "Configure", - pending: 0, - failed: 0, - running: false, - defaultOpen: true, - summary: config.url - ? `${config.handling}${config.platform ? ` · ${config.platform}` : ""}` - : "Channel has no URL — open to configure.", - tone: config.url ? "neutral" : "attention", - }; - - const playlistRunning = runningByStage.has("playlist"); - const playlist: StageStatus = { - id: "playlist", - title: "Playlist", - pending: 0, - failed: 0, - running: playlistRunning, - defaultOpen: true, - summary: playlistRunning - ? "Running…" - : config.lastSyncedAt - ? `Last sync ${new Date(config.lastSyncedAt).toLocaleString()}` - : "Never synced.", - tone: pickTone({ running: playlistRunning, pending: 0, failed: 0 }), - }; - - const downloadRunning = runningByStage.has("download"); - const downloadParts: string[] = []; - if (undownloadedIds.length > 0) { - downloadParts.push(pluralize(undownloadedIds.length, "undownloaded")); - } - if (actionableNoTranscript.length > 0) { - downloadParts.push( - pluralize( - actionableNoTranscript.length, - "dir missing transcript & audio", - "dirs missing transcript & audio", - ), - ); - } - if (buckets.partialDownloads.length > 0) { - downloadParts.push( - pluralize( - buckets.partialDownloads.length, - "partial download", - "partial downloads", - ), - ); - } - if (buckets.corruptSource.length > 0) { - downloadParts.push( - pluralize( - buckets.corruptSource.length, - "corrupt source (needs re-download)", - "corrupt sources (need re-download)", - ), - ); - } - if (buckets.corruptFullSource.length > 0) { - downloadParts.push( - pluralize( - buckets.corruptFullSource.length, - "corrupt full source (file kept)", - "corrupt full sources (files kept)", - ), - ); - } - const download: StageStatus = { - id: "download", - title: "Download", - pending: downloadPending, - failed: 0, - running: downloadRunning, - defaultOpen: true, - summary: downloadRunning - ? "Running…" - : downloadParts.length > 0 - ? downloadParts.join(" · ") - : "Nothing to download.", - tone: pickTone({ - running: downloadRunning, - pending: downloadPending, - failed: 0, - }), - }; - - const transcribeRunning = runningByStage.has("transcribe"); - const transcribeParts: string[] = []; - if (actionableDownloadedNoTranscript.length > 0) { - transcribeParts.push( - pluralize( - actionableDownloadedNoTranscript.length, - "video awaiting whisper", - "videos awaiting whisper", - ), - ); - } - if (failedVideoIds.length > 0) { - transcribeParts.push(pluralize(failedVideoIds.length, "failed")); - } - // Informational only — the replace-auto-captions lane is opt-in, so these are - // NOT counted as pending work (that would light every YouTube channel up - // amber forever). - const autoSubsCandidates = - buckets.autoSubsOnly.length + buckets.downloadedAutoSubsOnly.length; - if (autoSubsCandidates > 0) { - transcribeParts.push( - pluralize( - autoSubsCandidates, - "video with only auto-captions", - "videos with only auto-captions", - ), - ); - } - const transcribe: StageStatus = { - id: "transcribe", - title: "Transcribe", - pending: transcribePending, - failed: transcribeFailed, - running: transcribeRunning, - defaultOpen: true, - summary: transcribeRunning - ? "Running…" - : transcribeParts.length > 0 - ? transcribeParts.join(" · ") - : "All transcribed.", - tone: pickTone({ - running: transcribeRunning, - pending: transcribePending, - failed: transcribeFailed, - }), - }; - - // Videos whose digest is missing, stale or part-done against the local lane's - // current identity. Read from the operation registry via digestWorkOf, which - // is the one definition: a channel with no entry reports unknown coverage - // rather than reading as fully digested. - // - // Counted as pending work rather than merely informational: unlike the - // auto-captions lane, every transcribed video is eventually meant to have one. - // Videos BLOCKED on transcription are deliberately not in this number — there - // is nothing the digest lane can do about them — and the stage card names them - // separately. - const digestRunning = runningByStage.has("digest"); - const digestWork = digestWorkOf(snapshot); - const digestPending = digestWork.reachable; - const digest: StageStatus = { - id: "digest", - title: "Digest", - pending: digestPending, - failed: 0, - running: digestRunning, - defaultOpen: true, - summary: digestRunning - ? "Running…" - : digestPending > 0 - ? pluralize( - digestPending, - "transcript needs a digest", - "transcripts need a digest", - ) - : // "All digested" MUST NOT be said over a channel that simply has - // nothing to digest yet. Before the registry classified them, videos - // with no transcript were absent from every digest bucket, so a - // channel of untranscribed videos read as finished — the exact failure - // declaring the transcription dependency exists to end. - digestWork.blocked > 0 - ? pluralize( - digestWork.blocked, - "video is waiting on a transcript", - "videos are waiting on transcripts", - ) - : "All digested at the current settings.", - tone: pickTone({ - running: digestRunning, - pending: digestPending, - failed: 0, - fallback: "ok", - }), - }; - - // The backfill lane's work list, summed across every registered kind. - // - // `pending` counts ONLY the reachable half. The needs-re-acquiring population - // is reported in the summary line and never folded in: it is 91x larger on the - // measured corpus, so counting it would hold every channel permanently amber - // for work that cannot be done without an opt-in re-download — precisely the - // trap /api/widget/actionable documents for the digest work count. - const backfillRunning = runningByStage.has("speakers"); - // backfillLaneEntriesOf, not Object.values. The snapshot's per-kind map carries every - // operation in the catalog now, including digest — which runs on its own queue - // key, has its own stage card directly above, and would otherwise add ~75,000 - // videos to this instrument on the measured corpus. The filter is by the kind's - // declared lane rather than by its id, so the next operation registered on a - // lane of its own does not re-arm the same trap. - // A disabled lane has NO work, whatever the snapshot recorded before it was - // switched off — see `backfillEnabled` above. - const backfillEntries = backfillEnabled - ? backfillLaneEntriesOf(snapshot.backfill) - : []; - const backfillPending = backfillEntries.reduce( - (n, e) => n + reachableOperationWork(e), - 0, - ); - const backfillMissingInput = backfillEntries.reduce( - (n, e) => n + e.missingInput, - 0, - ); - // Same treatment as missingInput: reported in the summary line, never folded - // into `pending`. `?? 0` because snapshots written before the cap existed have - // no such field. - const backfillDeferred = backfillEntries.reduce( - (n, e) => n + (e.deferred ?? 0), - 0, - ); - // Same treatment again, and the wording matters: this number falls on its own - // as the prerequisite lane runs, so it must not read as something to fix. - const backfillBlocked = backfillEntries.reduce( - (n, e) => n + (e.blocked ?? 0), - 0, - ); - const backfillParts: string[] = []; - if (backfillPending > 0) { - backfillParts.push( - pluralize( - backfillPending, - "video needs derived data", - "videos need derived data", - ), - ); - } - if (backfillMissingInput > 0) { - backfillParts.push( - `${backfillMissingInput.toLocaleString()} needing media re-acquired`, - ); - } - if (backfillDeferred > 0) { - backfillParts.push( - `${backfillDeferred.toLocaleString()} deferred (too long to diarize)`, - ); - } - if (backfillBlocked > 0) { - backfillParts.push( - `${backfillBlocked.toLocaleString()} waiting on an earlier backfill`, - ); - } - const speakers: StageStatus = { - id: "speakers", - // NAMED AFTER THE OPERATIONS, NOT THE QUEUE. "Backfill" is a scheduler key - // that on this install stands for three different operations; nobody can - // arm, pause or run "a backfill". Derived, so a lane that gains a kind from - // another group degrades to "Derived data" rather than going stale. - title: operationsGroupLabel(backfillKindIds ?? []), - pending: backfillPending, - failed: 0, - running: backfillRunning, - defaultOpen: true, - summary: backfillRunning - ? "Running…" - : backfillEntries.length === 0 - ? "Nothing here is enabled." - : backfillParts.length > 0 - ? backfillParts.join(" · ") - : "Everything reachable is current.", - tone: pickTone({ - running: backfillRunning, - pending: backfillPending, - failed: 0, - // Neutral rather than "ok" when nothing is enabled: an empty work list - // because a feature is off is not the same as being finished. - fallback: backfillEntries.length === 0 ? "neutral" : "ok", - }), - }; - - const cleanupRunning = runningByStage.has("cleanup"); - const cleanupParts: string[] = []; - if (cleanupPending > 0) { - cleanupParts.push( - pluralize( - cleanupPending, - "dir has extra audio formats", - "dirs have extra audio formats", - ), - ); - } - // Kept auto-caption backups. Informational (not folded into `pending`): they - // are deliberately retained until purged by hand, so they are inventory, not - // a chore. - if (buckets.supersededAutoSubs.length > 0) { - cleanupParts.push( - pluralize( - buckets.supersededAutoSubs.length, - "superseded auto-caption backup", - "superseded auto-caption backups", - ), - ); - } - const cleanup: StageStatus = { - id: "cleanup", - title: "Cleanup", - pending: cleanupPending, - failed: 0, - running: cleanupRunning, - defaultOpen: true, - summary: cleanupRunning - ? "Running…" - : cleanupParts.length > 0 - ? cleanupParts.join(" · ") - : "Nothing to clean.", - tone: pickTone({ - running: cleanupRunning, - pending: cleanupPending, - failed: 0, - }), - }; - - const diagnosticsRunning = runningByStage.has("diagnostics"); - const diagnostics: StageStatus = { - id: "diagnostics", - title: "Diagnostics", - pending: diagnosticsPending, - failed: 0, - running: diagnosticsRunning, - defaultOpen: true, - summary: diagnosticsRunning - ? "Running…" - : diagnosticsPending > 0 - ? pluralize(diagnosticsPending, "anomaly", "anomalies") - : "All clear.", - tone: pickTone({ - running: diagnosticsRunning, - pending: diagnosticsPending, - failed: 0, - fallback: "ok", - }), - }; - - // WHERE THE MEDIA IS. The only stage whose tone comes from a filesystem fact - // rather than from a count: an unreachable channel is a channel whose numbers - // everywhere else on this page are about to be wrong (an unmounted drive reads - // as "nothing downloaded"), so this card is red the moment inspect() says so - // and neutral the rest of the time. "in-place" is not an achievement, so it is - // never "ok" — the fallback tone for a healthy relocation is neutral too. - const mediaStatus = media?.status ?? "in-place"; - const storage: StageStatus = { - id: "storage", - title: "Storage", - pending: 0, - failed: 0, - running: mediaStatus === "in-transition", - defaultOpen: true, - summary: - mediaStatus === "in-place" - ? "Media is in the channel directory." - : mediaStatus === "ok" - ? `Media relocated to ${media?.target ?? "another drive"}.` - : (media?.detail ?? mediaStatus), - tone: - mediaStatus === "in-transition" - ? "running" - : mediaStatus === "unreachable" || - mediaStatus === "inconsistent" - ? "danger" - : "neutral", - }; - - const danger: StageStatus = { - id: "danger", - title: "Danger zone", - pending: 0, - failed: 0, - running: false, - defaultOpen: true, - summary: "Delete this channel.", - tone: "neutral", - }; - - return { - configure, - playlist, - download, - transcribe, - digest, - speakers, - cleanup, - diagnostics, - storage, - danger, - }; -} diff --git a/editor/app/channels/[slug]/lib/videoRowsServer.ts b/editor/app/channels/[slug]/lib/videoRowsServer.ts @@ -3,7 +3,7 @@ import type { Dirent } from "node:fs"; import { readdir } from "node:fs/promises"; import type { ChannelSnapshot } from "yt-dlp-transcript-common/controller/channelSnapshot"; import type { JobRecord } from "yt-dlp-transcript-common/jobs/registry"; -import { normalizeBuckets } from "./stageStatus"; +import { normalizeBuckets } from "yt-dlp-transcript-common/views/pipeline/stageStatus"; import type { VideoRow, VideoRowStatus } from "./videoRows"; export async function readDataDirVideoIds( diff --git a/editor/app/channels/[slug]/page.tsx b/editor/app/channels/[slug]/page.tsx @@ -81,14 +81,14 @@ import { NextAction } from "./components/flow/NextAction"; import { AttentionStrip } from "./components/flow/AttentionStrip"; import { StageSwitcher } from "./components/flow/StageSwitcher"; import { OverviewPanel } from "./components/flow/OverviewPanel"; -import { computeChannelFlow } from "./lib/channelFlow"; +import { computeChannelFlow } from "yt-dlp-transcript-common/views/pipeline/channelFlow"; import { readChannelConfigCached } from "./lib/channelConfigCache"; import { computeStageStatuses, normalizeBuckets, GROUP_STAGES, type StageId, -} from "./lib/stageStatus"; +} from "yt-dlp-transcript-common/views/pipeline/stageStatus"; import { deleteChannelAction, renameChannelAction, diff --git a/editor/app/channels/[slug]/videos/page.tsx b/editor/app/channels/[slug]/videos/page.tsx @@ -28,7 +28,7 @@ import { VideoWorkspace } from "./components/VideoWorkspace"; import { readChannelConfigCached } from "../lib/channelConfigCache"; import { parseFilters, filterRows, serializeFilters } from "../lib/videoRows"; import { computeVideoRows, readDataDirVideoIds } from "../lib/videoRowsServer"; -import { normalizeBuckets } from "../lib/stageStatus"; +import { normalizeBuckets } from "yt-dlp-transcript-common/views/pipeline/stageStatus"; export const dynamic = "force-dynamic"; diff --git a/editor/app/channels/components/ChannelsTable.tsx b/editor/app/channels/components/ChannelsTable.tsx @@ -6,7 +6,7 @@ import type { ChannelStat } from "yt-dlp-transcript-common/controller/channels"; import { bandSentence, type OperationBand, -} from "../../components/pipelines/band"; +} from "yt-dlp-transcript-common/views/pipeline/band"; import { BandLegend, StateBand, diff --git a/editor/app/channels/lib/channelGroupSections.test.ts b/editor/app/channels/lib/channelGroupSections.test.ts @@ -9,7 +9,7 @@ import type { ChannelSnapshot } from "yt-dlp-transcript-common/controller/channe import type { OperationSnapshotEntry } from "yt-dlp-transcript-common/lib/operations"; import type { SiteSettings } from "yt-dlp-transcript-common/lib/settings"; import type { Site } from "yt-dlp-transcript-common/lib/site"; -import { normalizeBuckets } from "../[slug]/lib/stageStatus"; +import { normalizeBuckets } from "yt-dlp-transcript-common/views/pipeline/stageStatus"; import { buildChannelGroupSections, slugsInGroup, diff --git a/editor/app/channels/lib/channelGroupSections.ts b/editor/app/channels/lib/channelGroupSections.ts @@ -24,7 +24,7 @@ import { } from "yt-dlp-transcript-common/lib/channelPriority"; import type { SiteSettings } from "yt-dlp-transcript-common/lib/settings"; import type { Site } from "yt-dlp-transcript-common/lib/site"; -import { normalizeBuckets } from "../[slug]/lib/stageStatus"; +import { normalizeBuckets } from "yt-dlp-transcript-common/views/pipeline/stageStatus"; // Groups a site's channels into the sections /channels renders, and totals each // section's pipeline work off the SAME snapshot readers the channel page's diff --git a/editor/app/channels/page.tsx b/editor/app/channels/page.tsx @@ -30,8 +30,8 @@ import { getSettings, type SiteSettings, } from "yt-dlp-transcript-common/lib/settings"; -import { buildChannelBands } from "../components/pipelines/buildBands"; -import { EXTERNAL_BAND_IDS } from "../components/pipelines/buildBands"; +import { buildChannelBands } from "yt-dlp-transcript-common/views/pipeline/buildBands"; +import { EXTERNAL_BAND_IDS } from "yt-dlp-transcript-common/views/pipeline/buildBands"; import { ChannelsTable, type ChannelRow, diff --git a/editor/app/cleanup/components/HoldSieve.tsx b/editor/app/cleanup/components/HoldSieve.tsx @@ -4,8 +4,8 @@ import { STATION_DOT, STATION_TEXT, formatCount, -} from "../../channels/[slug]/components/flow/tone"; -import type { StageTone } from "../../channels/[slug]/lib/stageStatus"; +} from "yt-dlp-transcript-common/views/pipeline/tone"; +import type { StageTone } from "yt-dlp-transcript-common/views/pipeline/stageStatus"; import type { CleanupSummary } from "../lib/loadCleanup"; // THE SIEVE. The cleanup sweep is a cascade of `continue`s — a video leaves at diff --git a/editor/app/cleanup/components/ReleaseLedger.tsx b/editor/app/cleanup/components/ReleaseLedger.tsx @@ -4,7 +4,7 @@ import { useState } from "react"; import Link from "next/link"; import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog"; import { formatBytes } from "yt-dlp-transcript-common/lib/format"; -import { formatCount } from "../../channels/[slug]/components/flow/tone"; +import { formatCount } from "yt-dlp-transcript-common/views/pipeline/tone"; import { cancelJobAction } from "../../jobs/actions"; import { transcribeBucketAction, diff --git a/editor/app/components/lanes/laneState.ts b/editor/app/components/lanes/laneState.ts @@ -2,7 +2,7 @@ import { STATION_DOT, STATION_TEXT, formatCount, -} from "../../channels/[slug]/components/flow/tone"; +} from "yt-dlp-transcript-common/views/pipeline/tone"; // A lane's state, and the one derivation every surface reads it from. // diff --git a/editor/app/components/pipelines/StateBand.tsx b/editor/app/components/pipelines/StateBand.tsx @@ -5,7 +5,7 @@ import { SEGMENTS, segmentValue, type OperationBand, -} from "./band"; +} from "yt-dlp-transcript-common/views/pipeline/band"; // THE INSTRUMENT. One component, three scales, one vocabulary. // diff --git a/editor/app/components/pipelines/buildBands.test.ts b/editor/app/components/pipelines/buildBands.test.ts @@ -1,428 +0,0 @@ -import { test } from "node:test"; -import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; -import type { ChannelSnapshot } from "yt-dlp-transcript-common/controller/channelSnapshot"; -import type { OperationSnapshotEntry } from "yt-dlp-transcript-common/lib/operations"; -import { - bandCoverage, - buildChannelBands, - buildOperationBands, - sumOrNull, - type OperationBand, -} from "./buildBands"; - -// Run from this directory: -// cd editor/app/components/pipelines && ../../../../node_modules/.bin/tsx --test buildBands.test.ts - -function snapshotOf(patch: Partial<ChannelSnapshot> = {}): ChannelSnapshot { - return { - generatedAt: "2026-08-21T00:00:00.000Z", - totals: { videos: 100, transcribed: 40, downloaded: 60 }, - buckets: {} as ChannelSnapshot["buckets"], - ...patch, - } as ChannelSnapshot; -} - -function entryOf(patch: Partial<OperationSnapshotEntry>): OperationSnapshotEntry { - return { - missing: 0, - stale: 0, - partial: 0, - missingInput: 0, - deferred: 0, - blocked: 0, - ids: [], - ...patch, - } as OperationSnapshotEntry; -} - -const bandOf = (bands: OperationBand[], id: string): OperationBand => { - const found = bands.find((b) => b.id === id); - assert.ok(found, `no band for ${id}`); - return found; -}; - -test("the four work states are kept apart and never summed", () => { - // The measured shape of this corpus in miniature: diarization is dominated by - // missing media and attribution-diarized by blocked work. Any code that added - // them would report both lanes as busy. - const bands = buildOperationBands({ - snapshots: [ - snapshotOf({ - backfill: { - diarization: entryOf({ - missing: 647, - missingInput: 77_276, - eligible: 78_019, - }), - "attribution-diarized": entryOf({ - missing: 94, - blocked: 77_923, - eligible: 78_019, - }), - }, - }), - ], - operationIds: ["diarization", "attribution-diarized"], - }); - const dia = bandOf(bands, "diarization"); - assert.equal(dia.reachable, 647); - assert.equal(dia.missingInput, 77_276); - assert.equal(dia.blocked, 0); - const attr = bandOf(bands, "attribution-diarized"); - assert.equal(attr.reachable, 94); - assert.equal(attr.blocked, 77_923); - assert.equal(attr.missingInput, 0); -}); - -test("one channel that cannot report `eligible` voids the whole denominator", () => { - // The partial-sum trap. A snapshot predating `eligible` contributes videos to - // the corpus but nothing to the denominator, so summing what IS known gives a - // denominator smaller than its own numerator. - const bands = buildOperationBands({ - snapshots: [ - snapshotOf({ - backfill: { diarization: entryOf({ missing: 1, eligible: 500 }) }, - }), - snapshotOf({ - // No `eligible` — an older snapshot. - backfill: { diarization: entryOf({ missing: 2 }) }, - }), - ], - operationIds: ["diarization"], - }); - const dia = bandOf(bands, "diarization"); - assert.equal(dia.reachable, 3, "work counts still sum"); - assert.equal(dia.eligible, null, "the denominator does not"); - assert.equal(dia.present, null); - assert.equal(bandCoverage(dia), null, "and coverage renders as unknown"); -}); - -test("coverage is null, never 0, when the denominator is unknown", () => { - // A 0 here would read as "nothing digested" on a fully digested channel. - assert.equal( - bandCoverage({ present: null, eligible: 10 } as OperationBand), - null, - ); - assert.equal( - bandCoverage({ present: 5, eligible: null } as OperationBand), - null, - ); - assert.equal(bandCoverage({ present: 5, eligible: 0 } as OperationBand), null); - assert.equal(bandCoverage({ present: 5, eligible: 10 } as OperationBand), 0.5); -}); - -test("digest with no registry entry is an unfilled outline, never 0 %", () => { - // A channel with no `backfill.digest` must not read "all digested" — nor - // "none digested". Digest is a plain registry entry here, exactly like every - // other operation: no entry means no work KNOWN and coverage UNKNOWN. - const bands = buildOperationBands({ - snapshots: [snapshotOf({ backfill: {} })], - operationIds: ["digest"], - }); - const digest = bandOf(bands, "digest"); - assert.equal(digest.reachable, 0); - // Nothing to have an opinion about, and bandCoverage refuses to divide by it: - // the band draws as an empty outline rather than a filled 0 %. - assert.equal(digest.eligible, 0); - assert.equal(bandCoverage(digest), null); -}); - -test("the external pipelines get bands from totals and buckets", () => { - const bands = buildOperationBands({ - snapshots: [ - snapshotOf({ - totals: { videos: 100, transcribed: 40, downloaded: 60 }, - undownloadedIds: ["u1", "u2", "u3"], - buckets: { - downloadedNoTranscript: ["d1", "d2"], - noTranscript: Array.from({ length: 60 }, (_, i) => `n${i}`), - untranscribable: ["x1", "x2"], - partialDownloads: ["p1"], - } as unknown as ChannelSnapshot["buckets"], - }), - ], - operationIds: [], - }); - const download = bandOf(bands, "download"); - // Every video the playlist knows about, not just the dirs that exist. - assert.equal(download.eligible, 103); - assert.equal(download.present, 60); - assert.equal(download.reachable, 4, "3 never fetched + 1 partial"); - assert.equal(download.dispatched, false); - - const transcription = bandOf(bands, "transcription"); - assert.equal(transcription.eligible, 98, "100 videos less 2 untranscribable"); - assert.equal(transcription.present, 40); - assert.equal(transcription.reachable, 2, "audio in hand"); - // The rest of noTranscript is waiting on the DOWNLOAD lane — blocked on an - // operation this system produces, not reachable and not missing media. - assert.equal(transcription.blocked, 56); - assert.equal(transcription.missingInput, 0); -}); - -test("ids excluded from download are deferred, not reachable", () => { - // A channel deliberately not fetching members-only videos is not a lane with - // work to do, and the two must stay separable rather than one being netted - // off the other. - const bands = buildOperationBands({ - snapshots: [ - snapshotOf({ - totals: { videos: 0, transcribed: 0, downloaded: 0 }, - undownloadedIds: ["ok", "gone"], - excludedFromDownload: { deleted: ["gone"] }, - } as Partial<ChannelSnapshot>), - ], - operationIds: [], - }); - const download = bandOf(bands, "download"); - assert.equal(download.reachable, 1); - assert.equal(download.deferred, 1); -}); - -test("a switched-off operation gets no band at all", () => { - // Absent, not zero: an empty work list because nobody enabled the feature is - // not the same as being finished, and a full green bar would claim it was. - const bands = buildOperationBands({ - snapshots: [snapshotOf({ backfill: { diarization: entryOf({ missing: 5 }) } })], - operationIds: [], - }); - assert.equal( - bands.find((b) => b.id === "diarization"), - undefined, - ); -}); - -test("sumOrNull latches null and never returns a partial total", () => { - assert.equal(sumOrNull([1, 2, 3]), 6); - assert.equal(sumOrNull([1, null, 3]), null); - assert.equal(sumOrNull([]), 0); -}); - -// ── THE PER-CHANNEL PROJECTION ────────────────────────────────────────────── -// -// The /channels strip and the channel page's station foot are the same fold as -// the corpus rail, over one snapshot. These tests pin the three cases the strip -// actually meets on the live corpus: every operation present, one operation the -// snapshot has no entry for, and one that predates `eligible`. - -const CHANNEL_OPS = [ - "diarization", - "attribution-diarized", - "attribution-text", - "digest", -]; - -test("a channel's bands carry every operation, and the two external ones", () => { - const bands = buildChannelBands( - snapshotOf({ - totals: { videos: 11_344, transcribed: 11_339, downloaded: 11_340 }, - buckets: { - downloadedNoTranscript: ["a", "b"], - noTranscript: ["a", "b", "c"], - untranscribable: ["c"], - } as ChannelSnapshot["buckets"], - undownloadedIds: ["x", "y", "z", "w"], - backfill: { - // The measured shape of the-quartering, in miniature: digest is all - // reachable with nothing done, diarization is almost all media-gone, - // and attribution-diarized is almost all blocked behind it. - diarization: entryOf({ missing: 1, missingInput: 11_333, eligible: 11_338 }), - "attribution-diarized": entryOf({ - missing: 4, - blocked: 11_334, - eligible: 11_338, - }), - "attribution-text": entryOf({ missing: 11_337, eligible: 11_338 }), - digest: entryOf({ missing: 11_329, blocked: 2, eligible: 11_340 }), - }, - }), - CHANNEL_OPS, - ); - - assert.deepEqual( - bands.map((b) => b.id), - ["download", "transcription", ...CHANNEL_OPS], - ); - - // Digest: ALL accent, nothing done. This is the row that makes a percent bar - // useless and the state band useful — "0% complete" is true of every large - // channel and says nothing; "11,329 can run now" is the whole story. - const digest = bandOf(bands, "digest"); - assert.equal(digest.reachable, 11_329); - assert.equal(digest.blocked, 2); - assert.equal(digest.present, 9); - assert.equal(bandCoverage(digest), 9 / 11_340); - - // Diarization: all hollow. 11,333 with no media left is not work, and must - // never be added to the 1 video that is. - const diarize = bandOf(bands, "diarization"); - assert.equal(diarize.reachable, 1); - assert.equal(diarize.missingInput, 11_333); - - // Attribution-diarized: all hatched, waiting on the lane above it. - const named = bandOf(bands, "attribution-diarized"); - assert.equal(named.reachable, 4); - assert.equal(named.blocked, 11_334); - - // The external pipelines use the SAME definitions the transit line does, so a - // channel figure and a corpus figure cannot disagree about "downloaded". - const download = bandOf(bands, "download"); - assert.equal(download.eligible, 11_344 + 4); - assert.equal(download.present, 11_340); - assert.equal(download.reachable, 4); -}); - -test("an operation the snapshot has no entry for is an EMPTY band, not a missing column", () => { - // A channel whose report predates a kind still gets a cell — drawn empty, - // with a known denominator of 0, which bandCoverage reports as unknown rather - // than as 0% done. Dropping the column instead would make the table ragged - // and hide the fact that nothing has been measured yet. - const bands = buildChannelBands( - snapshotOf({ - backfill: { diarization: entryOf({ missing: 5, eligible: 10 }) }, - }), - CHANNEL_OPS, - ); - const text = bandOf(bands, "attribution-text"); - assert.equal(text.reachable, 0); - assert.equal(text.blocked, 0); - assert.equal(text.missingInput, 0); - assert.equal(text.eligible, 0); - assert.equal(bandCoverage(text), null); -}); - -test("an entry with no `eligible` draws an outline, never 0%", () => { - // UNKNOWN IS NOT ZERO, at channel scale. A snapshot written before the field - // existed — or one that has lapsed — has work counts but no denominator, and - // the band must say "we cannot tell you the coverage" rather than "none of it - // is done", which on a fully-diarized channel would be a lie. - const bands = buildChannelBands( - snapshotOf({ - backfill: { - diarization: entryOf({ missing: 3, missingInput: 90 }), - }, - }), - ["diarization"], - ); - const diarize = bandOf(bands, "diarization"); - assert.equal(diarize.eligible, null); - assert.equal(bandCoverage(diarize), null); - // The work counts survive the unknown denominator — they are separately - // known, and the cell still says how much can run now. - assert.equal(diarize.reachable, 3); - assert.equal(diarize.missingInput, 90); -}); - -test("a channel with no snapshot at all is every band empty", () => { - // A brand-new channel, before its first report. Every column present, every - // one empty, coverage unknown — the freshness note under the table is what - // explains why. - const bands = buildChannelBands(null, CHANNEL_OPS); - assert.equal(bands.length, 2 + CHANNEL_OPS.length); - for (const band of bands) { - assert.equal(band.reachable, 0); - assert.equal(bandCoverage(band), null); - } -}); - -// ── THE CLIENT/SERVER SPLIT, GUARDED ──────────────────────────────────────── - -test("band.ts stays directive-free, so a server component can call it", () => { - // band.ts carries the TYPE and every pure reading of a band. The channel - // page's station foot is a SERVER component and calls bandSentence() and - // bandHeadline() directly; adding "use client" here would break it at request - // time with "attempted to call bandSentence() from the server". - const src = readFileSync(new URL("./band.ts", import.meta.url), "utf8"); - // A DIRECTIVE, not the string — this file discusses "use client" in prose. - // A directive is a bare expression statement before any other code. - assert.ok( - !/^\s*(?:"use client"|'use client');?\s*$/m.test(src), - "band.ts must not be a client module", - ); - // And it must import nothing but types — buildBands.ts pulls in - // channelSnapshot → execa, which would put node:child_process in the browser - // bundle and fail `next build`. - const valueImports = [...src.matchAll(/^import\s+(?!type\b)/gm)]; - assert.equal( - valueImports.length, - 0, - "band.ts must not take a value import — see its header", - ); -}); - -test("StateBand.tsx exports only components, never callable helpers", () => { - // THE BUG THIS CAUGHT, ONCE. A plain function exported from a `"use client"` - // module cannot be CALLED by a server component — only rendered. bandSentence - // lived here, the server-rendered station foot called it, and every channel - // page 500'd at request time. - // - // `pnpm build` does NOT catch this: the route is force-dynamic, so nothing - // prerenders it and the error only appears on a request. e2e found it and the - // build did not, which is why this guard is a unit test and not a build step. - const src = readFileSync(new URL("./StateBand.tsx", import.meta.url), "utf8"); - assert.ok( - /^\s*(?:"use client"|'use client');?\s*$/m.test(src), - "StateBand.tsx is the client half", - ); - const exported = [...src.matchAll(/^export\s+(?:function|const)\s+(\w+)/gm)].map( - (m) => m[1], - ); - assert.ok(exported.length > 0, "found no exports to check — regex drifted"); - for (const name of exported) { - assert.ok( - /^[A-Z]/.test(name), - `${name} is exported from a client module but is not a component — a server component that calls it throws at request time. Move it to band.ts.`, - ); - } -}); - -test("a download/transcription entry in the snapshot is NOT folded twice", () => { - // Slice 1.5 gave the two bucket lanes a snapshot entry, and `/channels` passes - // `[...EXTERNAL_BAND_IDS, ...allOperations]` as operationIds — so without the - // filter in buildOperationBands the entry lands on top of addExternalBands and - // a three-video channel reads "6 done of 6". backfill.spec.ts caught it in the - // browser; this is the unit that pins it. - // - // The entries here are the ones generateChannelSnapshot writes: ids = the - // lane's default bucket union, eligible = present + the work counts. - const snapshot = snapshotOf({ - totals: { videos: 3, transcribed: 0, downloaded: 3 }, - buckets: { - downloadedNoTranscript: ["vidA", "vidB", "vidC"], - failedListed: [], - partialDownloads: [], - noTranscript: [], - untranscribable: [], - } as unknown as ChannelSnapshot["buckets"], - undownloadedIds: [], - backfill: { - download: entryOf({ missing: 0, ids: [], eligible: 3 }), - transcription: entryOf({ - missing: 3, - ids: ["vidA", "vidB", "vidC"], - eligible: 3, - }), - }, - }); - const bands = buildOperationBands({ - snapshots: [snapshot], - operationIds: ["download", "transcription", "diarization"], - }); - const dl = bandOf(bands, "download"); - assert.equal(dl.eligible, 3, "eligible must be the playlist, counted once"); - assert.equal(dl.present, 3); - assert.equal(dl.reachable, 0); - const tr = bandOf(bands, "transcription"); - assert.equal(tr.eligible, 3); - assert.equal(tr.present, 0); - assert.equal(tr.reachable, 3); - // And the band is still the BUCKET definition, not the entry's: the same - // numbers come back with the entries absent, which is every live snapshot. - const stripped = buildOperationBands({ - snapshots: [snapshotOf({ ...snapshot, backfill: {} })], - operationIds: ["download", "transcription", "diarization"], - }); - assert.deepEqual(bandOf(stripped, "download"), dl); - assert.deepEqual(bandOf(stripped, "transcription"), tr); -}); diff --git a/editor/app/components/pipelines/buildBands.ts b/editor/app/components/pipelines/buildBands.ts @@ -1,248 +0,0 @@ -import type { ChannelSnapshot } from "yt-dlp-transcript-common/controller/channelSnapshot"; -import { excludedDownloadIdSet } from "yt-dlp-transcript-common/controller/channelSnapshot"; -import { - EXTERNAL_OPERATIONS, - operationCostBasis, - operationLabel, - presentOperationWork, - reachableOperationWork, -} from "yt-dlp-transcript-common/lib/operations"; -import { sumOrNull, type OperationBand } from "./band"; - -export type { OperationBand } from "./band"; -export { bandCoverage, sumOrNull } from "./band"; - -// THE COMPARISON RAIL'S MODEL: one band per pipeline, summed across the corpus. -// -// This is the corpus-wide twin of channelFlow's transit line, and it holds the -// same two invariants for the same reasons — they are the two ways every earlier -// version of this number was wrong: -// -// 1. WORK THE LANE CAN DO IS NEVER SUMMED WITH WORK IT CANNOT. `reachable`, -// `blocked`, `missingInput` and `deferred` are four separate fields on four -// different axes, and nothing here adds them. On the live corpus that is not -// pedantry: attribution-diarized is 94 reachable against 77,923 blocked, and -// diarization is 647 against 77,276 with no media. A single "remaining" -// figure would say the same thing about a lane that is finished and a lane -// that cannot start. -// 2. UNKNOWN IS NOT ZERO. `eligible` and `present` are `number | null`, and one -// null poisons the whole sum deliberately — a third of the snapshots on disk -// predate `eligible`, and "three channels are done and the fourth is -// unknown" is not a number. The band renders a null denominator as an -// unfilled outline, never as 0% progress. -// -// WHY THE RATIO IS THE STORY, AND WHY EACH BAND KEEPS ITS OWN DENOMINATOR. -// Three of the four pipelines are dominated by a non-actionable state, so a -// count renders them as "94" and "647" and tells you nothing. And digest's -// eligible population is genuinely a different set from diarization's — sharing -// one denominator across the rail to make the bars comparable would be a lie -// about what is being compared. Each band states its own, in its own header. -// -// Pure and snapshot-only: common/controller/noCorpusWalkInRenderPaths.test.ts -// bans a corpus walk from a render path, and this feeds a 3-second poll. - -function emptyBand(id: string, dispatched: boolean): OperationBand { - return { - id, - label: operationLabel(id), - costBasis: operationCostBasis(id), - eligible: 0, - present: 0, - reachable: 0, - blocked: 0, - missingInput: 0, - deferred: 0, - dispatched, - }; -} - -// Fold one snapshot's entry for a registry operation into a band. `null` for -// either coverage half latches for the whole corpus. -function addRegistryEntry(band: OperationBand, snapshot: ChannelSnapshot): void { - const entry = snapshot.backfill?.[band.id]; - if (!entry) return; - band.reachable += reachableOperationWork(entry); - band.missingInput += entry.missingInput; - // `?? 0` at every read: snapshots written before these fields existed lack - // them, and undefined poisons the sum to NaN. - band.blocked += entry.blocked ?? 0; - band.deferred += entry.deferred ?? 0; - band.eligible = sumOrNull([band.eligible, entry.eligible ?? null]); - band.present = sumOrNull([band.present, presentOperationWork(entry)]); -} - -export type BuildOperationBandsInput = { - snapshots: ReadonlyArray<ChannelSnapshot | null>; - // Registry operations to build a band for, in rail order. Comes from - // allOperations(), so a switched-off feature is simply absent — which is - // the honest rendering: an empty work list because nobody enabled it is not - // the same as being finished. - operationIds: ReadonlyArray<string>; -}; - -// The two pipelines this system counts but does not dispatch through the -// operation registry. They are on the rail anyway, and deliberately: -// -// The rail exists so you never have to switch lanes to learn that THIS lane is -// idle because ANOTHER one is — and on this corpus that is the normal case, not -// the exception (diarization is 99.2% media-gone; attribution-diarized is 99.9% -// blocked behind diarization). Leaving transcription and download off it would -// remove exactly the two lanes whose state explains the other four. -// -// Their numbers do NOT come from Operation.state() — they have no registry -// entry, because EXTERNAL_OPERATIONS registers them for the dependency graph -// and not for dispatch. They come from `totals` and the buckets, using the SAME -// definitions the channel transit line already uses for its Download and -// Transcribe stations, so a corpus figure and a channel figure cannot disagree -// about what "downloaded" means. -// -// SLICE 1.5 GAVE THEM A SNAPSHOT ENTRY, AND THIS STILL DOES NOT READ IT. -// `snapshot.backfill.download` and `.transcription` are the DISPATCH work list — -// what the runner would hand out — and that is a different set from what these -// five numbers mean, in two places that both matter: -// -// * transcription's `reachable` here is downloadedNoTranscript ALONE. The -// lane's work list also carries `failedListed`, the retry bucket, which is -// 1,873 videos corpus-wide against 881 — reading the entry would nearly -// quadruple a rendered figure. -// * this band's `blocked` is "no audio yet", which the entry calls -// `missingInput`, and its `eligible`/`present` are the playlist and -// `totals.downloaded`/`totals.transcribed` — coverage measures the entry -// states rather than counts. -// -// So the entry and the band are two honest answers to two different questions, -// and the rail keeps its own. What DID stop being a special case is the id list -// below: it is the catalog's own, not a hand-written pair. -// -// Sync is catalogued beside them and still gets NO band, here or anywhere: its -// populations are channels, not videos (`scope: "channel"`), so every one of a -// band's five numbers would be a category error and the rail would draw -// "coverage unknown" over a hollow outline. Its rail row is composed instead — -// see SyncRailRow in OperationRail.tsx. -function addExternalBands( - bands: Map<string, OperationBand>, - snapshot: ChannelSnapshot, -): void { - const totals = snapshot.totals ?? { videos: 0, transcribed: 0, downloaded: 0 }; - const buckets = snapshot.buckets; - const undownloaded = snapshot.undownloadedIds ?? []; - const excluded = excludedDownloadIdSet(snapshot); - - const download = bands.get("download"); - if (download) { - // Eligible is every video the playlist knows about: the dirs that exist - // plus the ids that have never been fetched. `totals.videos` alone would be - // a denominator that grows only as work completes. - download.eligible = sumOrNull([ - download.eligible, - totals.videos + undownloaded.length, - ]); - download.present = sumOrNull([download.present, totals.downloaded]); - // Partial downloads are reachable work like any other — the same rule - // the dispatch decision applies to `partial`. - download.reachable += - undownloaded.filter((id) => !excluded.has(id)).length + - (buckets?.partialDownloads?.length ?? 0); - // Excluded ids have LEFT the line: a channel deliberately not fetching - // them is not a lane with work to do. They are deferred, not reachable — - // and never subtracted from anything, so the two stay separable. - download.deferred += undownloaded.filter((id) => excluded.has(id)).length; - } - - const transcription = bands.get("transcription"); - if (transcription) { - // A video marked untranscribable is not eligible — it is not work anyone is - // waiting on, and counting it would put a permanent ceiling under 100%. - const untranscribable = buckets?.untranscribable?.length ?? 0; - transcription.eligible = sumOrNull([ - transcription.eligible, - Math.max(0, totals.videos - untranscribable), - ]); - transcription.present = sumOrNull([ - transcription.present, - totals.transcribed, - ]); - // Reachable = the audio is in hand. Everything else without a transcript is - // waiting on the DOWNLOAD lane, which is exactly what `blocked` means here - // — an operation this system produces, one station upstream. - const downloadedNoTranscript = ( - buckets?.downloadedNoTranscript ?? [] - ).filter((id) => !excluded.has(id)).length; - const noTranscript = buckets?.noTranscript?.length ?? 0; - transcription.reachable += downloadedNoTranscript; - transcription.blocked += Math.max( - 0, - noTranscript - untranscribable - downloadedNoTranscript, - ); - } -} - -// The rail, left to right. Download and transcription lead because everything -// else depends on them; digest and the backfill kinds follow in registry order. -// -// OFF THE CATALOG, not a literal pair. EXTERNAL_OPERATIONS is the declaration of -// exactly this set — the media-derived pipelines this system counts and does not -// dispatch through the registry — and sync is deliberately not in it (its -// populations are channels, not videos). A third external pipeline would join -// the rail by being declared, the way pauseLaneFor and bucketLaneOperationId -// already read that same field. -export const EXTERNAL_BAND_IDS: readonly string[] = EXTERNAL_OPERATIONS.map( - (op) => op.id, -); - -export function buildOperationBands({ - snapshots, - operationIds, -}: BuildOperationBandsInput): OperationBand[] { - const bands = new Map<string, OperationBand>(); - for (const id of EXTERNAL_BAND_IDS) bands.set(id, emptyBand(id, false)); - for (const id of operationIds) { - if (!bands.has(id)) bands.set(id, emptyBand(id, true)); - } - // EXACTLY ONE FOLD PER BAND, and the two external ids are addExternalBands'. - // - // `/channels` passes `[...EXTERNAL_BAND_IDS, ...allOperations]` in — it wants - // a column for every band — and until slice 1.5 that was harmless here - // because `snapshot.backfill.download` did not exist and addRegistryEntry - // returned early. Now it does exist, and folding it on top of - // addExternalBands DOUBLES a channel's Download and Transcribe coverage. - // (Caught by backfill.spec.ts, which read the row as "6 done of 6" on a - // three-video channel.) - // - // This is the same trap backfillLaneOperationEntriesOf was written for, at - // the one surface that does not go through it: the moment the snapshot map - // stopped being one lane, "every id in this map is mine" stopped being true. - const registryIds = operationIds.filter( - (id) => !EXTERNAL_BAND_IDS.includes(id), - ); - - for (const snapshot of snapshots) { - if (!snapshot) continue; - addExternalBands(bands, snapshot); - for (const id of registryIds) { - const band = bands.get(id); - if (!band) continue; - // Digest is a plain registry entry here, like every other operation: a - // snapshot with no entry contributes no work and no coverage, and the - // band stays an unfilled outline rather than reading 0 %. - addRegistryEntry(band, snapshot); - } - } - - return [...bands.values()]; -} - -// ONE CHANNEL'S BANDS — the strip on /channels and the station foot on a -// channel page. -// -// A thin wrapper and deliberately not a second implementation: a channel figure -// and the corpus figure it contributes to MUST agree about what "downloaded" -// or "reachable" means, and the only way to guarantee that is for both to be -// the same fold over the same snapshot. buildOperationBands already takes an -// array; a channel is an array of one. -export function buildChannelBands( - snapshot: ChannelSnapshot | null, - operationIds: ReadonlyArray<string>, -): OperationBand[] { - return buildOperationBands({ snapshots: [snapshot], operationIds }); -} diff --git a/editor/app/operations/components/OperationRail.tsx b/editor/app/operations/components/OperationRail.tsx @@ -5,7 +5,7 @@ import { bandCoverage, bandDenominator, type OperationBand, -} from "../../components/pipelines/band"; +} from "yt-dlp-transcript-common/views/pipeline/band"; import { BandLegend, StateBand, diff --git a/editor/app/operations/lanes.ts b/editor/app/operations/lanes.ts @@ -12,7 +12,7 @@ import { getChannelBriefs } from "../lib/requestCache"; import { buildOperationBands, type OperationBand, -} from "../components/pipelines/buildBands"; +} from "yt-dlp-transcript-common/views/pipeline/buildBands"; // WHAT THE CONSOLE KNOWS ABOUT A LANE THAT IS NOT ITS RUNNER'S STATE. //