import type { ChannelSnapshot } from "../../controller/channelSnapshot"; import { excludedDownloadIdSet } from "../../controller/channelSnapshot"; import { EXTERNAL_OPERATIONS, operationCostBasis, operationLabel, presentOperationWork, reachableOperationWork, } from "../../lib/operations"; import { sumOrNull, type OperationBand } from "./band"; export type { OperationBand } from "./band"; export { bandCoverage, sumOrNull } from "./band"; // THE COMPARISON RAIL'S MODEL: one band per pipeline, summed across the corpus. // // This is the corpus-wide twin of channelFlow's transit line, and it holds the // same two invariants for the same reasons — they are the two ways every earlier // version of this number was wrong: // // 1. WORK THE LANE CAN DO IS NEVER SUMMED WITH WORK IT CANNOT. `reachable`, // `blocked`, `missingInput` and `deferred` are four separate fields on four // different axes, and nothing here adds them. On the live corpus that is not // pedantry: attribution-diarized is 94 reachable against 77,923 blocked, and // diarization is 647 against 77,276 with no media. A single "remaining" // figure would say the same thing about a lane that is finished and a lane // that cannot start. // 2. UNKNOWN IS NOT ZERO. `eligible` and `present` are `number | null`, and one // null poisons the whole sum deliberately — a third of the snapshots on disk // predate `eligible`, and "three channels are done and the fourth is // unknown" is not a number. The band renders a null denominator as an // unfilled outline, never as 0% progress. // // WHY THE RATIO IS THE STORY, AND WHY EACH BAND KEEPS ITS OWN DENOMINATOR. // Three of the four pipelines are dominated by a non-actionable state, so a // count renders them as "94" and "647" and tells you nothing. And digest's // eligible population is genuinely a different set from diarization's — sharing // one denominator across the rail to make the bars comparable would be a lie // about what is being compared. Each band states its own, in its own header. // // Pure and snapshot-only: common/controller/noCorpusWalkInRenderPaths.test.ts // bans a corpus walk from a render path, and this feeds a 3-second poll. function emptyBand(id: string, dispatched: boolean): OperationBand { return { id, label: operationLabel(id), costBasis: operationCostBasis(id), eligible: 0, present: 0, reachable: 0, blocked: 0, missingInput: 0, deferred: 0, dispatched, }; } // Fold one snapshot's entry for a registry operation into a band. `null` for // either coverage half latches for the whole corpus. function addRegistryEntry(band: OperationBand, snapshot: ChannelSnapshot): void { const entry = snapshot.backfill?.[band.id]; if (!entry) return; band.reachable += reachableOperationWork(entry); band.missingInput += entry.missingInput; // `?? 0` at every read: snapshots written before these fields existed lack // them, and undefined poisons the sum to NaN. band.blocked += entry.blocked ?? 0; band.deferred += entry.deferred ?? 0; band.eligible = sumOrNull([band.eligible, entry.eligible ?? null]); band.present = sumOrNull([band.present, presentOperationWork(entry)]); } export type BuildOperationBandsInput = { snapshots: ReadonlyArray; // Registry operations to build a band for, in rail order. Comes from // allOperations(), so a switched-off feature is simply absent — which is // the honest rendering: an empty work list because nobody enabled it is not // the same as being finished. operationIds: ReadonlyArray; }; // The two pipelines this system counts but does not dispatch through the // operation registry. They are on the rail anyway, and deliberately: // // The rail exists so you never have to switch lanes to learn that THIS lane is // idle because ANOTHER one is — and on this corpus that is the normal case, not // the exception (diarization is 99.2% media-gone; attribution-diarized is 99.9% // blocked behind diarization). Leaving transcription and download off it would // remove exactly the two lanes whose state explains the other four. // // Their numbers do NOT come from Operation.state() — they have no registry // entry, because EXTERNAL_OPERATIONS registers them for the dependency graph // and not for dispatch. They come from `totals` and the buckets, using the SAME // definitions the channel transit line already uses for its Download and // Transcribe stations, so a corpus figure and a channel figure cannot disagree // about what "downloaded" means. // // SLICE 1.5 GAVE THEM A SNAPSHOT ENTRY, AND THIS STILL DOES NOT READ IT. // `snapshot.backfill.download` and `.transcription` are the DISPATCH work list — // what the runner would hand out — and that is a different set from what these // five numbers mean, in two places that both matter: // // * transcription's `reachable` here is downloadedNoTranscript ALONE. The // lane's work list also carries `failedListed`, the retry bucket, which is // 1,873 videos corpus-wide against 881 — reading the entry would nearly // quadruple a rendered figure. // * this band's `blocked` is "no audio yet", which the entry calls // `missingInput`, and its `eligible`/`present` are the playlist and // `totals.downloaded`/`totals.transcribed` — coverage measures the entry // states rather than counts. // // So the entry and the band are two honest answers to two different questions, // and the rail keeps its own. What DID stop being a special case is the id list // below: it is the catalog's own, not a hand-written pair. // // Sync is catalogued beside them and still gets NO band, here or anywhere: its // populations are channels, not videos (`scope: "channel"`), so every one of a // band's five numbers would be a category error and the rail would draw // "coverage unknown" over a hollow outline. Its rail row is composed instead — // see SyncRailRow in OperationRail.tsx. function addExternalBands( bands: Map, snapshot: ChannelSnapshot, ): void { const totals = snapshot.totals ?? { videos: 0, transcribed: 0, downloaded: 0 }; const buckets = snapshot.buckets; const undownloaded = snapshot.undownloadedIds ?? []; const excluded = excludedDownloadIdSet(snapshot); const download = bands.get("download"); if (download) { // Eligible is every video the playlist knows about: the dirs that exist // plus the ids that have never been fetched. `totals.videos` alone would be // a denominator that grows only as work completes. download.eligible = sumOrNull([ download.eligible, totals.videos + undownloaded.length, ]); download.present = sumOrNull([download.present, totals.downloaded]); // Partial downloads are reachable work like any other — the same rule // the dispatch decision applies to `partial`. download.reachable += undownloaded.filter((id) => !excluded.has(id)).length + (buckets?.partialDownloads?.length ?? 0); // Excluded ids have LEFT the line: a channel deliberately not fetching // them is not a lane with work to do. They are deferred, not reachable — // and never subtracted from anything, so the two stay separable. download.deferred += undownloaded.filter((id) => excluded.has(id)).length; } const transcription = bands.get("transcription"); if (transcription) { // A video marked untranscribable is not eligible — it is not work anyone is // waiting on, and counting it would put a permanent ceiling under 100%. const untranscribable = buckets?.untranscribable?.length ?? 0; transcription.eligible = sumOrNull([ transcription.eligible, Math.max(0, totals.videos - untranscribable), ]); transcription.present = sumOrNull([ transcription.present, totals.transcribed, ]); // Reachable = the audio is in hand. Everything else without a transcript is // waiting on the DOWNLOAD lane, which is exactly what `blocked` means here // — an operation this system produces, one station upstream. const downloadedNoTranscript = ( buckets?.downloadedNoTranscript ?? [] ).filter((id) => !excluded.has(id)).length; const noTranscript = buckets?.noTranscript?.length ?? 0; transcription.reachable += downloadedNoTranscript; transcription.blocked += Math.max( 0, noTranscript - untranscribable - downloadedNoTranscript, ); } } // The rail, left to right. Download and transcription lead because everything // else depends on them; digest and the backfill kinds follow in registry order. // // OFF THE CATALOG, not a literal pair. EXTERNAL_OPERATIONS is the declaration of // exactly this set — the media-derived pipelines this system counts and does not // dispatch through the registry — and sync is deliberately not in it (its // populations are channels, not videos). A third external pipeline would join // the rail by being declared, the way pauseLaneFor and bucketLaneOperationId // already read that same field. export const EXTERNAL_BAND_IDS: readonly string[] = EXTERNAL_OPERATIONS.map( (op) => op.id, ); export function buildOperationBands({ snapshots, operationIds, }: BuildOperationBandsInput): OperationBand[] { const bands = new Map(); for (const id of EXTERNAL_BAND_IDS) bands.set(id, emptyBand(id, false)); for (const id of operationIds) { if (!bands.has(id)) bands.set(id, emptyBand(id, true)); } // EXACTLY ONE FOLD PER BAND, and the two external ids are addExternalBands'. // // `/channels` passes `[...EXTERNAL_BAND_IDS, ...allOperations]` in — it wants // a column for every band — and until slice 1.5 that was harmless here // because `snapshot.backfill.download` did not exist and addRegistryEntry // returned early. Now it does exist, and folding it on top of // addExternalBands DOUBLES a channel's Download and Transcribe coverage. // (Caught by backfill.spec.ts, which read the row as "6 done of 6" on a // three-video channel.) // // This is the same trap backfillLaneOperationEntriesOf was written for, at // the one surface that does not go through it: the moment the snapshot map // stopped being one lane, "every id in this map is mine" stopped being true. const registryIds = operationIds.filter( (id) => !EXTERNAL_BAND_IDS.includes(id), ); for (const snapshot of snapshots) { if (!snapshot) continue; addExternalBands(bands, snapshot); for (const id of registryIds) { const band = bands.get(id); if (!band) continue; // Digest is a plain registry entry here, like every other operation: a // snapshot with no entry contributes no work and no coverage, and the // band stays an unfilled outline rather than reading 0 %. addRegistryEntry(band, snapshot); } } return [...bands.values()]; } // ONE CHANNEL'S BANDS — the strip on /channels and the station foot on a // channel page. // // A thin wrapper and deliberately not a second implementation: a channel figure // and the corpus figure it contributes to MUST agree about what "downloaded" // or "reachable" means, and the only way to guarantee that is for both to be // the same fold over the same snapshot. buildOperationBands already takes an // array; a channel is an array of one. export function buildChannelBands( snapshot: ChannelSnapshot | null, operationIds: ReadonlyArray, ): OperationBand[] { return buildOperationBands({ snapshots: [snapshot], operationIds }); }