commit 20402534b9892adbf113584f905fa04a0534d62e parent 77f579b3e9fef356fb6d9c4e7e1c3fd1798a69ab Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st> Date: Wed, 26 Aug 2026 18:46:57 -0400 snapshot: the digest work list has one definition `buckets.noDigest` is gone: not written, not typed, not defaulted, not read. The digest work list is `snapshot.backfill.digest` — the operation registry's entry, folded from the same `kind.state()` / `kind.resolveTarget()` pair the digest runner dispatches from. The fallback in `digestWorkOf` existed for a migration that is over. Measured read-only against the real corpus on 2026-08-26: 68 of 68 `transcripts/channels/*/snapshot.json` carry `backfill.digest`, all 68 with a numeric `eligible`; `generatedAt` spans 2026-08-11 to 2026-08-26. The `"bucket"` branch was reading nothing, so no rendered number moves — every surface already went through `digestWorkOf` and got the registry answer. The two counters could not have been reconciled anyway. Corpus sums today: bucket 59,159 vs entry ids 47,382, a disagreement of 11,777 videos. On 65 of 68 channels the delta is exactly `deferred` (the registry gates on a current cues.json; the bucket does not). The other three — HasanAbiVODs3, chibi-reviews, destiny — are videos with a fresh digest but no current cues.json, which the registry calls deferred and the bucket called done. The bucket also had no transcript gate (a channel of untranscribed videos read as fully digested) and no `partial`. Two definitions of "digested" is the failure the registry entry exists to end; this deletes the second one. Also gone: the `source` field on `DigestWork` (nothing to disambiguate), the snapshot writer's second digest identity (`resolveDigestTarget` + `isSectionFresh` per video), and the `noDigest` accumulator and write. No regeneration is needed and none is scripted: old snapshots keep a stray `noDigest` key until their next routine regen, no reader validates bucket keys, and an offline writer would race the live editor's snapshotScheduler. The `noDigest` view-model names (DashboardChannel.noDigest, the widget's wire field, actionableNoDigestCount, DigestStage's noDigestIds) are deliberately kept: they mean "videos with no current digest", which is still what they hold. Diffstat:
20 files changed, 137 insertions(+), 196 deletions(-)
diff --git a/common/controller/channelSnapshot.test.ts b/common/controller/channelSnapshot.test.ts @@ -83,13 +83,14 @@ test("ids are sorted, because a snapshot is compared byte-for-byte", () => { }); // --------------------------------------------------------------------------- -// digestWorkOf: one reader, three generations of snapshot. +// digestWorkOf: one reader, one definition. -test("digestWorkOf prefers the registry entry over the legacy bucket", () => { - // When both exist the registry wins — it is the classification the RUNNER - // uses, and the bucket has no cues-staleness or transcript gate, so the two - // genuinely disagree. Two definitions of "digested" is the failure this whole - // change exists to end. +test("digestWorkOf reads the registry entry, the split included", () => { + // The registry's classification is the one the RUNNER dispatches from, and it + // is now the only one the snapshot carries: the `buckets.noDigest` list that + // used to sit beside it had no cues-staleness or transcript gate, so the two + // genuinely disagreed (11,777 videos corpus-wide on 2026-08-26). Two + // definitions of "digested" is the failure this whole change exists to end. const work = digestWorkOf({ backfill: { digest: { @@ -102,9 +103,7 @@ test("digestWorkOf prefers the registry entry over the legacy bucket", () => { eligible: 10, }, }, - buckets: { noDigest: ["x", "y", "z", "w", "v", "u", "t"] }, }); - assert.equal(work.source, "registry"); assert.equal(work.reachable, 3); assert.deepEqual(work.ids, ["a", "b", "c"]); assert.equal(work.blocked, 4); @@ -112,26 +111,9 @@ test("digestWorkOf prefers the registry entry over the legacy bucket", () => { assert.equal(work.partial, 1); }); -test("digestWorkOf falls back to noDigest so a stale snapshot is not 'all done'", () => { - // A THIRD of the channels on disk carry snapshots months old. Without this - // every one of them would report zero digest work until the regen reached it, - // which is the single most dangerous way for a coverage number to be wrong. - const work = digestWorkOf({ buckets: { noDigest: ["a", "b"] } }); - assert.equal(work.source, "bucket"); - assert.equal(work.reachable, 2); - assert.deepEqual(work.ids, ["a", "b"]); - // The bucket cannot know these, and says so via `source` rather than by - // implying a split it never computed. - assert.equal(work.blocked, 0); - assert.equal(work.deferred, 0); - assert.equal(work.present, null); - assert.equal(work.eligible, null); -}); - test("digestWorkOf reports UNKNOWN, not zero coverage, when it cannot tell", () => { - for (const snapshot of [null, undefined, {}, { buckets: {} }]) { + for (const snapshot of [null, undefined, {}, { backfill: {} }]) { const work = digestWorkOf(snapshot); - assert.equal(work.source, "none"); assert.equal(work.reachable, 0); // The distinction that matters: no work outstanding AND no idea how much is // done. A 0 here would render as "0% digested" on every surface. diff --git a/common/controller/channelSnapshot.ts b/common/controller/channelSnapshot.ts @@ -25,7 +25,6 @@ import { } from "../lib/availability-server"; import { isDoNotClean } from "../lib/doNotClean-server"; import { loadDigest } from "../lib/digest-server"; -import { isSectionFresh } from "../lib/digest"; import { addOperationState, allOperations, @@ -37,7 +36,6 @@ import { type OperationClassification, type OperationSnapshotEntry, } from "../lib/operations"; -import { resolveDigestTarget } from "./digestTarget"; import { isExcludedFromTruncatedCheck } from "../lib/excludeTruncatedCheck-server"; import { loadDownloadOutcome } from "../lib/downloadOutcome-server"; import type { Paths } from "../lib/paths"; @@ -92,7 +90,7 @@ export type ChannelSnapshot = { // lane can do today; `missingInput` needs the media re-acquired and, measured // on this corpus, is 91x larger. A single "remaining" figure here would put // every channel permanently at the top of every list — which is the documented - // reason /api/widget/actionable refuses to filter on `noDigest`. + // reason /api/widget/actionable refuses to filter on the digest work count. // // THIS MAP IS NO LONGER THE BACKFILL LANE, and anything reading it generically // must say which lane it means. It is written from allOperations — every @@ -192,18 +190,6 @@ export type ChannelSnapshot = { // (retry-bucket with forceCookies). Optional: older snapshots lack it; // readers must default to []. needsCookies: string[]; - // Transcribed videos whose digest is missing or STALE against the local - // lane's current freshness target (engine, requested model, prompt version, - // prompt shape, context hash) — the AI digest layer's work list, and the - // denominator for corpus coverage during the backfill. This is the same - // target countMissingDigests and runDigestBatch compute, deliberately: a - // bucket that silently meant something weaker than its name is how a sweep - // reports "nothing to do" on a corpus that needs redoing. - // - // Only TRANSCRIBED videos are listed: a video without a transcript is a - // transcription problem, not a digest one. Optional: older snapshots lack - // it; readers must default to []. - noDigest: string[]; // Videos whose digest pass recorded something a human should look at: // either warnings alongside a section that WAS written, or a total failure // that wrote no section at all (DigestRecord.failures). @@ -214,7 +200,8 @@ export type ChannelSnapshot = { // log. A review queue keyed on written warnings alone would have been blind // to precisely the worst outputs. // - // Costs no extra I/O: the sidecar is already loaded here for noDigest. + // Costs no extra I/O: the sidecar is already loaded here for + // `digestEngines` and `digestWarnings`. // Optional: older snapshots lack it; readers must default to []. digestWarnings?: string[]; }; @@ -443,30 +430,30 @@ export function foldBackfillEntry( return { ...counts, ids: ids.sort(), eligible }; } -// The digest work list, from whichever of the two places this snapshot has it. +// The digest work list. ONE DEFINITION: the operation registry's entry. +// +// The digest layer once grew its own counter (`buckets.noDigest`) beside the +// registry, and the two disagreed by an entire channel — the stage card read +// "All digested" while the batch reported everything stale. They could not be +// reconciled: the bucket had no cues-staleness gate (so it called `deferred` +// work done), no transcript gate, and no `partial`. Measured 2026-08-26, the +// corpus-wide disagreement was 11,777 videos (bucket 59,159 vs entry 47,382). +// The bucket is gone; `snapshot.backfill.digest` — folded from the same +// `kind.state()` the digest runner dispatches from — is the definition. // -// ONE READER, because the alternative is what this whole change exists to undo. -// The digest layer grew its own counter (`buckets.noDigest`) before the -// operation registry existed, and for a while the two disagreed by an entire -// channel — the stage card read "All digested" while the batch reported -// everything stale. There is now one classification, in the registry, and this -// is the only function allowed to decide where a surface reads it from. +// THE FALLBACK THE MIGRATION NEEDED IS RETIRED. On 2026-08-26 all 68 +// `transcripts/channels/*/snapshot.json` carried `backfill.digest` (and +// `eligible`), so the bucket branch was reading nothing. A snapshot with no +// entry is now a channel whose FIRST snapshot has not been written yet, not an +// old generation of one — and it reports unknown coverage, never zero. // -// THE FALLBACK IS NOT DEFENSIVENESS, IT IS THE MIGRATION. A third of the -// channels on disk carry snapshots months old, and 11 of them predate the -// `noDigest` bucket itself — so there are three generations in play at once: -// snapshots with `backfill.digest` (written after this change), snapshots with -// only `noDigest`, and snapshots with neither. Without the middle case every -// channel that has not been regenerated yet would read as fully digested, which -// is the single most dangerous way for a coverage number to be wrong. `source` -// is returned so a surface can say which generation it is looking at rather -// than quietly presenting a guess as a measurement. +// Left in this module rather than moved after commit 2 left it with no callers +// in `common/`: it is the editor's one adapter from an `OperationSnapshotEntry` +// to the digest surfaces, and moving it would be churn without a deletion. export type DigestWork = { // Videos needing digest work that the lane can do right now. ids: string[]; reachable: number; - // Zero for a `noDigest`-derived answer: that bucket has no cues-staleness - // gate and no transcript gate, so it cannot distinguish these. blocked: number; deferred: number; partial: number; @@ -474,16 +461,10 @@ export type DigestWork = { // snapshot cannot say — never 0, which would render as "none digested". present: number | null; eligible: number | null; - source: "registry" | "bucket" | "none"; }; export function digestWorkOf( - snapshot: - | Pick<ChannelSnapshot, "backfill"> & { - buckets?: Partial<ChannelSnapshot["buckets"]>; - } - | null - | undefined, + snapshot: Pick<ChannelSnapshot, "backfill"> | null | undefined, ): DigestWork { const entry = snapshot?.backfill?.[DIGEST_OPERATION_ID]; if (entry) { @@ -495,25 +476,11 @@ export function digestWorkOf( partial: entry.partial ?? 0, present: presentOperationWork(entry), eligible: entry.eligible ?? null, - source: "registry", - }; - } - const bucket = snapshot?.buckets?.noDigest; - if (bucket) { - return { - ids: bucket, - reachable: bucket.length, - // The bucket genuinely does not know these. Reporting 0 is not a guess: - // it is the honest statement that this snapshot carries no such split, - // and `source` says why. - blocked: 0, - deferred: 0, - partial: 0, - present: null, - eligible: null, - source: "bucket", }; } + // No entry: no work known, and coverage UNKNOWN rather than zero. Null is the + // whole point — 0 would render as "none digested" on a channel nobody has + // snapshotted yet. return { ids: [], reachable: 0, @@ -522,7 +489,6 @@ export function digestWorkOf( partial: 0, present: null, eligible: null, - source: "none", }; } @@ -604,8 +570,14 @@ export async function generateChannelSnapshot( // Which operations are live, and what identity each would produce right now. // Resolved ONCE per channel — a settings read and some string work — so the - // per-video probe below is a comparison rather than a derivation, exactly as - // digestTarget is above. + // per-video probe below is a comparison rather than a derivation. + // + // This is the ONLY digest identity the snapshot resolves. It used to resolve a + // second one (`resolveDigestTarget`, for the `noDigest` bucket) and the two + // definitions disagreed; the property that mattered — the stage's count and + // the batch runner's target cannot differ — is now preserved by construction, + // because the entry is folded from `kind.state()` against this + // `kind.resolveTarget(...)`, the same identity the runner dispatches from. // // allOperations, NOT backfillLaneOperations. The snapshot's job is to carry a // work list for every operation in the catalog, not for one lane: nothing can @@ -804,27 +776,8 @@ export async function generateChannelSnapshot( const autoSubsOnly: string[] = []; const downloadedAutoSubsOnly: string[] = []; const supersededAutoSubs: string[] = []; - const noDigest: string[] = []; const digestWarnings: string[] = []; const digestEngines: Record<string, number> = {}; - // The SAME freshness target countMissingDigests and runDigestBatch use, so - // the Digest stage's count and the batch runner's progress target cannot - // disagree. This bucket used to ask only "is there an ai-digest.json with - // items?", which meant that after any prompt/model/context change the stage - // read "All digested." while the batch reported the whole channel as stale. - // - // Affordable in this hot path because the per-video cost is ZERO extra I/O: - // the digest sidecar is already loaded above, and isSectionFresh is a pure - // comparison. Only the target itself is new work, and it is resolved ONCE per - // channel (a settings read, a registry lookup, and one small context file). - // - // The LOCAL lane is the target on purpose: it is the lane that carries the - // corpus, and the metered lane exists only for the long tail. - const digestTarget = await resolveDigestTarget({ - paths, - channelSlug: slug, - lane: "local", - }); let transcribedWithAudioBytes = 0; let multipleAudioFormatsBytes = 0; let foreignAudioBytes = 0; @@ -1060,20 +1013,10 @@ export async function generateChannelSnapshot( if (engine && hasItems) { digestEngines[engine] = (digestEngines[engine] ?? 0) + 1; } - // The WORK LIST is identity-aware: a video whose digest predates the - // current identity is work, not coverage. A digest shared from a - // duplicate cluster's canonical member counts as done — the canonical - // member's own freshness is what drives regeneration, and the share is - // re-applied from it (isSharedFrom's contract). - const fresh = - digest?.derivedFrom != null || - digestTarget.sections.every((section) => - isSectionFresh(digest, section, digestTarget.target), - ); - if (!fresh) noDigest.push(id); // Reviewable regardless of freshness: a video that failed outright is - // ALSO in noDigest (it has no section), and a video whose section landed - // with warnings is fresh and would otherwise never be surfaced again. + // also reachable work in `backfill.digest` (it has no section), and a + // video whose section landed with warnings is fresh and would otherwise + // never be surfaced again. if ( (digest?.warnings?.length ?? 0) > 0 || (digest?.failures?.length ?? 0) > 0 @@ -1222,7 +1165,6 @@ export async function generateChannelSnapshot( downloadedAutoSubsOnly: downloadedAutoSubsOnly.sort(), supersededAutoSubs: supersededAutoSubs.sort(), needsCookies: needsCookies.sort(), - noDigest: noDigest.sort(), digestWarnings: digestWarnings.sort(), }, digestEngines, diff --git a/common/controller/digestBatch.ts b/common/controller/digestBatch.ts @@ -156,8 +156,9 @@ export async function runDigestBatch( const settings = getSettings(); const digestSettings = settings.digest; const lane = opts.lane ?? "local"; - // ONE derivation, shared with countMissingDigests and the channel snapshot's - // noDigest bucket — see digestTarget.ts. It must match what digestVideo will + // ONE derivation, shared with countMissingDigests and the digest operation + // kind the snapshot folds `backfill.digest` from — see digestTarget.ts. It + // must match what digestVideo will // actually chunk with, or the batch's freshness check and the writer would // disagree on the identity and every video would look stale forever. const resolved = await resolveDigestTarget({ diff --git a/common/controller/digestTarget.ts b/common/controller/digestTarget.ts @@ -1,13 +1,14 @@ // The ONE place a digest freshness target is derived. // -// Three callers need to answer "would we regenerate this section?" — the batch -// runner (to skip), countMissingDigests (to size a progress bar), and the -// channel snapshot (to fill the noDigest bucket). They MUST agree, and before -// this module they did not: the snapshot asked only "does an ai-digest.json with -// items exist?", so after any config change the Digest stage read "All digested" -// while the batch reported the whole channel as stale. Two counters that -// disagree by an entire channel is how a sweep reports "nothing to do" on a -// corpus that needs redoing. +// Callers need to answer "would we regenerate this section?" — the batch runner +// (to skip), countMissingDigests (to size a progress bar), and the digest +// operation kind's resolveTarget, which the snapshot folds its work list from. +// This is the ONE identity, and the snapshot no longer carries a second one: +// it used to keep a `buckets.noDigest` list that asked only "does an +// ai-digest.json with items exist?", so after any config change the Digest stage +// read "All digested" while the batch reported the whole channel as stale. Two +// counters that disagree by an entire channel is how a sweep reports "nothing +// to do" on a corpus that needs redoing. // // The lane is a PARAMETER, never assumed. countMissingDigests used to resolve // settings.localAppId unconditionally, so a metered-lane job's progress target diff --git a/common/lib/digest-server.ts b/common/lib/digest-server.ts @@ -82,8 +82,10 @@ export async function loadDigest( } } -// Cheap existence/coverage check for the snapshot's noDigest bucket and the -// channel digestCount, which run over every video dir in a channel. Reads the +// Cheap existence/coverage check for the snapshot's `digestEngines` split and +// the channel digestCount, which run over every video dir in a channel. It is +// identity-BLIND on purpose — it answers "is there a digest at all?", never +// "is it current"; the work list is the digest operation's `state()`. Reads the // file (a few KB) rather than statting, because a digest whose sections are all // empty is not coverage. export async function hasDigest(videoDir: string): Promise<boolean> { diff --git a/common/lib/operations.test.ts b/common/lib/operations.test.ts @@ -1134,8 +1134,8 @@ test("a kind that can defer says WHY, and digest's reason is not 'it clears itse }); test("digest: a digest shared from a duplicate cluster counts as done", async () => { - // Worth ~11% of the sweep. If this entry disagreed with the snapshot's - // noDigest bucket here, every mirror would be regenerated. + // Worth ~11% of the sweep. This entry IS the snapshot's digest work list, so + // getting the share rule wrong here regenerates every mirror. assert.equal( await classifyDigest({ sidecar: digestSidecar({ derivedFrom: { videoId: "canonical" } }), diff --git a/common/lib/operations.ts b/common/lib/operations.ts @@ -32,8 +32,9 @@ // diarized. A single "remaining" number would therefore read 77,105 — and 91x of // that is unreachable without re-downloading. The repo has already been burned by // exactly this once: editor/app/api/widget/actionable/route.ts deliberately -// refuses to filter on `noDigest` because during the backfill that is 99.87% of -// the corpus and counting it would put every channel in the list forever. So +// refuses to filter on the digest work count (historically the `noDigest` +// bucket) because during the backfill that is 99.87% of the corpus and counting +// it would put every channel in the list forever. So // `missing` (reachable now) and `missing-input` (needs re-acquiring) are // SEPARATE numbers, everywhere, and no surface is allowed to add them together. // @@ -1113,9 +1114,9 @@ const digest: Operation = { // A digest SHARED from a duplicate cluster's canonical member counts as // done. The canonical member's own freshness drives regeneration and the // share is re-applied from it (isSharedFrom's contract) — so re-deriving it - // here would undo ~11% of the sweep's saving. Same rule as the snapshot's - // noDigest bucket, deliberately, because two definitions of "digested" is - // the exact failure this entry exists to stop. + // here would undo ~11% of the sweep's saving. This entry IS the definition + // of "digested" — the snapshot's second one (`buckets.noDigest`) is gone, + // because two definitions is the exact failure this entry exists to stop. if (record?.derivedFrom != null) return "present"; // EVERY configured section must be fresh to count as done, matching // countMissingDigests and the batch. What is new is that "not all of them" diff --git a/editor/app/actionable/lib/loadActionable.ts b/editor/app/actionable/lib/loadActionable.ts @@ -135,12 +135,10 @@ export function actionableCleanExtraFormatsCount(row: ActionableRow): number { // produced something a human should look at", which includes the total failures // that write no section and so are invisible to any count of files. // -// Read through digestWorkOf rather than off `buckets.noDigest` directly: the -// registry's classification is the one the runner uses, and it additionally -// excludes videos with no transcript (blocked) and videos whose cues.json is -// stale (deferred) — work that was being offered here and then declined by the -// runner. The bucket remains the fallback for a snapshot written before the -// registry entry existed. +// Read through digestWorkOf: the registry's classification is the one the runner +// uses, and it excludes videos with no transcript (blocked) and videos whose +// cues.json is stale (deferred) — work the old `noDigest` bucket offered here +// and the runner then declined. export function actionableNoDigestCount(row: ActionableRow): number { return digestWorkOf(row.snapshot).reachable; } @@ -157,13 +155,13 @@ export function actionableDigestWarningsCount(row: ActionableRow): number { // is the population that needs its media re-acquired first — measured at ~91x // the reachable count corpus-wide, so filtering on it would put every channel in // the list forever. That is not a hypothetical: it is the documented reason -// /api/widget/actionable refuses to filter on `noDigest`. +// /api/widget/actionable refuses to filter on the digest work count. // backfillLaneEntriesOf, not Object.values: the snapshot map is every catalog operation // now, and digest is one of them. These two functions decide whether a channel // appears in the BACKFILL section at all, so folding a ~75,000-video operation // that runs on another queue into them would put every channel in the list -// forever — the same trap /api/widget/actionable documents for `noDigest`, hit -// from the other direction. +// forever — the same trap /api/widget/actionable documents for the digest work +// count, hit from the other direction. export function actionableBackfillCount(row: ActionableRow): number { return backfillLaneEntriesOf(row.snapshot?.backfill).reduce( (n, e) => n + reachableOperationWork(e), diff --git a/editor/app/api/widget/actionable/route.ts b/editor/app/api/widget/actionable/route.ts @@ -13,10 +13,10 @@ export type WidgetActionableChannel = { slug: string; undownloaded: number; untranscribed: number; - // Videos with no current digest — missing, stale or part-done — now read from - // the operation registry rather than the `noDigest` bucket. Reported but STILL - // NOT used to decide whether a channel "needs work", and the reasoning is - // unchanged by the swap: during the backfill this is ~99.87% of the corpus, so + // Videos with no current digest — missing, stale or part-done — read from the + // operation registry. Reported but STILL NOT used to decide whether a channel + // "needs work", and the reasoning is unchanged: during the backfill this is + // ~99.87% of the corpus, so // counting it would put every channel in the list forever and drown the two // buckets a human can actually act on today. The registry's classification // removes untranscribed and cues-stale videos from the number, which makes it diff --git a/editor/app/api/widget/sync/route.ts b/editor/app/api/widget/sync/route.ts @@ -50,12 +50,13 @@ export type WidgetSyncPayload = { // will, so a coverage percentage against it can never reach 100%. // // NULL UNTIL EVERY CHANNEL CAN REPORT IT, deliberately. This is summed - // across channels, and a channel whose snapshot predates the digest registry - // entry contributes videos to `digested` but nothing to this — so a partial - // sum would be a denominator smaller than its own numerator, which is a - // worse lie than the one it replaces. A null says "not yet knowable" and the - // band falls back to `videos`; it becomes a number when the snapshots have - // all been regenerated. + // across channels, and a channel with no `backfill.digest` entry contributes + // videos to `digested` but nothing to this — so a partial sum would be a + // denominator smaller than its own numerator, which is a worse lie than the + // one it replaces. A null says "not yet knowable" and the band falls back to + // `videos`. Every channel on the measured corpus reports it (68/68 on + // 2026-08-26); the remaining case is a channel added since, whose first + // snapshot has not been written yet. eligible: number | null; channelsWithAny: number; // channels the layer has reached at all // The DENOMINATOR for channelsWithAny. Without it "66 channels reached" is a diff --git a/editor/app/channels/[slug]/components/stages/SpeakersStage.tsx b/editor/app/channels/[slug]/components/stages/SpeakersStage.tsx @@ -17,7 +17,8 @@ // actually do; the muted sub-line reports what it cannot reach without a // re-download. Measured on the real corpus those are 835 and ~76,270, and a // single "77,105 remaining" here would make the card useless the day it shipped -// — the same failure /api/widget/actionable already documents for `noDigest`. +// — the same failure /api/widget/actionable already documents for the digest +// work count (historically the `noDigest` bucket). import Link from "next/link"; import { useState } from "react"; diff --git a/editor/app/channels/[slug]/lib/channelFlow.test.ts b/editor/app/channels/[slug]/lib/channelFlow.test.ts @@ -269,12 +269,13 @@ test("deferred, blocked and missing-input never enter a gap's reachable count", } }); -test("the digest station reads through the registry, not the legacy bucket alone", () => { - // A snapshot old enough to have only `noDigest`. digestWorkOf falls back to - // it, so the work is still reported rather than reading as fully digested. +test("the digest station reads the registry entry", () => { + // The operation registry's entry is the one definition of the digest work + // list. `eligible` is absent here, so the station reports the work and still + // refuses to say how many are done. const flow = flowOf( snapshotOf({ - buckets: { ...normalizeBuckets(undefined), noDigest: ["a", "b", "c"] }, + backfill: { digest: entry({ missing: 3, ids: ["a", "b", "c"] }) }, }), ); assert.equal(flow.gaps.find((g) => g.to === "digest")?.reachable, 3); diff --git a/editor/app/channels/[slug]/lib/channelFlow.ts b/editor/app/channels/[slug]/lib/channelFlow.ts @@ -243,9 +243,9 @@ export function computeChannelFlow( (id) => !excludedIds.has(id), ); - // Read the digest operation through digestWorkOf, never buckets.noDigest: the - // bucket has no cues-staleness gate and no transcript gate, so a channel that - // has not been re-reported since the registry landed would read as done. + // Read the digest operation through digestWorkOf — the operation registry's + // entry. The old `noDigest` bucket had no cues-staleness gate and no + // transcript gate, so it called deferred and blocked videos done. const digestWork = digestWorkOf(snapshot); // backfillLaneEntriesOf, never Object.values: the per-kind map now carries EVERY diff --git a/editor/app/channels/[slug]/lib/stageStatus.ts b/editor/app/channels/[slug]/lib/stageStatus.ts @@ -42,7 +42,6 @@ export function normalizeBuckets( downloadedAutoSubsOnly: raw?.downloadedAutoSubsOnly ?? [], supersededAutoSubs: raw?.supersededAutoSubs ?? [], needsCookies: raw?.needsCookies ?? [], - noDigest: raw?.noDigest ?? [], }; } @@ -403,8 +402,7 @@ export function computeStageStatuses( // Videos whose digest is missing, stale or part-done against the local lane's // current identity. Read from the operation registry via digestWorkOf, which - // falls back to the legacy `noDigest` bucket for a snapshot written before - // this — so a channel that has not been regenerated yet still reports its work + // is the one definition: a channel with no entry reports unknown coverage // rather than reading as fully digested. // // Counted as pending work rather than merely informational: unlike the @@ -456,7 +454,7 @@ export function computeStageStatuses( // is reported in the summary line and never folded in: it is 91x larger on the // measured corpus, so counting it would hold every channel permanently amber // for work that cannot be done without an opt-in re-download — precisely the - // trap /api/widget/actionable documents for `noDigest`. + // trap /api/widget/actionable documents for the digest work count. const backfillRunning = runningByStage.has("speakers"); // backfillLaneEntriesOf, not Object.values. The snapshot's per-kind map carries every // operation in the catalog now, including digest — which runs on its own queue diff --git a/editor/app/channels/[slug]/lib/videoRows.ts b/editor/app/channels/[slug]/lib/videoRows.ts @@ -41,8 +41,9 @@ export type VideoRow = { incompleteTranscript: boolean; // The digest pass recorded warnings, or failed outright and wrote no section // at all. Independent flag (it composes with `transcribed`, and a total - // failure is also in noDigest) so the review queue can ask "what did the model - // do badly here?" separately from "what has no digest yet?". + // failure is also reachable work in `backfill.digest`) so the review queue can + // ask "what did the model do badly here?" separately from "what has no digest + // yet?". digestWarnings: boolean; // Download completed but the audio was far shorter than the video — the source // served a truncated stream (download-outcome "failed-short-audio"). The stub diff --git a/editor/app/channels/[slug]/page.tsx b/editor/app/channels/[slug]/page.tsx @@ -416,10 +416,9 @@ export default async function ChannelDetailPage({ /> ); case "digest": { - // From the operation registry, falling back to the legacy `noDigest` - // bucket for a snapshot written before it (digestWorkOf). A third of the - // channels on disk are months old, so the fallback is what stops them - // reading as fully digested until the regen reaches them. + // From the operation registry (digestWorkOf) — the one definition of + // the digest work list, the same one the runner dispatches from. A + // channel with no entry reports unknown coverage, never "all digested". const digestWork = digestWorkOf(snapshot); return ( <DigestStage diff --git a/editor/app/channels/lib/channelGroupSections.test.ts b/editor/app/channels/lib/channelGroupSections.test.ts @@ -317,17 +317,27 @@ test("digest and speakers totals are disjoint", () => { assert.equal(sections[0].speakers.total, 88); }); -test("digest counts reachable work, never the noDigest bucket", () => { +test("digest counts reachable work from the registry entry", () => { const sections = build( - siteOf({ channels: [{ slug: "stale-report" }] }), + siteOf({ channels: [{ slug: "some-work" }] }), [ channel( - "stale-report", + "some-work", {}, - // A snapshot from before the registry: noDigest is populated, and there - // is no per-kind entry. digestWorkOf falls back to the bucket rather - // than reading the channel as fully digested. - snapshotOf({ buckets: normalizeBuckets({ noDigest: ["a", "b", "c"] }) }), + // The per-kind entry is the work list. `blocked` and `deferred` are work + // the lane cannot do today and are deliberately not in the total. + snapshotOf({ + backfill: { + digest: entry({ + missing: 2, + stale: 1, + blocked: 4, + deferred: 5, + ids: ["a", "b", "c"], + eligible: 3, + }), + }, + }), ), ], ); diff --git a/editor/app/channels/lib/channelGroupSections.ts b/editor/app/channels/lib/channelGroupSections.ts @@ -165,9 +165,9 @@ export function stationWorkFor( if (snapshot.totals.transcribed <= 0) { return { eligible: false, work: 0, reason: "no transcripts yet" }; } - // digestWorkOf, NEVER buckets.noDigest: the bucket has no cues-staleness - // gate and no transcript gate, so a channel not re-reported since the - // operation registry landed reads as fully digested. + // digestWorkOf — the operation registry's entry is the one definition of + // "digested". The `noDigest` bucket it replaced had no cues-staleness gate + // and no transcript gate, so it called deferred and blocked videos done. return { eligible: true, work: digestWorkOf(snapshot).reachable }; } diff --git a/editor/app/components/pipelines/buildBands.test.ts b/editor/app/components/pipelines/buildBands.test.ts @@ -112,20 +112,16 @@ test("coverage is null, never 0, when the denominator is unknown", () => { assert.equal(bandCoverage({ present: 5, eligible: 10 } as OperationBand), 0.5); }); -test("digest falls back to the bucket when a snapshot predates its registry entry", () => { - // Do NOT stop reading buckets.noDigest until every snapshot has regenerated: - // the fallback is all that keeps a stale channel from reading "all digested". +test("digest with no registry entry is an unfilled outline, never 0 %", () => { + // A channel with no `backfill.digest` must not read "all digested" — nor + // "none digested". Digest is a plain registry entry here, exactly like every + // other operation: no entry means no work KNOWN and coverage UNKNOWN. const bands = buildOperationBands({ - snapshots: [ - snapshotOf({ - buckets: { noDigest: ["a", "b", "c"] } as ChannelSnapshot["buckets"], - }), - ], + snapshots: [snapshotOf({ backfill: {} })], operationIds: ["digest"], }); const digest = bandOf(bands, "digest"); - assert.equal(digest.reachable, 3); - // The bucket cannot say how many are done, and it must not pretend to. + assert.equal(digest.reachable, 0); assert.equal(digest.eligible, null); assert.equal(digest.present, null); }); diff --git a/editor/e2e/backfill.spec.ts b/editor/e2e/backfill.spec.ts @@ -857,7 +857,14 @@ test("a digest entry in the snapshot does not move the backfill instrument", asy string, { missing?: number; ids?: string[]; eligible?: number; blocked?: number } >; + buckets?: Record<string, unknown>; }>(`test-transcripts/channels/${SLUG}/snapshot.json`); + // ONE definition of the digest work list. The writer used to keep a second + // one, `buckets.noDigest`, whose answer disagreed with this entry by 11,777 + // videos corpus-wide (it had no cues-staleness gate and no transcript gate). + // This is the only place the writer's output is asserted end-to-end, so + // without this line the deletion is pinned by nothing. + expect(snapshot.buckets).not.toHaveProperty("noDigest"); const digestEntry = snapshot.backfill?.digest; expect(digestEntry, "snapshot.backfill.digest must exist").toBeTruthy(); expect(digestEntry?.missing).toBe(2);