commit a145483403af6cac7a8c07665244b35242abe73e
parent ac744ed118f3389b5edce6193272fa2e75e5443e
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 11 Aug 2026 19:32:36 -0400
Break the backfill figure down by kind, and give every lane a second line
The backfill card read "77,952 reachable · 77,134 need media", which looks
inverted and is not. Summed read-only from the 67 channel snapshots:
diarization 329 reachable / 77,133 needing media, attribution-diarized 84 /
77,462 blocked, attribution-text 77,538. Both labels were on the right number.
What changed is the corpus — both attribution kinds are switched on now, so the
lane has three kinds where it used to have one.
The sum is the problem. 99.5% of "reachable" is attribution-text at about one
model call per transcript CHUNK, against 329 audio passes for diarization, so
adding them yields a number in no unit at all — the mistake backfillKinds.ts's
own header forbids for `missing` vs `missing-input`, one level up. The two
figures also overlap, so the corpus read as simultaneously all-actionable and
all-blocked, and the largest single fact was invisible: 77,462 videos are
BLOCKED on diarization's output, which is exactly why attribution can only run
text-only today.
So the card prints a line per kind, each clause dropped at 0, never summed —
the rule BackfillStage already applies per channel. The other three lanes stop
being a status word over a sentence: transcription and downloads name their
backlog and its spread, transcription adds worker occupancy, downloads adds
free disk against the floor, digest adds its reach plus the videos waiting on
a transcript or held for want of a normalized one.
Also: the downloads gate is TWO switches. disk.low stops downloads exactly as
the manual pause does, but the lane derived its state from the pause alone, so
a disk stop left the card reading "Idle" while nothing could move. The lane is
now held by either, and the detail names which. The disk and manual-pause
instruments moved off the pipeline band onto the card — aria-labels and
conditions verbatim, since disk-space.spec exact-matches both and does not care
where on the page they live. The band keeps only what is pipeline-wide.
laneKindEntriesOf() is laneEntriesOf() with the ids kept; the latter is now
defined in terms of it so the "filtered by the declaration, not by id" rule
stays in one place rather than being re-derived by the first surface that wants
a breakdown. The sync payload's "SCALARS ONLY" note is revised, not ignored:
`kinds` is bounded by the REGISTRY (three entries of five numbers off the same
sums), and the constraint it protects — no per-channel breakdown on a polled
endpoint — still holds.
Verified: tsc clean in both packages; pnpm build clean; 53/53 registry unit
tests; the per-kind and digest figures reproduced against the real snapshots
with a read-only script (no editor booted); full e2e 473 passed with the two
known reds only — jobs-batch-tasks-drain's documented drain/cancel race (dump
confirms `done`) and a cookies-mode download that times out under load and
passes in isolation.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Diffstat:
11 files changed, 453 insertions(+), 98 deletions(-)
diff --git a/common/lib/backfillKinds.test.ts b/common/lib/backfillKinds.test.ts
@@ -20,6 +20,7 @@ import {
type BackfillClassification,
type BackfillKind,
laneEntriesOf,
+ laneKindEntriesOf,
presentBackfillWork,
} from "./backfillKinds";
import { candidateAction } from "../controller/backfillBatch";
@@ -1345,6 +1346,35 @@ test("laneEntriesOf filters by the DECLARATION, not by a hardcoded id", () => {
assert.deepEqual(laneEntriesOf({}), []);
});
+test("laneKindEntriesOf applies the SAME filter, keyed by kind", () => {
+ // The keyed form exists so the corpus-wide backfill card can say WHICH kind a
+ // number came from: summed, this lane reads "77,952 reachable · 77,134 need
+ // media", where 99.5% of the first is attribution-text at ~1 model call per
+ // transcript CHUNK and all of the second is diarization at a few hundred audio
+ // passes. A breakdown that re-derived its own filter is how a "Digest" row
+ // ends up on the backfill card contradicting the figure above it — so the two
+ // are one function, and this asserts they cannot drift.
+ const laneIds = laneBackfillKinds(settingsWithDiarization()).map((k) => k.id);
+ const backfill: Record<
+ string,
+ ReturnType<typeof emptyBackfillCounts> & { ids: string[] }
+ > = {};
+ for (const id of [...laneIds, "digest", "some-kind-from-a-newer-build"]) {
+ backfill[id] = { ...emptyBackfillCounts(), missing: 1, ids: [] };
+ }
+ assert.deepEqual(
+ laneKindEntriesOf(backfill).map(([id]) => id).sort(),
+ [...laneIds].sort(),
+ );
+ // Exactly the entries laneEntriesOf returns, in the same order.
+ assert.deepEqual(
+ laneKindEntriesOf(backfill).map(([, e]) => e),
+ laneEntriesOf(backfill),
+ );
+ assert.deepEqual(laneKindEntriesOf(undefined), []);
+ assert.deepEqual(laneKindEntriesOf({}), []);
+});
+
test("presentBackfillWork says UNKNOWN rather than zero on an old snapshot", () => {
// A 0 here would render as "nothing digested" on a fully digested channel,
// which is the most dangerous direction for a coverage number to be wrong.
diff --git a/common/lib/backfillKinds.ts b/common/lib/backfillKinds.ts
@@ -1014,7 +1014,16 @@ export function laneBackfillKinds(settings: SiteSettings): BackfillKind[] {
}
// The READ-SIDE twin of laneBackfillKinds: given a snapshot's per-kind map,
-// return only the entries belonging to the shared backfill lane.
+// return only the entries belonging to the shared backfill lane — KEYED, so a
+// surface can say WHICH kind a number came from. laneEntriesOf below is this
+// with the ids dropped, for the callers that only sum.
+//
+// The keyed form is what the corpus-wide backfill card needs. Summed, this lane
+// reads "77,952 reachable · 77,134 need media" — both figures correct, and
+// together meaningless: 99.5% of the first is attribution-text (one model call
+// per transcript CHUNK) and all of the second is diarization (329 runs). Adding
+// kinds gives a number in no unit at all, which is the mistake the header
+// forbids one level up for `missing` vs `missing-input`.
//
// THIS EXISTS BECAUSE THE SNAPSHOT MAP STOPPED BEING THE LANE. It used to be
// written from laneBackfillKinds, so `Object.values(snapshot.backfill)` and "the
@@ -1038,13 +1047,23 @@ export function laneBackfillKinds(settings: SiteSettings): BackfillKind[] {
// `enabled(settings)` — these are counts already written to disk, and a feature
// switched off after a snapshot was taken does not retroactively unmake the work
// it recorded.
+export function laneKindEntriesOf<T>(
+ backfill: Record<string, T> | undefined | null,
+): [string, T][] {
+ if (!backfill) return [];
+ return Object.entries(backfill).filter(
+ ([id]) => getBackfillKind(id)?.lane.queueKey === BACKFILL_QUEUE,
+ );
+}
+
+// The same set with the ids dropped, for the callers that only ever sum. Defined
+// in terms of the above rather than beside it: the filter and every word of the
+// rule above it must stay in ONE place, or the next surface that wants per-kind
+// detail copies a `key !== "digest"` in and re-arms the trap.
export function laneEntriesOf<T>(
backfill: Record<string, T> | undefined | null,
): T[] {
- if (!backfill) return [];
- return Object.entries(backfill)
- .filter(([id]) => getBackfillKind(id)?.lane.queueKey === BACKFILL_QUEUE)
- .map(([, entry]) => entry);
+ return laneKindEntriesOf(backfill).map(([, entry]) => entry);
}
// Resolve a caller-supplied list of kind ids against the registry. An empty or
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,8 @@
# Changelog
## [Unreleased]
+- **The backfill card now breaks its figure down by kind, and every lane card gained a second line.** Backfill read "77,952 reachable · 77,134 need media", which looks inverted and is not: three kinds are running, and summing them produces a number in no unit at all. 99.5% of that "reachable" is attribution-from-text, which costs about one model call per transcript *chunk*; the whole of "need media" is diarization, which is a few hundred audio passes. Worse, the two overlap — most of the videos needing media for diarization are also inside the text-attribution total — so the corpus read as simultaneously all-actionable and all-blocked, and the biggest single fact was invisible: **77,463 videos are blocked on diarization's output**, which is precisely why attribution can only run text-only. The card now prints a line per kind (`Speaker diarization 329 · 77,134 need media`), each clause dropped at zero, and never adds them together. The other three lanes stopped being a status word over a sentence: transcription and downloads name their backlog and how many channels it spans, transcription adds worker occupancy, downloads adds free disk against the floor, and digest names how many channels the layer has reached plus the videos waiting on a transcript or held for want of a normalized one.
+- **A full disk now stops the downloads lane visibly.** `disk.low` blocks downloads exactly as the manual pause does, but the lane derived its state from the manual toggle alone — so a real disk stop left the card reading "Idle" while nothing could move. The lane is now held by either switch and the card says which one is shut. The disk and manual-pause readouts moved off the pipeline band and onto the downloads card, so each fact is stated once, next to the button that changes it; the band keeps only what is pipeline-wide (running/queued, the sync heartbeat, the scheduler).
- **Cleanup now says what is holding the audio it can't reclaim, and what to run to get it back.** The page led with one number — how much space you can free right now — and said nothing about the rest of the disk. The sweep skips videos for four different reasons and reported them only as a line in a job log after the fact, with no bytes attached and nothing ranked. A **sieve** now runs down the page: all the audio on disk enters at the top, each gate siphons off its share (no transcript yet, the keep-latest window, do-not-clean pins, awaiting diarization), and the remainder steps down to the reclaimable figure the page already led with. A video leaves at the *first* gate it hits, exactly as the sweep's own cascade does, so the five figures are an attribution and never overlap — a pinned, undiarized video is counted once, under the pin. Below it, the **release ledger** ranks the channels holding the most, split by what it costs to get the space back: a run of a lane that is already weeks deep, or a setting that frees it the moment it changes. "Not counted" sits in the second group and is not called a hold — excluding a channel hides its bytes from the total, it never protected them, which makes it the fastest win on the page.
- **Some of that audio is held forever, and nothing anywhere said so.** The cleanup guard fires on `diarization.enabled` alone, but the diarize lane will never produce a sidecar for a video that is over `maxAudioHours`, has no diarizable input, or belongs to a channel whose diarization models were never configured — the kind reports itself disabled and no job is ever queued. Those videos were held from cleaning permanently, and running *Diarize speakers* until the end of time would not have moved the number. The sieve now draws that slice hatched inside gate ④ with its own figure and says it in words: *will not clear on its own*. Deciding what to do about it — raise the cap, configure the models, or let the sweep past the guard — is deliberately left to you: audio is the one input in this pipeline that cannot be regenerated. Channel is the honest granularity throughout; per-video byte sizes exist nowhere outside the report, and inventing them would have cost a corpus walk on every render.
- **Every lane now has one card, and the card shows both of its switches.** Transcription, downloads, digest and backfill each get a card on the dashboard *and* in the monitor widget: the lane's name, its state, its figure, and every control it has, in one place. A hairline under the name reads left → right as feed → gate → lane, so a *held* lane — sweep armed, gate shut — draws as a lit feed running into a break, which is the state that used to look identical to "wedged". The figures that belong to a lane (digest coverage, backfill reachable / needs-media) moved onto their own card, out of the instrument row several elements away from the buttons that move them. The widget gains what it never had: starting and stopping a sweep, and any digest control at all.
diff --git a/editor/app/api/widget/sync/route.ts b/editor/app/api/widget/sync/route.ts
@@ -6,7 +6,8 @@ import { getChannelBriefs } from "../../../lib/requestCache";
import { getSettings } from "yt-dlp-transcript-common/lib/settings";
import {
laneBackfillKinds,
- laneEntriesOf,
+ laneKindEntriesOf,
+ operationLabel,
reachableBackfillWork,
} from "yt-dlp-transcript-common/lib/backfillKinds";
import { buildScheduleView } from "yt-dlp-transcript-common/jobs/syncScheduler";
@@ -54,18 +55,36 @@ export type WidgetSyncPayload = {
// all been regenerated.
eligible: number | null;
channelsWithAny: number; // channels the layer has reached at all
+ // The DENOMINATOR for channelsWithAny. Without it "66 channels reached" is a
+ // count with nothing to be a fraction of, and the card cannot tell a corpus
+ // where the sweep has touched everything from one where it has barely begun.
+ channels: number;
+ // Videos with no transcript yet, and videos held back for want of a current
+ // normalized one. NEVER summed with `digested` nor with each other — they are
+ // the two reasons a digest cannot happen, and each has a different fix (wait
+ // for transcription; run Normalize transcripts).
+ blocked: number;
+ deferred: number;
paused: boolean; // settings.digest.digestsPaused
sweeping: boolean; // a corpus-wide sweep is armed
};
- // Corpus-wide backfill state. Same constraint as `digest` above — scalars only
- // — and free for the same reason: every brief already carries its channel's
- // snapshot, so this is a sum rather than a corpus walk (which cost 4.4 s on an
- // endpoint the widget polls).
+ // Corpus-wide backfill state, and free for the same reason as `digest`: every
+ // brief already carries its channel's snapshot, so this is a sum rather than a
+ // corpus walk (which cost 4.4 s on an endpoint the widget polls).
//
// `reachable` and `needsMedia` are separate FIELDS, not a total, because on the
// measured corpus they are 835 and ~76,270. A single number here would report
// a backfill as barely begun forever, no matter how much of the reachable work
// was finished.
+ //
+ // THE "SCALARS ONLY" CONSTRAINT IS REVISED, NOT DROPPED. `kinds` is an array,
+ // and it is bounded by the REGISTRY — three entries of five numbers today, off
+ // exactly the sums beside it, not a row per channel. What the constraint
+ // protects is that this endpoint is POLLED: a per-channel breakdown would grow
+ // with the corpus (67 channels and climbing) and put a corpus walk back on a
+ // 15-second timer. A per-KIND breakdown cannot, because adding a kind means
+ // adding an entry to backfillKinds.ts. That distinction still holds, and it is
+ // the line to keep: bounded by the code, never by the data.
backfill: {
reachable: number; // missing + stale: what the lane can do now
needsMedia: number; // missing-input: needs an opt-in re-download first
@@ -76,6 +95,26 @@ export type WidgetSyncPayload = {
// lane: with no feature on there is nothing to report at all, and that must
// not look like "all caught up".
anyKind: boolean;
+ // The same numbers, kept apart by kind — which is the only form of them that
+ // means anything. Summed, this lane reads "77,952 reachable · 77,134 need
+ // media": both correct, and together a figure in no unit, since 99.5% of the
+ // first is attribution-text at ~1 model call per transcript CHUNK and all of
+ // the second is diarization at 329 runs. The split also surfaces the largest
+ // single fact about this corpus, which the sums hide completely — 77,463
+ // videos BLOCKED on diarization's output, which is why attribution can only
+ // run text-only.
+ //
+ // Sorted by `reachable` descending, tie-broken by id, so the order is stable
+ // across polls rather than following whatever order the snapshots were
+ // written in.
+ kinds: {
+ id: string;
+ label: string; // operationLabel(id) — resolved here, not in the client
+ reachable: number;
+ needsMedia: number;
+ blocked: number;
+ deferred: number;
+ }[];
};
};
@@ -131,6 +170,17 @@ export async function buildWidgetSyncPayload(): Promise<WidgetSyncPayload> {
// through — see the payload type.
let backfillReachable = 0;
let backfillNeedsMedia = 0;
+ // The per-kind breakdown, accumulated in the SAME pass rather than a second
+ // one. A Map keyed by kind id, so a snapshot naming a kind another snapshot
+ // does not is simply added rather than dropped.
+ const backfillByKind = new Map<
+ string,
+ { reachable: number; needsMedia: number; blocked: number; deferred: number }
+ >();
+ // Digest's two non-work counters, off the digestWorkOf() call already made
+ // below — no extra read.
+ let digestBlocked = 0;
+ let digestDeferred = 0;
for (const c of channels) {
videos += c.snapshot?.totals.videos ?? 0;
const n = digestCountOf(c.snapshot);
@@ -143,13 +193,28 @@ export async function buildWidgetSyncPayload(): Promise<WidgetSyncPayload> {
const work = digestWorkOf(c.snapshot);
if (work.eligible == null) digestEligibleKnown = false;
else digestEligible += Math.max(0, work.eligible - work.blocked);
+ digestBlocked += work.blocked;
+ digestDeferred += work.deferred;
// The backfill LANE only. The snapshot's per-kind map now carries every
// catalog operation, digest included, and the widget's backfill strip has
// only ever meant diarization plus attribution — the digest coverage figure
// it shows beside this one is computed separately, from digestCountOf above.
- for (const entry of laneEntriesOf(c.snapshot?.backfill)) {
+ for (const [id, entry] of laneKindEntriesOf(c.snapshot?.backfill)) {
backfillReachable += reachableBackfillWork(entry);
backfillNeedsMedia += entry.missingInput;
+ const acc = backfillByKind.get(id) ?? {
+ reachable: 0,
+ needsMedia: 0,
+ blocked: 0,
+ deferred: 0,
+ };
+ acc.reachable += reachableBackfillWork(entry);
+ acc.needsMedia += entry.missingInput;
+ // `?? 0` at every read site: every snapshot written before these fields
+ // existed lacks them, and undefined poisons the sum to NaN.
+ acc.blocked += entry.blocked ?? 0;
+ acc.deferred += entry.deferred ?? 0;
+ backfillByKind.set(id, acc);
}
}
@@ -167,6 +232,9 @@ export async function buildWidgetSyncPayload(): Promise<WidgetSyncPayload> {
eligible: digestEligibleKnown ? digestEligible : null,
videos,
channelsWithAny,
+ channels: channels.length,
+ blocked: digestBlocked,
+ deferred: digestDeferred,
paused: settings.digest.digestsPaused,
sweeping: settings.digest.sweepEnabled,
},
@@ -177,6 +245,11 @@ export async function buildWidgetSyncPayload(): Promise<WidgetSyncPayload> {
enabled: settings.backfill.enabled,
sweeping: settings.backfill.sweepEnabled,
anyKind: laneBackfillKinds(settings).length > 0,
+ kinds: [...backfillByKind]
+ .map(([id, counts]) => ({ id, label: operationLabel(id), ...counts }))
+ // Biggest reachable first, so the kind an operator can act on most leads.
+ // Tie-broken by id so the order does not shuffle between polls.
+ .sort((a, b) => b.reachable - a.reachable || a.id.localeCompare(b.id)),
},
} satisfies WidgetSyncPayload;
}
diff --git a/editor/app/components/dashboard/DashboardCockpit.tsx b/editor/app/components/dashboard/DashboardCockpit.tsx
@@ -69,6 +69,7 @@ export function DashboardCockpit({
jobs={jobs}
workers={workers}
sync={sync}
+ actionable={actionable}
now={now}
onWorkersChange={refetchWorkers}
onSynced={refetchSync}
diff --git a/editor/app/components/dashboard/PipelineBand.tsx b/editor/app/components/dashboard/PipelineBand.tsx
@@ -5,11 +5,11 @@ import { useState } from "react";
import type { ActiveJobsPayload } from "../../jobs/active/buildActiveJobs";
import type { WorkersPayload } from "../../workers/components/WorkersView";
import type { WidgetSyncPayload } from "../../api/widget/sync/route";
+import type { WidgetActionablePayload } from "../../api/widget/actionable/route";
import { ActiveJobsLive } from "../../jobs/components/ActiveJobsLive";
import { LaneDeck } from "../lanes/LaneDeck";
import { syncAllChannelsAction, type SyncAllResult } from "../../channels/actions";
import { fmtTime } from "../../widget/lib/relativeTime";
-import { formatBytes } from "yt-dlp-transcript-common/lib/format";
// The hero: a single live instrument readout of the whole pipeline — running/
// queued jobs, worker-pool occupancy + pause state, sync heartbeat + scheduler —
@@ -19,6 +19,7 @@ export function PipelineBand({
jobs,
workers,
sync,
+ actionable,
now,
onWorkersChange,
onSynced,
@@ -26,6 +27,9 @@ export function PipelineBand({
jobs: ActiveJobsPayload | null;
workers: WorkersPayload | null;
sync: WidgetSyncPayload | null;
+ // Passed straight through to the lane cards, which state each lane's backlog
+ // under its figure. The cockpit already polls it for the Needs-work panel.
+ actionable: WidgetActionablePayload | null;
now: number | null;
onWorkersChange: () => void | Promise<void>;
onSynced: () => void | Promise<void>;
@@ -33,22 +37,18 @@ export function PipelineBand({
const jobList = jobs?.jobs ?? [];
const running = jobList.filter((j) => j.status === "running").length;
const queued = jobList.filter((j) => j.status === "queued").length;
- const workerList = workers?.workers ?? [];
- const busy = workerList.filter((w) => w.busy).length;
- const paused = workers?.paused ?? false;
- const downloadsPaused = workers?.downloadsPaused ?? false;
- // THE TWO REASONS DOWNLOADS STOP, AND THEY ARE NOT THE SAME THING.
- // `downloadsPaused` is the manual toggle; `disk.low` is the gate deciding on
- // its own. This band used to render one red "downloads paused" that could
- // only ever mean the toggle — so a real disk stop said nothing at all, and if
- // both were true the operator would un-pause and watch nothing happen. They
- // now get separate instruments with their own wording.
- const disk = jobs?.disk ?? null;
- const diskLow = disk?.low ?? false;
- // The digest and backfill figures used to be two more instruments up here,
- // several elements away from the buttons that moved them. They now sit inside
- // their own lane cards — state, figure and both switches in one place — which
- // is the whole point of LaneDeck.
+ // WHAT IS LEFT UP HERE IS PIPELINE-WIDE, AND ONLY THAT. The digest and
+ // backfill figures went to their own lane cards first; the workers gauge, the
+ // manual downloads pause and the disk gate have now followed them, because
+ // each is one lane's fact and a card that states its own numbers next to its
+ // own switches makes the chip a second, more distant copy.
+ //
+ // `running · queued` stays because it is the whole queue rather than any one
+ // lane, and so do the sync heartbeat and the scheduler.
+ //
+ // The two disk/pause nodes are not deleted — they are CARRIED, aria-labels and
+ // conditions intact, onto the downloads card. disk-space.spec asserts both by
+ // exact label on this page and does not care where on it they are.
const lastSyncText =
sync == null
@@ -87,26 +87,6 @@ export function PipelineBand({
}
/>
<Instrument
- dotClass={
- paused
- ? "bg-destructive"
- : busy > 0
- ? "bg-success"
- : "bg-success/40"
- }
- label={
- <>
- workers{" "}
- <span className="font-medium">
- {busy}/{workerList.length}
- </span>
- {paused && (
- <span className="ml-1 text-destructive">· paused</span>
- )}
- </>
- }
- />
- <Instrument
dotClass={sync?.scheduler.overdue ? "bg-warning" : "bg-muted-foreground/40"}
label={
<>
@@ -118,60 +98,14 @@ export function PipelineBand({
dotClass={schedulerOn ? "bg-success" : "bg-muted-foreground/40"}
label={<>sched {schedulerOn ? "on" : "off"}</>}
/>
- {downloadsPaused && (
- <Instrument
- dotClass="bg-destructive"
- label={
- <span
- aria-label="downloads paused manually"
- className="text-destructive"
- >
- downloads paused
- <span className="text-muted-foreground"> · manual</span>
- </span>
- }
- />
- )}
- {disk?.enabled && (
- <Instrument
- dotClass={diskLow ? "bg-destructive" : "bg-success/40"}
- label={
- diskLow ? (
- <span
- aria-label="disk low"
- className="text-destructive"
- title={disk.message}
- >
- downloads stopped · disk{" "}
- <span className="font-medium">
- {formatBytes(disk.freeBytes)}
- </span>{" "}
- free, floor {formatBytes(disk.thresholdBytes)}
- {disk.reason === "below-resume-margin" && (
- <span className="text-muted-foreground">
- {" "}
- · resumes at {formatBytes(disk.resumeBytes)}
- </span>
- )}
- </span>
- ) : (
- <>
- disk{" "}
- <span className="font-medium">
- {formatBytes(disk.freeBytes)}
- </span>{" "}
- free
- </>
- )
- }
- />
- )}
</div>
</div>
<LaneDeck
workers={workers}
sync={sync}
+ jobs={jobs}
+ actionable={actionable}
onWorkersChange={onWorkersChange}
onSynced={onSynced}
/>
diff --git a/editor/app/components/lanes/LaneCard.tsx b/editor/app/components/lanes/LaneCard.tsx
@@ -38,6 +38,7 @@ export function LaneCard({
name,
state,
figure,
+ detail,
note,
controls,
}: {
@@ -46,6 +47,16 @@ export function LaneCard({
// The lane's own number, in the house's tabular mono. For a lane with no sweep
// this is the sentence saying so.
figure: ReactNode;
+ // About two lines under the figure: what the figure is MADE OF. The backfill
+ // lane's is a per-kind breakdown, because its sum is a number in no unit at
+ // all — see the sync payload's `kinds`.
+ //
+ // SUPPRESSED IN THE DENSE FRAME, deliberately, and this is the whole reason it
+ // is a separate prop from `note`. A rail cell is one line; `note` survives
+ // there because it is one short clause ("the sweep is holding") that the rail-
+ // as-edge cannot draw. Two or three lines of breakdown would either wrap the
+ // cell to three times its height or be truncated into a lie.
+ detail?: ReactNode;
// Anything the card must keep saying in words — the backfill lane's "the sweep
// is holding", which the rail now also draws.
note?: ReactNode;
@@ -117,6 +128,11 @@ export function LaneCard({
<p className="font-mono text-xs tabular-nums text-muted-foreground">
{figure}
</p>
+ {detail && (
+ <div className="flex flex-col gap-0.5 text-[11px] text-muted-foreground">
+ {detail}
+ </div>
+ )}
{note}
{controls.length > 0 && (
<div className="flex flex-wrap items-center gap-1.5">
diff --git a/editor/app/components/lanes/LaneDeck.tsx b/editor/app/components/lanes/LaneDeck.tsx
@@ -1,7 +1,11 @@
"use client";
+import type { ReactNode } from "react";
import type { WidgetSyncPayload } from "../../api/widget/sync/route";
+import type { WidgetActionablePayload } from "../../api/widget/actionable/route";
+import type { ActiveJobsPayload } from "../../jobs/active/buildActiveJobs";
import type { WorkersPayload } from "../../workers/components/WorkersView";
+import { formatBytes } from "yt-dlp-transcript-common/lib/format";
import { useSectionFrame } from "../../widget/components/WidgetSection";
import { LaneCard, type LaneControl } from "./LaneCard";
import { deriveLaneState, formatCount } from "./laneState";
@@ -37,11 +41,24 @@ import {
export function LaneDeck({
workers,
sync,
+ jobs,
+ actionable,
onWorkersChange,
onSynced,
}: {
workers: WorkersPayload | null;
sync: WidgetSyncPayload | null;
+ // The two payloads the DETAIL lines read. Both are already polled by every
+ // surface that renders this deck — the dashboard cockpit unconditionally, the
+ // widget behind its own section flags — so no new request exists because of
+ // this.
+ //
+ // NULL DROPS THE LINE, never renders a zero. Same rule as formatCount's
+ // "—, never 0": in the widget these are null whenever their flag is off, and
+ // "0 videos awaiting transcription" on a corpus with 12,486 of them is a
+ // worse answer than saying nothing.
+ jobs: ActiveJobsPayload | null;
+ actionable: WidgetActionablePayload | null;
onWorkersChange: () => void | Promise<void>;
onSynced: () => void | Promise<void>;
}) {
@@ -49,9 +66,20 @@ export function LaneDeck({
const paused = workers?.paused ?? false;
const downloadsPaused = workers?.downloadsPaused ?? false;
- const busy = (workers?.workers ?? []).filter((w) => w.busy).length;
+ const workerList = workers?.workers ?? [];
+ const busy = workerList.filter((w) => w.busy).length;
const digest = sync?.digest ?? null;
const backfill = sync?.backfill ?? null;
+ const disk = jobs?.disk ?? null;
+ const diskLow = disk?.low ?? false;
+
+ // The two backlogs, summed over the channels that HAVE one — /actionable
+ // already filters to those, so the channel count is the number of channels
+ // carrying that particular bucket rather than the array length (a channel with
+ // downloads outstanding and nothing to transcribe is in the array, and must
+ // not be counted in the transcription sentence).
+ const untranscribed = sumBacklog(actionable, (c) => c.untranscribed);
+ const undownloaded = sumBacklog(actionable, (c) => c.undownloaded);
// THE DENOMINATOR IS ELIGIBLE VIDEOS, NOT EVERY VIDEO DIRECTORY. `videos`
// counts every directory in the corpus, ~1,700 of which have no transcript or
@@ -81,6 +109,20 @@ export function LaneDeck({
name="Transcription"
state={deriveLaneState({ gateHeld: paused, activeCount: busy })}
figure="no sweep — work arrives from jobs"
+ detail={
+ <>
+ <BacklogLine
+ backlog={untranscribed}
+ verb="awaiting transcription"
+ empty="nothing awaiting transcription"
+ />
+ {workers && (
+ <span>
+ {busy}/{workerList.length} workers busy
+ </span>
+ )}
+ </>
+ }
controls={[
paused
? {
@@ -112,11 +154,65 @@ export function LaneDeck({
// gates the auto-download runner on its next loop iteration and makes manual
// download-bearing pipeline actions return a "Downloads are paused" notice;
// store-playlist/enumeration stay allowed.
+ //
+ // THE GATE IS TWO SWITCHES, NOT ONE. `disk.low` stops downloads exactly as the
+ // manual pause does — the auto-download runner idles and download-bearing
+ // actions are refused — so a lane state derived from `downloadsPaused` alone
+ // read "Idle" while nothing could move. The detail line then names WHICH one is
+ // shut, because un-pausing a disk stop does nothing and an operator has to be
+ // able to tell them apart.
const downloads = (
<LaneCard
name="Downloads"
- state={deriveLaneState({ gateHeld: downloadsPaused })}
+ state={deriveLaneState({ gateHeld: downloadsPaused || diskLow })}
figure="no sweep — work arrives from jobs"
+ detail={
+ <>
+ <BacklogLine
+ backlog={undownloaded}
+ verb="not downloaded"
+ empty="everything downloaded"
+ />
+ {(downloadsPaused || disk?.enabled) && (
+ <span className="flex flex-wrap items-center gap-x-2">
+ {/* Carried here VERBATIM from the pipeline band, aria-label and
+ condition unchanged: disk-space.spec exact-matches both labels
+ and asserts each disappears when its condition is false. It does
+ not care where on the page they are. */}
+ {downloadsPaused && (
+ <span
+ aria-label="downloads paused manually"
+ className="text-destructive"
+ >
+ paused manually
+ </span>
+ )}
+ {disk?.enabled &&
+ (diskLow ? (
+ <span
+ aria-label="disk low"
+ className="text-destructive"
+ title={disk.message}
+ >
+ stopped · disk {formatBytes(disk.freeBytes)} free, floor{" "}
+ {formatBytes(disk.thresholdBytes)}
+ {disk.reason === "below-resume-margin" && (
+ <span className="text-muted-foreground">
+ {" "}
+ · resumes at {formatBytes(disk.resumeBytes)}
+ </span>
+ )}
+ </span>
+ ) : (
+ <span>
+ disk: {formatBytes(disk.freeBytes)} free, floor{" "}
+ {formatBytes(disk.thresholdBytes)}
+ </span>
+ ))}
+ </span>
+ )}
+ </>
+ }
controls={[
downloadsPaused
? {
@@ -235,6 +331,39 @@ export function LaneDeck({
</>
)
}
+ detail={
+ digest === null ? undefined : (
+ <>
+ {/* The coverage percentage stays on the FIGURE rather than being
+ restated here — a card that says a thing twice is the accessory
+ to remove. What the detail adds is the reach (a sweep can be a
+ third of the way through the corpus and have touched every
+ channel, or the reverse) and the two reasons a video is not in
+ the numerator at all. */}
+ <span>
+ {digest.channelsWithAny.toLocaleString()} of{" "}
+ {digest.channels.toLocaleString()}{" "}
+ {digest.channels === 1 ? "channel" : "channels"} reached
+ </span>
+ {(digest.blocked > 0 || digest.deferred > 0) && (
+ <span>
+ {/* NEVER summed — with each other or with `digested`. One is
+ waiting on another lane, the other needs Normalize
+ transcripts run by hand. */}
+ {digest.blocked > 0 && (
+ <>
+ {digest.blocked.toLocaleString()} waiting on a transcript
+ </>
+ )}
+ {digest.blocked > 0 && digest.deferred > 0 && " · "}
+ {digest.deferred > 0 && (
+ <>{digest.deferred.toLocaleString()} deferred</>
+ )}
+ </span>
+ )}
+ </>
+ )
+ }
controls={digestControls}
/>
);
@@ -266,6 +395,9 @@ export function LaneDeck({
const backfillSweeping = backfill?.sweeping ?? false;
const laneEnabled = backfill?.enabled ?? false;
const backfillAvailable = backfill?.anyKind ?? false;
+ // Empty when the payload has not arrived, or when it predates `kinds` — which
+ // drops the breakdown rather than rendering a row of zeros.
+ const backfillKinds = backfillAvailable ? (backfill?.kinds ?? []) : [];
const backfillControls: LaneControl[] = backfillAvailable
? [
backfillSweeping
@@ -343,6 +475,34 @@ export function LaneDeck({
</>
)
}
+ // THE PER-KIND BREAKDOWN, and it REPLACES both detail lines rather than
+ // joining them: this lane's figure is the one that cannot survive being
+ // summed, so the detail's whole job is to take it apart.
+ //
+ // Only when there is more than one kind — the rule BackfillStage already
+ // applies per channel. A single-kind corpus would otherwise be shown a
+ // breakdown of itself, restating the figure one line lower.
+ detail={
+ backfillKinds.length > 1 ? (
+ <>
+ {backfillKinds.map((k) => (
+ <span
+ key={k.id}
+ aria-label={`backfill lane kind ${k.id}`}
+ data-kind={k.id}
+ >
+ <span className="font-medium">{k.label}</span>{" "}
+ <KindCounts
+ reachable={k.reachable}
+ needsMedia={k.needsMedia}
+ blocked={k.blocked}
+ deferred={k.deferred}
+ />
+ </span>
+ ))}
+ </>
+ ) : undefined
+ }
note={
backfillSweeping && !laneEnabled ? (
// Kept in words as well as in the rail. A sweep armed with the lane
@@ -375,3 +535,89 @@ export function LaneDeck({
</div>
);
}
+
+// One backlog: how many videos, across how many channels that actually have any.
+// Null when the payload is absent, which is what drops the line.
+type Backlog = { videos: number; channels: number } | null;
+
+function sumBacklog(
+ actionable: WidgetActionablePayload | null,
+ pick: (c: WidgetActionablePayload["channels"][number]) => number,
+): Backlog {
+ if (!actionable) return null;
+ let videos = 0;
+ let channels = 0;
+ for (const c of actionable.channels) {
+ const n = pick(c);
+ if (n <= 0) continue;
+ videos += n;
+ channels++;
+ }
+ return { videos, channels };
+}
+
+// "12,486 videos awaiting transcription across 31 channels".
+//
+// A backlog of zero is a real, useful answer here — unlike formatCount's dash,
+// which stands for a measurement nobody took. The distinction is that /actionable
+// reported and found nothing, versus not having reported at all.
+//
+// `empty` is its own phrase rather than "nothing " + verb, because the verbs are
+// not all positive: the downloads lane's is "not downloaded", and the composed
+// form reads "nothing not downloaded".
+function BacklogLine({
+ backlog,
+ verb,
+ empty,
+}: {
+ backlog: Backlog;
+ verb: ReactNode;
+ empty: string;
+}): ReactNode {
+ if (!backlog) return null;
+ if (backlog.videos === 0) return <span>{empty}</span>;
+ return (
+ <span>
+ {backlog.videos.toLocaleString()}{" "}
+ {backlog.videos === 1 ? "video" : "videos"} {verb} across{" "}
+ {backlog.channels.toLocaleString()}{" "}
+ {backlog.channels === 1 ? "channel" : "channels"}
+ </span>
+ );
+}
+
+// One kind's numbers: "329 · 77,134 need media".
+//
+// THESE ARE NEVER SUMMED — not with each other, and not across kinds. Reachable
+// diarization work is 329 audio passes; reachable attribution-text work is
+// ~77,539 videos at about one model call per transcript CHUNK; `needsMedia` is
+// gated behind an opt-in re-download and `blocked` behind another kind finishing.
+// Four different units and four different things an operator would have to do, so
+// a total of any two of them means nothing. This is the same rule
+// backfillKinds.ts states for `missing` vs `missing-input`, one level down.
+//
+// Each clause is omitted at 0, so a kind with nothing outstanding says so in
+// words rather than showing a row of zeros.
+function KindCounts({
+ reachable,
+ needsMedia,
+ blocked,
+ deferred,
+}: {
+ reachable: number;
+ needsMedia: number;
+ blocked: number;
+ deferred: number;
+}): ReactNode {
+ const clauses: ReactNode[] = [];
+ // The bare number is the reachable one — what the lane can do today, which is
+ // the figure this whole card leads with.
+ if (reachable > 0) clauses.push(reachable.toLocaleString());
+ if (needsMedia > 0) {
+ clauses.push(`${needsMedia.toLocaleString()} need media`);
+ }
+ if (blocked > 0) clauses.push(`${blocked.toLocaleString()} blocked`);
+ if (deferred > 0) clauses.push(`${deferred.toLocaleString()} deferred`);
+ if (clauses.length === 0) return <>nothing outstanding</>;
+ return <>{clauses.join(" · ")}</>;
+}
diff --git a/editor/app/widget/components/MonitorWidget.tsx b/editor/app/widget/components/MonitorWidget.tsx
@@ -186,6 +186,8 @@ export function MonitorWidget({
confirmSyncAll={config.syncConfirm}
workersData={workersPayload}
syncData={syncData}
+ jobsData={jobsPayload}
+ actionableData={actionablePayload}
onWorkersChange={refetchWorkers}
onSynced={refetchSync}
/>
diff --git a/editor/app/widget/components/WidgetControls.tsx b/editor/app/widget/components/WidgetControls.tsx
@@ -2,6 +2,8 @@
import { useState } from "react";
import type { WidgetSyncPayload } from "../../api/widget/sync/route";
+import type { WidgetActionablePayload } from "../../api/widget/actionable/route";
+import type { ActiveJobsPayload } from "../../jobs/active/buildActiveJobs";
import type { WorkersPayload } from "../../workers/components/WorkersView";
import { LaneDeck } from "../../components/lanes/LaneDeck";
import { DrainAllButton } from "../../jobs/components/DrainAllButton";
@@ -30,6 +32,8 @@ export function WidgetControls({
confirmSyncAll,
workersData,
syncData,
+ jobsData,
+ actionableData,
onWorkersChange,
onSynced,
}: {
@@ -44,6 +48,12 @@ export function WidgetControls({
// widget's 15s-floor poll, and a click refetches immediately.
workersData: WorkersPayload | null;
syncData: WidgetSyncPayload | null;
+ // The lanes' DETAIL lines read these two. They ride the widget's own section
+ // flags (`jobs`/`disk` and `actionable`), so in a widget pinned without them
+ // they are null — and a null drops the line rather than rendering a zero,
+ // which is the rule LaneDeck states.
+ jobsData: ActiveJobsPayload | null;
+ actionableData: WidgetActionablePayload | null;
onWorkersChange: () => void | Promise<void>;
onSynced: () => void | Promise<void>;
}) {
@@ -62,6 +72,8 @@ export function WidgetControls({
<LaneDeck
workers={workersData}
sync={syncData}
+ jobs={jobsData}
+ actionable={actionableData}
onWorkersChange={onWorkersChange}
onSynced={onSynced}
/>
diff --git a/editor/e2e/backfill.spec.ts b/editor/e2e/backfill.spec.ts
@@ -800,4 +800,24 @@ test("a digest entry in the snapshot does not move the backfill instrument", asy
const body = await sync.json();
expect(body.backfill.reachable).toBe(1);
expect(body.backfill.needsMedia).toBe(1);
+ // And the PER-KIND breakdown the lane card renders is filtered by the same
+ // rule as the sums it splits. This is the half a sum cannot guard: a
+ // breakdown built off Object.entries would list a "Digest" row on the backfill
+ // card while `reachable` above stayed correct, so the card would contradict
+ // its own figure.
+ const kindIds = (body.backfill.kinds as { id: string }[]).map((k) => k.id);
+ expect(kindIds).not.toContain("digest");
+ expect(kindIds).toContain("diarization");
+ // Off the same entries as the sums, so the two can never disagree.
+ const kinds = body.backfill.kinds as {
+ id: string;
+ reachable: number;
+ needsMedia: number;
+ }[];
+ expect(kinds.reduce((n, k) => n + k.reachable, 0)).toBe(
+ body.backfill.reachable,
+ );
+ expect(kinds.reduce((n, k) => n + k.needsMedia, 0)).toBe(
+ body.backfill.needsMedia,
+ );
});
