commit 5215b918d701fd00c025feee84149853dad83771 parent 1184ba988be42eb9907163068e66f7623774da84 Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st> Date: Sat, 22 Aug 2026 19:24:16 -0400 naming: one word was three things, and hiding a fourth "Backfill" named a lane, a per-channel action, and — on the download stage — an unrelated subtitle DOWNLOAD job. It is a queue key; nobody arms, pauses or runs one. Every surface where an operator reads a FIGURE now reads an operation name. The transit-line station stopped summing three operations into one coverage number. On the live corpus that was adding diarization (one audio pass per video, 4 done of 11,338) to attribution-text (~1 model call per transcript CHUNK, 1 done of 11,338) and printing the result under the queue's name. The numeral belongs to one operation now; the others state themselves in the foot, each with its own band and its own denominator. The helper that made the sum is deleted, not just unused. The lane name survives in exactly one place — the settings fieldset and the lane card, where the one shared pause and the one shared sweep live — and there it lists its members, because "these share a queue and a pause" is load-bearing: the sweep and the pause are two controls, and conflating them is how an operator loses a week of GPU time. And the fact that hid: attribution-text is ON, reaches 11,337 videos of one channel (~194,000 model calls corpus-wide), has completed ONE video, and read as a quiet row everywhere — because "11,337 reachable" is the same shape of number whether the unit is an audio pass or a per-chunk model call. costBasis is now declared per operation and printed beside its backlog wherever it is armed. No threshold, no editorialising, and no setting is changed. Specs move only where visible copy moved. The internal addressing hooks (aria-label="backfill section", "backfill kind <id>", the widget's region name) are untouched — they confuse nobody and changing them is churn with no reader. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Diffstat:
26 files changed, 669 insertions(+), 131 deletions(-)
diff --git a/common/lib/backfillKinds.test.ts b/common/lib/backfillKinds.test.ts @@ -13,6 +13,7 @@ import { operationCostBasis, operationGroup, operationLabel, + operationsActionLabel, operationsGroupLabel, digestLaneFor, diarizationLaneFor, @@ -1246,6 +1247,23 @@ test("a set's label is derived, so a mixed lane cannot claim one member's name", assert.equal(operationsGroupLabel(["digest", "no-such-op"]), "Digest"); }); +test("a group has a station name AND a name you can put a verb in front of", () => { + // "Run speakers work" is why these are two declarations rather than one + // lower-cased derivation. The station eyebrow needs a noun; the button needs + // an object. + assert.equal( + operationsActionLabel([ + "diarization", + "attribution-diarized", + "attribution-text", + ]), + "speaker work", + ); + assert.equal(operationsActionLabel(["digest"]), "digests"); + assert.equal(operationsActionLabel([]), "derived data"); + assert.equal(operationsActionLabel(["diarization", "digest"]), "derived data"); +}); + test("attribution-text's cost basis states the CHUNK unit, not the video", () => { // The measured fact that hid behind the word "backfill": this lane's unit is // the transcript chunk, so its 11,337 reachable videos are on the order of diff --git a/common/lib/backfillKinds.ts b/common/lib/backfillKinds.ts @@ -1054,6 +1054,37 @@ export function operationCostBasis(id: string): string { // DERIVED, never hardcoded, which is the point: a lane that gains a kind from a // different group degrades to the honest generic name instead of continuing to // advertise a label that now describes two thirds of what it holds. +// The same set, named as a THING AN OPERATOR RUNS rather than as a stage on a +// line — "Run speaker work", "N videos are waiting on digests". +// +// Two labels for one group is not duplication: "Speakers" is a station on a +// transit line and has to be a noun at eyebrow width; "speaker work" is the +// object of a verb and has to survive being lower-cased into a sentence. The +// alternative — deriving one from the other — produces "Run speakers work", +// which is why this is declared. +export function groupActionLabel(group: OperationGroup): string { + switch (group) { + case "media": + return "downloads"; + case "transcript": + return "transcripts"; + case "digest": + return "digests"; + case "speakers": + return "speaker work"; + } +} + +export function operationsActionLabel(ids: ReadonlyArray<string>): string { + const groups = new Set<OperationGroup>(); + for (const id of ids) { + const group = operationGroup(id); + if (group) groups.add(group); + } + if (groups.size !== 1) return "derived data"; + return groupActionLabel([...groups][0]); +} + export function operationsGroupLabel(ids: ReadonlyArray<string>): string { const groups = new Set<OperationGroup>(); for (const id of ids) { diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog ## [Unreleased] +- **One word was standing in for three different things, and hiding a fourth.** "Backfill" named a *lane* (three operations sharing a CPU queue), a *per-channel action*, and — on the download stage — a completely unrelated **download job** for subtitle tracks. Nobody arms, pauses or runs "a backfill"; it is a queue key. Every screen where an operator reads a **figure** now reads an **operation name**: the channel transit line's station is *Speakers*, its stage panel is *Speaker work · 3 operations* with a *Run speaker work* button, the download stage says *Fetch missing subtitle tracks*, and the lane card and `/auto-queue` name their members. The lane name survives in exactly one place — the settings fieldset and the lane card, where the one shared pause and the one shared sweep live — and there it now **lists what it holds**, because "these operations share a queue and a pause" is a true and load-bearing fact rather than a leaked implementation detail. +- **The transit line's Speakers station stopped summing three operations into one number.** It set its coverage by adding every operation on the lane together — the one thing this codebase forbids everywhere else — which on the live corpus meant adding diarization (one audio pass per video, 4 done of 11,338) to speaker-names-from-the-transcript (**~1 model call per transcript chunk**, 1 done of 11,338) and printing the total under a label that named the queue. The numeral now belongs to exactly one operation, and each member states itself, with its own band and its own denominator, in the station foot. + - **Every pipeline is now visible on /channels, not just two of them.** The table printed two bare integers — `Downloads` and `Transcripts` — and said nothing whatsoever about the four derived pipelines, which were collapsed everywhere else behind the word *backfill*. All six now draw the same **state band** the comparison rail on /auto-queue uses, at table scale: one fill per population, `can run now` the only saturated colour on the page, and pattern (solid / hatched / dotted / hollow) carrying the meaning ahead of hue, because no four-colour palette clears all-pairs colour-blindness. **Not a percent bar, deliberately** — digest sits at 0 done on every large channel and diarization is 99.96% media-gone, so "% complete" renders `0%` on all 68 rows and says nothing; what varies, and what an operator needs, is the *shape* of the remainder. Every pipeline column **sorts by what can run now**, which answers a question the page has never been able to answer: *which channel has the most diarizable audio left right now* previously meant opening 68 channel pages one at a time. The costs nothing extra to draw — `/channels` was already reading every channel's full snapshot for its counts and throwing the rest away. - **`docker compose up -d` now stands up a working archive.** The repo had two Dockerfiles and neither ran the app: one fans per-site export builds out across containers, the other runs sharded e2e. So the only way to host this was to install the whole Unix toolchain by hand, which is why the Windows instructions said "use WSL2 and follow the Linux steps". There is now a runtime image and a compose stack — the editor plus Caddy by default, with the published site, the project homepage and umtool behind compose **profiles**, so somebody who only wants an archive runs two containers rather than five. First boot creates the volumes, downloads a speech model, and seeds a `settings.json` **carrying one enabled worker**: the defaults ship `workers: []`, zero workers means zero transcription slots, and a fresh container that looks healthy and silently transcribes nothing is the worst possible first run. Two things are deliberately not baked into the image and cannot be: the corpus, and the export site — that site is a static render *of* a corpus, and there is no corpus at image-build time, so `docker/publish-site.sh` builds it at run time into the volume the `site` service serves. diff --git a/editor/app/actionable/components/InlineActionButton.tsx b/editor/app/actionable/components/InlineActionButton.tsx @@ -47,7 +47,10 @@ const LABEL: Record<Variant["kind"], { idle: string; running: string }> = { redownloadShortAudio: { idle: "Re-download (corrected format)", running: "Queuing…" }, cleanTranscribedAudio: { idle: "Clean audio", running: "Queuing…" }, cleanExtraFormats: { idle: "Clean extra formats", running: "Queuing…" }, - backfillChannel: { idle: "Backfill", running: "Queuing…" }, + // "Backfill" is a queue key, not a thing anyone asked for. The real name + // depends on which operations are enabled, which only the server knows — so + // this is the fallback and `label` overrides it. See /actionable's page. + backfillChannel: { idle: "Run derived data", running: "Queuing…" }, digestChannel: { idle: "Digest channel", running: "Queuing…" }, refreshReport: { idle: "Refresh report", running: "Refreshing…" }, }; @@ -114,13 +117,19 @@ export function InlineActionButton({ // appears EXACTLY ONCE per page — that restraint is what pays for the channel // line's boldness — so this is a prop rather than a new default. tone = "default", + label, }: { variant: Variant; tone?: "default" | "brand"; + // Overrides the idle label (and with it the accessible name) for an action + // whose real name is resolved on the server — the derived-data lane, whose + // name comes from the operations that are actually enabled. + label?: string; }) { const [status, setStatus] = useState<Status>({ kind: "idle" }); const [pending, startTransition] = useTransition(); - const labels = LABEL[variant.kind]; + const base = LABEL[variant.kind]; + const labels = label ? { ...base, idle: label } : base; const slug = variant.slug; const ariaLabel = `${labels.idle.toLowerCase()} ${slug}`; diff --git a/editor/app/actionable/page.tsx b/editor/app/actionable/page.tsx @@ -2,6 +2,11 @@ import type { Metadata } from "next"; import Link from "next/link"; import { getPaths } from "yt-dlp-transcript-common/lib/paths"; import { formatBytes } from "yt-dlp-transcript-common/lib/format"; +import { getSettings } from "yt-dlp-transcript-common/lib/settings"; +import { + laneBackfillKinds, + operationsActionLabel, +} from "yt-dlp-transcript-common/lib/backfillKinds"; import { actionableCleanExtraFormatsBytes, actionableCleanExtraFormatsCount, @@ -63,6 +68,12 @@ type SectionConfig = { export default async function ActionablePage() { const paths = getPaths(); const summary = await loadActionableSummary(paths); + // What the derived-data lane is actually called, given which operations are + // enabled. "Backfill" is its queue key; on this install it stands for three + // operations, and no button anyone presses should be named after a queue. + const laneAction = `Run ${operationsActionLabel( + laneBackfillKinds(getSettings()).map((k) => k.id), + )}`; const nothingPending = summary.undownloaded.length === 0 && summary.missingNeverFetched.length === 0 && @@ -194,7 +205,7 @@ export default async function ActionablePage() { id: "backfill", title: "Channels missing derived data the corpus predates", description: - "Videos an enabled backfill feature has nothing on disk for — no record, or one produced by a different engine/model/threshold than the current settings. Run \u201cBackfill\u201d to catch them up. The count is what the lane can do TODAY; the second column is the separate population whose source media has already been deleted, which needs the opt-in re-download to reach at all.", + `Videos an enabled derived-data operation has nothing on disk for — no record, or one produced by a different engine/model/threshold than the current settings. Run \u201c${laneAction}\u201d to catch them up. The count is what the lane can do TODAY; the second column is the separate population whose source media has already been deleted, which needs the opt-in re-download to reach at all.`, countLabel: "to backfill", emptyLabel: "Nothing pending.", getCount: actionableBackfillCount, @@ -209,6 +220,10 @@ export default async function ActionablePage() { primaryAction: (r) => ( <InlineActionButton variant={{ kind: "backfillChannel", slug: r.channel.slug }} + // Named after the operations that are actually on, resolved here on + // the server. The button used to read "Backfill", which is the + // queue key three unrelated operations happen to share. + label={laneAction} /> ), }, diff --git a/editor/app/api/widget/sync/route.ts b/editor/app/api/widget/sync/route.ts @@ -7,7 +7,9 @@ import { getSettings } from "yt-dlp-transcript-common/lib/settings"; import { laneBackfillKinds, laneKindEntriesOf, + operationCostBasis, operationLabel, + operationsGroupLabel, presentBackfillWork, reachableBackfillWork, } from "yt-dlp-transcript-common/lib/backfillKinds"; @@ -96,6 +98,14 @@ export type WidgetSyncPayload = { // lane: with no feature on there is nothing to report at all, and that must // not look like "all caught up". anyKind: boolean; + // The lane's name in the operator's terms, derived on the server from the + // GROUP its enabled kinds declare — "Speakers" today. The client must not + // resolve this itself: backfillKinds.ts reaches the filesystem. + // + // "Backfill" is a queue key. Nobody arms, pauses or runs "a backfill"; the + // word survives only on the two controls that genuinely act on the shared + // queue, and even there the card now lists what is in it. + groupLabel: string; // The same numbers, kept apart by kind — which is the only form of them that // means anything. Summed, this lane reads "77,952 reachable · 77,134 need // media": both correct, and together a figure in no unit, since 99.5% of the @@ -119,6 +129,12 @@ export type WidgetSyncPayload = { kinds: { id: string; label: string; // operationLabel(id) — resolved here, not in the client + // What ONE UNIT of this operation costs, in words. The fact that hid + // behind the lane name: attribution-text is armed on ~194,000 model calls + // corpus-wide because its unit is the transcript CHUNK, has completed one + // video, and read as a quiet row on every screen. Stated flat, with no + // threshold — what is affordable is the operator's call. + costBasis: string; reachable: number; needsMedia: number; blocked: number; @@ -279,8 +295,16 @@ export async function buildWidgetSyncPayload(): Promise<WidgetSyncPayload> { enabled: settings.backfill.enabled, sweeping: settings.backfill.sweepEnabled, anyKind: laneBackfillKinds(settings).length > 0, + groupLabel: operationsGroupLabel( + laneBackfillKinds(settings).map((k) => k.id), + ), kinds: [...backfillByKind] - .map(([id, counts]) => ({ id, label: operationLabel(id), ...counts })) + .map(([id, counts]) => ({ + id, + label: operationLabel(id), + costBasis: operationCostBasis(id), + ...counts, + })) // Biggest reachable first, so the kind an operator can act on most leads. // Tie-broken by id so the order does not shuffle between polls. .sort((a, b) => b.reachable - a.reachable || a.id.localeCompare(b.id)), diff --git a/editor/app/auto-queue/components/AutoQueueView.tsx b/editor/app/auto-queue/components/AutoQueueView.tsx @@ -24,7 +24,7 @@ import { leafOrder, } from "./dispatch"; import { deriveLaneState } from "../../components/lanes/laneState"; -import type { SweepLaneStatus } from "../lanes"; +import type { AutoQueueLanesPayload, SweepLaneStatus } from "../lanes"; // THE CONSOLE. One grammar for four pipelines. // @@ -61,12 +61,20 @@ import type { SweepLaneStatus } from "../lanes"; type LaneId = AutoQueueKind | "digest" | "backfill"; -const LANE_OPTIONS: { id: LaneId; label: string }[] = [ - { id: "transcription", label: "Auto-transcribe" }, - { id: "download", label: "Auto-download" }, - { id: "digest", label: "Digest" }, - { id: "backfill", label: "Backfill" }, -]; +// The two RUNNER lanes have fixed names; the two sweep-fed ones are named after +// the operations they hold, which the server resolves and puts on the payload. +// A hardcoded "Backfill" here would be a fourth copy of a list that already +// drifted once. +function laneOptions( + lanes: AutoQueueLanesPayload | null, +): { id: LaneId; label: string }[] { + return [ + { id: "transcription", label: "Auto-transcribe" }, + { id: "download", label: "Auto-download" }, + { id: "digest", label: lanes?.digest.label ?? "Digest" }, + { id: "backfill", label: lanes?.backfill.label ?? "Derived data" }, + ]; +} export function AutoQueueView({ initial, @@ -138,7 +146,7 @@ export function AutoQueueView({ value={lane} onChange={(e) => setLane(e.target.value as LaneId)} > - {LANE_OPTIONS.map((o) => ( + {laneOptions(data.lanes).map((o) => ( <option key={o.id} value={o.id}> {o.label} </option> diff --git a/editor/app/auto-queue/components/OperationRail.tsx b/editor/app/auto-queue/components/OperationRail.tsx @@ -137,6 +137,22 @@ function RailRow({ ? "coverage unknown" : `${(coverage * 100).toFixed(coverage < 0.1 ? 2 : 0)}% of ${denominator?.toLocaleString()}`} </span> + {/* WHAT THIS BACKLOG COSTS, only where there IS one and only for an + operation this system actually dispatches. The sentence the console + was missing: attribution-text sits at 77,000-odd reachable on a lane + whose unit is the transcript CHUNK — the order of 194,000 model + calls — and read exactly like diarization's 647 audio passes. + Stated flat. A "this is a lot" threshold would be a magic number the + next operation gets wrong, and what is affordable is not this page's + call to make. */} + {band.dispatched && band.reachable > 0 && band.costBasis && ( + <span + aria-label={`${band.label} cost`} + className="italic text-muted-foreground/80" + > + {band.costBasis} + </span> + )} </span> </li> ); diff --git a/editor/app/auto-queue/lanes.ts b/editor/app/auto-queue/lanes.ts @@ -1,7 +1,11 @@ import { getPaths } from "yt-dlp-transcript-common/lib/paths"; import { getSettings } from "yt-dlp-transcript-common/lib/settings"; import type { AutoQueueOrder, AutoQueueReach } from "yt-dlp-transcript-common/jobs/autoQueuePolicy"; -import { allBackfillKinds } from "yt-dlp-transcript-common/lib/backfillKinds"; +import { + allBackfillKinds, + operationsActionLabel, + operationsGroupLabel, +} from "yt-dlp-transcript-common/lib/backfillKinds"; import { getDigestSweepJobId } from "yt-dlp-transcript-common/controller/digestSweep"; import { getBackfillSweepJobId } from "yt-dlp-transcript-common/controller/backfillSweep"; import { getRegistry } from "yt-dlp-transcript-common/jobs/registry"; @@ -129,7 +133,15 @@ export async function buildAutoQueueLanes(): Promise<AutoQueueLanesPayload> { }, backfill: { id: "backfill", - label: "Backfill", + // NAMED AFTER WHAT IT HOLDS. "Backfill" is this lane's queue key, and on + // this install it stands for three operations with different inputs and + // different costs. Derived, so a lane that gains a kind from another + // group degrades to "Derived data" rather than going stale. + label: operationsGroupLabel( + kinds + .filter((k) => k.lane.queueKey === BACKFILL_QUEUE) + .map((k) => k.id), + ), sweeping: settings.backfill.sweepEnabled, // INVERTED: backfill's gate is `enabled`, where digest's is `paused`. // Normalised here rather than at every reader, exactly as LaneDeck does. diff --git a/editor/app/channels/[slug]/components/flow/FlowStation.tsx b/editor/app/channels/[slug]/components/flow/FlowStation.tsx @@ -1,6 +1,14 @@ import Link from "next/link"; import { Progress } from "yt-dlp-transcript-common/components/ui/progress"; -import type { FlowStation as Station } from "../../lib/channelFlow"; +import type { + FlowStation as Station, + StationOperation, +} from "../../lib/channelFlow"; +import { + bandHeadline, + bandSentence, + StateBand, +} from "../../../../components/pipelines/StateBand"; import { formatCount, formatCoverage, STATION_DOT } from "./tone"; // One station on the line, emitted as THREE siblings so the parent grid can put @@ -51,7 +59,7 @@ export function StationRail({ station }: { station: Station }) { export function StationFoot({ station }: { station: Station }) { return ( - <div className="flex flex-col gap-1 min-w-[5rem]"> + <div className="flex flex-col gap-1 min-w-[5rem] max-w-[8rem]"> <p aria-label={`${station.label} coverage`} className="text-[11px] text-muted-foreground tabular-nums" @@ -63,9 +71,29 @@ export function StationFoot({ station }: { station: Station }) { "—" : `${formatCoverage(station.coverage)} of ${formatCount(station.denominator)}`} </p> - {station.coverage != null && ( - // Only drawn when there is a real ratio behind it. An empty meter for an - // unknown denominator would say "none of it is done". + {station.operations.length > 0 ? ( + // ONE BAND PER OPERATION, never one bar for the station. + // + // The station used to draw a single meter over a number that summed + // three unrelated pipelines. Its members are three different + // populations in two different units, and the only honest picture is + // three bands — the same instrument, at the smallest of its three + // scales, so a reader coming from /channels or /auto-queue already + // knows what the fills mean. + station.operations.map((op) => ( + <StationPipeline + key={op.id} + operation={op} + // Named only when the station holds more than one. Under Digest the + // name would just repeat the eyebrow a line above; under Speakers + // it is the whole point, because three operations were hiding + // behind one word and this is where they are finally listed. + named={station.operations.length > 1} + /> + )) + ) : station.coverage != null ? ( + // The two stations that are not registry operations — playlist and + // transcode — keep the plain meter. // // aria-hidden, and no label: the <p> above states the same ratio in // words, so the bar is decorative. Naming it "<station> coverage meter" @@ -76,7 +104,48 @@ export function StationFoot({ station }: { station: Station }) { aria-hidden="true" className="h-1 max-w-[7rem]" /> - )} + ) : null} + </div> + ); +} + +// One pipeline under a station: its name, its band, and the one population that +// dominates what is left. +// +// The name is only drawn when the station holds MORE THAN ONE — under Digest it +// would just repeat the station eyebrow a line above. Under Speakers it is the +// whole point: three operations were hiding behind one word, and this is where +// they are finally listed. +function StationPipeline({ + operation, + named, +}: { + operation: StationOperation; + named: boolean; +}) { + const sentence = bandSentence(operation.band); + return ( + <div + className="flex flex-col gap-0.5" + // The exact figures, and the unit one of them is counted in. Never summed + // and never rounded — the band is the shape, this is the arithmetic. + title={`${operation.label} — ${sentence}. ${operation.costBasis}.`} + > + <span className="sr-only"> + {operation.label}: {sentence}. {operation.costBasis}. + </span> + <span + aria-hidden="true" + className="flex items-baseline gap-1 text-[10px] leading-none text-muted-foreground" + > + {named && ( + <span className="font-mono uppercase tracking-[0.1em]"> + {operation.shortLabel} + </span> + )} + <span className="tabular-nums">{bandHeadline(operation.band)}</span> + </span> + <StateBand band={operation.band} size="station" className="max-w-[7rem]" /> </div> ); } diff --git a/editor/app/channels/[slug]/components/stages/BackfillStage.tsx b/editor/app/channels/[slug]/components/stages/BackfillStage.tsx @@ -1,8 +1,15 @@ "use client"; -// The channel-level view of the backfill lane: what is reachable, what would +// The channel-level view of one derived-data lane: what is reachable, what would // need its media re-acquired, and a button to run it. // +// IT NAMES THE WORK, NOT THE QUEUE. This card used to be headed "Backfill +// derived data" with a button reading "Backfill channel", and on this install +// that one word was standing in for THREE operations with different inputs, +// different costs and different reasons to be armed — a fact no screen could +// state. The heading now comes from the group its kinds declare, and every +// kind's own row states what one unit of it costs. +// // Modelled on DigestStage — same bucket-count heading, same StreamActionLog + // QueueControl shape — so the stage rail reads as one system. // @@ -46,6 +53,10 @@ export type BackfillKindView = { // dependsOn on the server so the copy can name them. dependsOnLabels: string[]; stale: number; + // What one unit of this operation costs, from the registry. Printed on the + // kind's own row, because an armed operation with a five-figure backlog and + // a per-CHUNK unit reads exactly like a quiet one without it. + costBasis: string; }; type Props = { @@ -56,6 +67,12 @@ type Props = { // settings.backfill.allowRedownload — surfaced because it is the difference // between the sub-line being informational and being actionable. allowRedownload: boolean; + // What this lane's operations are, as a group — "Speakers" for the eyebrow, + // "speaker work" for the button. Derived on the server from the registry, so + // a lane holding a mix of groups says "derived data" rather than naming one + // member. See operationsGroupLabel / operationsActionLabel. + groupLabel: string; + actionLabel: string; // No backfill feature is switched on at all. The card still renders (the rail // is fixed) but says so rather than reporting an empty work list as "done". anyEnabled: boolean; @@ -68,7 +85,13 @@ export function BackfillStage({ kinds, allowRedownload, anyEnabled, + groupLabel, + actionLabel, }: Props) { + // "speaker work" → "Speaker work". Sentence case, not title case: it is a + // phrase, and "Speaker Work" is a proper noun this system does not have. + const heading = actionLabel.charAt(0).toUpperCase() + actionLabel.slice(1); + const runLabel = `Run ${actionLabel}`; const [queue, setQueue] = useState(defaultQueueKey); const reachable = kinds.reduce((n, k) => n + k.reachableIds.length, 0); @@ -94,12 +117,25 @@ export function BackfillStage({ <div aria-label="backfill section" className="flex flex-col gap-3"> <div> <h3 className="text-base font-semibold"> - Backfill derived data ({reachable}) + {heading} · {kinds.length}{" "} + {kinds.length === 1 ? "operation" : "operations"} </h3> - <p className="text-sm text-muted-foreground"> - {anyEnabled - ? "Catch-up for derived data this channel's videos predate. A re-run does only what is still missing or stale, so running it twice costs nothing the second time." - : "No backfill is enabled. Turn one on in Settings and this card will report what the existing corpus is missing."} + <p + aria-label="backfill reachable" + className="text-sm text-muted-foreground" + > + {anyEnabled ? ( + <> + {reachable.toLocaleString()}{" "} + {reachable === 1 ? "video" : "videos"} can be worked on now. + {kinds.length > 1 + ? " These operations share one queue and one pause, which is the only sense in which they are one lane." + : ""} A re-run does only what is still missing or stale, so running + it twice costs nothing the second time. + </> + ) : ( + "Nothing here is enabled. Turn an operation on in Settings and this card will report what the existing corpus is missing." + )} </p> {missingInput > 0 && ( <p @@ -145,21 +181,32 @@ export function BackfillStage({ )} </div> - {kinds.length > 1 && - kinds.map((k) => ( - <p - key={k.id} - aria-label={`backfill kind ${k.id}`} - className="text-xs text-muted-foreground" - > - <span className="font-medium">{k.label}</span>:{" "} - {k.reachableIds.length} reachable - {k.stale > 0 && ` (${k.stale} stale)`} ·{" "} - {k.missingInput.toLocaleString()} needing media - {k.deferred > 0 && ` · ${k.deferred.toLocaleString()} deferred`} - {k.blocked > 0 && ` · ${k.blocked.toLocaleString()} blocked`} - </p> - ))} + {/* ONE ROW PER OPERATION, always — not only when there are several. + A single-operation lane still has to say WHICH operation, because the + card above it no longer does: the heading names a group. */} + {kinds.map((k) => ( + <p + key={k.id} + aria-label={`backfill kind ${k.id}`} + className="text-xs text-muted-foreground" + > + <span className="font-medium">{k.label}</span>:{" "} + {k.reachableIds.length} reachable + {k.stale > 0 && ` (${k.stale} stale)`} ·{" "} + {k.missingInput.toLocaleString()} needing media + {k.deferred > 0 && ` · ${k.deferred.toLocaleString()} deferred`} + {k.blocked > 0 && ` · ${k.blocked.toLocaleString()} blocked`} + {/* WHAT ONE UNIT COSTS, beside the backlog and never as a judgement. + 11,337 reachable videos means ~194,000 model calls when the unit + is the transcript chunk and ~11,337 when it is the video, and no + other line on this page can tell you which. Stated flat: a "this + is a lot" threshold would be a magic number the next operation + gets wrong, and what is affordable is the operator's call. */} + {k.costBasis && ( + <span className="italic"> — {k.costBasis}</span> + )} + </p> + ))} <VideoIdList slug={slug} @@ -168,8 +215,8 @@ export function BackfillStage({ emptyAriaLabel="videos needing a backfill empty" emptyMessage={ anyEnabled - ? "Nothing reachable to backfill." - : "No backfill is enabled." + ? `Nothing reachable for ${actionLabel}.` + : "Nothing here is enabled." } itemAriaLabel={(id) => `video needing a backfill ${id}`} /> @@ -177,16 +224,16 @@ export function BackfillStage({ <StreamActionLog trigger={() => backfillChannelAction(slug, queue)} cancelAction={cancelJobAction} - buttonLabel="Backfill channel" - runningLabel="Backfilling…" - label="Backfill channel" + buttonLabel={runLabel} + runningLabel="Running…" + label={runLabel} extraControls={ <QueueControl value={queue} onChange={setQueue} defaultQueueKey={defaultQueueKey} existingQueues={existingQueues} - actionLabel="Backfill channel" + actionLabel={runLabel} /> } /> diff --git a/editor/app/channels/[slug]/components/stages/DownloadStage.tsx b/editor/app/channels/[slug]/components/stages/DownloadStage.tsx @@ -175,7 +175,11 @@ export function DownloadStage({ <div className="flex flex-col gap-2"> <Heading title="Download missing subs" - desc="Backfill sub tracks (live_chat, alt-language captions) for already-downloaded videos. Reads each video's metadata.info.json for advertised tracks, then re-invokes yt-dlp with --skip-download for any whose expected sub file is missing on disk. Uses the channel's subLangs config (defaults to en.*,live_chat)." + // "Backfill" here meant a DOWNLOAD JOB — a third, unrelated sense of + // the word, on a page where the other two were a lane and a + // per-channel action. Nothing about this shares a queue, a pause or a + // counter with the speaker lane. + desc="Fetch missing subtitle tracks (live_chat, alt-language captions) for already-downloaded videos. Reads each video's metadata.info.json for advertised tracks, then re-invokes yt-dlp with --skip-download for any whose expected sub file is missing on disk. Uses the channel's subLangs config (defaults to en.*,live_chat)." /> <label className="flex items-center gap-2 text-sm"> <input diff --git a/editor/app/channels/[slug]/lib/channelFlow.test.ts b/editor/app/channels/[slug]/lib/channelFlow.test.ts @@ -97,13 +97,22 @@ test("a pre-`eligible` snapshot reports unknown coverage, not zero", () => { assert.equal(station(flow, "backfill").coverage, null); }); -test("one unknown term poisons the whole coverage sum rather than under-reporting", () => { +test("the lane station does NOT sum its operations — it reads the lead one", () => { + // THE BUG THIS STATION USED TO BE. `through` and `denominator` were the sum + // across every kind on the lane, which on the live corpus added diarization's + // coverage (one audio pass per video, 4 done of 11,338) to attribution-text's + // (~1 model call per transcript CHUNK, 1 done of 11,338) and printed the + // result under a label that named the queue. Two different populations in two + // different units, added, and no screen said so. + // + // The numeral now belongs to exactly ONE operation — the first in dependency + // order — and the rest state themselves separately in the station foot. const flow = flowOf( snapshotOf({ backfill: { diarization: entry({ eligible: 10 }), - // Same lane, no `eligible` — written by an older build. - "attribution-diarized": entry({}), + // Same lane, a different population. Nothing may fold it in. + "attribution-diarized": entry({ eligible: 1000, missing: 400 }), }, }), { @@ -114,8 +123,57 @@ test("one unknown term poisons the whole coverage sum rather than under-reportin }, ); - assert.equal(station(flow, "backfill").through, null); - assert.equal(station(flow, "backfill").denominator, null); + const lane = station(flow, "backfill"); + assert.equal(lane.through, 10); + assert.equal(lane.denominator, 10); + // The sum would be 1,010. Asserting the negative is the point. + assert.notEqual(lane.denominator, 1010); + // Both operations are carried, each with its own band and its own + // denominator, so nothing is hidden by not being summed. + assert.deepEqual( + lane.operations.map((o) => o.id), + ["diarization", "attribution-diarized"], + ); + assert.equal(lane.operations[1].band.eligible, 1000); + assert.equal(lane.operations[1].band.reachable, 400); +}); + +test("the lane station is named after its operations, not its queue key", () => { + // "Backfill" is a scheduler key. An operator cannot arm, pause or run "a + // backfill" — they can run speaker work. The label is DERIVED from the group + // its kinds declare, so a lane holding a mix degrades to the generic name + // rather than advertising one member's. + const speakers = flowOf(snapshotOf(), { + backfillKinds: [ + kind({ id: "diarization" }), + kind({ id: "attribution-text" }), + ], + }); + assert.equal(station(speakers, "backfill").label, "Speakers"); + + const mixed = flowOf(snapshotOf(), { + backfillKinds: [kind({ id: "diarization" }), kind({ id: "digest" })], + }); + assert.equal(station(mixed, "backfill").label, "Derived data"); + + // Nothing enabled: the generic name, and no operations to state. + const off = flowOf(snapshotOf(), { backfillKinds: [] }); + assert.equal(station(off, "backfill").label, "Derived data"); + assert.deepEqual(station(off, "backfill").operations, []); +}); + +test("an unknown `eligible` on the lead operation still renders unknown, not zero", () => { + // Invariant 2, at the one station whose denominator moved. A snapshot written + // before `eligible` existed has work counts and no denominator, and the + // station must say "—" rather than 0%. + const flow = flowOf( + snapshotOf({ backfill: { diarization: entry({ missing: 3 }) } }), + { backfillKinds: [kind({ id: "diarization" })] }, + ); + const lane = station(flow, "backfill"); + assert.equal(lane.through, null); + assert.equal(lane.denominator, null); + assert.equal(lane.coverage, null); }); test("coverage is a real ratio once the snapshot can say", () => { diff --git a/editor/app/channels/[slug]/lib/channelFlow.ts b/editor/app/channels/[slug]/lib/channelFlow.ts @@ -7,11 +7,16 @@ import { import { digestCountOf } from "yt-dlp-transcript-common/controller/channels"; import { laneEntriesOf, + operationCatalog, operationLabel, + operationsActionLabel, + operationsGroupLabel, presentBackfillWork, reachableBackfillWork, type BackfillKind, } from "yt-dlp-transcript-common/lib/backfillKinds"; +import type { OperationBand } from "../../../components/pipelines/band"; +import { buildChannelBands } from "../../../components/pipelines/buildBands"; import { normalizeBuckets, type StageId, @@ -44,6 +49,16 @@ import { // 2. UNKNOWN IS NOT ZERO. A third of the snapshots on disk predate `eligible`, // so `present`/`coverage` are `number | null` and a reader must render "—". // A 0 there reads as "nothing digested" on a fully digested channel. +// +// AND THE ONE THIS FILE USED TO BREAK. The `backfill` station set `through` and +// `denominator` by SUMMING THREE OPERATIONS — the one thing invariant 1 forbids +// everywhere else, hidden behind a station label that named the queue rather +// than the work. On the live corpus it was adding audio passes (diarization: 4 +// done of 11,338) to per-chunk model calls (attribution-text: 1 done of 11,338, +// and its unit is the transcript CHUNK, not the video), and calling the result +// "Backfill". A station now carries its group's OPERATIONS, each with its own +// band and its own denominator, and the numeral above them belongs to exactly +// one of them — see leadOf. export type FlowStationId = | "playlist" @@ -53,8 +68,24 @@ export type FlowStationId = | "digest" | "backfill"; +// One pipeline drawn under a station. The band is the same instrument the +// /channels strip and the /auto-queue rail draw, at station scale — which is +// what makes a figure here and a figure there impossible to disagree. +export type StationOperation = { + id: string; + label: string; + shortLabel: string; + // What one unit costs, in words. Printed wherever the operation is armed, so + // an 11,337-video backlog of per-chunk model calls cannot read as a quiet row. + costBasis: string; + band: OperationBand; +}; + export type FlowStation = { id: FlowStationId; + // DERIVED for a station that holds several operations, never hardcoded: a + // lane holding a mix of groups falls back to "Derived data" rather than + // advertising one member's name. See operationsGroupLabel. label: string; // Videos that have cleared this station. NULL when the snapshot cannot say — // see invariant 2 above. Only the digest and backfill stations can be null; @@ -69,6 +100,10 @@ export type FlowStation = { tone: StageTone; // Which stage panel this station opens (?stage=). stage: StageId; + // The pipelines that run at this station, in dependency order. Empty for + // playlist and transcode, which are not operations the registry dispatches or + // counts — they keep the plain coverage meter. + operations: StationOperation[]; }; // A population that has LEFT the line: it is not work the lane can pick up, and @@ -138,6 +173,50 @@ function siding( return count > 0 ? [{ label, count, stage, ...(hint ? { hint } : {}) }] : []; } +// THE PIPELINES DRAWN UNDER EACH STATION, off the registry. +// +// The band for a channel is the same fold over the same snapshot the corpus +// rail uses, so a channel figure and a corpus figure cannot disagree about what +// "downloaded" or "reachable" means. Everything else here — the label, the +// column-width label, the cost basis — is read from the operation catalog +// rather than restated, which is what stops this file drifting from the two +// other surfaces that group the same operations. +function stationOperations( + snapshot: ChannelSnapshot, + laneKindIds: ReadonlyArray<string>, +): Map<string, StationOperation> { + const catalog = new Map(operationCatalog().map((o) => [o.id, o])); + const bands = buildChannelBands(snapshot, laneKindIds); + const out = new Map<string, StationOperation>(); + for (const band of bands) { + const op = catalog.get(band.id); + if (!op) continue; + out.set(band.id, { + id: band.id, + label: op.label, + shortLabel: op.shortLabel, + costBasis: op.costBasis, + band, + }); + } + return out; +} + +// The operation whose coverage the station's big numeral belongs to: the FIRST +// in dependency order, which is the one every other member of the group either +// consumes or runs beside. +// +// Explicitly NOT a sum, and not an average either. The three speaker operations +// are three different populations measured in two different units — one audio +// pass per video against ~1 model call per transcript chunk — and any single +// figure over all three is the mistake this station used to make. One member +// owns the numeral; the rest state themselves, separately, in the foot. +function leadOf( + ops: ReadonlyArray<StationOperation>, +): StationOperation | null { + return ops[0] ?? null; +} + export function computeChannelFlow( input: ComputeChannelFlowInput, ): ChannelFlow { @@ -174,6 +253,17 @@ export function computeChannelFlow( // an empty work list in the "finished" sense, and the tone rule below says so. const laneOff = backfillKinds.length === 0; + const laneKindIds = backfillKinds.map((k) => k.id); + const operationsById = stationOperations(snapshot, laneKindIds); + const opsFor = (...ids: string[]): StationOperation[] => + ids + .map((id) => operationsById.get(id)) + .filter((o): o is StationOperation => o != null); + // The lane's own operations, in dependency order — the group the station is + // named after, and the members its foot states one by one. + const laneOps = laneOff ? [] : opsFor(...laneKindIds); + const laneLead = leadOf(laneOps); + const laneReachable = laneOff ? 0 : laneEntries.reduce((n, e) => n + reachableBackfillWork(e), 0); @@ -183,14 +273,15 @@ export function computeChannelFlow( const laneDeferred = laneEntries.reduce((n, e) => n + (e.deferred ?? 0), 0); const laneBlocked = laneEntries.reduce((n, e) => n + (e.blocked ?? 0), 0); - // presentBackfillWork returns null on a pre-`eligible` snapshot. One null - // poisons the whole sum — deliberately: "3 of the 4 kinds are done and the - // fourth is unknown" is not a number, and rendering the partial sum would - // understate coverage without saying so. - const lanePresent = sumOrNull(laneEntries.map((e) => presentBackfillWork(e))); - const laneEligible = sumOrNull( - laneEntries.map((e) => (e.eligible == null ? null : e.eligible)), - ); + // NO CROSS-OPERATION `present` OR `eligible` SUM LIVES HERE ANY MORE, and the + // helper that made one is gone with it. It used to add diarization's coverage + // to attribution-text's, which is an audio pass plus a per-chunk model call + // over two different populations. The station reads its LEAD operation and + // the foot states each member on its own — see leadOf. + // + // The four WORK counts above are still summed, and legitimately: a siding is + // "how many videos have left the line for this reason", and that reason is + // the same reason whichever operation reported it. // digestCountOf sums `digestEngines`, which 11 of the 65 live snapshots lack // entirely — it returns 0 for those, which would read as "nothing digested". @@ -208,6 +299,7 @@ export function computeChannelFlow( running: stages.playlist.running, tone: stages.playlist.tone, stage: "playlist", + operations: [], }, download: { id: "download", @@ -218,6 +310,7 @@ export function computeChannelFlow( running: stages.download.running, tone: stages.download.tone, stage: "download", + operations: opsFor("download"), }, transcode: { id: "transcode", @@ -234,6 +327,7 @@ export function computeChannelFlow( running: stages.transcode.running, tone: stages.transcode.tone, stage: "transcode", + operations: [], }, transcribe: { id: "transcribe", @@ -244,6 +338,7 @@ export function computeChannelFlow( running: stages.transcribe.running, tone: stages.transcribe.tone, stage: "transcribe", + operations: opsFor("transcription"), }, digest: { id: "digest", @@ -254,19 +349,32 @@ export function computeChannelFlow( running: stages.digest.running, tone: stages.digest.tone, stage: "digest", + operations: opsFor("digest"), }, backfill: { id: "backfill", - label: "Backfill", - through: laneOff ? null : lanePresent, - denominator: laneOff ? null : laneEligible, - coverage: laneOff ? null : ratio(lanePresent, laneEligible), + // NAMES THE WORK, NOT THE QUEUE. "Backfill" is a scheduler key — three + // operations happen to share it — and an operator cannot control, arm or + // pause "a backfill". They can pause speaker work. The name is derived + // from the group its members declare, so a lane that gains a kind from + // another group degrades to "Derived data" instead of lying. + label: operationsGroupLabel(laneKindIds), + // THE NUMERAL BELONGS TO ONE OPERATION, not to a sum of three. See leadOf. + // Read off the band rather than recomputed: the band IS presentBackfill- + // Work over this snapshot, and a second derivation is a second thing that + // can disagree with the strip on /channels. + through: laneLead?.band.present ?? null, + denominator: laneLead?.band.eligible ?? null, + coverage: laneLead + ? ratio(laneLead.band.present, laneLead.band.eligible) + : null, running: stages.backfill.running, // A station whose lane is DISABLED is neutral — never "ok" and never // amber. An empty work list because a feature is off is not the same as // being finished, and colouring it green claims a thing nobody checked. tone: laneOff ? "neutral" : stages.backfill.tone, stage: "backfill", + operations: laneOps, }, }; @@ -432,7 +540,7 @@ export function computeChannelFlow( stations, gaps, bottleneck: biggest ? biggest.from : null, - next: pickNext(gaps), + next: pickNext(gaps, laneKindIds), }; } @@ -442,12 +550,18 @@ export function computeChannelFlow( // start — clearing the upstream gap is what makes the downstream one shrink. // The bottleneck is still reported separately; it is the thing to LOOK at, not // necessarily the thing to press. -function pickNext(gaps: FlowGap[]): ChannelFlow["next"] { +function pickNext( + gaps: FlowGap[], + laneKindIds: ReadonlyArray<string>, +): ChannelFlow["next"] { const ACTIONABLE: Partial<Record<StageId, string>> = { download: "Download missing", transcribe: "Transcribe pending", digest: "Digest channel", - backfill: "Backfill", + // A verb and its object, derived from the group — "Run speaker work". The + // button used to read "Backfill", which is a queue key with no object and + // nothing an operator recognises as a thing they wanted done. + backfill: `Run ${operationsActionLabel(laneKindIds)}`, }; for (const gap of gaps) { if (gap.reachable <= 0) continue; @@ -464,18 +578,6 @@ function pickNext(gaps: FlowGap[]): ChannelFlow["next"] { return null; } -// Null when ANY term is unknown. See invariant 2: a partial sum reported as a -// whole is worse than saying nothing. -function sumOrNull(values: (number | null)[]): number | null { - if (values.length === 0) return null; - let n = 0; - for (const v of values) { - if (v == null) return null; - n += v; - } - return n; -} - // What a deferred video of this kind is waiting for, from the registry rather // than hardcoded here. BackfillStage used to say "too long to diarize", which // was true only while diarization was the sole kind that could defer. diff --git a/editor/app/channels/[slug]/lib/stageStatus.ts b/editor/app/channels/[slug]/lib/stageStatus.ts @@ -7,6 +7,7 @@ import { import type { JobRecord } from "yt-dlp-transcript-common/jobs/registry"; import { laneEntriesOf, + operationsGroupLabel, reachableBackfillWork, } from "yt-dlp-transcript-common/lib/backfillKinds"; @@ -132,6 +133,11 @@ export type ComputeStageStatusesInput = { // and nothing would ever run the work it is advertising. Defaults to true so // an omitted flag behaves as it always did. backfillEnabled?: boolean; + // The ids of the lane's enabled kinds, so the stage can be titled after what + // it HOLDS rather than after its queue key. Optional and defaulting to the + // generic name, because a caller that only needs tone and counts should not + // have to resolve the registry. + backfillKindIds?: ReadonlyArray<string>; }; export function computeStageStatuses( @@ -144,6 +150,7 @@ export function computeStageStatuses( config, runningJobs, backfillEnabled = true, + backfillKindIds, } = input; const buckets = normalizeBuckets(snapshot.buckets); @@ -477,7 +484,11 @@ export function computeStageStatuses( } const backfill: StageStatus = { id: "backfill", - title: "Backfill", + // NAMED AFTER THE OPERATIONS, NOT THE QUEUE. "Backfill" is a scheduler key + // that on this install stands for three different operations; nobody can + // arm, pause or run "a backfill". Derived, so a lane that gains a kind from + // another group degrades to "Derived data" rather than going stale. + title: operationsGroupLabel(backfillKindIds ?? []), pending: backfillPending, failed: 0, running: backfillRunning, @@ -485,7 +496,7 @@ export function computeStageStatuses( summary: backfillRunning ? "Running…" : backfillEntries.length === 0 - ? "No backfill is enabled." + ? "Nothing here is enabled." : backfillParts.length > 0 ? backfillParts.join(" · ") : "Everything reachable is current.", diff --git a/editor/app/channels/[slug]/page.tsx b/editor/app/channels/[slug]/page.tsx @@ -70,6 +70,8 @@ import { BackfillStage } from "./components/stages/BackfillStage"; import { getBackfillKind, laneBackfillKinds, + operationsActionLabel, + operationsGroupLabel, } from "yt-dlp-transcript-common/lib/backfillKinds"; import { ChannelLine } from "./components/flow/ChannelLine"; import { NextAction } from "./components/flow/NextAction"; @@ -250,6 +252,7 @@ export default async function ChannelDetailPage({ config, runningJobs, backfillEnabled: backfillKinds.length > 0, + backfillKindIds: backfillKinds.map((k) => k.id), }); const stageOrder: StageId[] = [ @@ -427,6 +430,10 @@ export default async function ChannelDetailPage({ return { id: kind.id, label: kind.label, + // What one unit of this operation costs. Declared on the + // registry so the card can print it beside the backlog without + // knowing anything about diarization or model calls. + costBasis: kind.costBasis, reachableIds: entry?.ids ?? [], missingInput: entry?.missingInput ?? 0, // ?? 0 is load-bearing, not defensive: snapshots written before @@ -448,6 +455,9 @@ export default async function ChannelDetailPage({ })} allowRedownload={settings.backfill.allowRedownload} anyEnabled={backfillKinds.length > 0} + // The lane's name comes from what it HOLDS, not from its queue key. + groupLabel={operationsGroupLabel(backfillKinds.map((k) => k.id))} + actionLabel={operationsActionLabel(backfillKinds.map((k) => k.id))} /> ); case "cleanup": { diff --git a/editor/app/components/lanes/LaneDeck.tsx b/editor/app/components/lanes/LaneDeck.tsx @@ -398,6 +398,10 @@ export function LaneDeck({ // Empty when the payload has not arrived, or when it predates `kinds` — which // drops the breakdown rather than rendering a row of zeros. const backfillKinds = backfillAvailable ? (backfill?.kinds ?? []) : []; + // The lane's name in the operator's terms, from the server. "Backfill" is a + // queue key; it survives below only on the two controls that genuinely act on + // the shared queue, and the card now says what is in it. + const backfillGroup = backfill?.groupLabel ?? "Derived data"; const backfillControls: LaneControl[] = backfillAvailable ? [ backfillSweeping @@ -451,7 +455,7 @@ export function LaneDeck({ const backfillLane = ( <LaneCard - name="Backfill" + name={backfillGroup} // `anyKind` false is UNAVAILABLE, not idle: with no backfill feature // registered there is nothing to hold, and an empty work list because a // feature is switched off must not read as "all caught up". @@ -498,23 +502,57 @@ export function LaneDeck({ blocked={k.blocked} deferred={k.deferred} /> + {/* WHAT AN ARMED OPERATION COSTS. The sentence that was + missing: this lane can be armed on ~194,000 model calls and + look exactly like one armed on 329 audio passes, because + "11,337 reachable" is the same shape of number either way. + Stated flat, beside the backlog, with no threshold — a "this + is a lot" cutoff would be a magic number the next operation + gets wrong. */} + {k.costBasis && ( + <span className="text-muted-foreground"> + {" "} + — {k.costBasis} + </span> + )} </span> ))} </> ) : undefined } + // TWO NOTES, NOT ONE SENTENCE. "Here is what this lane holds" and "the + // sweep is armed while the lane is off" are different conditions, and + // folding them together loses the one that is a problem. note={ - backfillSweeping && !laneEnabled ? ( - // Kept in words as well as in the rail. A sweep armed with the lane - // off holds at a zero limit rather than doing work, and "wedged" is - // what that looks like to anyone who does not already know. - <span - aria-label="backfill lane off" - className="text-xs text-muted-foreground" - > - the sweep is holding - </span> - ) : undefined + <> + {backfillAvailable && backfillKinds.length > 1 && ( + // THE ONE PLACE THE LANE IS STILL NAMED AS A LANE, and it earns it: + // that these operations share a queue and a pause is a true, + // load-bearing fact — the sweep and the pause are two controls, and + // conflating them is how an operator loses a week of GPU time. So + // it is said once, here, where both controls are, and the members + // are LISTED rather than hidden behind the queue key. Everywhere an + // operator reads a figure, they read an operation name instead. + <span + aria-label="backfill lane members" + className="block text-xs text-muted-foreground" + > + {backfillKinds.length} operations share one backfill queue and one + pause: {backfillKinds.map((k) => k.label).join(", ")}. + </span> + )} + {backfillSweeping && !laneEnabled && ( + // Kept in words as well as in the rail. A sweep armed with the lane + // off holds at a zero limit rather than doing work, and "wedged" is + // what that looks like to anyone who does not already know. + <span + aria-label="backfill lane off" + className="block text-xs text-muted-foreground" + > + the sweep is holding + </span> + )} + </> } controls={backfillControls} /> diff --git a/editor/app/components/pipelines/StateBand.tsx b/editor/app/components/pipelines/StateBand.tsx @@ -125,6 +125,32 @@ export function bandSentence(band: OperationBand): string { return parts.join(" · "); } +// THE ONE POPULATION THAT DOMINATES WHAT IS LEFT, in three words. +// +// Not a total and not a percentage — the shape of the remainder is the thing +// that varies across this corpus, and naming its largest part is the shortest +// true sentence about a pipeline. "11,333 no media" and "11,329 to do" are the +// same size of number and mean opposite things; a single "remaining" figure +// would render them identically. +export function bandHeadline(band: OperationBand): string { + const candidates: Array<[number, string]> = [ + [band.reachable, "to do"], + [band.blocked, "blocked"], + [band.missingInput, "no media"], + [band.deferred, "held"], + ]; + let best: [number, string] | null = null; + for (const c of candidates) { + if (c[0] > (best?.[0] ?? 0)) best = c; + } + if (!best) { + // Nothing outstanding. Whether that is "finished" depends on a denominator + // we may not have, so say what is known and no more. + return bandDenominator(band) === null ? "coverage unknown" : "nothing to do"; + } + return `${best[0].toLocaleString()} ${best[1]}`; +} + export function StateBand({ band, size, diff --git a/editor/app/components/pipelines/band.ts b/editor/app/components/pipelines/band.ts @@ -44,6 +44,15 @@ export type OperationBand = { missingInput: number; // Held by a gate — stale cues, a duration window. Dotted. deferred: number; + // WHAT ONE UNIT OF THIS OPERATION COSTS, in words, from the registry. + // + // The fact that hid behind a shared lane name: an operation can be armed at + // enormous cost and read as a quiet row, because "11,337 reachable" is the + // same shape of number whether the unit is one audio pass per video or ~1 + // model call per transcript CHUNK — a ~17x difference on the same figure. + // Carried on the band so every surface that draws a backlog can state its + // unit beside it, with no threshold and no editorialising. + costBasis: string; // Whether the operation is dispatched by this system at all. False for the // two external pipelines, which are here because the rail's whole point is // that you never have to switch lanes to learn that this one is idle BECAUSE diff --git a/editor/app/components/pipelines/buildBands.ts b/editor/app/components/pipelines/buildBands.ts @@ -5,6 +5,7 @@ import { } from "yt-dlp-transcript-common/controller/channelSnapshot"; import { DIGEST_KIND_ID, + operationCostBasis, operationLabel, presentBackfillWork, reachableBackfillWork, @@ -47,6 +48,7 @@ function emptyBand(id: string, dispatched: boolean): OperationBand { return { id, label: operationLabel(id), + costBasis: operationCostBasis(id), eligible: 0, present: 0, reachable: 0, diff --git a/editor/app/settings/components/SettingsForm.tsx b/editor/app/settings/components/SettingsForm.tsx @@ -870,7 +870,7 @@ export function SettingsForm({ initial, apps, digestApps }: Props) { hint="Videos longer than this are reported as deferred instead of being diarized, and are never counted as work still to do. This is a stopgap for an out-of-memory crash on very long recordings: the engine's memory use grows with the SQUARE of the number of speaker turns, so a dense six-hour stream can exhaust 16 GB after 40 minutes of work and produce nothing. 0 turns the limit off." /> <Field - label="Backfill concurrency" + label="Diarization concurrency" name="diarizationConcurrency" defaultValue={String(initial.diarization.concurrency)} type="number" @@ -878,7 +878,14 @@ export function SettingsForm({ initial, apps, digestApps }: Props) { /> </fieldset> <fieldset className="flex flex-col gap-3 border border-border rounded p-3"> - <legend className="px-1 text-sm font-medium">Backfill lane</legend> + {/* THE ONE FIELDSET THAT STILL SAYS "LANE", and it is the right one: + this is where the shared pause and the shared sweep live, and that + these operations share a queue is a load-bearing fact rather than a + leaked implementation detail. Everywhere an operator reads a FIGURE + they now read an operation name instead. */} + <legend className="px-1 text-sm font-medium"> + Speaker work lane + </legend> {/* The marker again, and here it guards two things an unrelated save must never touch: `allowRedownload` (which writes media to a nearly-full @@ -888,9 +895,12 @@ export function SettingsForm({ initial, apps, digestApps }: Props) { */} <input type="hidden" name="backfillFormPresent" value="1" readOnly /> <p className="text-xs text-muted-foreground"> - Catch-up for derived data the existing corpus predates. Each feature - declares what it needs and how to tell whether a video has it; this - section decides how much of the machine the catch-up may use. + Catch-up for derived data the existing corpus predates — speaker + diarization and the two speaker-name lanes below. These operations + share one queue and one pause: the settings here decide how much of + the machine all of them together may use, not any one of them + individually. Each feature declares what it needs and how to tell + whether a video already has it. </p> <label className="flex items-start gap-2 text-sm"> <input @@ -956,8 +966,9 @@ export function SettingsForm({ initial, apps, digestApps }: Props) { <input type="hidden" name="attributionFormPresent" value="1" readOnly /> <p className="text-xs text-muted-foreground"> Putting names to the speakers. Nothing here runs on its own — it - registers two backfills, and the Backfill lane above decides when they - get the machine. + registers two operations, and the Speaker work lane above decides when + they get the machine. Both are on the same shared pause as speaker + diarization. </p> <label className="flex items-start gap-2 text-sm"> <input diff --git a/editor/app/widget/components/MonitorWidget.tsx b/editor/app/widget/components/MonitorWidget.tsx @@ -697,7 +697,10 @@ function BackfillStrip({ data }: { data: WidgetSyncPayload }) { }`} /> <span className="tabular-nums"> - {compactCount(b.reachable)} backfill + {/* The visible word names the WORK; the section's aria-label stays + "Backfill" because it is how the section registry and the widget + specs address this strip, and nobody reads it. */} + {compactCount(b.reachable)} {b.groupLabel.toLowerCase()} {/* Still a separate figure, never summed into the one beside it. */} {b.needsMedia > 0 && ` \u00b7 ${compactCount(b.needsMedia)} need media`} </span> @@ -719,7 +722,7 @@ function BackfillStrip({ data }: { data: WidgetSyncPayload }) { }`} /> <span className="tabular-nums"> - Backfill {b.reachable.toLocaleString()} reachable + {b.groupLabel} {b.reachable.toLocaleString()} reachable {b.needsMedia > 0 && <> · {b.needsMedia.toLocaleString()} need media</>} {!b.enabled && " · lane off"} {b.enabled && b.sweeping && " · sweeping"} diff --git a/editor/e2e/attribution.spec.ts b/editor/e2e/attribution.spec.ts @@ -150,11 +150,11 @@ async function seedChannel(opts: { diarization?: boolean } = {}) { async function runBackfill(page: Page): Promise<void> { await page.goto(channelStage(CHANNEL, "backfill")); await page - .getByRole("button", { name: "Backfill channel", exact: true }) + .getByRole("button", { name: "Run speaker work", exact: true }) .click(); // The batch's closing summary line — emitted after the last write, so it is // the happens-before edge for the sidecar reads below. - await expect(page.getByLabel("Backfill channel output")).toContainText( + await expect(page.getByLabel("Run speaker work output")).toContainText( "already current", { timeout: 90_000 }, ); @@ -337,10 +337,14 @@ test("the stage card and /actionable show attribution beside diarization, with t section.getByLabel("backfill kind attribution-text"), ).toContainText("0 needing media"); // Reachable work across kinds: 1 + 2. The needs-re-acquiring figure is on its - // own line and is never folded into the heading. - await expect(section.getByRole("heading")).toContainText( - "Backfill derived data (3)", + // own line and is never folded into it. + await expect(section.getByLabel("backfill reachable")).toContainText( + "3 videos can be worked on now", ); + // The card names the WORK, not the queue: three operations were hiding behind + // the word "backfill", and the heading now says which group they are and how + // many of them there are. + await expect(section.getByRole("heading")).toContainText("Speaker work"); // AND THE RE-ACQUIRE LINE IS GONE, which is the point of the whole change. // Nothing in this fixture needs media fetched: the one video that cannot be // attributed from audio is waiting for the diarization lane, and no download diff --git a/editor/e2e/backfill.spec.ts b/editor/e2e/backfill.spec.ts @@ -153,8 +153,8 @@ test("the stage card separates reachable work from what needs its media back", a await page.goto(channelStage(SLUG, "backfill")); const section = page.getByLabel("backfill section"); - await expect(section.getByRole("heading")).toContainText( - "Backfill derived data (1)", + await expect(section.getByLabel("backfill reachable")).toContainText( + "1 video can be worked on now", ); await expect(section.getByLabel("backfill needs re-acquiring")).toContainText( "1", @@ -183,7 +183,7 @@ test("running the lane captures the reachable video and skips the one with no me await page.goto(channelStage(SLUG, "backfill")); await page - .getByRole("button", { name: "Backfill channel", exact: true }) + .getByRole("button", { name: "Run speaker work", exact: true }) .click(); await expect @@ -194,7 +194,7 @@ test("running the lane captures the reachable video and skips the one with no me // Never touched: re-download is off, so a video whose input is gone is // COUNTED, not attempted. expect(await pathExists(dataRel("vidB", "diarization.json"))).toBe(false); - await expect(page.getByLabel("Backfill channel output")).toContainText( + await expect(page.getByLabel("Run speaker work output")).toContainText( "need their media re-acquired", { timeout: 30_000 }, ); @@ -227,7 +227,7 @@ test("a sidecar from a different threshold is regenerated", async ({ page }) => await page.goto(channelStage(SLUG, "backfill")); await page - .getByRole("button", { name: "Backfill channel", exact: true }) + .getByRole("button", { name: "Run speaker work", exact: true }) .click(); await expect @@ -260,7 +260,7 @@ test("re-acquired media is deleted after a successful backfill", async ({ await page.goto(channelStage(SLUG, "backfill")); await page - .getByRole("button", { name: "Backfill channel", exact: true }) + .getByRole("button", { name: "Run speaker work", exact: true }) .click(); // The work landed… @@ -294,10 +294,10 @@ test("re-acquired media is deleted even when the backfill fails", async ({ await page.goto(channelStage(SLUG, "backfill")); await page - .getByRole("button", { name: "Backfill channel", exact: true }) + .getByRole("button", { name: "Run speaker work", exact: true }) .click(); - await expect(page.getByLabel("Backfill channel output")).toContainText( + await expect(page.getByLabel("Run speaker work output")).toContainText( /failed|Removed re-acquired/, { timeout: 60_000 }, ); @@ -330,7 +330,7 @@ test("re-acquired media is KEPT when the video is marked do-not-clean", async ({ await page.goto(channelStage(SLUG, "backfill")); await page - .getByRole("button", { name: "Backfill channel", exact: true }) + .getByRole("button", { name: "Run speaker work", exact: true }) .click(); await expect @@ -340,7 +340,7 @@ test("re-acquired media is KEPT when the video is marked do-not-clean", async ({ .toBe(true); // Kept, and SAID SO in the log — an unexplained file on a full disk is how a // leak gets discovered the hard way. - await expect(page.getByLabel("Backfill channel output")).toContainText( + await expect(page.getByLabel("Run speaker work output")).toContainText( "do not clean", { timeout: 30_000 }, ); @@ -366,10 +366,10 @@ test("the disk floor refuses to re-acquire anything", async ({ page }) => { await page.goto(channelStage(SLUG, "backfill")); await page - .getByRole("button", { name: "Backfill channel", exact: true }) + .getByRole("button", { name: "Run speaker work", exact: true }) .click(); - await expect(page.getByLabel("Backfill channel output")).toContainText( + await expect(page.getByLabel("Run speaker work output")).toContainText( "below the", { timeout: 60_000 }, ); @@ -510,7 +510,7 @@ test("a backfill runs concurrently with a transcription", async ({ await page.goto(channelStage(SLUG, "backfill")); await page - .getByRole("button", { name: "Backfill channel", exact: true }) + .getByRole("button", { name: "Run speaker work", exact: true }) .click(); // Both running at once. This is the assertion the separate queueKey exists for. @@ -594,7 +594,7 @@ test("a digest runs concurrently with a backfill, not behind it", async ({ await generateReport(page, SLUG); await page.goto(channelStage(SLUG, "backfill")); await page - .getByRole("button", { name: "Backfill channel", exact: true }) + .getByRole("button", { name: "Run speaker work", exact: true }) .click(); await expect .poll( @@ -784,8 +784,8 @@ test("a digest entry in the snapshot does not move the backfill instrument", asy const backfillSection = page.getByLabel("backfill section"); // 1 — vidA's diarization. NOT 3, which is what folding the digest entry in // would produce here. - await expect(backfillSection.getByRole("heading")).toContainText( - "Backfill derived data (1)", + await expect(backfillSection.getByLabel("backfill reachable")).toContainText( + "1 video can be worked on now", ); // The digest card reports its own work, off the same snapshot entry. diff --git a/editor/e2e/channel-line.spec.ts b/editor/e2e/channel-line.spec.ts @@ -138,7 +138,9 @@ test("a pre-`eligible` snapshot renders an em dash, not 0%", async ({ const line = page.getByLabel("channel line"); await expect(line.getByLabel("Digest through")).toHaveText("—"); await expect(line.getByLabel("Digest coverage", { exact: true })).toHaveText("—"); - await expect(line.getByLabel("Backfill through")).toHaveText("—"); + // "Speakers", not "Backfill": the station is named after the operations it + // holds, not after the queue key three of them happen to share. + await expect(line.getByLabel("Speakers through")).toHaveText("—"); // The negative assertion is the point: a 0% here is the bug. await expect(line.getByLabel("Digest coverage", { exact: true })).not.toContainText("0%"); }); @@ -230,7 +232,7 @@ test("a lane that is switched off reads neutral, not finished", async ({ page.getByLabel("channel line").getByLabel("to backfill shortfall"), ).toContainText("7"); await expect( - page.getByLabel("channel line").getByLabel("Backfill through"), + page.getByLabel("channel line").getByLabel("Speakers through"), ).toHaveText("43"); // Lane OFF, same snapshot: nothing is offered, and coverage is unknown rather @@ -239,12 +241,15 @@ test("a lane that is switched off reads neutral, not finished", async ({ await writeSettings(laneSettings(false)); await page.goto(`/channels/${SLUG}`); const line = page.getByLabel("channel line"); - await expect(line.getByLabel("Backfill through")).toHaveText("—"); + // With nothing enabled the label falls back to the derived generic name — + // operationsGroupLabel over an empty set. It cannot claim "Speakers" when no + // speaker operation is on. + await expect(line.getByLabel("Derived data through")).toHaveText("—"); await expect(line.getByLabel("to backfill shortfall")).toContainText("0"); - // …and the Backfill stage says why rather than claiming completion. + // …and the stage says why rather than claiming completion. await page.goto(channelStage(SLUG, "backfill")); - await expect(page.getByLabel("Backfill stage summary")).toContainText( - "No backfill is enabled", + await expect(page.getByLabel("Derived data stage summary")).toContainText( + "Nothing here is enabled", ); }); diff --git a/editor/e2e/channel-stage-selection.spec.ts b/editor/e2e/channel-stage-selection.spec.ts @@ -64,7 +64,10 @@ test("no ?stage= lands on the overview, which lists every stage without opening "Download", "Transcribe", "Digest", - "Backfill", + // Named after the operations it holds, not after the queue key. With the + // default test settings no speaker operation is enabled, so the derived + // generic name is what an honest label falls back to. + "Derived data", "Cleanup", "Diagnostics", ]) {