commit 30eb2e1577ccbb29b4b1313ab4f0ff9d49141419
parent cb76eb6cd85938fc65083d4f5148b71bed59dea7
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 24 Aug 2026 00:46:30 -0400
sweeps: a scope console — pick the operations and channels before arming
The /auto-queue sweep cards gain a scope control and a live channel
itinerary: which operations an armed sweep will run and which channels it
will walk, chosen before arming instead of discovered from the job log.
Scope is written by the ARM action only, and the run's own planner stays
out of every render path (the guard test now names the new modules too).
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Diffstat:
17 files changed, 1577 insertions(+), 43 deletions(-)
diff --git a/common/controller/backfillSweep.ts b/common/controller/backfillSweep.ts
@@ -44,6 +44,7 @@ import { countBackfillWork, runBackfillBatch } from "./backfillBatch";
import type { AutoQueueOrder } from "../jobs/autoQueuePolicy";
import { buildRecencyKeys } from "./recencyIndex";
import { orderPlanByRecency } from "./planOrder";
+import type { SweepPlanEntry } from "../lib/sweepPlan";
export const BACKFILL_SWEEP_KIND = "backfill-sweep";
export const BACKFILL_CHANNEL_KIND = "backfill-channel";
@@ -94,17 +95,16 @@ export type BackfillSweepOptions = {
channelSlugs?: string[];
};
-export type BackfillPlanEntry = {
- channelSlug: string;
- reachable: number;
- missingInput: number;
- // Newest / oldest upload date (YYYYMMDD) among this channel's REACHABLE ids,
- // present only when the plan was asked to order by recency. "" is "no dated
- // reachable video": last under "newest", first under "oldest", the same rule
- // makeRecencyComparator applies to a single video.
- newestPending: string;
- oldestPending: string;
-};
+// One channel's row in the plan: how much is left, and the newest / oldest
+// upload date (YYYYMMDD) among its REACHABLE ids — the dates present only when
+// the plan was asked to order by recency. "" is "no dated reachable video":
+// last under "newest", first under "oldest", the same rule makeRecencyComparator
+// applies to a single video.
+//
+// ONE TYPE, TWO NAMES. The definition lives in lib/sweepPlan.ts because the
+// console's preview folds the same shape in the browser, and a second
+// declaration here is how the two would drift into disagreeing about a channel.
+export type BackfillPlanEntry = SweepPlanEntry;
// What is left, per channel, ordered heaviest-first by REACHABLE work.
//
@@ -239,7 +239,12 @@ export async function buildBackfillSweepPlan(opts: {
// The snapshot's per-kind counts, summed over the kinds in scope. `missing` and
// `stale` are the reachable half; `missingInput` stays its own number and is
// never added to them — see lib/backfillKinds.ts's header.
-function countFromSnapshot(
+//
+// EXPORTED so the console's preview counts with the RUN'S OWN ARITHMETIC rather
+// than a second copy of it. A preview that disagrees with the run about how much
+// work a channel holds is worse than no preview: it is the number an operator
+// arms a multi-day commitment against. See ./sweepPreview.ts.
+export function countFromSnapshot(
backfill: Record<string, BackfillSnapshotEntry>,
kinds: string[],
): { reachable: number; missingInput: number } {
diff --git a/common/controller/noCorpusWalkInRenderPaths.test.ts b/common/controller/noCorpusWalkInRenderPaths.test.ts
@@ -27,7 +27,27 @@ import { fileURLToPath } from "node:url";
const HERE = path.dirname(fileURLToPath(import.meta.url));
const EDITOR_APP = path.resolve(HERE, "..", "..", "editor", "app");
-const BANNED = "listChannelStatsFromDisk";
+// EVERY identifier that reaches the corpus walk, not just the walk itself.
+//
+// The guard grepped for one name and that was not enough: `buildBackfillSweepPlan`
+// CALLS listChannelStatsFromDisk, so importing it into a page would have walked
+// 474,559 files on a 3-second poll and passed this test with room to spare.
+// `buildDigestSweepPlan` is the same hazard from the other sweep — an LMDB scan
+// plus a per-video freshness check, priced for a run that happens hourly.
+//
+// If you are here because you want one of these on a screen: the snapshot-only
+// preview is ../controller/sweepPreview.ts, which is the same counting off the
+// channel snapshots and is what the /auto-queue console draws.
+//
+// THE MATCH IS TEXTUAL AND CONTEXT-BLIND, so a mere MENTION in a comment fails
+// it too. That is deliberate and not worth softening: a guard that skipped
+// comments and strings is a guard an offending call can hide behind, and the
+// cost of the false positive is one reworded comment.
+const BANNED = [
+ "listChannelStatsFromDisk",
+ "buildBackfillSweepPlan",
+ "buildDigestSweepPlan",
+];
async function walk(dir: string): Promise<string[]> {
const out: string[] = [];
@@ -57,8 +77,10 @@ test("the corpus walk never reaches a render path", async () => {
await Promise.all(
files.map(async (file) => {
const source = await readFile(file, "utf8");
- if (source.includes(BANNED)) {
- offenders.push(path.relative(EDITOR_APP, file));
+ for (const banned of BANNED) {
+ if (source.includes(banned)) {
+ offenders.push(`${banned} in ${path.relative(EDITOR_APP, file)}`);
+ }
}
}),
);
@@ -66,8 +88,8 @@ test("the corpus walk never reaches a render path", async () => {
assert.deepEqual(
offenders.sort(),
[],
- `${BANNED} walks the whole corpus and must not be reachable from a page, ` +
- `layout, API route or server action. Found in: ${offenders.join(", ")}`,
+ `these walk the whole corpus and must not be reachable from a page, ` +
+ `layout, API route or server action. Found: ${offenders.join(", ")}`,
);
});
diff --git a/common/controller/sweepPreview.test.ts b/common/controller/sweepPreview.test.ts
@@ -0,0 +1,309 @@
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test common/controller/sweepPreview.test.ts
+//
+// THE ONE THING WORTH TESTING HERE is that the preview and the run agree. The
+// console is a commitment device: an operator reads the plan, sees "11,342 to do
+// across 3 of 68 channels", and arms a run that may take days. A preview that
+// counts differently from the planner is not a cosmetic defect — it is the
+// number the decision was made on.
+//
+// So the first test builds a fixture corpus and asserts buildSweepPreview()
+// equals buildBackfillSweepPlan() over it, entry for entry. The rest pin the
+// four rules the fold has to get right on its own: an empty scope is EVERY kind
+// (not none), an unknown kind id contributes nothing (sanitizeBackfill keeps
+// them), a channel with no snapshot is unknown rather than zero, and the order
+// is the one controller/planOrder documents.
+
+import { mkdtempSync, writeFileSync, mkdirSync } from "node:fs";
+import { rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { test, after } from "node:test";
+import assert from "node:assert/strict";
+
+// Set BEFORE anything can call getPaths(), which memoizes its first answer at
+// module scope. node:test runs each file in its own process, so this is scoped
+// to this file alone.
+const ROOT = mkdtempSync(path.join(os.tmpdir(), "sweep-preview-"));
+process.env.TRANSCRIPTS_DIR = ROOT;
+// SETTINGS_FILE is its own env key — it defaults to the MONOREPO root, not the
+// corpus, so without this the fixture would silently resolve its kinds against
+// the live install's settings.json.
+process.env.SETTINGS_FILE = path.join(ROOT, "settings.json");
+
+const { getPaths } = await import("../lib/paths");
+const { buildBackfillSweepPlan } = await import("./backfillSweep");
+const { buildSweepChannelCounts, buildSweepPreview } = await import(
+ "./sweepPreview"
+);
+const { foldSweepPlan, sweepPlanTotals } = await import("../lib/sweepPlan");
+const { listChannelBriefs } = await import("./channels");
+
+after(() => rm(ROOT, { recursive: true, force: true }));
+
+const KINDS = ["attribution-diarized", "attribution-text"];
+
+// A snapshot entry for one kind. `ids` is EXACTLY the reachable set, which is
+// the invariant channelSnapshot.test.ts pins and this file relies on.
+function entry(missing: number, missingInput: number, prefix: string) {
+ return {
+ missing,
+ stale: 0,
+ partial: 0,
+ missingInput,
+ deferred: 0,
+ blocked: 0,
+ eligible: missing + missingInput + 5,
+ ids: Array.from({ length: missing }, (_, i) => `${prefix}${i}`),
+ };
+}
+
+function writeChannel(
+ slug: string,
+ backfill: Record<string, ReturnType<typeof entry>> | null,
+): void {
+ const dir = path.join(ROOT, "channels", slug);
+ mkdirSync(path.join(dir, "data"), { recursive: true });
+ writeFileSync(
+ path.join(dir, "config.json"),
+ // `handling` is what parseChannelConfig requires; without it the config is
+ // null and the channel is invisible to every reader here.
+ JSON.stringify({
+ handling: "transcribe",
+ name: slug,
+ url: `https://example.test/${slug}`,
+ }),
+ );
+ if (backfill === null) return;
+ writeFileSync(
+ path.join(dir, "snapshot.json"),
+ JSON.stringify({
+ generatedAt: new Date(0).toISOString(),
+ totals: { videos: 10, transcribed: 10, downloaded: 10 },
+ backfill,
+ buckets: {},
+ undownloadedIds: [],
+ }),
+ );
+}
+
+writeFileSync(
+ path.join(ROOT, "settings.json"),
+ JSON.stringify({
+ attribution: { enabled: true, textOnlyEnabled: true, diarizedEnabled: true },
+ }),
+);
+writeChannel("alpha", {
+ "attribution-diarized": entry(4, 1, "a"),
+ "attribution-text": entry(11, 0, "t"),
+});
+writeChannel("beta", {
+ "attribution-diarized": entry(2, 7, "b"),
+ "attribution-text": entry(0, 0, "u"),
+});
+writeChannel("gamma", {
+ "attribution-diarized": entry(0, 0, "g"),
+ "attribution-text": entry(0, 0, "h"),
+});
+// No snapshot at all — the case the run WALKS and the preview must not silently
+// report as zero.
+writeChannel("delta", null);
+
+test("the preview counts what the run's own planner counts", async () => {
+ const paths = getPaths();
+ const briefs = await listChannelBriefs(paths);
+ const plan = await buildBackfillSweepPlan({ paths });
+ const preview = buildSweepPreview({ briefs, kindIds: KINDS });
+ // `delta` has no snapshot, so the planner walks it (and finds an empty data
+ // dir); the preview reports it unknown. Both therefore say zero for it here —
+ // what the test pins is that every channel the snapshots DO cover agrees.
+ assert.deepEqual(preview, plan);
+ const totals = sweepPlanTotals(preview);
+ assert.equal(totals.reachable, 4 + 11 + 2);
+ assert.equal(totals.missingInput, 1 + 7);
+ assert.equal(totals.channels, 2);
+ assert.equal(totals.idle, 2);
+});
+
+test("scoping to one kind is the run's own scope, counted the same way", async () => {
+ const paths = getPaths();
+ const briefs = await listChannelBriefs(paths);
+ const scope = ["attribution-diarized"];
+ const plan = await buildBackfillSweepPlan({ paths, kindIds: scope });
+ const preview = buildSweepPreview({
+ briefs,
+ kindIds: KINDS,
+ scopeKindIds: scope,
+ });
+ assert.deepEqual(preview, plan);
+ // And the point of the whole console: the expensive lane dropping out is
+ // visible as a smaller number, not as a different-looking screen.
+ assert.equal(sweepPlanTotals(preview).reachable, 4 + 2);
+});
+
+test("an empty scope is EVERY kind, never none", () => {
+ const counts = buildSweepChannelCounts({
+ briefs: [
+ {
+ slug: "alpha",
+ snapshot: {
+ backfill: {
+ "attribution-diarized": entry(4, 0, "a"),
+ "attribution-text": entry(11, 0, "t"),
+ },
+ },
+ } as never,
+ ],
+ kindIds: KINDS,
+ });
+ assert.equal(sweepPlanTotals(foldSweepPlan({ counts })).reachable, 15);
+ assert.equal(
+ sweepPlanTotals(foldSweepPlan({ counts, kindIds: [] })).reachable,
+ 15,
+ );
+});
+
+test("an unknown kind id contributes nothing — it does not match everything", () => {
+ // sanitizeBackfill keeps an id that no longer names a registered kind, so a
+ // stored scope can name one. Counting it as zero is the safe reading; the
+ // console says so out loud, and this pins that it cannot silently widen.
+ const counts = buildSweepChannelCounts({
+ briefs: [
+ {
+ slug: "alpha",
+ snapshot: { backfill: { "attribution-text": entry(11, 0, "t") } },
+ } as never,
+ ],
+ kindIds: [...KINDS, "operation-that-was-removed"],
+ });
+ assert.equal(
+ sweepPlanTotals(
+ foldSweepPlan({ counts, kindIds: ["operation-that-was-removed"] }),
+ ).reachable,
+ 0,
+ );
+});
+
+test("a channel with no snapshot is unknown, not zero", () => {
+ const counts = buildSweepChannelCounts({
+ briefs: [{ slug: "delta", snapshot: null }],
+ kindIds: KINDS,
+ });
+ assert.equal(counts[0].unknown, true);
+ assert.equal(counts[0].byKind["attribution-text"].reachable, 0);
+ const reported = buildSweepChannelCounts({
+ briefs: [
+ {
+ slug: "gamma",
+ snapshot: { backfill: { "attribution-text": entry(0, 0, "h") } },
+ } as never,
+ ],
+ kindIds: KINDS,
+ });
+ // The distinction the flag exists for: both read 0, and only one of them is a
+ // finding.
+ assert.equal(reported[0].unknown, false);
+});
+
+test("dates come from the reachable ids, per kind, and fold as a max of maxima", () => {
+ const counts = buildSweepChannelCounts({
+ briefs: [
+ {
+ slug: "alpha",
+ snapshot: {
+ backfill: {
+ "attribution-diarized": entry(2, 0, "a"),
+ "attribution-text": entry(2, 0, "t"),
+ },
+ },
+ } as never,
+ ],
+ kindIds: KINDS,
+ dates: new Map([
+ ["a0", "20240101"],
+ ["a1", "20240202"],
+ ["t0", "20260819"],
+ // t1 deliberately undated: an id nothing could date must not become the
+ // extreme, in either direction.
+ ]),
+ });
+ const byKind = counts[0].byKind;
+ assert.deepEqual(
+ [byKind["attribution-diarized"].newestPending, byKind["attribution-diarized"].oldestPending],
+ ["20240202", "20240101"],
+ );
+ assert.deepEqual(
+ [byKind["attribution-text"].newestPending, byKind["attribution-text"].oldestPending],
+ ["20260819", "20260819"],
+ );
+ // Both kinds in scope: the union's extremes.
+ const both = foldSweepPlan({ counts })[0];
+ assert.deepEqual(
+ [both.newestPending, both.oldestPending],
+ ["20260819", "20240101"],
+ );
+ // Drop the expensive lane and the row re-dates itself — no second lookup.
+ const one = foldSweepPlan({ counts, kindIds: ["attribution-diarized"] })[0];
+ assert.deepEqual(
+ [one.newestPending, one.oldestPending],
+ ["20240202", "20240101"],
+ );
+});
+
+test("order is planOrder's rule: newest first, weight as the tiebreak", () => {
+ const counts = [
+ {
+ channelSlug: "heavy-and-old",
+ unknown: false,
+ byKind: {
+ k: {
+ reachable: 900,
+ missingInput: 0,
+ newestPending: "20200101",
+ oldestPending: "20190101",
+ },
+ },
+ },
+ {
+ channelSlug: "light-and-fresh",
+ unknown: false,
+ byKind: {
+ k: {
+ reachable: 3,
+ missingInput: 0,
+ newestPending: "20260819",
+ oldestPending: "20260819",
+ },
+ },
+ },
+ {
+ channelSlug: "undated",
+ unknown: false,
+ byKind: {
+ k: {
+ reachable: 50,
+ missingInput: 0,
+ newestPending: "",
+ oldestPending: "",
+ },
+ },
+ },
+ ];
+ assert.deepEqual(
+ foldSweepPlan({ counts, order: "newest" }).map((e) => e.channelSlug),
+ // Undated sorts LAST under newest — the same thing makeRecencyComparator
+ // does to an undatable video.
+ ["light-and-fresh", "heavy-and-old", "undated"],
+ );
+ assert.deepEqual(
+ foldSweepPlan({ counts, order: "oldest" }).map((e) => e.channelSlug),
+ // ...and FIRST under oldest.
+ ["undated", "heavy-and-old", "light-and-fresh"],
+ );
+ assert.deepEqual(
+ foldSweepPlan({ counts, order: "listed" }).map((e) => e.channelSlug),
+ // "listed" consults no date at all: heaviest first, byte-for-byte the
+ // historical plan.
+ ["heavy-and-old", "undated", "light-and-fresh"],
+ );
+});
diff --git a/common/controller/sweepPreview.ts b/common/controller/sweepPreview.ts
@@ -0,0 +1,184 @@
+// The sweep plan, built from SNAPSHOTS ONLY — the half of ./sweepPlan.ts that
+// has to touch the corpus, kept apart from the half that has to run in a
+// browser.
+//
+// WHY THIS IS NOT buildBackfillSweepPlan. That function is the run's planner and
+// it is right for the run: an absent snapshot makes it WALK the channel's
+// videos, because planning an unreported channel as zero would exclude it from
+// the sweep forever. One walk is ~474,559 file touches and ~4 seconds, which is
+// why common/controller/noCorpusWalkInRenderPaths.test.ts bans it from anything
+// that renders — and this console re-plans on a 3-second poll. So the preview
+// takes the other trade: snapshots only, and a channel with no snapshot is
+// reported as UNKNOWN rather than as zero or as a walk.
+//
+// What it does NOT re-implement is the counting. countFromSnapshot is exported
+// from ./backfillSweep and used verbatim here, because a preview that disagrees
+// with the run about how much work a channel holds is worse than no preview: it
+// is the number an operator arms a multi-day commitment against.
+
+import type { AutoQueueOrder } from "../jobs/autoQueuePolicy";
+import {
+ DIGEST_KIND_ID,
+ presentBackfillWork,
+} from "../lib/backfillKinds";
+import {
+ foldSweepPlan,
+ type SweepChannelCounts,
+ type SweepKindCounts,
+ type SweepPlanEntry,
+} from "../lib/sweepPlan";
+import { countFromSnapshot } from "./backfillSweep";
+import { digestWorkOf, type ChannelSnapshot } from "./channelSnapshot";
+
+// Exactly what listChannelBriefs already returns, narrowed to the two fields
+// this needs. Structural, so the editor's per-request brief cache feeds it with
+// no mapping.
+export type SweepPreviewBrief = {
+ slug: string;
+ snapshot: ChannelSnapshot | null;
+};
+
+export type BuildSweepChannelCountsInput = {
+ briefs: ReadonlyArray<SweepPreviewBrief>;
+ // Every operation the console can offer — the CATALOG, not the scope. Counts
+ // are carried for all of them so deselecting one re-plans in the browser with
+ // no round trip; which of them are actually in scope is the fold's business.
+ kindIds: ReadonlyArray<string>;
+ // videoId -> YYYYMMDD, for the reachable ids. Absent (or an id absent from
+ // it) leaves the date "", which controller/planOrder sorts last under
+ // "newest" and first under "oldest" — the same thing it does for a corpus
+ // with no dates at all, which is a documented fall-through to weight order
+ // rather than an arbitrary shuffle. See ./sweepRecency.ts for who fills it.
+ dates?: ReadonlyMap<string, string>;
+};
+
+function emptyKindCounts(): SweepKindCounts {
+ return {
+ reachable: 0,
+ missingInput: 0,
+ blocked: 0,
+ deferred: 0,
+ // 0, not null, for an operation with no entry at all: the channel is not
+ // withholding a number, there is simply no population. A snapshot that
+ // HAS an entry but predates `eligible` is the null case, below.
+ eligible: 0,
+ present: 0,
+ newestPending: "",
+ oldestPending: "",
+ };
+}
+
+// The reachable ids for one operation on one channel.
+//
+// snapshot.backfill[id].ids IS the reachable set and nothing else — never
+// missing-input, deferred or blocked (a test in lib/backfillKinds keeps it equal
+// to reachableBackfillWork). So dating these ids dates exactly the work the
+// sweep would do, which is the only population the order is about.
+function reachableIdsFor(
+ snapshot: ChannelSnapshot,
+ id: string,
+): ReadonlyArray<string> {
+ const entry = snapshot.backfill?.[id];
+ if (entry) return entry.ids ?? [];
+ // Digest joined the operation registry after most snapshots on disk were
+ // written, and digestWorkOf is the fallback that reads the same population off
+ // the buckets with the transcript and cues-staleness gates applied — the same
+ // one components/pipelines/buildBands.ts uses, for the same reason: without it
+ // a stale channel reads "all digested".
+ if (id === DIGEST_KIND_ID) return digestWorkOf(snapshot).ids;
+ return [];
+}
+
+type WorkCounts = Omit<SweepKindCounts, "newestPending" | "oldestPending">;
+
+function countsFor(snapshot: ChannelSnapshot, id: string): WorkCounts | null {
+ const entry = snapshot.backfill?.[id];
+ if (entry) {
+ // The RUN'S OWN arithmetic for the two figures the plan is about, on one
+ // kind at a time.
+ const { reachable, missingInput } = countFromSnapshot(snapshot.backfill!, [
+ id,
+ ]);
+ return {
+ reachable,
+ missingInput,
+ // `?? 0` at every read: every snapshot on disk predates one or other of
+ // these fields, and undefined poisons a sum to NaN.
+ blocked: entry.blocked ?? 0,
+ deferred: entry.deferred ?? 0,
+ eligible: entry.eligible ?? null,
+ present: presentBackfillWork(entry),
+ };
+ }
+ if (id === DIGEST_KIND_ID) {
+ const work = digestWorkOf(snapshot);
+ return {
+ reachable: work.reachable,
+ // The bucket fallback genuinely does not know a missing-input count; 0
+ // here is the honest statement that this snapshot carries no such split,
+ // not a claim that no media is gone.
+ missingInput: 0,
+ blocked: work.blocked,
+ deferred: work.deferred,
+ eligible: work.eligible,
+ present: work.present,
+ };
+ }
+ return null;
+}
+
+export function buildSweepChannelCounts({
+ briefs,
+ kindIds,
+ dates,
+}: BuildSweepChannelCountsInput): SweepChannelCounts[] {
+ return briefs.map((brief) => {
+ const snapshot = brief.snapshot;
+ const byKind: Record<string, SweepKindCounts> = {};
+ for (const id of kindIds) {
+ const counts = emptyKindCounts();
+ byKind[id] = counts;
+ if (!snapshot) continue;
+ const work = countsFor(snapshot, id);
+ if (work) Object.assign(counts, work);
+ if (!dates) continue;
+ for (const videoId of reachableIdsFor(snapshot, id)) {
+ const key = dates.get(videoId);
+ if (!key) continue;
+ if (key > counts.newestPending) counts.newestPending = key;
+ if (!counts.oldestPending || key < counts.oldestPending) {
+ counts.oldestPending = key;
+ }
+ }
+ }
+ return {
+ channelSlug: brief.slug,
+ // No snapshot at all — see the field's comment in lib/sweepPlan.ts. NOT
+ // the same as a snapshot that reports zero work, which is a real finding.
+ unknown: snapshot === null,
+ byKind,
+ };
+ });
+}
+
+// Counts + fold in one call, for a server that wants the plan directly (the
+// offline sanity script, and the tests that compare this against the run's own
+// planner).
+export function buildSweepPreview({
+ briefs,
+ kindIds,
+ scopeKindIds,
+ order,
+ dates,
+}: BuildSweepChannelCountsInput & {
+ // The scope, when it differs from the catalog. Absent = every catalog kind,
+ // which is what an unscoped sweep runs.
+ scopeKindIds?: ReadonlyArray<string>;
+ order?: AutoQueueOrder;
+}): SweepPlanEntry[] {
+ return foldSweepPlan({
+ counts: buildSweepChannelCounts({ briefs, kindIds, dates }),
+ kindIds: scopeKindIds,
+ order,
+ });
+}
diff --git a/common/controller/sweepRecency.ts b/common/controller/sweepRecency.ts
Binary files differ.
diff --git a/common/lib/sweepPlan.ts b/common/lib/sweepPlan.ts
@@ -0,0 +1,163 @@
+// THE PLAN: the ordered channel itinerary a sweep will actually walk.
+//
+// A sweep is a commitment, not a toggle — on this corpus arming one can mean
+// ~194,000 model calls — and until now the only way to read what was being
+// committed to was to arm it and watch the log. This module is the shape that
+// makes the commitment legible BEFORE the click: the same counting the run
+// does, folded over whichever operations are in scope, ordered by the same
+// comparator, so the itinerary on screen is the itinerary that runs.
+//
+// PURE, AND IN lib/ RATHER THAN controller/ ON PURPOSE. The console re-plans on
+// every checkbox — deselect an operation and the totals fall, the rows re-sort
+// and the button re-labels itself — and a round trip per keystroke would make
+// that feel like a form rather than an instrument. So the fold has to run in
+// the browser, which means it may not import anything that touches disk:
+// controller/channelSnapshot reaches runYtdlp reaches execa, and a client
+// component importing that fails `next build` on `node:child_process`. This is
+// the same split components/pipelines/band.ts already documents, for the same
+// reason and in the same direction.
+//
+// COUNTS ARE CARRIED PER OPERATION AND NEVER PRE-SUMMED. The server hands over
+// one entry per kind per channel; the client sums only what is selected. That
+// is not merely a convenience: the reason the scope control exists at all is
+// that "11,337 reachable" is the same shape of number whether one unit is a
+// single audio pass or ~1 model call per transcript CHUNK, so a payload that
+// arrived pre-summed would have already thrown away the distinction the console
+// is being built to show.
+
+import { orderPlanByRecency } from "../controller/planOrder";
+import type { AutoQueueOrder } from "../jobs/autoQueuePolicy";
+
+// One operation's outstanding work on one channel.
+//
+// `reachable` and `missingInput` stay apart here as they do everywhere else —
+// see lib/backfillKinds.ts's header. On the measured corpus they are four orders
+// of magnitude apart on diarization, and one "remaining" number would say the
+// same thing about a lane that is finished and a lane that cannot start.
+export type SweepKindCounts = {
+ reachable: number;
+ missingInput: number;
+ // The band's other two work populations, carried so a plan row can draw the
+ // same five-fill instrument /channels and the comparison rail draw — and
+ // NEVER added to `reachable`. Waiting on an upstream operation and held by a
+ // gate are not work this run can do.
+ blocked: number;
+ deferred: number;
+ // The band's coverage halves. `null` is "this snapshot cannot say", and it
+ // poisons a sum deliberately: a partial denominator is smaller than its own
+ // numerator, which is a worse lie than admitting the number is not knowable.
+ eligible: number | null;
+ present: number | null;
+ // Newest / oldest upload date (YYYYMMDD) among THIS operation's reachable
+ // ids, or "" when none of them could be dated. "" sorts last under "newest"
+ // and first under "oldest", exactly as controller/planOrder documents for a
+ // channel and makeRecencyComparator does for a single video.
+ //
+ // Per KIND rather than per channel, because the fold below is a max over the
+ // selected kinds and a max of maxima is the max of the union — so toggling an
+ // operation off re-dates the row correctly with no second lookup.
+ newestPending: string;
+ oldestPending: string;
+};
+
+export type SweepChannelCounts = {
+ channelSlug: string;
+ // This channel has never been reported on, so every count below is 0 BECAUSE
+ // NOTHING HAS LOOKED — not because there is nothing to do. The run's own
+ // planner walks such a channel's videos rather than planning it as zero
+ // (see buildBackfillSweepPlan), which a 3-second poll cannot do; so the
+ // console says "not reported yet" where it would otherwise print a confident
+ // zero. Unknown is not zero, here as everywhere else on these surfaces.
+ unknown: boolean;
+ // Keyed by operation id. An operation with no entry contributes nothing,
+ // which is also what an UNKNOWN id does — sanitizeBackfill keeps an id that
+ // no longer names a registered kind, and a stored scope naming one must
+ // count as zero rather than as everything.
+ byKind: Record<string, SweepKindCounts>;
+};
+
+// One channel's row in the plan. The shape controller/backfillSweep's own
+// planner returns, so preview and run cannot describe a channel differently.
+export type SweepPlanEntry = {
+ channelSlug: string;
+ reachable: number;
+ missingInput: number;
+ newestPending: string;
+ oldestPending: string;
+};
+
+// Fold the per-kind counts down to one row per channel, ordered.
+//
+// `kindIds` empty means EVERY kind on the payload — the same rule
+// resolveBackfillKinds applies to an absent scope, and the same rule
+// startBackfillSweep persists. It is deliberately not "no kinds": an unscoped
+// sweep is the corpus-wide one, and rendering it as an empty plan would say the
+// opposite of what arming it would do.
+export function foldSweepPlan({
+ counts,
+ kindIds,
+ order = "listed",
+}: {
+ counts: ReadonlyArray<SweepChannelCounts>;
+ kindIds?: ReadonlyArray<string>;
+ order?: AutoQueueOrder;
+}): SweepPlanEntry[] {
+ const scope = kindIds && kindIds.length > 0 ? new Set(kindIds) : null;
+ const plan: SweepPlanEntry[] = counts.map((channel) => {
+ const entry: SweepPlanEntry = {
+ channelSlug: channel.channelSlug,
+ reachable: 0,
+ missingInput: 0,
+ newestPending: "",
+ oldestPending: "",
+ };
+ for (const [id, kind] of Object.entries(channel.byKind)) {
+ if (scope && !scope.has(id)) continue;
+ entry.reachable += kind.reachable;
+ entry.missingInput += kind.missingInput;
+ if (kind.newestPending > entry.newestPending) {
+ entry.newestPending = kind.newestPending;
+ }
+ if (
+ kind.oldestPending &&
+ (!entry.oldestPending || kind.oldestPending < entry.oldestPending)
+ ) {
+ entry.oldestPending = kind.oldestPending;
+ }
+ }
+ return entry;
+ });
+ // The SAME comparator and the SAME weight tiebreak buildBackfillSweepPlan
+ // passes. Not a copy of the rule: the rule itself.
+ return orderPlanByRecency(
+ plan,
+ order,
+ (a, b) =>
+ b.reachable - a.reachable || a.channelSlug.localeCompare(b.channelSlug),
+ );
+}
+
+export type SweepPlanTotals = {
+ reachable: number;
+ missingInput: number;
+ // Channels the sweep would actually visit — those holding reachable work.
+ channels: number;
+ // Channels in the corpus that hold none. Stated separately rather than
+ // subtracted at the call site so "65 channels with nothing to do" and "3 of 68"
+ // come from one place and cannot disagree.
+ idle: number;
+};
+
+export function sweepPlanTotals(
+ plan: ReadonlyArray<SweepPlanEntry>,
+): SweepPlanTotals {
+ let reachable = 0;
+ let missingInput = 0;
+ let channels = 0;
+ for (const entry of plan) {
+ reachable += entry.reachable;
+ missingInput += entry.missingInput;
+ if (entry.reachable > 0) channels++;
+ }
+ return { reachable, missingInput, channels, idle: plan.length - channels };
+}
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,13 @@
# Changelog
## [Unreleased]
+- **A corpus sweep is a commitment, and you can finally read it before making it.** Arming the speaker-work sweep was one unlabelled button meaning *every enabled operation, every channel* — and on this archive that includes speaker-names-from-the-transcript at **~1 model call per transcript chunk**, on the order of 194,000 calls. The one setting that would have bounded it, `sweepKinds`, has been honoured by the run since the sweep was written and settable by **nothing but a hand-edit of `settings.json`**; "diarization and names-from-the-audio only" had no expression anywhere in the product. The lane now opens on a console: tick which **operations** the run covers, each stating its own backlog *and what one unit of it costs*, and read **THE PLAN** underneath — the ordered channel itinerary the sweep will actually walk, built by the same counting the run does, with the position marker on the channel it is working now. Untick the expensive lane and the total falls, the rows re-sort, and the button re-labels itself. *Newest first, across all channels* stops being a phrase in a dropdown and becomes a consequence you can see.
+- **The channel rows in the plan are the channel scope.** Rather than a second picker, **Choose channels** turns the itinerary into checkboxes — off by default, so the resting state is a clean list — and the button then says *Sweep 3 channels* instead of *Sweep every channel*. Scope, plan and progress are deliberately **one view rather than three**, because the alternative has a bug in it: the scope has to be written at the instant the sweep is armed. The boot hook re-launches a sweep from settings alone, so a scope saved separately (or saved and then not armed) comes back after a restart as the corpus-wide run it was meant to replace. The control that sets the scope is therefore the control that arms.
+- **The digest sweep got the same console, and a scope it could not previously be given at all.** Its channel scope was reachable in the controller and unreachable from the application — the action that armed it took no arguments. It takes one now. It offers no operation list, because it *is* one operation and a permanently-ticked checkbox would imply a choice that does not exist.
+- **A scope naming an operation that no longer exists no longer arms a sweep that runs forever doing nothing.** Settings sanitation keeps an unknown operation id, and the resolver then matches nothing with it — so the sweep starts, reports itself armed, and holds at zero. Unknown ids are now dropped at the moment of arming, with the console saying which; a scope that names *only* unknown operations is refused outright rather than started.
+- **The feed and the gate stopped sounding alike.** *Start sweep* and *Pause Backfill* were the same shape of phrase for two acts whose costs to undo differ by a week of GPU time. The feed **sweeps** — *Sweep every channel* / *Stop sweeping* — and the gate **holds** — *Hold the lane* / *Resume the lane*. The lane heading says what it holds and what this panel does with it: *Speakers · sweep*. The settings fieldset no longer implies scope lives there, and points at the console instead.
+- **The plan costs nothing to draw.** The page was already reading every channel's snapshot for the comparison rail and throwing the rest away — third cycle running that this has been true. The run's own planner could not be used: it *walks the corpus* when a channel has never been reported (~474,559 file touches, ~4 seconds) and this refreshes every three seconds. So the preview counts off the snapshots with the run's own arithmetic — verified against the live archive at 68 channels, 78,757 outstanding: **identical totals, identical per-channel counts, identical ordering** — and a channel nothing has reported on is drawn as *not reported yet* rather than as a confident zero. The guard test that keeps the corpus walk out of render paths grepped for one identifier and would have let the sweep planner straight through; it now names every function that reaches it.
+
- **One word was standing in for three different things, and hiding a fourth.** "Backfill" named a *lane* (three operations sharing a CPU queue), a *per-channel action*, and — on the download stage — a completely unrelated **download job** for subtitle tracks. Nobody arms, pauses or runs "a backfill"; it is a queue key. Every screen where an operator reads a **figure** now reads an **operation name**: the channel transit line's station is *Speakers*, its stage panel is *Speaker work · 3 operations* with a *Run speaker work* button, the download stage says *Fetch missing subtitle tracks*, and the lane card and `/auto-queue` name their members. The lane name survives in exactly one place — the settings fieldset and the lane card, where the one shared pause and the one shared sweep live — and there it now **lists what it holds**, because "these operations share a queue and a pause" is a true and load-bearing fact rather than a leaked implementation detail.
- **The transit line's Speakers station stopped summing three operations into one number.** It set its coverage by adding every operation on the lane together — the one thing this codebase forbids everywhere else — which on the live corpus meant adding diarization (one audio pass per video, 4 done of 11,338) to speaker-names-from-the-transcript (**~1 model call per transcript chunk**, 1 done of 11,338) and printing the total under a label that named the queue. The numeral now belongs to exactly one operation, and each member states itself, with its own band and its own denominator, in the station foot.
- **An armed operation now says what one unit of it costs.** The fact that hid: speaker-names-from-the-transcript is switched **on**, can reach 11,337 videos of one channel — on the order of **194,000 model calls corpus-wide** — has completed one video, and read as a quiet row on every screen, because "11,337 reachable" is the same shape of number whether the unit is an audio pass or a per-chunk model call. Each operation declares its cost basis, printed beside its backlog wherever it is armed. Deliberately **no threshold and no editorialising**: what is affordable is the operator's call, and a "this is a lot" cutoff would be a magic number the next operation gets wrong. Nothing here changes a setting.
diff --git a/editor/app/auto-queue/components/SweepLane.tsx b/editor/app/auto-queue/components/SweepLane.tsx
@@ -23,6 +23,8 @@ import { saveLaneOrderAction } from "../actions";
import type { SweepLaneStatus } from "../lanes";
import type { OperationBand } from "../../components/pipelines/band";
import { OrderReach } from "./OrderReach";
+import { SweepPlan } from "./SweepPlan";
+import { SweepScope } from "./SweepScope";
import { formatElapsed } from "./dispatch";
// A sweep-fed lane: digest or backfill.
@@ -58,6 +60,52 @@ export function SweepLane({
const [pending, startTransition] = useTransition();
const [busy, setBusy] = useState(false);
const digest = lane.id === "digest";
+ // THE SCOPE THE OPERATOR IS COMPOSING, seeded from what is ARMED.
+ //
+ // Seeded once and then owned locally, deliberately: the payload re-arrives
+ // every 3 seconds, and re-seeding on each poll would undo a tick the moment
+ // it was made. It is a draft until the arm button writes it — and it is
+ // written BY the arm action, never as a separate settings save, so a restart
+ // between the two cannot resurrect a bounded run as a corpus-wide one.
+ const [scopeKinds, setScopeKinds] = useState<string[]>(lane.scopeKinds);
+ const [channelScope, setChannelScope] = useState<ReadonlySet<string> | null>(
+ lane.scopeChannels.length > 0 ? new Set(lane.scopeChannels) : null,
+ );
+ // WHILE A SWEEP IS RUNNING THE CONSOLE SHOWS WHAT IS ARMED, NOT A DRAFT.
+ //
+ // The scope is read once, at arm time, and persisted there; nothing re-reads
+ // it mid-run. So leaving the checkboxes live during a sweep would let an
+ // operator untick the expensive lane, watch the plan total fall, and believe
+ // they had changed a run that is still doing every operation. The controls go
+ // read-only and show the armed scope, and the panel says how to change it.
+ const armed = lane.sweeping;
+ // A stored scope of `[]` means "every enabled operation" — the rule
+ // resolveBackfillKinds applies and the rule an unscoped sweep runs — so it
+ // renders as every box ticked rather than none.
+ const allKindIds = lane.operations.map((op) => op.id);
+ const draftKinds = scopeKinds.length > 0 ? scopeKinds : allKindIds;
+ const selectedKinds = armed
+ ? lane.scopeKinds.length > 0
+ ? lane.scopeKinds
+ : allKindIds
+ : draftKinds;
+ // Ticking every box is not the same as pinning today's three: it re-arms the
+ // UNSCOPED sweep, which tracks the registry, so an operation enabled later is
+ // picked up by the resumed run instead of being silently excluded forever.
+ const armKinds = draftKinds.length === allKindIds.length ? [] : draftKinds;
+ const unknownScopeIds = lane.scopeKinds.filter(
+ (id) => !lane.operations.some((op) => op.id === id),
+ );
+ const armChannels = channelScope ? [...channelScope] : undefined;
+ const shownChannels = armed
+ ? lane.scopeChannels.length > 0
+ ? new Set(lane.scopeChannels)
+ : null
+ : channelScope;
+ const canArm =
+ lane.available &&
+ (digest || draftKinds.length > 0) &&
+ (channelScope === null || channelScope.size > 0);
const state = deriveLaneState({
available: lane.available,
gateHeld: lane.gateHeld,
@@ -82,20 +130,38 @@ export function SweepLane({
return (
<div className="flex flex-col gap-4">
<div className="flex flex-wrap items-center gap-x-3 gap-y-2">
+ {/* NAMED AFTER WHAT IT HOLDS, then after what this panel does with it.
+ "Backfill" was a queue key standing in for three operations with
+ different inputs and different costs; lanes.ts derives the group
+ name, and "· sweep" says which of the lane's two controls this panel
+ is about. */}
<h2 className="font-display text-lg font-semibold tracking-tight">
- {lane.label}
+ {lane.label} <span className="text-muted-foreground">· sweep</span>
</h2>
<span className="flex items-center gap-2 text-sm">
<span aria-hidden="true" className={`size-2 rounded-full ${LANE_DOT[state]}`} />
<span className={LANE_TEXT[state]}>{LANE_WORD[state]}</span>
</span>
<span className="ml-auto flex flex-wrap gap-2">
- {/* THE FEED. Named "sweep" on both surfaces. */}
+ {/* THE FEED, and it now says WHAT IT WILL DO rather than naming the
+ mechanism. "Start sweep" and "Pause Backfill" are the same shape
+ of phrase for two acts whose costs to undo differ by a week of GPU
+ time; different verbs are the cheapest thing that keeps them
+ apart. The label is derived from the scope composed below, so the
+ button and the plan cannot disagree about what is being armed.
+
+ THE SCOPE TRAVELS THROUGH THIS ACTION, never through a separate
+ settings save — startBackfillSweep persists the flag and the scope
+ in one awaited write, which is what lets the boot hook resume the
+ same run rather than a corpus-wide one.
+
+ aria-label deliberately UNCHANGED: it is internal addressing that
+ confuses nobody, and the e2e suite finds these buttons by it. */}
<Button
type="button"
size="sm"
variant={lane.sweeping ? "outline" : "default"}
- disabled={working || !lane.available}
+ disabled={working || !lane.available || (!lane.sweeping && !canArm)}
aria-label={`${lane.sweeping ? "Stop" : "Start"} ${lane.label} sweep`}
onClick={() =>
run(
@@ -104,15 +170,16 @@ export function SweepLane({
? stopDigestSweepAction
: stopBackfillSweepAction
: digest
- ? startDigestSweepAction
- // Unscoped on purpose: every enabled lane kind, whole
- // corpus. A scope belongs to the sweep controls that own
- // one, not to a console button whose label says "sweep".
- : () => startBackfillSweepAction(),
+ ? () => startDigestSweepAction(armChannels)
+ : () => startBackfillSweepAction(armKinds, armChannels),
)
}
>
- {lane.sweeping ? "Stop sweep" : "Start sweep"}
+ {lane.sweeping
+ ? "Stop sweeping"
+ : channelScope
+ ? `Sweep ${channelScope.size.toLocaleString()} channel${channelScope.size === 1 ? "" : "s"}`
+ : "Sweep every channel"}
</Button>
{/* THE GATE. Backfill's is inverted on the wire (`enabled`); it is
normalised in lanes.ts so this button means the same thing on both
@@ -140,7 +207,7 @@ export function SweepLane({
: "border-warning/30 text-warning hover:bg-warning-soft hover:text-warning"
}
>
- {lane.gateHeld ? "Resume" : "Pause"}
+ {lane.gateHeld ? "Resume the lane" : "Hold the lane"}
</Button>
</span>
</div>
@@ -183,6 +250,53 @@ export function SweepLane({
run(() => saveLaneOrderAction(lane.id, next))
}
/>
+
+ {/* SCOPE, ORDER, PLAN — in the order an operator composes them, and all
+ three above the button that commits them.
+
+ AN UNAVAILABLE LANE GETS NO CONSOLE. With no operation registered
+ there is nothing to scope, and a plan of 68 zeroed rows reads as "all
+ caught up" when the truth is "switched off" — which the sentence at
+ the top of the panel says instead. */}
+ {lane.available && (
+ <>
+ {armed && (
+ <p className="text-xs text-muted-foreground">
+ This is the scope the running sweep was armed with. Stop sweeping
+ to change it — the scope is read once, when the sweep starts, so
+ an edit made now would not reach the run.
+ </p>
+ )}
+ <SweepScope
+ operations={lane.operations}
+ selected={new Set(selectedKinds)}
+ unknownScopeIds={unknownScopeIds}
+ busy={working || armed}
+ onToggle={(id, next) =>
+ setScopeKinds(
+ next
+ ? [...new Set([...draftKinds, id])]
+ : draftKinds.filter((k) => k !== id),
+ )
+ }
+ />
+ <SweepPlan
+ plan={lane.plan}
+ operations={lane.operations}
+ scopeKinds={selectedKinds}
+ // The order the RUN will use. Only "corpus" reach reaches the
+ // CHANNEL order — under "channel" the sweep stays heaviest-first
+ // and each per-channel batch applies the order itself — so the plan
+ // is drawn the way runSweepLoop resolves it, not the way the
+ // dropdown reads.
+ order={lane.reach === "corpus" ? lane.order : "listed"}
+ runningChannel={lane.inFlight[0]?.channelSlug ?? null}
+ channelScope={shownChannels}
+ busy={working || armed}
+ onChannelScope={setChannelScope}
+ />
+ </>
+ )}
</div>
);
}
diff --git a/editor/app/auto-queue/components/SweepPlan.tsx b/editor/app/auto-queue/components/SweepPlan.tsx
@@ -0,0 +1,264 @@
+"use client";
+
+import { useMemo, useState } from "react";
+import type { AutoQueueOrder } from "yt-dlp-transcript-common/jobs/autoQueuePolicy";
+import {
+ foldSweepPlan,
+ sweepPlanTotals,
+ type SweepChannelCounts,
+} from "yt-dlp-transcript-common/lib/sweepPlan";
+import { bandForScope } from "../../components/pipelines/band";
+import { StateBand } from "../../components/pipelines/StateBand";
+import type { SweepOperation } from "../lanes";
+
+// THE PLAN — the ordered channel itinerary the sweep will actually walk, drawn
+// before it is armed.
+//
+// This is the whole point of the console. "Newest first, across all channels"
+// was a phrase in a dropdown; here it is a consequence you can see, because the
+// list re-sorts under your hand when you change the order, and re-totals when
+// you change the scope. You cannot arm blind, because the thing you are arming
+// is the thing you are reading.
+//
+// SCOPE, PLAN AND PROGRESS ARE ONE VIEW, NOT THREE. The conventional split
+// would put scope in a settings form, a preview behind a modal and progress in
+// a status card. Merging them is justified twice: it is the direct fix for a
+// real bug — a scope that is not recorded at arm time resurrects a bounded run
+// as a corpus-wide one after a restart, so the control that sets the scope must
+// BE the control that arms — and it is the argument OrderReach.tsx already
+// makes for living at the foot of the lane rather than in /settings.
+//
+// NO ANIMATION, and it is a subtraction rather than an oversight: 68 rows
+// easing every 3 seconds is a light show, not an instrument. StateBand's
+// `strip` size is the non-animating one for exactly this reason.
+//
+// NO PER-ROW PERCENTAGE. Digest sits at 0 done on every large channel and
+// diarization is 99.96% media-gone, so "% complete" renders the same number on
+// every row and says nothing; what varies — and what an operator needs — is the
+// SHAPE of the remainder, which is what the band draws.
+
+// How many working rows are drawn before the list is folded. Explicit, with the
+// remainder named and one click away: a silent top-N reads as "this is all of
+// it" when it is not.
+const VISIBLE_ROWS = 12;
+
+function formatPlanDate(key: string): string {
+ if (!/^\d{8}$/.test(key)) return "undated";
+ return `${key.slice(0, 4)}-${key.slice(4, 6)}-${key.slice(6)}`;
+}
+
+export function SweepPlan({
+ plan,
+ operations,
+ scopeKinds,
+ order,
+ // The channel this sweep's per-channel job is on right now, for the position
+ // marker. Read from the lane's in-flight list rather than stored, so it can
+ // never be stale in a way the rest of the lane is not.
+ runningChannel,
+ // The channel scope, or null for "every channel" — the resting state. Not a
+ // separate picker: the plan rows ARE the channel scope, which is why turning
+ // it on turns these rows into checkboxes instead of opening a second list.
+ channelScope,
+ busy,
+ onChannelScope,
+}: {
+ plan: SweepChannelCounts[];
+ operations: SweepOperation[];
+ scopeKinds: string[];
+ order: AutoQueueOrder;
+ runningChannel: string | null;
+ channelScope: ReadonlySet<string> | null;
+ busy: boolean;
+ onChannelScope: (next: ReadonlySet<string> | null) => void;
+}) {
+ const [showAll, setShowAll] = useState(false);
+
+ const rows = useMemo(() => {
+ const byKind = new Map(plan.map((row) => [row.channelSlug, row]));
+ const ordered = foldSweepPlan({ counts: plan, kindIds: scopeKinds, order });
+ return ordered.map((entry) => {
+ const counts = byKind.get(entry.channelSlug);
+ // An empty scope is EVERY operation — the same rule foldSweepPlan and
+ // resolveBackfillKinds apply, and the same rule an unscoped sweep runs.
+ const inScope = scopeKinds;
+ const selected = Object.entries(counts?.byKind ?? {}).filter(
+ ([id]) => inScope.length === 0 || inScope.includes(id),
+ );
+ return {
+ entry,
+ unknown: counts?.unknown === true,
+ band: bandForScope({
+ id: "sweep-scope",
+ // The band's own label and unit are the SCOPE's, not one operation's.
+ // With more than one operation selected the unit is "one operation on
+ // one video", which is exactly what the sweep dispatches — see
+ // bandForScope's header for why that is the one legitimate sum.
+ label: operations
+ .filter((op) => inScope.length === 0 || inScope.includes(op.id))
+ .map((op) => op.label)
+ .join(" · "),
+ costBasis: "",
+ counts: selected.map(([, c]) => c),
+ }),
+ };
+ });
+ }, [plan, scopeKinds, order, operations]);
+
+ const totals = useMemo(
+ () => sweepPlanTotals(rows.map((r) => r.entry)),
+ [rows],
+ );
+ const working = rows.filter((r) => r.entry.reachable > 0);
+ const unreported = rows.filter((r) => r.entry.reachable === 0 && r.unknown);
+ const idle = rows.length - working.length - unreported.length;
+ const visible = showAll ? working : working.slice(0, VISIBLE_ROWS);
+ const choosing = channelScope !== null;
+
+ const toggleChannel = (slug: string, next: boolean) => {
+ const set = new Set(channelScope ?? []);
+ if (next) set.add(slug);
+ else set.delete(slug);
+ onChannelScope(set);
+ };
+
+ return (
+ <div className="flex flex-col gap-2">
+ <div className="flex flex-wrap items-baseline justify-between gap-x-4 gap-y-1">
+ <p className="font-mono text-xs uppercase tracking-[0.14em] text-muted-foreground">
+ The plan
+ </p>
+ <p className="text-sm text-muted-foreground">
+ {/* THE ONE LARGE NUMERAL on this panel, mirroring the transit line's
+ one station numeral so the two consoles rhyme. It is the plan
+ total — the number the commitment is actually about. */}
+ <span className="font-display text-3xl font-semibold tabular-nums text-foreground">
+ {totals.reachable.toLocaleString()}
+ </span>{" "}
+ to do across {totals.channels.toLocaleString()} of{" "}
+ {rows.length.toLocaleString()} channels
+ {totals.missingInput > 0 && (
+ // STATED SEPARATELY, NEVER SUMMED. On this corpus the two are
+ // orders of magnitude apart, and one "remaining" figure would say
+ // the same thing about a lane that is finished and a lane that
+ // cannot start.
+ <>
+ {" · "}
+ <span className="tabular-nums">
+ {totals.missingInput.toLocaleString()}
+ </span>{" "}
+ need their media back first
+ </>
+ )}
+ </p>
+ </div>
+
+ {working.length === 0 ? (
+ <p className="text-sm text-muted-foreground">
+ Nothing reachable in this scope. That is not the same as finished —
+ widen the operations above, or check what is waiting on media.
+ </p>
+ ) : (
+ <ul className="flex flex-col gap-1">
+ {visible.map(({ entry, band }) => {
+ const here = entry.channelSlug === runningChannel;
+ return (
+ <li
+ key={entry.channelSlug}
+ className="grid grid-cols-[1.25rem_minmax(6rem,1fr)_minmax(4rem,8rem)_auto] items-center gap-x-3 gap-y-1 text-sm"
+ >
+ <span className="flex items-center justify-center">
+ {choosing ? (
+ <input
+ type="checkbox"
+ aria-label={`sweep ${entry.channelSlug}`}
+ checked={channelScope?.has(entry.channelSlug) === true}
+ disabled={busy}
+ onChange={(e) =>
+ toggleChannel(entry.channelSlug, e.target.checked)
+ }
+ />
+ ) : (
+ // THE POSITION MARKER. Only meaningful while something is
+ // running, and deliberately the same arrow the sweep's own
+ // log prints for the channel it is entering.
+ <span
+ aria-hidden="true"
+ className={here ? "text-info" : "text-transparent"}
+ >
+ →
+ </span>
+ )}
+ </span>
+ <span
+ className={`truncate font-mono text-xs ${
+ here ? "text-foreground" : "text-muted-foreground"
+ }`}
+ title={entry.channelSlug}
+ >
+ {entry.channelSlug}
+ </span>
+ <StateBand band={band} size="strip" />
+ <span className="flex flex-wrap items-baseline justify-end gap-x-3">
+ <span className="tabular-nums">
+ {entry.reachable.toLocaleString()} to do
+ </span>
+ <span className="w-28 text-right text-xs tabular-nums text-muted-foreground">
+ {order === "oldest"
+ ? `oldest ${formatPlanDate(entry.oldestPending)}`
+ : `newest ${formatPlanDate(entry.newestPending)}`}
+ </span>
+ </span>
+ </li>
+ );
+ })}
+ </ul>
+ )}
+
+ {working.length > visible.length && (
+ <button
+ type="button"
+ className="self-start text-xs underline underline-offset-2 text-muted-foreground hover:text-foreground"
+ onClick={() => setShowAll(true)}
+ >
+ + {(working.length - visible.length).toLocaleString()} more channels
+ with work
+ </button>
+ )}
+
+ <div className="flex flex-wrap items-baseline gap-x-4 gap-y-1 text-xs text-muted-foreground">
+ {idle > 0 && (
+ <span>
+ {idle.toLocaleString()} channel{idle === 1 ? "" : "s"} with nothing
+ to do
+ </span>
+ )}
+ {unreported.length > 0 && (
+ // UNKNOWN IS NOT ZERO. These channels have never been reported on, so
+ // 0 here would be a claim nothing has checked. The run's own planner
+ // walks them; this list cannot, so it says so.
+ <span className="text-warning">
+ {unreported.length.toLocaleString()} not reported yet (
+ {unreported
+ .slice(0, 3)
+ .map((r) => r.entry.channelSlug)
+ .join(", ")}
+ {unreported.length > 3 ? "…" : ""}) — the sweep will still visit them
+ </span>
+ )}
+ <button
+ type="button"
+ className="ml-auto underline underline-offset-2 hover:text-foreground"
+ disabled={busy}
+ onClick={() =>
+ onChannelScope(
+ choosing ? null : new Set(working.map((r) => r.entry.channelSlug)),
+ )
+ }
+ >
+ {choosing ? "Use every channel" : "Choose channels"}
+ </button>
+ </div>
+ </div>
+ );
+}
diff --git a/editor/app/auto-queue/components/SweepScope.tsx b/editor/app/auto-queue/components/SweepScope.tsx
@@ -0,0 +1,113 @@
+"use client";
+
+import { useId } from "react";
+import type { SweepOperation } from "../lanes";
+
+// WHICH OPERATIONS THIS SWEEP RUNS — the axis that had no control at all.
+//
+// `backfill.sweepKinds` has existed and been honoured by the run since the sweep
+// was written, and no screen has ever set it: arming was one unlabelled button
+// meaning "every enabled operation, whole corpus". On this install that includes
+// speaker-names-from-the-transcript at ~1 model call per transcript CHUNK, on the
+// order of 194,000 calls corpus-wide. "Diarization and names-from-audio only" was
+// a settings.json hand-edit or nothing.
+//
+// EVERY ROW STATES ITS UNIT. The three operations here have backlogs of 1, 4 and
+// 11,337 — and the last is not 11,337 times the first in cost, it is far more,
+// because its unit is not the video. A checkbox list without the cost basis
+// beside it would be asking for a decision with the deciding fact left out.
+//
+// NO NEW COLOUR. An operation that is out of scope drops to muted text; nothing
+// gains an accent. `can run now` stays the only saturated fill on any pipeline
+// surface — the rule the rail, the /channels strip and the transit line all hold
+// — and a second accent here would break that reading on four surfaces at once.
+//
+// Native <input type=checkbox> like every other policy control on this page; see
+// the note at the foot of dispatch.ts for why that is load-bearing rather than
+// stylistic.
+
+export function SweepScope({
+ operations,
+ selected,
+ // Ids the STORED scope names that no operation answers to. sanitizeBackfill
+ // keeps an unknown id and resolveBackfillKinds then matches nothing with it,
+ // so a sweep armed on one runs forever doing nothing. Reported here rather
+ // than swallowed.
+ unknownScopeIds,
+ busy,
+ onToggle,
+}: {
+ operations: SweepOperation[];
+ selected: ReadonlySet<string>;
+ unknownScopeIds: string[];
+ busy: boolean;
+ onToggle: (id: string, next: boolean) => void;
+}) {
+ const groupId = useId();
+ // One operation is not a choice. The digest sweep is exactly this case, and
+ // drawing it a single permanently-ticked checkbox would imply a scope that
+ // cannot be varied.
+ if (operations.length < 2) return null;
+
+ return (
+ <div className="flex flex-col gap-2 rounded-md border border-border bg-card px-3 py-2">
+ <div className="flex flex-wrap items-baseline justify-between gap-x-4 gap-y-1">
+ <p
+ id={groupId}
+ className="font-mono text-xs uppercase tracking-[0.14em] text-muted-foreground"
+ >
+ Run which operations
+ </p>
+ <p className="text-xs tabular-nums text-muted-foreground">
+ {selected.size} of {operations.length} selected
+ </p>
+ </div>
+ <ul className="flex flex-col gap-1" aria-labelledby={groupId}>
+ {operations.map((op) => {
+ const on = selected.has(op.id);
+ return (
+ <li key={op.id}>
+ <label
+ className={`flex flex-wrap items-baseline gap-x-3 gap-y-0.5 text-sm ${
+ on ? "" : "text-muted-foreground"
+ }`}
+ >
+ <input
+ type="checkbox"
+ className="self-center"
+ checked={on}
+ disabled={busy}
+ onChange={(e) => onToggle(op.id, e.target.checked)}
+ />
+ <span className={on ? "text-foreground" : ""}>{op.label}</span>
+ <span className="tabular-nums">
+ {op.reachable.toLocaleString()}
+ </span>
+ {/* THE UNIT, from the registry. No threshold and no
+ editorialising: what is affordable is the operator's call,
+ and a "this is a lot" cutoff would be a magic number the next
+ operation gets wrong. */}
+ <span className="text-xs text-muted-foreground">
+ {op.costBasis}
+ </span>
+ </label>
+ </li>
+ );
+ })}
+ </ul>
+ {selected.size === 0 && (
+ <p className="text-xs text-warning">
+ Nothing selected — there is no sweep to arm. Tick at least one
+ operation.
+ </p>
+ )}
+ {unknownScopeIds.length > 0 && (
+ <p className="text-xs text-warning">
+ The stored scope also names {unknownScopeIds.join(", ")}, which no
+ enabled operation answers to. It counts as nothing and will be dropped
+ when you re-arm.
+ </p>
+ )}
+ </div>
+ );
+}
diff --git a/editor/app/auto-queue/lanes.ts b/editor/app/auto-queue/lanes.ts
@@ -3,9 +3,16 @@ import { getSettings } from "yt-dlp-transcript-common/lib/settings";
import type { AutoQueueOrder, AutoQueueReach } from "yt-dlp-transcript-common/jobs/autoQueuePolicy";
import {
allBackfillKinds,
+ DIGEST_KIND_ID,
+ laneBackfillKinds,
+ operationCostBasis,
+ operationLabel,
operationsActionLabel,
operationsGroupLabel,
} from "yt-dlp-transcript-common/lib/backfillKinds";
+import { buildSweepChannelCounts } from "yt-dlp-transcript-common/controller/sweepPreview";
+import { loadSweepDates } from "yt-dlp-transcript-common/controller/sweepRecency";
+import type { SweepChannelCounts } from "yt-dlp-transcript-common/lib/sweepPlan";
import { getDigestSweepJobId } from "yt-dlp-transcript-common/controller/digestSweep";
import { getBackfillSweepJobId } from "yt-dlp-transcript-common/controller/backfillSweep";
import { getRegistry } from "yt-dlp-transcript-common/jobs/registry";
@@ -62,6 +69,43 @@ export type SweepLaneStatus = {
inFlight: { id: string; channelSlug: string | null; startedAt: number | null }[];
order: AutoQueueOrder;
reach: AutoQueueReach;
+ // ── THE SWEEP'S SCOPE AND ITS PLAN ──────────────────────────────────────
+ //
+ // A sweep is a commitment, not a toggle: arming the backfill one on this
+ // corpus can mean ~194,000 model calls. Everything below exists so the
+ // commitment can be READ before it is made, and it is on the same payload as
+ // the state above for the reason the rail is: two polls would let the plan
+ // and the switch that arms it disagree about the same moment.
+ //
+ // The operations this sweep can run, in registry order. Exactly one entry for
+ // the digest lane, which is why the console offers it no operation list —
+ // there is nothing to choose.
+ operations: SweepOperation[];
+ // What is ARMED right now, read back out of settings rather than held in the
+ // client. Empty means unscoped: every operation, every channel — which is
+ // what an unscoped sweep has always meant and what the boot hook resumes.
+ scopeKinds: string[];
+ scopeChannels: string[];
+ // The itinerary: one entry per channel, counts per operation, undated where
+ // nothing could date them. The client folds this to a plan on every scope
+ // change; see common/lib/sweepPlan.ts for why the fold is not done here.
+ plan: SweepChannelCounts[];
+};
+
+// One operation an operator can put in or out of a sweep's scope.
+//
+// `costBasis` is carried because it is the fact that hid behind a shared lane
+// name: "11,337 reachable" is the same shape of number whether one unit is a
+// single pass over the audio or ~1 model call per transcript CHUNK, a ~17x
+// difference on the same figure. A scope control that did not state it would be
+// asking the operator to choose blind.
+export type SweepOperation = {
+ id: string;
+ label: string;
+ costBasis: string;
+ // Corpus-wide reachable work for this operation alone, so the checkbox says
+ // what ticking it costs before the plan below re-folds.
+ reachable: number;
};
// The unified dispatcher's state. Not a lane: it does not do work, it decides
@@ -119,6 +163,63 @@ export async function buildAutoQueueLanes(): Promise<AutoQueueLanesPayload> {
operationIds: kinds.map((k) => k.id),
});
+ // ── THE PLAN, off the briefs already in hand ────────────────────────────
+ //
+ // FREE, and that is the third cycle running that this has been true: this
+ // function already read every channel's snapshot for the rail and threw the
+ // rest away. What it costs now is a fold over data already in memory.
+ //
+ // NOT THE SWEEP'S OWN PLANNER, which walks the corpus when a snapshot is
+ // absent (~474,559 file touches) and is banned from render paths by
+ // common/controller/noCorpusWalkInRenderPaths.test.ts. The preview is the
+ // same counting off the snapshots, with an unreported channel reported as
+ // unknown rather than as zero.
+ //
+ // (That guard greps for the NAME, in any context — so it is not written here
+ // even in prose. Dumb on purpose: a guard that skipped comments would be a
+ // guard an offending call could hide behind.)
+ const laneKinds = laneBackfillKinds(settings);
+ const backfillKindIds = laneKinds.map((k) => k.id);
+ // Digest is one operation on its own queue, always catalogued whether or not
+ // the feature is switched on — the lane is always available (see below), so
+ // its plan must be too.
+ const digestKindIds = [DIGEST_KIND_ID];
+ const planKindIds = [...new Set([...backfillKindIds, ...digestKindIds])];
+ // Dates for the ids both plans will order by. Cached for a minute — see
+ // controller/sweepRecency.ts for the measured cost and why a stale date
+ // cannot mis-plan a sweep that stores no cursor.
+ const dates = await loadSweepDates({
+ paths,
+ briefs,
+ kindIds: planKindIds,
+ });
+ const counts = buildSweepChannelCounts({
+ briefs,
+ kindIds: planKindIds,
+ dates,
+ });
+ // Each lane sees only its own operations. Handing the backfill console a
+ // digest column would offer a scope its arm action cannot express — digest's
+ // sweep takes channels and nothing else, because it IS one operation.
+ const planFor = (ids: ReadonlyArray<string>): SweepChannelCounts[] =>
+ counts.map((row) => ({
+ channelSlug: row.channelSlug,
+ unknown: row.unknown,
+ byKind: Object.fromEntries(
+ ids.filter((id) => row.byKind[id]).map((id) => [id, row.byKind[id]]),
+ ),
+ }));
+ const operationsFor = (ids: ReadonlyArray<string>): SweepOperation[] =>
+ ids.map((id) => ({
+ id,
+ label: operationLabel(id),
+ costBasis: operationCostBasis(id),
+ reachable: counts.reduce(
+ (n, row) => n + (row.byKind[id]?.reachable ?? 0),
+ 0,
+ ),
+ }));
+
return {
digest: {
id: "digest",
@@ -130,6 +231,13 @@ export async function buildAutoQueueLanes(): Promise<AutoQueueLanesPayload> {
inFlight: inFlightOn([DIGEST_LOCAL_QUEUE, DIGEST_REMOTE_QUEUE]),
order: settings.digest.recencyOrder,
reach: settings.digest.recencyReach,
+ // ONE ENTRY, deliberately. The digest sweep runs exactly one operation,
+ // so there is no operation scope to offer and the console renders none —
+ // the channel itinerary is the whole of its scope.
+ operations: operationsFor(digestKindIds),
+ scopeKinds: [],
+ scopeChannels: settings.digest.sweepChannels,
+ plan: planFor(digestKindIds),
},
backfill: {
id: "backfill",
@@ -153,6 +261,10 @@ export async function buildAutoQueueLanes(): Promise<AutoQueueLanesPayload> {
inFlight: inFlightOn([BACKFILL_QUEUE]),
order: settings.backfill.order,
reach: settings.backfill.reach,
+ operations: operationsFor(backfillKindIds),
+ scopeKinds: settings.backfill.sweepKinds,
+ scopeChannels: settings.backfill.sweepChannels,
+ plan: planFor(backfillKindIds),
},
bands,
arbiter: {
diff --git a/editor/app/components/lanes/LaneDeck.tsx b/editor/app/components/lanes/LaneDeck.tsx
@@ -257,7 +257,7 @@ export function LaneDeck({
digestSweeping
? {
key: "digest-sweep",
- label: "Stop Digest Sweep",
+ label: "Stop sweeping",
glyph: "■",
ariaLabel: "stop digest sweep",
title:
@@ -268,7 +268,7 @@ export function LaneDeck({
}
: {
key: "digest-sweep",
- label: "Start Digest Sweep",
+ label: "Sweep every channel",
glyph: "⟳",
ariaLabel: "start digest sweep",
title:
@@ -285,7 +285,7 @@ export function LaneDeck({
digestPaused
? {
key: "digest-gate",
- label: "Resume Digests",
+ label: "Resume the lane",
glyph: "▶",
ariaLabel: "resume digests",
title:
@@ -296,7 +296,7 @@ export function LaneDeck({
}
: {
key: "digest-gate",
- label: "Pause Digests",
+ label: "Hold the lane",
glyph: "❙❙",
ariaLabel: "pause digests",
title:
@@ -392,6 +392,13 @@ export function LaneDeck({
// flight finishes instead of being thrown away, and the stop reaches the
// per-channel job the sweep is waiting on rather than meaning "after this
// channel". Pause when you want it back; stop when you don't.
+ //
+ // THE TWO NOW USE DIFFERENT VERBS, which is the cheapest fix for the hazard
+ // this comment has been describing in prose. "Start sweep" and "Pause
+ // Backfill" are the same shape of phrase for two acts whose costs to undo
+ // differ by a week of GPU time. The feed SWEEPS; the gate HOLDS. The
+ // aria-labels are untouched — they are internal addressing that confuses
+ // nobody, and the e2e suite finds these buttons by them.
const backfillSweeping = backfill?.sweeping ?? false;
const laneEnabled = backfill?.enabled ?? false;
const backfillAvailable = backfill?.anyKind ?? false;
@@ -407,7 +414,7 @@ export function LaneDeck({
backfillSweeping
? {
key: "backfill-sweep",
- label: "Stop Backfill Sweep",
+ label: "Stop sweeping",
glyph: "■",
ariaLabel: "stop backfill sweep",
title:
@@ -418,7 +425,7 @@ export function LaneDeck({
}
: {
key: "backfill-sweep",
- label: "Start Backfill Sweep",
+ label: "Sweep every channel",
glyph: "⟳",
ariaLabel: "start backfill sweep",
title:
@@ -430,7 +437,7 @@ export function LaneDeck({
laneEnabled
? {
key: "backfill-gate",
- label: "Pause Backfill",
+ label: "Hold the lane",
glyph: "❙❙",
ariaLabel: "pause backfill",
title:
@@ -441,7 +448,7 @@ export function LaneDeck({
}
: {
key: "backfill-gate",
- label: "Resume Backfill",
+ label: "Resume the lane",
glyph: "▶",
ariaLabel: "resume backfill",
title:
diff --git a/editor/app/components/pipelines/band.ts b/editor/app/components/pipelines/band.ts
@@ -209,3 +209,66 @@ export function bandHeadline(band: OperationBand): string {
}
return `${best[0].toLocaleString()} ${best[1]}`;
}
+
+// ── A BAND FOR THE OPERATIONS A SWEEP HAS IN SCOPE ──────────────────────────
+//
+// The plan on /auto-queue draws one strip per channel, and the strip has to
+// re-draw when an operation is ticked off — in the browser, with no round trip.
+// So the fold lives here, on the directive-free side, beside the type.
+//
+// THIS IS THE ONE PLACE SUMMING ACROSS OPERATIONS IS LEGITIMATE, and it is worth
+// being precise about why, because every other surface is forbidden from doing
+// it. The rule that forbids it (buildBands' header, laneEntriesOf's) is about a
+// FIGURE IN NO UNIT: adding diarization's videos to attribution-text's videos
+// gives a number that is neither, because one unit of the first is an audio pass
+// and one unit of the second is ~1 model call per transcript chunk.
+//
+// A sweep plan is not that number. The sweep dispatches one operation on one
+// video at a time, so the unit here IS "one operation on one video" — and every
+// segment of this band is counted in it. The rows are directly comparable
+// because they all sit under the SAME scope, which is the control immediately
+// above them. Change the scope and every row changes together.
+//
+// What it still must not do is invent a denominator: `eligible` and `present`
+// fold with sumOrNull, so one operation that cannot say makes the whole band
+// say so — an outline, never 0%.
+export function bandForScope({
+ id,
+ label,
+ costBasis,
+ counts,
+}: {
+ id: string;
+ label: string;
+ costBasis: string;
+ counts: ReadonlyArray<{
+ reachable: number;
+ blocked: number;
+ missingInput: number;
+ deferred: number;
+ eligible: number | null;
+ present: number | null;
+ }>;
+}): OperationBand {
+ const band: OperationBand = {
+ id,
+ label,
+ costBasis,
+ eligible: 0,
+ present: 0,
+ reachable: 0,
+ blocked: 0,
+ missingInput: 0,
+ deferred: 0,
+ dispatched: true,
+ };
+ for (const c of counts) {
+ band.reachable += c.reachable;
+ band.blocked += c.blocked;
+ band.missingInput += c.missingInput;
+ band.deferred += c.deferred;
+ band.eligible = sumOrNull([band.eligible, c.eligible]);
+ band.present = sumOrNull([band.present, c.present]);
+ }
+ return band;
+}
diff --git a/editor/app/jobs/actions.ts b/editor/app/jobs/actions.ts
@@ -3,6 +3,7 @@
import { revalidatePath } from "next/cache";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import { getSettings, writeSettings } from "yt-dlp-transcript-common/lib/settings";
+import { laneBackfillKinds } from "yt-dlp-transcript-common/lib/backfillKinds";
import { pruneJobLogs } from "yt-dlp-transcript-common/jobs/listJobs";
import { getRegistry } from "yt-dlp-transcript-common/jobs/registry";
import { readJobMeta } from "yt-dlp-transcript-common/jobs/jobMeta";
@@ -218,9 +219,17 @@ export async function resumeDigestsAction(): Promise<DigestPauseResult> {
// all — and it persists, so the boot hook resumes it.
export type DigestSweepResult = { ok: boolean; jobId?: string; error?: string };
-export async function startDigestSweepAction(): Promise<DigestSweepResult> {
+// `channelSlugs` is the scope, and it goes THROUGH THIS ACTION rather than
+// through a settings save. startDigestSweep persists the flag and the scope in
+// one write, which is what lets the boot hook resume the same run; a scope
+// written separately would be a second writer for one decision, and a restart
+// between the two writes would resurrect a bounded run as a corpus-wide one.
+// Absent = the whole corpus, exactly as before.
+export async function startDigestSweepAction(
+ channelSlugs?: string[],
+): Promise<DigestSweepResult> {
try {
- const jobId = await startDigestSweep();
+ const jobId = await startDigestSweep({ channelSlugs });
revalidatePath("/jobs");
return jobId
? { ok: true, jobId }
@@ -254,11 +263,54 @@ export async function startBackfillSweepAction(
channelSlugs?: string[],
): Promise<BackfillSweepResult> {
try {
- const jobId = await startBackfillSweep({ kindIds, channelSlugs });
+ // VALIDATED HERE, because sanitizeBackfill does not.
+ //
+ // An unknown kind id survives a settings write and then matches nothing:
+ // resolveBackfillKinds filters the registry BY the list, so a scope naming
+ // one operation that has since been renamed arms a sweep that does exactly
+ // no work while reporting itself armed. That is the worst kind of failure
+ // this console can have — the operator reads a plan, clicks, and watches a
+ // sweep hold at zero forever.
+ //
+ // So: drop the unknowns, arm what is left, and SAY which were dropped. Not
+ // a refusal — the remaining operations are still what the operator asked
+ // for — and not silence either.
+ const known = new Set(laneBackfillKinds(getSettings()).map((k) => k.id));
+ const wanted = kindIds ?? [];
+ const scope = wanted.filter((id) => known.has(id));
+ const dropped = wanted.filter((id) => !known.has(id));
+ if (wanted.length > 0 && scope.length === 0) {
+ return {
+ ok: false,
+ error:
+ `No enabled operation is named by this scope (${dropped.join(", ")}). ` +
+ `A sweep armed on it would run forever without doing anything.`,
+ };
+ }
+ const jobId = await startBackfillSweep({
+ // `[]` AND `undefined` ARE NOT THE SAME THING HERE, and the difference is
+ // a bug the console would otherwise hit on its first click.
+ // startBackfillSweep does `opts.kindIds ?? settings.backfill.sweepKinds`,
+ // so `undefined` INHERITS whatever scope is on disk — which is right for
+ // the boot hook and wrong for an operator who has just ticked every
+ // operation and pressed the button. An explicit `[]` overrides it, and
+ // means what an unscoped sweep has always meant: every enabled lane kind,
+ // tracked as the registry changes rather than pinned to today's three.
+ kindIds: kindIds === undefined ? undefined : scope,
+ channelSlugs,
+ });
revalidatePath("/jobs");
- return jobId
- ? { ok: true, jobId }
- : { ok: false, error: "The sweep could not be started (see job logs)." };
+ if (!jobId) {
+ return { ok: false, error: "The sweep could not be started (see job logs)." };
+ }
+ // Armed, and still worth saying what was thrown away.
+ return dropped.length > 0
+ ? {
+ ok: true,
+ jobId,
+ error: `Ignored ${dropped.length} operation(s) this build does not have enabled: ${dropped.join(", ")}.`,
+ }
+ : { ok: true, jobId };
} catch (e) {
return { ok: false, error: (e as Error).message };
}
diff --git a/editor/app/jobs/components/PauseBackfillButton.tsx b/editor/app/jobs/components/PauseBackfillButton.tsx
@@ -29,7 +29,7 @@ export function PauseBackfillButton({
}) {
return paused ? (
<LaneActionButton
- label="Resume Backfill"
+ label="Resume the lane"
glyph="▶"
ariaLabel="resume backfill"
title="Resume the backfill lane. A held job picks up within a few seconds — it was holding, not stopped."
@@ -39,7 +39,7 @@ export function PauseBackfillButton({
/>
) : (
<LaneActionButton
- label="Pause Backfill"
+ label="Hold the lane"
glyph="❙❙"
ariaLabel="pause backfill"
title="Hold the backfill lane without ending anything. A running job idles at zero and keeps its place; nothing is re-derived on resume. Survives a restart."
diff --git a/editor/app/settings/components/SettingsForm.tsx b/editor/app/settings/components/SettingsForm.tsx
@@ -1,5 +1,6 @@
"use client";
+import Link from "next/link";
import { useActionState, useState } from "react";
import {
saveSettingsAction,
@@ -902,6 +903,22 @@ export function SettingsForm({ initial, apps, digestApps }: Props) {
individually. Each feature declares what it needs and how to tell
whether a video already has it.
</p>
+ <p className="text-xs text-muted-foreground">
+ <strong className="font-medium text-foreground">
+ Which operations a corpus sweep runs, and over which channels, is
+ not set here.
+ </strong>{" "}
+ That scope has to be written at the moment the sweep is armed — a
+ scope saved separately would be resurrected as a corpus-wide run by
+ the next restart — so it lives with the plan it produces, on{" "}
+ <Link
+ href="/auto-queue"
+ className="underline underline-offset-2 hover:text-foreground"
+ >
+ Auto-queue
+ </Link>
+ , where you can read the itinerary before committing to it.
+ </p>
<label className="flex items-start gap-2 text-sm">
<input
type="checkbox"
diff --git a/editor/e2e/backfill.spec.ts b/editor/e2e/backfill.spec.ts
@@ -433,6 +433,108 @@ test("the sweep arms with its scope and disarms clearing it", async ({
.toEqual({ enabled: false, channels: 0, kinds: 0 });
});
+// (8c) THE SCOPE IS SET FROM A SCREEN, AND IT IS THE ARM ACTION THAT WRITES IT.
+//
+// `backfill.sweepKinds` has been honoured by the run since the sweep was written
+// and settable by nothing but a hand-edit of settings.json. That mattered here
+// rather than merely being untidy: an unscoped sweep runs EVERY enabled
+// operation, and on the live corpus that includes speaker-names-from-the-
+// transcript at ~1 model call per transcript chunk — on the order of 194,000
+// calls. "Diarization only" had no expression in the product.
+//
+// Two halves are pinned, and the second is the one with a bug behind it. Arming
+// a scoped sweep must RECORD the scope, because the boot hook re-launches from
+// settings alone and a scope that was not recorded resurrects a bounded run as
+// a corpus-wide one. Stopping must CLEAR it, because a stale scope silently
+// narrows the next sweep.
+test("a sweep can be scoped to one operation from the console, and the scope is persisted", async ({
+ page,
+}) => {
+ test.setTimeout(SLOW);
+ await resetData("one-transcribe-channel-with-audio");
+ await writeSettings({
+ ...backfillSettings(),
+ // Three operations, so there is a choice to make. With diarization alone
+ // the console offers no scope control at all — one operation is not a
+ // choice — which is itself the right behaviour and not what this pins.
+ attribution: {
+ enabled: true,
+ diarizedEnabled: true,
+ textOnlyEnabled: true,
+ },
+ });
+
+ await page.goto("/auto-queue");
+ const lane = page.locator('section[data-lane="backfill"]');
+ // RE-SELECTED UNTIL IT TAKES. The lane switcher only swaps panes once React
+ // has attached; a selectOption before hydration sets the DOM value and fires
+ // nothing at all — no state change, no error, no clue. Same class of failure
+ // as the pre-hydration lost click this suite has been bitten by before.
+ await expect
+ .poll(
+ async () => {
+ await page
+ .getByLabel("Lane", { exact: true })
+ .selectOption("backfill");
+ return lane.isVisible();
+ },
+ { timeout: 30_000 },
+ )
+ .toBe(true);
+
+ // THE PLAN IS DRAWN BEFORE ANYTHING IS ARMED. That is the whole point: the
+ // commitment is readable before it is made.
+ await expect(lane.getByText("The plan")).toBeVisible();
+
+ // Leave diarization ticked and drop the two expensive lanes.
+ await lane.getByLabel("Speaker names (from the audio)").uncheck();
+ await lane.getByLabel("Speaker names (from the transcript)").uncheck();
+ await expect(lane.getByText("1 of 3 selected")).toBeVisible();
+
+ // Addressed by its ARIA name, which is unchanged internal addressing — and on
+ // this panel it is derived from the lane's group ("Start Speakers sweep"),
+ // not from the queue key the dashboard card uses. The regex keeps this spec
+ // from re-breaking if a fourth operation moves the group label.
+ const arm = lane.getByRole("button", { name: /sweep$/i });
+ await expect(arm).toBeEnabled({ timeout: 30_000 });
+ await arm.click();
+
+ await expect
+ .poll(
+ async () => {
+ const s = await readJson<{
+ backfill?: { sweepEnabled?: boolean; sweepKinds?: string[] };
+ }>("test-settings.json").catch(() => ({}) as Record<string, never>);
+ return {
+ enabled: s.backfill?.sweepEnabled ?? false,
+ kinds: s.backfill?.sweepKinds ?? [],
+ };
+ },
+ { timeout: 30_000 },
+ )
+ .toEqual({ enabled: true, kinds: ["diarization"] });
+
+ const stop = lane.getByRole("button", { name: /sweep$/i });
+ await expect(stop).toBeEnabled({ timeout: 30_000 });
+ await expect(stop).toHaveText(/stop sweeping/i);
+ await stop.click();
+
+ await expect
+ .poll(
+ async () => {
+ const s = await readJson<{
+ backfill?: { sweepEnabled?: boolean; sweepKinds?: string[] };
+ }>("test-settings.json").catch(() => ({}) as Record<string, never>);
+ return {
+ enabled: s.backfill?.sweepEnabled ?? false,
+ kinds: (s.backfill?.sweepKinds ?? []).length,
+ };
+ },
+ { timeout: 30_000 },
+ )
+ .toEqual({ enabled: false, kinds: 0 });
+});
+
// (8b) …AND SURVIVES A RESTART. This is the reason the flag is persisted at all:
// a sweep is days of work and will outlive several restarts by construction.
// The suite cannot restart the dev server mid-run, so /api/test/
