commit 4ed2bd9510c85d7a50fb637741001a1ba78ac1d9
parent 79031ffc55b487302964943a8f3d20fa3bca3ddb
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 7 Sep 2026 22:30:36 -0400
editor: one console per lane, and the sweep panel goes
Slice 1.3, third commit — the editor half. `SweepLane.tsx` (422),
`SweepScope.tsx` (154), `SweepPlan.tsx` (264) and `ArbiterBar.tsx` (113) are
deleted, with `/api/test/resume-backfill-sweep`, `saveLaneOrderAction`, the two
arbiter actions and the four sweep actions. `operations/lanes.ts` goes from 370
lines to 131: what a lane needs that its runner cannot answer is the rail's
BANDS and which operations the lane dispatches, and that is all this builds now.
An operation page had TWO consoles for the length of slice 1.2 so an operator
mid-migration could see which one was driving. It has one, and it is the
runner's: `<section data-lane>` is the contract all four lanes share, and
`data-sweep-lane` is deleted rather than renamed back — nothing but the runner
console carries a lane attribute now.
What the panel said that was worth keeping moved rather than died. The band, the
"this operation is switched off" warning and the "shares one lane, one runner
and one pause with…" sentence are a `LaneOperationContext` div ABOVE the console
(a nested <section> would break every scoped lookup under strict mode). The
"also runs on <endpoint>" line is in `LaneHeader`, per LANE rather than per
operation row — the slots belong to the machine, so two operations served by one
endpoint are one row, not two. `NextUp` is the plan the itinerary used to be.
`armLaneAction(lane, scope)` replaces `startDigestSweepAction(channels)` and
`startBackfillSweepAction(kinds, channels)`. It writes `enabled: true` and a
root in ONE settings write and then starts the runner — the single write is the
property the sweeps' own start/stop pair documented at length, because a scope
written separately from the switch means a restart between the two resurrects a
bounded run as a corpus-wide one. The root is built by `laneRootFromScope`, the
same function the settings migration uses, so arming a lane and migrating a
sweep armed on the same scope produce byte-identical trees. An absent scope
keeps the stored tree; `disarmLaneAction` writes `enabled: false` and DRAINS,
which is what the sweep's stop did, and leaves the tree alone.
`laneState.ts` loses its `feedRunning` axis and `sweepLaneNote`. The axis meant
"a corpus-wide sweep is armed", a fact only two lanes ever had; the note was the
sweep's vocabulary for states the runner already names through `idleReason`.
So `buildActiveJobs.buildLanes` is one loop over LANES with four runner rows
(the two sweep rows are gone, and no label moved), and `railStates` is one fold
instead of two.
The dashboard and widget cards keep two controls each, and the arm one now says
what it does: "Run every channel" / "Stop the lane", `arm digest lane` /
`disarm digest lane`. The widget sync payload carries `armed`
(`autoQueue[lane].enabled`) where it carried `sweeping`. The runner console
stays the only surface that can SCOPE a lane; from the dashboard the switch
means the whole corpus, exactly as "Sweep every channel" did.
Also retired here: the `lane-blocked` idle wording in both its copies, the
`backfill-sweep` entries in `stageStatus` and `JOB_KINDS_BY_LANE` (which is
rekeyed by lane and kept — those are the MANUAL per-channel jobs, and without
them a hand-started digest is invisible on the digest page), and the "Resource
share" field with `backfill.weight` behind it.
tsc clean in editor; the e2e rewrites are the next commit.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
27 files changed, 511 insertions(+), 1923 deletions(-)
diff --git a/editor/app/api/test/resume-backfill-sweep/route.ts b/editor/app/api/test/resume-backfill-sweep/route.ts
@@ -1,35 +0,0 @@
-import { NextResponse } from "next/server";
-import { getRegistry } from "yt-dlp-transcript-common/jobs/registry";
-import {
- getBackfillSweepJobId,
- resumeBackfillSweepIfEnabled,
-} from "yt-dlp-transcript-common/controller/backfillSweep";
-
-export const dynamic = "force-dynamic";
-
-// E2E test harness only. Simulates a SERVER RESTART for the backfill sweep,
-// which is the one property of an armed sweep that cannot otherwise be tested:
-// the sweep is process state, a restart destroys it, and instrumentation.ts's
-// boot hook is what brings it back. An e2e suite cannot restart the dev server
-// mid-run, so it does the two halves a restart does — drop the in-process
-// singleton, then call the hook — and asserts a sweep comes back.
-//
-// Mounted unconditionally, like the other /api/test routes: the editor is a
-// localhost admin tool, not a deployed service.
-export async function GET() {
- // Half one: lose the live sweep, the way a restart does. Cancelling the job
- // clears the singleton through the sweep's own `finally`, so this leaves
- // exactly the state a fresh process starts in — the persisted settings flag
- // and nothing running.
- const before = getBackfillSweepJobId();
- if (before) getRegistry().cancel(before);
-
- // Half two: the boot hook. Reads settings.backfill.sweepEnabled and its
- // persisted scope, and re-launches from those alone.
- await resumeBackfillSweepIfEnabled();
-
- return NextResponse.json({
- cancelled: before,
- resumed: getBackfillSweepJobId(),
- });
-}
diff --git a/editor/app/api/test/resume-lane/route.ts b/editor/app/api/test/resume-lane/route.ts
@@ -16,8 +16,8 @@ export const dynamic = "force-dynamic";
// restart the dev server mid-run, so it does the two halves a restart does —
// drop the in-process runner, then call the hook — and asserts one comes back.
//
-// The same shape as /api/test/resume-backfill-sweep, which does this for the
-// sweep. Slice 1.3 deletes that one and leaves this.
+// It replaced /api/test/resume-backfill-sweep, which did this for the sweep;
+// that route retired with the sweep in slice 1.3.
//
// Mounted unconditionally, like the other /api/test routes: the editor is a
// localhost admin tool, not a deployed service.
diff --git a/editor/app/api/widget/sync/route.ts b/editor/app/api/widget/sync/route.ts
@@ -62,7 +62,7 @@ export type WidgetSyncPayload = {
channelsWithAny: number; // channels the layer has reached at all
// The DENOMINATOR for channelsWithAny. Without it "66 channels reached" is a
// count with nothing to be a fraction of, and the card cannot tell a corpus
- // where the sweep has touched everything from one where it has barely begun.
+ // where the lane has touched everything from one where it has barely begun.
channels: number;
// Videos with no transcript yet, and videos held back for want of a current
// normalized one. NEVER summed with `digested` nor with each other — they are
@@ -72,7 +72,11 @@ export type WidgetSyncPayload = {
deferred: number;
// held — computed by isGateHeld; nothing downstream inverts anything.
held: boolean;
- sweeping: boolean; // a corpus-wide sweep is armed
+ // The lane is ARMED — `autoQueue.digest.enabled`, the switch its runner
+ // resumes from at boot. It was `sweeping` (a corpus-wide sweep is armed)
+ // until slice 1.3; the sweeps are gone and a lane's tree is its scope, so
+ // this is the same question asked of the thing that now answers it.
+ armed: boolean;
};
// Corpus-wide backfill state, and free for the same reason as `digest`: every
// brief already carries its channel's snapshot, so this is a sum rather than a
@@ -101,7 +105,8 @@ export type WidgetSyncPayload = {
// `enabled` here until slice 7, and a pinned widget tab that predates the
// rename reads `undefined ?? false` — not held — until it is reloaded.
held: boolean;
- sweeping: boolean; // a corpus-wide backfill sweep is armed
+ // The lane is ARMED — see `digest.armed` above.
+ armed: boolean;
// Whether any backfill FEATURE is on. Distinct from the gate: with no
// feature on there is nothing to report at all, and that must not look like
// "all caught up".
@@ -111,8 +116,8 @@ export type WidgetSyncPayload = {
// resolve this itself: operations.ts reaches the filesystem.
//
// "Backfill" is a queue key. Nobody arms, pauses or runs "a backfill"; the
- // word survives only on the two controls that genuinely act on the shared
- // queue, and even there the card now lists what is in it.
+ // word survives only on the controls that genuinely act on the shared queue,
+ // and even there the card now lists what is in it.
groupLabel: string;
// The same numbers, kept apart by kind — which is the only form of them that
// means anything. Summed, this lane reads "77,952 reachable · 77,134 need
@@ -294,14 +299,14 @@ export async function buildWidgetSyncPayload(): Promise<WidgetSyncPayload> {
blocked: digestBlocked,
deferred: digestDeferred,
held: isGateHeld(settings, "digest"),
- sweeping: settings.digest.sweepEnabled,
+ armed: settings.autoQueue.digest.enabled,
},
backfill: {
reachable: backfillReachable,
needsMedia: backfillNeedsMedia,
videos,
held: isGateHeld(settings, "backfill"),
- sweeping: settings.backfill.sweepEnabled,
+ armed: settings.autoQueue.backfill.enabled,
anyKind: backfillLaneOperations(settings).length > 0,
groupLabel: operationsGroupLabel(
backfillLaneOperations(settings).map((k) => k.id),
diff --git a/editor/app/channels/[slug]/digestActions.ts b/editor/app/channels/[slug]/digestActions.ts
@@ -56,7 +56,7 @@ export type DigestLaneChoice = CommonDigestLaneChoice;
// the card's live log.
//
// The remoteEnabled fail-fast MOVED INTO THE RUNNER rather than staying here.
-// The arbiter can ask for the metered lane too, and a guard only one of two
+// The lane runner can ask for the metered lane too, and a guard only one of two
// callers performs is not a guard.
export async function digestChannelAction(
slug: string,
diff --git a/editor/app/channels/[slug]/lib/stageStatus.ts b/editor/app/channels/[slug]/lib/stageStatus.ts
@@ -120,10 +120,6 @@ const JOB_KIND_TO_STAGE: Record<string, StageId> = {
"digest-channel-remote": "digest",
"digest-share-cluster": "digest",
"backfill-channel": "speakers",
- // The sweep is corpus-wide and carries no channelSlug, so it will not normally
- // appear in a channel's running jobs — mapped anyway so that if one ever is
- // attributed here it lights the right card rather than none.
- "backfill-sweep": "speakers",
// The pre-registry per-channel diarization button lands on the channel queue
// but is the same work this card is about, so it lights this card too.
"diarize-channel": "speakers",
diff --git a/editor/app/components/lanes/LaneDeck.tsx b/editor/app/components/lanes/LaneDeck.tsx
@@ -10,12 +10,7 @@ import { useSectionFrame } from "../../widget/components/WidgetSection";
import { LaneCard, type LaneControl } from "./LaneCard";
import { deriveLaneState, formatCount } from "./laneState";
import { pauseLaneControl } from "./pauseControl";
-import {
- startBackfillSweepAction,
- startDigestSweepAction,
- stopBackfillSweepAction,
- stopDigestSweepAction,
-} from "../../jobs/actions";
+import { armLaneAction, disarmLaneAction } from "../../jobs/actions";
// The four lanes, and the ONLY place lane → server-action wiring lives. Rendered
// identically by the dashboard band and the monitor widget's controls section,
@@ -24,10 +19,16 @@ import {
// digest control whatsoever).
//
// Every lane's gate reaches the wire as `held`, computed once by isGateHeld; no
-// reader inverts. What still differs between lanes is their FEED — transcription
-// and downloads have none (work arrives from jobs), digest and backfill each
-// have a sweep — so this is where `held` and `sweeping` are folded onto
-// { gateHeld, feedRunning }. See laneState.ts for why those two are the axes.
+// reader inverts.
+//
+// WHAT DIFFERS BETWEEN THE LANES HERE IS WHERE THEIR WORK COMES FROM, not how
+// they are dispatched: all four have a runner. Transcription and downloads are
+// fed by arrivals, so their cards carry a pause and nothing else; digest and
+// backfill are catch-up over a corpus that already exists, so theirs also carry
+// the switch that arms the lane — which since slice 1.3 writes
+// `autoQueue[lane].enabled` and starts the runner, where it used to arm a sweep.
+// The SCOPE is the lane's tree and is authored on its console; from here the
+// switch means the whole corpus, exactly as "Sweep every channel" did.
export function LaneDeck({
workers,
@@ -99,7 +100,7 @@ export function LaneDeck({
<LaneCard
name="Transcription"
state={deriveLaneState({ gateHeld: paused, activeCount: busy })}
- figure="no sweep — work arrives from jobs"
+ figure="work arrives from jobs"
detail={
<>
<BacklogLine
@@ -142,7 +143,7 @@ export function LaneDeck({
<LaneCard
name="Downloads"
state={deriveLaneState({ gateHeld: downloadsPaused || diskLow })}
- figure="no sweep — work arrives from jobs"
+ figure="work arrives from jobs"
detail={
<>
<BacklogLine
@@ -212,36 +213,36 @@ export function LaneDeck({
//
// Stopping the sweep drains rather than cancels: the channel in flight
// finishes instead of losing a part-generated video.
- const digestSweeping = digest?.sweeping ?? false;
+ const digestArmed = digest?.armed ?? false;
const digestPaused = digest?.held ?? false;
const digestControls: LaneControl[] = [
- digestSweeping
+ digestArmed
? {
- key: "digest-sweep",
- label: "Stop sweeping",
+ key: "digest-arm",
+ label: "Stop the lane",
glyph: "■",
- ariaLabel: "stop digest sweep",
+ ariaLabel: "disarm digest lane",
title:
- "Stop the corpus-wide digest backfill. The channel in flight finishes first; nothing already generated is lost.",
+ "Switch the digest lane off and let its runner finish what it is holding. The video in flight completes; nothing already generated is lost, and the lane's rules are left alone.",
variant: "active",
- action: stopDigestSweepAction,
+ action: () => disarmLaneAction("digest"),
onChange: onSynced,
}
: {
- key: "digest-sweep",
- label: "Sweep every channel",
+ key: "digest-arm",
+ label: "Run every channel",
glyph: "⟳",
- ariaLabel: "start digest sweep",
+ ariaLabel: "arm digest lane",
title:
- "Start the corpus-wide digest backfill: every channel in turn, heaviest first by remaining audio-hours. Survives a restart.",
+ "Switch the digest lane on across every channel and start its runner. Survives a restart. To scope it to particular channels, use the rules on the digest operation page.",
variant: "idle",
- action: startDigestSweepAction,
+ action: () => armLaneAction("digest", {}),
onChange: onSynced,
},
];
// The pause only appears once there is something to hold — or once it is
// already holding, which is the state you have to be able to get out of.
- if (digestSweeping || digestPaused) {
+ if (digestArmed || digestPaused) {
digestControls.push(
pauseLaneControl({
lane: "digest",
@@ -254,10 +255,7 @@ export function LaneDeck({
const digestLane = (
<LaneCard
name="Digest"
- state={deriveLaneState({
- gateHeld: digestPaused,
- feedRunning: digestSweeping,
- })}
+ state={deriveLaneState({ gateHeld: digestPaused })}
figure={
digest === null ? (
"—"
@@ -342,7 +340,7 @@ export function LaneDeck({
// differ by a week of GPU time. The feed SWEEPS; the gate HOLDS. The
// aria-labels are untouched — they are internal addressing that confuses
// nobody, and the e2e suite finds these buttons by them.
- const backfillSweeping = backfill?.sweeping ?? false;
+ const backfillArmed = backfill?.armed ?? false;
const backfillHeld = backfill?.held ?? false;
const backfillAvailable = backfill?.anyKind ?? false;
// Empty when the payload has not arrived, or when it predates `kinds` — which
@@ -354,27 +352,27 @@ export function LaneDeck({
const backfillGroup = backfill?.groupLabel ?? "Derived data";
const backfillControls: LaneControl[] = backfillAvailable
? [
- backfillSweeping
+ backfillArmed
? {
- key: "backfill-sweep",
- label: "Stop sweeping",
+ key: "backfill-arm",
+ label: "Stop the lane",
glyph: "■",
- ariaLabel: "stop backfill sweep",
+ ariaLabel: "disarm backfill lane",
title:
- "Stop the corpus-wide backfill. The channel in flight finishes first; nothing already written is lost.",
+ "Switch the backfill lane off and let its runner finish what it is holding. The video in flight completes; nothing already written is lost, and the lane's rules are left alone.",
variant: "active",
- action: stopBackfillSweepAction,
+ action: () => disarmLaneAction("backfill"),
onChange: onSynced,
}
: {
- key: "backfill-sweep",
- label: "Sweep every channel",
+ key: "backfill-arm",
+ label: "Run every channel",
glyph: "⟳",
- ariaLabel: "start backfill sweep",
+ ariaLabel: "arm backfill lane",
title:
- "Start the corpus-wide backfill: every channel in turn, heaviest first by reachable work. Survives a restart.",
+ "Switch the backfill lane on across every channel and start its runner. Survives a restart. To scope it to particular channels or operations, use the rules on one of its operation pages.",
variant: "idle",
- action: () => startBackfillSweepAction(),
+ action: () => armLaneAction("backfill", {}),
onChange: onSynced,
},
pauseLaneControl({
@@ -394,7 +392,6 @@ export function LaneDeck({
state={deriveLaneState({
available: backfillAvailable,
gateHeld: backfillHeld,
- feedRunning: backfillSweeping,
})}
figure={
backfill === null || !backfillAvailable ? (
@@ -473,15 +470,15 @@ export function LaneDeck({
pause: {laneOperations.map((k) => k.label).join(", ")}.
</span>
)}
- {backfillSweeping && backfillHeld && (
- // Kept in words as well as in the rail. A sweep armed with the lane
- // off holds at a zero limit rather than doing work, and "wedged" is
+ {backfillArmed && backfillHeld && (
+ // Kept in words as well as in the rail. An armed lane behind a shut
+ // gate holds at a zero limit rather than doing work, and "wedged" is
// what that looks like to anyone who does not already know.
<span
aria-label="backfill lane off"
className="block text-xs text-muted-foreground"
>
- the sweep is holding
+ the lane is holding
</span>
)}
</>
diff --git a/editor/app/components/lanes/laneState.test.ts b/editor/app/components/lanes/laneState.test.ts
@@ -1,64 +1,50 @@
import test from "node:test";
import assert from "node:assert/strict";
-import { deriveLaneState, sweepLaneNote } from "./laneState";
+import { deriveLaneState, LANE_WORD } from "./laneState";
-// THE FOUR STRINGS, AS LITERALS. They were a table in railStates.ts (the
-// operations rail) and a byte-identical second table in buildActiveJobs.ts (the
-// Active Jobs lane strip), and nothing pinned either — so the two could drift
-// and no test would say. This is the pin: both consumers read this function
-// now, so these literals are what both of them print.
+// THE FOUR STATES, AS LITERALS, and the precedence between them.
+//
+// It used to pin a second function beside this one — `sweepLaneNote`, the
+// sentence a sweep lane gave for not working. That retired with the sweeps in
+// slice 1.3: every lane is dispatched by a runner now, so the reason a lane is
+// not working is the runner's own `idleReason`, which the console and the
+// Active Jobs strip both map through `idleReasonText` / `autoIdleNote`. What is
+// left here is the state itself, which all four lanes still share.
-test("sweepLaneNote names why a sweep lane is not working", () => {
- assert.equal(
- sweepLaneNote({ available: false, gateHeld: false }),
- "no operation switched on",
- );
+test("the gate decides before anything else", () => {
+ // A running lane behind a shut gate is HOLDING, not running and not stopped:
+ // its limit() returns 0 and the pool idle-waits, which looks exactly like
+ // wedged unless something says otherwise. That state is the reason this is
+ // four strings rather than a boolean.
+ assert.equal(deriveLaneState({ gateHeld: true }), "holding");
+ assert.equal(deriveLaneState({ gateHeld: true, activeCount: 3 }), "holding");
+});
+
+test("unavailable beats the gate", () => {
+ // With no operation switched on there is nothing to hold, and "Holding" would
+ // send an operator to the wrong control.
+ assert.equal(deriveLaneState({ available: false, gateHeld: true }), "unavailable");
assert.equal(
- sweepLaneNote({ gateHeld: true, feedRunning: true }),
- "sweep armed, lane paused",
+ deriveLaneState({ available: false, gateHeld: false, activeCount: 2 }),
+ "unavailable",
);
- assert.equal(sweepLaneNote({ gateHeld: true }), "lane paused");
- assert.equal(sweepLaneNote({ gateHeld: false }), "no sweep armed");
});
-test("a working lane has nothing to explain", () => {
- // Null, not "" and not a cheerful sentence: the rail renders the note only
- // when there is one, and a lane that IS working needs no excuse.
- assert.equal(sweepLaneNote({ gateHeld: false, feedRunning: true }), null);
- assert.equal(sweepLaneNote({ gateHeld: false, activeCount: 3 }), null);
+test("work in flight is the only thing that reads as running", () => {
+ assert.equal(deriveLaneState({ gateHeld: false, activeCount: 1 }), "running");
+ assert.equal(deriveLaneState({ gateHeld: false, activeCount: 0 }), "idle");
+ assert.equal(deriveLaneState({ gateHeld: false }), "idle");
});
-test("the note follows deriveLaneState's precedence, fact for fact", () => {
- // `available: false` beats the gate: with no operation switched on there is
- // nothing to hold, and "lane paused" would send an operator to the wrong
- // control.
+test("every state has a word, and Off is not Idle", () => {
+ // The word beside the dot is what assistive tech reads, so it has to stand on
+ // its own — and "Off" must never be spelled "Idle": an idle-looking lane
+ // claims "all caught up" where the truth is "nothing would run this".
+ assert.equal(LANE_WORD[deriveLaneState({ available: false, gateHeld: false })], "Off");
+ assert.equal(LANE_WORD[deriveLaneState({ gateHeld: true })], "Holding");
+ assert.equal(LANE_WORD[deriveLaneState({ gateHeld: false })], "Idle");
assert.equal(
- sweepLaneNote({ available: false, gateHeld: true, feedRunning: true }),
- "no operation switched on",
+ LANE_WORD[deriveLaneState({ gateHeld: false, activeCount: 1 })],
+ "Running",
);
- // The gate beats the feed: a sweep armed behind a shut gate is HOLDING, and
- // the note has to say the gate, not the sweep.
- assert.equal(
- sweepLaneNote({ gateHeld: true, feedRunning: true, activeCount: 2 }),
- "sweep armed, lane paused",
- );
-});
-
-test("the word and the note cannot disagree: holding <=> a paused note", () => {
- // The whole reason sweepLaneNote takes deriveLaneState's input rather than
- // its output — one object, one precedence, two readings.
- const cases = [
- { available: false, gateHeld: false },
- { available: false, gateHeld: true },
- { gateHeld: true },
- { gateHeld: true, feedRunning: true },
- { gateHeld: false },
- { gateHeld: false, feedRunning: true },
- { gateHeld: false, activeCount: 1 },
- ];
- for (const input of cases) {
- const holding = deriveLaneState(input) === "holding";
- const paused = sweepLaneNote(input)?.endsWith("lane paused") === true;
- assert.equal(holding, paused, JSON.stringify(input));
- }
});
diff --git a/editor/app/components/lanes/laneState.ts b/editor/app/components/lanes/laneState.ts
@@ -7,59 +7,43 @@ import {
// A lane's state, and the one derivation every surface reads it from.
//
// THE POINT OF THIS TYPE IS THAT A LANE HAS TWO SWITCHES, NOT ONE. There is the
-// SWEEP — the producer that feeds the lane work — and the GATE — the pause that
-// decides whether the lane consumes it. They are independent, and the state
-// nobody had a word for is the combination "sweep armed, gate shut": the lane is
-// not stopped and it is not working. That is `holding`, and it is the whole
-// reason this is four states rather than a boolean.
+// RUNNER — is anything dispatching at all — and the GATE — the pause that
+// decides whether the lane may consume what it picks. They are independent, and
+// the state nobody had a word for is the combination "runner up, gate shut":
+// the lane is not stopped and it is not working. That is `holding`, and it is
+// the whole reason this is four states rather than a boolean.
//
// Every lane's gate reaches the wire as `held`, computed once by isGateHeld
-// (common/lib/pauseGates.ts); no reader inverts. What still differs is the FEED:
-// transcription and downloads have none — work arrives from jobs — while digest
-// and backfill each have a sweep. Folding those two facts into one state happens
-// in LaneDeck, and everything downstream reads only this.
+// (common/lib/pauseGates.ts); no reader inverts.
+//
+// THE `feedRunning` AXIS RETIRED WITH THE SWEEPS in slice 1.3. It meant "a
+// corpus-wide sweep is armed", which was a fact only two of the four lanes had;
+// all four are dispatched by a runner now, so a caller with no work in flight is
+// `idle` on the same terms auto-transcribe always was.
export type LaneState = "running" | "holding" | "idle" | "unavailable";
export function deriveLaneState({
available = true,
gateHeld,
- feedRunning = false,
activeCount = 0,
}: {
- // The lane exists at all. `false` is not "idle": with no backfill feature
- // registered there is nothing to hold, and an idle-looking lane would read as
- // "all caught up" when the truth is "switched off".
+ // The lane exists at all. `false` is not "idle": with no operation registered
+ // there is nothing to hold, and an idle-looking lane would read as "all caught
+ // up" when the truth is "switched off".
available?: boolean;
gateHeld: boolean;
- // A corpus-wide sweep is armed. Absent for the lanes fed by jobs instead.
- feedRunning?: boolean;
- // Work in flight right now, for lanes with no sweep of their own.
+ // Work in flight right now.
activeCount?: number;
}): LaneState {
if (!available) return "unavailable";
- // The gate decides first. A sweep armed behind a shut gate is HOLDING, not
- // running — the batch's limit() returns 0 and the pool idle-waits, which looks
- // exactly like wedged unless something says otherwise.
+ // The gate decides first. A running lane behind a shut gate is HOLDING, not
+ // running — its limit() returns 0 and the pool idle-waits, which looks exactly
+ // like wedged unless something says otherwise.
if (gateHeld) return "holding";
- if (feedRunning || activeCount > 0) return "running";
+ if (activeCount > 0) return "running";
return "idle";
}
-// WHY A SWEEP LANE IS NOT WORKING, in words; null when it is, or when there is
-// nothing to say. ONE COPY: the operations rail (railStates.ts) and the Active
-// Jobs lane strip (buildActiveJobs.ts) carried this table twice, byte for
-// byte, and the sweep panel says the same two facts in sentences. Takes
-// deriveLaneState's input and follows its precedence, so the word and the
-// note cannot disagree about which fact wins.
-export function sweepLaneNote(
- input: Parameters<typeof deriveLaneState>[0],
-): string | null {
- if (input.available === false) return "no operation switched on";
- if (input.gateHeld) return input.feedRunning ? "sweep armed, lane paused" : "lane paused";
- if (input.feedRunning || (input.activeCount ?? 0) > 0) return null;
- return "no sweep armed";
-}
-
// No new palette. These map onto the station tones the channel line already
// uses (see flow/tone.ts): four theme families × light/dark means a bespoke hue
// here would be wrong in eight palettes at once. Deliberately NOT a second copy
diff --git a/editor/app/jobs/actions.ts b/editor/app/jobs/actions.ts
@@ -2,21 +2,23 @@
import { revalidatePath } from "next/cache";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
-import { getSettings } from "yt-dlp-transcript-common/lib/settings";
-import { backfillLaneOperations } from "yt-dlp-transcript-common/lib/operations";
+import {
+ getSettings,
+ writeSettings,
+} from "yt-dlp-transcript-common/lib/settings";
+import { laneRootFromScope } from "yt-dlp-transcript-common/lib/laneMigration";
+import { operationsForLane } from "yt-dlp-transcript-common/lib/operations";
+import type { AutoQueueKind } from "yt-dlp-transcript-common/lib/autoQueueTypes";
+import {
+ drainAutoRunner,
+ startAutoRunner,
+ startAutoRunnerBlockedReason,
+} from "yt-dlp-transcript-common/controller/autoRunner";
import { pruneJobLogs } from "yt-dlp-transcript-common/jobs/listJobs";
import { getRegistry } from "yt-dlp-transcript-common/jobs/registry";
import { readJobMeta } from "yt-dlp-transcript-common/jobs/jobMeta";
import type { JobSpec } from "yt-dlp-transcript-common/jobs/jobSpec";
import type { StreamActionResult } from "yt-dlp-transcript-common/jobs/streamCommand";
-import {
- startDigestSweep,
- stopDigestSweep,
-} from "yt-dlp-transcript-common/controller/digestSweep";
-import {
- startBackfillSweep,
- stopBackfillSweep,
-} from "yt-dlp-transcript-common/controller/backfillSweep";
import { runJobSpec } from "./runJobSpec";
import { stuckJobIds } from "./active/buildActiveJobs";
@@ -146,96 +148,90 @@ export async function retryAllFailedAction(): Promise<{ count: number }> {
return { count };
}
-// Arm / disarm the corpus-wide sweep. Distinct from the pause: a pause holds a
-// running sweep at zero throughput, this decides whether there is a sweep at
-// all — and it persists, so the boot hook resumes it.
-export type DigestSweepResult = { ok: boolean; jobId?: string; error?: string };
-
-// `channelSlugs` is the scope, and it goes THROUGH THIS ACTION rather than
-// through a settings save. startDigestSweep persists the flag and the scope in
-// one write, which is what lets the boot hook resume the same run; a scope
-// written separately would be a second writer for one decision, and a restart
-// between the two writes would resurrect a bounded run as a corpus-wide one.
-// Absent = the whole corpus, exactly as before.
-export async function startDigestSweepAction(
- channelSlugs?: string[],
-): Promise<DigestSweepResult> {
- try {
- const jobId = await startDigestSweep({ channelSlugs });
- revalidatePath("/jobs");
- return jobId
- ? { ok: true, jobId }
- : { ok: false, error: "The sweep could not be started (see job logs)." };
- } catch (e) {
- return { ok: false, error: (e as Error).message };
- }
-}
-
-export async function stopDigestSweepAction(): Promise<DigestSweepResult> {
- try {
- await stopDigestSweep();
- revalidatePath("/jobs");
- return { ok: true };
- } catch (e) {
- return { ok: false, error: (e as Error).message };
- }
-}
-
-// Arm / disarm the corpus-wide BACKFILL sweep. Same pair as the digest sweep
-// above and for the same reason it is a pair: the flag persists, so the boot
-// hook resumes it, and stopping has to clear the scope as well as the flag.
-export type BackfillSweepResult = {
- ok: boolean;
- jobId?: string;
- error?: string;
-};
+// ARM A LANE: write its scope as a tree, switch it on, and start its runner.
+//
+// This replaces `startDigestSweepAction(channels)` and
+// `startBackfillSweepAction(kinds, channels)`. What those did was persist a
+// flag plus a scope in one write and then launch a corpus walk; what this does
+// is persist `enabled: true` plus a `root` in one write and then start the lane
+// runner. The one-write property is why it is still a single action: a scope
+// written separately from the switch means a restart between the two writes
+// resurrects a deliberately-bounded run as a corpus-wide one, which is the bug
+// the sweeps' own start/stop pair documented at length.
+//
+// THE SCOPE IS THE TREE, and it is built by the SAME function the settings
+// migration uses (`laneRootFromScope`), so arming a lane here and migrating a
+// sweep that was armed with the same scope produce byte-identical roots. There
+// is one leaf builder; a second one is how the two would drift.
+//
+// An ABSENT scope keeps the stored tree — that is the difference between "arm
+// this lane on these channels" and "switch this lane back on". An empty scope
+// object is NOT absent: `{}` means the whole corpus, exactly as an empty
+// `sweepChannels` did.
+export type ArmLaneResult = { ok: boolean; jobId?: string; error?: string };
-export async function startBackfillSweepAction(
- kindIds?: string[],
- channelSlugs?: string[],
-): Promise<BackfillSweepResult> {
+export async function armLaneAction(
+ lane: AutoQueueKind,
+ scope?: { channels?: string[]; operations?: string[] },
+): Promise<ArmLaneResult> {
try {
- // VALIDATED HERE, because sanitizeBackfill does not.
+ const settings = getSettings();
+ // VALIDATED HERE, because the sanitizer does not.
//
- // An unknown kind id survives a settings write and then matches nothing:
- // resolveBackfillLaneOperations filters the registry BY the list, so a scope naming
- // one operation that has since been renamed arms a sweep that does exactly
- // no work while reporting itself armed. That is the worst kind of failure
- // this console can have — the operator reads a plan, clicks, and watches a
- // sweep hold at zero forever.
- //
- // So: drop the unknowns, arm what is left, and SAY which were dropped. Not
- // a refusal — the remaining operations are still what the operator asked
- // for — and not silence either.
- const known = new Set(backfillLaneOperations(getSettings()).map((k) => k.id));
- const wanted = kindIds ?? [];
- const scope = wanted.filter((id) => known.has(id));
+ // An unknown operation id survives a settings write and then matches
+ // nothing: a leaf naming it draws no list at all, so a lane armed on it
+ // reports itself running and does exactly no work. That is the worst
+ // failure this console can have — the operator clicks, and watches a lane
+ // hold at zero forever. So: drop the unknowns, arm what is left, and SAY
+ // which were dropped.
+ const known = new Set(operationsForLane(lane, settings).map((o) => o.id));
+ const wanted = scope?.operations ?? [];
+ const operations = wanted.filter((id) => known.has(id));
const dropped = wanted.filter((id) => !known.has(id));
- if (wanted.length > 0 && scope.length === 0) {
+ if (wanted.length > 0 && operations.length === 0) {
return {
ok: false,
error:
`No enabled operation is named by this scope (${dropped.join(", ")}). ` +
- `A sweep armed on it would run forever without doing anything.`,
+ `A lane armed on it would run forever without doing anything.`,
};
}
- const jobId = await startBackfillSweep({
- // `[]` AND `undefined` ARE NOT THE SAME THING HERE, and the difference is
- // a bug the console would otherwise hit on its first click.
- // startBackfillSweep does `opts.kindIds ?? settings.backfill.sweepKinds`,
- // so `undefined` INHERITS whatever scope is on disk — which is right for
- // the boot hook and wrong for an operator who has just ticked every
- // operation and pressed the button. An explicit `[]` overrides it, and
- // means what an unscoped sweep has always meant: every enabled lane kind,
- // tracked as the registry changes rather than pinned to today's three.
- kindIds: kindIds === undefined ? undefined : scope,
- channelSlugs,
+ // TICKING EVERY OPERATION IS NOT THE SAME AS NAMING TODAY'S THREE. A leaf
+ // that names none draws the lane's whole union, so it tracks the registry:
+ // an operation enabled later is picked up rather than silently excluded
+ // forever. Only a STRICT SUBSET goes onto the leaves.
+ const scoped =
+ operations.length > 0 && operations.length < known.size ? operations : [];
+ const policy = settings.autoQueue[lane];
+ await writeSettings({
+ ...settings,
+ autoQueue: {
+ ...settings.autoQueue,
+ [lane]: {
+ ...policy,
+ enabled: true,
+ root: scope
+ ? laneRootFromScope(lane, {
+ channels: scope.channels,
+ operations: scoped,
+ })
+ : policy.root,
+ },
+ },
});
+ // ARM = ENABLE + START, because the two were one act on the button this
+ // replaces. `startAutoRunner` is a no-op on a lane already running.
+ const jobId = await startAutoRunner(lane);
revalidatePath("/jobs");
- if (!jobId) {
- return { ok: false, error: "The sweep could not be started (see job logs)." };
+ revalidatePath("/operations/[id]", "page");
+ if (jobId === null) {
+ return {
+ ok: false,
+ error:
+ startAutoRunnerBlockedReason(lane) ??
+ "The lane could not be started (see job logs).",
+ };
}
- // Armed, and still worth saying what was thrown away.
return dropped.length > 0
? {
ok: true,
@@ -248,10 +244,30 @@ export async function startBackfillSweepAction(
}
}
-export async function stopBackfillSweepAction(): Promise<BackfillSweepResult> {
+// DISARM: switch the lane off and let the runner finish what it is holding.
+//
+// A DRAIN, not a stop, and that is the sweeps' behaviour kept: the unit in
+// flight completes instead of being thrown away. The `enabled: false` is what
+// stops a restart bringing it back — the boot hook resumes every enabled lane.
+//
+// The TREE IS LEFT ALONE. A disarm that cleared the scope would be the bug the
+// old `stopDigestSweep` had to grow a second clause for; here the scope is a
+// tree an operator authored, and it must survive being switched off.
+export async function disarmLaneAction(
+ lane: AutoQueueKind,
+): Promise<ArmLaneResult> {
try {
- await stopBackfillSweep();
+ const settings = getSettings();
+ await writeSettings({
+ ...settings,
+ autoQueue: {
+ ...settings.autoQueue,
+ [lane]: { ...settings.autoQueue[lane], enabled: false },
+ },
+ });
+ drainAutoRunner(lane);
revalidatePath("/jobs");
+ revalidatePath("/operations/[id]", "page");
return { ok: true };
} catch (e) {
return { ok: false, error: (e as Error).message };
diff --git a/editor/app/jobs/active/buildActiveJobs.ts b/editor/app/jobs/active/buildActiveJobs.ts
@@ -36,23 +36,15 @@ import {
// Directive-free and value-import-free by construction (laneState.ts imports
// only tone maps and a type), so a server module can read the same derivation
// the client rail does.
+import { deriveLaneState } from "../../components/lanes/laneState";
import {
- deriveLaneState,
- sweepLaneNote,
-} from "../../components/lanes/laneState";
-import {
- AUTO_DOWNLOAD_KIND,
- AUTO_TRANSCRIBE_KIND,
+ autoRunnerJobKind,
getAutoRunnerStatus,
} from "yt-dlp-transcript-common/controller/autoRunner";
-import { getDigestSweepJobId } from "yt-dlp-transcript-common/controller/digestSweep";
-import { getBackfillSweepJobId } from "yt-dlp-transcript-common/controller/backfillSweep";
-import { backfillLaneOperations } from "yt-dlp-transcript-common/lib/operations";
import {
- BACKFILL_QUEUE,
- DIGEST_LOCAL_QUEUE,
- DIGEST_REMOTE_QUEUE,
-} from "yt-dlp-transcript-common/lib/queueKeys";
+ LANES,
+ type AutoQueueKind,
+} from "yt-dlp-transcript-common/lib/autoQueueTypes";
export type DiskStatusView = {
// Whether the low-disk gate is configured (minFreeDiskGB > 0). When false the
@@ -376,7 +368,7 @@ export async function buildActiveJobsPayload(): Promise<ActiveJobsPayload> {
workerCount: pool.summary().length,
},
disk,
- lanes: buildLanes(jobs),
+ lanes: buildLanes(),
};
}
@@ -389,108 +381,56 @@ export async function stuckJobIds(): Promise<string[]> {
}
// The four lanes, from state this process already holds — getAutoRunnerStatus
-// is an in-memory read and the sweep job ids are registry lookups, so the strip
-// costs nothing on a 1-second poll.
+// is an in-memory read, so the strip costs nothing on a 1-second poll.
+//
+// ONE LOOP FOR FOUR LANES since slice 1.3. Two of these rows used to be SWEEP
+// rows, derived from an armed flag and a registry lookup for the orchestrator
+// job, while the two above them were runner rows — the same strip describing
+// two different mechanisms in one vocabulary. Every lane has a runner now.
//
-// The wording comes from the SAME two helpers the console uses. A runner's
-// reason is its own idleReason, mapped to words here rather than on the client
-// so /jobs, the widget and the dashboard cannot describe one state three
-// ways.
-function buildLanes(jobs: JobRowView[]): ActiveLaneView[] {
+// The wording comes from the SAME helpers the console uses. A runner's reason
+// is its own idleReason, mapped to words here rather than on the client so
+// /jobs, the widget and the dashboard cannot describe one state three ways.
+function buildLanes(): ActiveLaneView[] {
const settings = getSettings();
- const runningOn = (keys: ReadonlyArray<string>): number => {
- const set = new Set(keys);
- return jobs.filter(
- (j) => j.status === "running" && j.queueKey !== undefined && set.has(j.queueKey),
- ).length;
- };
-
- const lanes: ActiveLaneView[] = [];
-
- for (const [kind, label, jobKind] of [
- ["transcription", "Auto-transcribe", AUTO_TRANSCRIBE_KIND],
- ["download", "Auto-download", AUTO_DOWNLOAD_KIND],
- ] as const) {
+ return LANES.map((kind) => {
const status = getAutoRunnerStatus(kind);
const inFlight = status.inFlight.length;
- lanes.push({
- kind: jobKind,
- label,
+ return {
+ kind: autoRunnerJobKind(kind),
+ label: LANE_LABEL[kind],
// A STOPPED runner is "unavailable", not "idle". It will never pick
// anything up, and an idle-looking lane reads as "all caught up".
state: !status.running
? "unavailable"
- : inFlight > 0
- ? "running"
- : "idle",
- note: status.running ? autoIdleNote(status.idleReason, kind) : "not running",
+ : deriveLaneState({
+ gateHeld: isGateHeld(settings, kind),
+ activeCount: inFlight,
+ }),
+ note: status.running
+ ? autoIdleNote(status.idleReason, kind)
+ : "not running",
inFlight,
- });
- }
-
- const digestInFlight = runningOn([DIGEST_LOCAL_QUEUE, DIGEST_REMOTE_QUEUE]);
- lanes.push(
- sweepLane({
- kind: "digest-sweep",
- label: "Digest",
- sweeping: settings.digest.sweepEnabled || getDigestSweepJobId() !== null,
- gateHeld: isGateHeld(settings, "digest"),
- available: true,
- inFlight: digestInFlight,
- }),
- );
-
- const backfillInFlight = runningOn([BACKFILL_QUEUE]);
- lanes.push(
- sweepLane({
- kind: "backfill-sweep",
- label: "Backfill",
- sweeping:
- settings.backfill.sweepEnabled || getBackfillSweepJobId() !== null,
- gateHeld: isGateHeld(settings, "backfill"),
- available: backfillLaneOperations(settings).length > 0,
- inFlight: backfillInFlight,
- }),
- );
-
- return lanes;
+ };
+ });
}
-function sweepLane(l: {
- kind: string;
- label: string;
- sweeping: boolean;
- gateHeld: boolean;
- available: boolean;
- inFlight: number;
-}): ActiveLaneView {
- // THE SAME DERIVATION THE RAIL READS, not a server-side restatement of it.
- // This file used to spell out deriveLaneState's precedence and carry a
- // byte-identical copy of its note table, which is two chances for the strip
- // and the rail to describe one lane differently. HOLDING is the state that
- // has no other name — a sweep armed behind a shut gate is not stopped and is
- // not working, and it looked identical to wedged.
- const input = {
- available: l.available,
- gateHeld: l.gateHeld,
- feedRunning: l.sweeping,
- activeCount: l.inFlight,
- };
- return {
- kind: l.kind,
- label: l.label,
- state: deriveLaneState(input),
- note: sweepLaneNote(input),
- inFlight: l.inFlight,
- };
-}
+// The strip's name for each lane. The two runner lanes keep the labels the
+// strip has always shown; the two operation lanes keep the ones their sweep
+// rows had, so nothing on the dashboard is renamed by this slice.
+const LANE_LABEL: Record<AutoQueueKind, string> = {
+ transcription: "Auto-transcribe",
+ download: "Auto-download",
+ digest: "Digest",
+ backfill: "Backfill",
+};
// The runner idle reasons, in words. A copy of the console's idleReasonText,
// kept server-side because this payload is consumed by three clients and the
// sentence must be the same in all three.
function autoIdleNote(
reason: string | null,
- kind: "transcription" | "download",
+ kind: AutoQueueKind,
): string | null {
switch (reason) {
case "no-pending":
@@ -509,14 +449,12 @@ function autoIdleNote(
return "downloads paused globally";
case "lane-held":
return "lane held";
- case "lane-blocked":
- return "its sweep is armed";
case "engine-unreachable":
return "engine unreachable";
case "snoozed":
return "snoozed";
case "disabled":
- return `auto-${kind === "transcription" ? "transcribe" : "download"} is switched off`;
+ return `the ${LANE_LABEL[kind].toLowerCase()} lane is switched off`;
default:
return null;
}
diff --git a/editor/app/operations/[id]/page.tsx b/editor/app/operations/[id]/page.tsx
@@ -9,7 +9,6 @@ import { pauseLaneFor } from "yt-dlp-transcript-common/lib/pauseGates";
import {
operationCatalog,
operationLabel,
- operationsForLane,
type OperationSettingsBlock,
} from "yt-dlp-transcript-common/lib/operations";
import { getSettings, type SiteSettings } from "yt-dlp-transcript-common/lib/settings";
@@ -22,7 +21,6 @@ import { liveJobRows } from "../../jobs/active/buildActiveJobs";
import { buildAutoQueueStatusPayload } from "../status";
import { buildSyncRow } from "../syncRow";
import { buildSchedulerStatusPayload } from "../../scheduler/status";
-import { sweepLaneIdFor, type SweepLaneId } from "../lanes";
import { SyncConsole } from "../components/sync/SyncConsole";
import { SchedulerSettingsForm } from "../components/sync/SchedulerSettingsForm";
import { resolveHeartbeatSeconds } from "../../scheduler/heartbeat";
@@ -53,20 +51,28 @@ const BUCKETS_BY_KIND = Object.fromEntries(
LANES.map((lane) => [lane, [...selectableBucketsForKind(lane)]]),
) as Record<AutoQueueKind, string[]>;
-// The job kinds that ARE this operation, for the running-jobs list. Only the
-// sweep-fed operations need one: a runner lane lists its own units in flight,
-// and a second list beside it would be the pile this redesign is undoing.
+// The MANUAL per-channel jobs on this operation's lane, for the running-jobs
+// list. The runner lists its own in-flight units, which are in-process and make
+// no job record; these are the jobs a channel page's Run button starts, and
+// without them a digest running on a channel is invisible on the digest page.
//
-// Keyed by LANE rather than by operation, because that is the truth: one job
-// on the shared queue is doing whichever kinds the sweep armed.
-const JOB_KINDS_BY_LANE: Record<SweepLaneId, readonly string[]> = {
+// Keyed by LANE rather than by operation, because that is the truth: one job on
+// the shared queue is doing whichever operations the run was scoped to. The two
+// bucket lanes have none — auto-transcribe and auto-download already list their
+// own units, and a second list beside them would be the pile this redesign is
+// undoing.
+//
+// The two SWEEP orchestrator kinds are gone from it; they retired with the
+// sweeps in slice 1.3.
+const JOB_KINDS_BY_LANE: Record<AutoQueueKind, readonly string[]> = {
+ transcription: [],
+ download: [],
digest: [
- "digest-sweep",
"digest-channel-local",
"digest-channel-remote",
"digest-share-cluster",
],
- backfill: ["backfill-sweep", "backfill-channel", "diarize-channel"],
+ backfill: ["backfill-channel", "diarize-channel"],
};
function descriptorFor(id: string) {
@@ -75,7 +81,7 @@ function descriptorFor(id: string) {
// THE OPERATION'S OWN SETTINGS FORM, chosen by the block the descriptor
// DECLARES — not by a table keyed by operation id. That is the same rule
-// `runner` and `sweepLaneIdFor` follow, and it is what lets both attribution
+// `pauseLaneFor` follows, and it is what lets both attribution
// operations render one form: they name one block, so they get one switch arm.
//
// Exhaustive on purpose (the union is closed): a settings block added to the
@@ -162,9 +168,6 @@ export default async function OperationPage({
// (so an external operation sharing a queue with no runner of its own still
// gets null) and falls back to the declared queue key.
const runnerKind: AutoQueueKind | null = pauseLaneFor(op.id);
- // Likewise for the sweep lane. Null for an operation neither sweep dispatches;
- // OperationDetail renders that as "no console here" rather than borrowing one.
- const laneId = sweepLaneIdFor(op);
// THE SCHEDULER PAYLOAD, READ ONCE PER RENDER. The rail's sync row is a fold
// of it and the cadence console below is the whole of it, so the sync page
@@ -203,25 +206,10 @@ export default async function OperationPage({
);
// PER LANE, not per operation: `settings.backfill` governs BACKFILL_QUEUE,
// which three operations share, so this same form is drawn on all three of
- // their pages beside the pause and sweep they also share. The digest lane has
- // no equivalent — its lane facts (`sweepEnabled`, order and reach) live in the
- // digest block and the OrderReach control.
- // THE OPERATIONS EACH LANE DISPATCHES, as ids, for the ladder's operation
- // select. Server-side because it is a settings-dependent registry read
- // (`digest.remoteEnabled` moves the digest operation between two queue keys
- // that are both the digest lane), and the ladder is a client component.
- //
- // A lane with 0 or 1 operations draws no select: there is nothing to choose,
- // and an operation leaf and a bucket-less leaf claim the same work.
- const operationsByLane = Object.fromEntries(
- LANES.map((lane) => [
- lane,
- operationsForLane(lane, settings).map((o) => o.id),
- ]),
- ) as Record<AutoQueueKind, string[]>;
-
+ // their pages beside the pause they also share. The digest lane has no
+ // equivalent — its order lives on the lane policy the console already edits.
const laneSettings =
- laneId === "backfill" ? (
+ runnerKind === "backfill" ? (
<LaneSettingsForm key="backfill-lane" initial={settings.backfill} />
) : null;
@@ -237,10 +225,7 @@ export default async function OperationPage({
<SyncConsole key="sync" initial={await schedulerPayload} />
) : null;
- // The sweep panel's own in-flight list. Keyed off the SWEEP lane, not off the
- // runner: since 1.2 a digest page has both consoles, and the runner lists its
- // own units in flight while these are the per-channel jobs the sweep starts.
- const laneJobKinds = laneId ? JOB_KINDS_BY_LANE[laneId] : [];
+ const laneJobKinds = runnerKind ? JOB_KINDS_BY_LANE[runnerKind] : [];
// Through the one builder, so this list has the same progress bars /jobs does
// (it used to drop `progress`, `tasks`, `drainable` and the reorder bounds).
const activeJobs =
@@ -285,10 +270,8 @@ export default async function OperationPage({
channels={channelOptions}
platforms={[...PLATFORM_VALUES]}
bucketsByKind={BUCKETS_BY_KIND}
- operationsByLane={operationsByLane}
activeJobs={activeJobs}
runnerKind={runnerKind}
- laneId={laneId}
sync={sync}
cadenceConsole={cadenceConsole}
dependsOn={[...(op.dependsOn ?? [])].map((depId) => ({
diff --git a/editor/app/operations/actions.ts b/editor/app/operations/actions.ts
@@ -22,7 +22,6 @@ import {
type AutoQueueGroup,
type AutoQueueNode,
type AutoQueueOrder,
- type AutoQueueReach,
} from "yt-dlp-transcript-common/jobs/autoQueuePolicy";
// EVERY OPERATIONS SURFACE, not just the board. The console is a board plus one
@@ -249,82 +248,3 @@ export async function prioritizeChannelDownloadAction(
revalidateOperations();
return { ok: true };
}
-
-// Order and reach for the two SWEEP-fed lanes.
-//
-// A separate action from saveAutoQueueAction because these lanes have no policy
-// tree to save alongside: their order lives in settings.digest / settings.
-// backfill, not in settings.autoQueue. Routing them through the policy form
-// would mean an operator could not change an order without also committing a
-// pending tree edit for a different lane.
-//
-// SPREAD-AND-OVERRIDE, never a rebuilt literal. settings.digest carries
-// `sweepEnabled` and its scope, and settings.backfill carries both plus
-// `allowRedownload`; a literal here that omitted one would disarm a multi-week
-// sweep, or write media to a 97%-full disk, on an unrelated save. That exact
-// bug is documented in the settings form's own digest block.
-//
-// It takes effect on the sweep's NEXT PASS — both sweeps re-read the setting per
-// pass rather than at launch — and on the next per-channel batch, which resolves
-// the order itself. Nothing needs restarting.
-export async function saveLaneOrderAction(
- lane: "digest" | "backfill",
- input: { order: AutoQueueOrder; reach: AutoQueueReach },
-): Promise<SaveResult> {
- const current = getSettings();
- const next: SiteSettings =
- lane === "digest"
- ? {
- ...current,
- digest: {
- ...current.digest,
- recencyOrder: input.order,
- recencyReach: input.reach,
- },
- }
- : {
- ...current,
- backfill: {
- ...current.backfill,
- order: input.order,
- reach: input.reach,
- },
- };
- try {
- await writeSettings(next);
- } catch (e) {
- return { ok: false, error: (e as Error).message };
- }
- revalidateOperations();
- return { ok: true };
-}
-
-// The arbiter: start / stop the unified dispatcher.
-//
-// Kept out of saveLaneOrderAction and out of the policy form: starting a
-// dispatcher is an operational act, not a setting, and routing it through a
-// form would mean a pending tree edit had to be saved or discarded first.
-//
-// It REFUSES rather than silently no-opping when a sweep is armed, and the
-// refusal carries the reason — two dispatchers on one lane would start the same
-// channel twice, and an operator who clicks Start and sees nothing happen
-// deserves to be told which switch is in the way.
-export async function startArbiterAction(): Promise<SaveResult> {
- const { startArbiter } = await import(
- "yt-dlp-transcript-common/controller/arbiter"
- );
- const result = await startArbiter();
- revalidateOperations();
- return result.jobId
- ? { ok: true }
- : { ok: false, error: result.error ?? "The arbiter could not be started." };
-}
-
-export async function stopArbiterAction(): Promise<SaveResult> {
- const { stopArbiter } = await import(
- "yt-dlp-transcript-common/controller/arbiter"
- );
- stopArbiter();
- revalidateOperations();
- return { ok: true };
-}
diff --git a/editor/app/operations/components/ArbiterBar.tsx b/editor/app/operations/components/ArbiterBar.tsx
@@ -1,113 +0,0 @@
-"use client";
-
-import Link from "next/link";
-import { useState, useTransition } from "react";
-import { Button } from "yt-dlp-transcript-common/components/ui/button";
-import { startArbiterAction, stopArbiterAction } from "../actions";
-import type { ArbiterStatus } from "../lanes";
-
-// THE ARBITER, on one line, under the rail.
-//
-// It sits above the lane switcher and outside every lane because it is not a
-// lane — it is the thing that decides which lane runs. Putting it inside one
-// would say the opposite.
-//
-// A <div>, never a <section>: nesting one inside this page's structure breaks
-// every heading-scoped lookup in the e2e suite. The name is plain text for the
-// same reason the rail's names are.
-//
-// IT SAYS WHY IT WILL NOT START, before the click rather than after. The two
-// reasons are an armed sweep (two dispatchers on one lane would start the same
-// channel twice) and a tree with no operation rule in it (it would come up,
-// find nothing to dispatch, and stop). Both come from the server, from the same
-// function that would refuse the start, so the page cannot promise something
-// the action then declines.
-
-export function ArbiterBar({
- arbiter,
- onRefresh,
-}: {
- arbiter: ArbiterStatus;
- onRefresh: () => void | Promise<void>;
-}) {
- const [pending, startTransition] = useTransition();
- const [error, setError] = useState<string | null>(null);
-
- const run = (action: () => Promise<{ ok: boolean; error?: string }>) => {
- startTransition(async () => {
- const result = await action();
- setError(result.ok ? null : (result.error ?? "Failed."));
- await onRefresh();
- });
- };
-
- const noRules = arbiter.operationRules === 0;
- const blocked = arbiter.blockedReason ?? (noRules ? NO_RULES : null);
-
- return (
- <div className="flex flex-col gap-1.5 rounded-md border border-border bg-card px-3 py-2">
- <div className="flex flex-wrap items-center gap-x-3 gap-y-1 text-sm">
- <span className="font-mono text-xs uppercase tracking-[0.14em] text-muted-foreground">
- Arbiter
- </span>
- <span
- aria-hidden="true"
- className={`size-2 shrink-0 rounded-full ${
- arbiter.running
- ? "bg-info animate-pulse motion-reduce:animate-none"
- : "bg-muted-foreground/40"
- }`}
- />
- <span className={arbiter.running ? "text-foreground" : "text-muted-foreground"}>
- {arbiter.running ? "Running" : "Stopped"}
- </span>
- <span className="text-xs text-muted-foreground">
- {arbiter.operationRules === 0
- ? "no rule names an operation"
- : `${arbiter.operationRules} rule${arbiter.operationRules === 1 ? "" : "s"} name an operation`}
- </span>
- {arbiter.jobId && (
- <Link
- href={`/jobs/${arbiter.jobId}`}
- className="text-xs underline underline-offset-2 text-muted-foreground hover:text-foreground"
- >
- log
- </Link>
- )}
- <span className="ml-auto">
- {arbiter.running ? (
- <Button
- type="button"
- size="sm"
- variant="outline"
- aria-label="Stop the arbiter"
- disabled={pending}
- onClick={() => run(stopArbiterAction)}
- >
- Stop
- </Button>
- ) : (
- <Button
- type="button"
- size="sm"
- aria-label="Start the arbiter"
- disabled={pending || blocked !== null}
- onClick={() => run(startArbiterAction)}
- >
- Start
- </Button>
- )}
- </span>
- </div>
- <p className="text-xs text-muted-foreground">
- {blocked ??
- "One dispatcher for every operation a rule names: one job per channel, on each operation's own lane."}
- </p>
- {error && <p className="text-xs text-warning">{error}</p>}
- </div>
- );
-}
-
-const NO_RULES =
- "No rule names an operation yet, so there would be nothing to dispatch. " +
- "Point a rule at Digest or a speaker operation first.";
diff --git a/editor/app/operations/components/LaneHeader.tsx b/editor/app/operations/components/LaneHeader.tsx
@@ -3,6 +3,7 @@
import Link from "next/link";
import { Button } from "yt-dlp-transcript-common/components/ui/button";
import type { AutoQueueKindStatus } from "../status";
+import type { LaneWorker } from "../lanes";
import { PauseLaneButton } from "../../components/lanes/pauseControl";
import { formatElapsed, idleReasonText } from "./dispatch";
@@ -19,6 +20,7 @@ import { formatElapsed, idleReasonText } from "./dispatch";
export function LaneHeader({
title,
status,
+ runsOn,
now,
busy,
onControl,
@@ -26,6 +28,9 @@ export function LaneHeader({
}: {
title: string;
status: AutoQueueKindStatus;
+ // Delegate workers that can take this lane's work besides this machine. See
+ // WhereItRuns below.
+ runsOn: LaneWorker[];
now: number | null;
busy: boolean;
onControl: (action: "start" | "stop" | "drain") => void;
@@ -91,8 +96,8 @@ export function LaneHeader({
Stop all act on the RUNNER; this holds the LANE, which is a
different thing with a different lifetime — a held lane stays held
across a runner restart, and a stopped runner can still be holding.
- The same button the dashboard, the widget and both sweep panels
- draw, so there is one idea of what holding looks like.
+ The same button the dashboard and the widget draw, so there is
+ one idea of what holding looks like.
A plain <button> plus a role="alert" span on error only: no nested
<section>, and role="status" stays reserved for "Saved." */}
@@ -136,6 +141,8 @@ export function LaneHeader({
)}
</p>
+ <WhereItRuns runsOn={runsOn} />
+
{/* The page's answer to the most common question an operator arrives
with. Only rendered while the runner is UP: for a stopped runner the
pill above already says everything, and a second line repeating it
@@ -159,3 +166,45 @@ function Sep() {
</span>
);
}
+
+// WHERE THE LANE'S WORK CAN RUN besides this machine: the delegate workers (LLM
+// endpoints, tagged unit executors) whose tags match one of its operations,
+// folded to one row per configured worker with its slot count.
+//
+// READ-ONLY, and never part of any policy — delegation is a worker-pool concern
+// decided per item at dispatch, and this only reports it. Derived through the
+// SAME workerMatches rule the dispatchers grant by (operations/lanes.ts), so
+// this line and the actual routing cannot disagree.
+//
+// It moved here from the sweep console's scope list, which drew it per
+// operation row. A lane is the right scope: the slots belong to the machine, so
+// two operations served by one endpoint are one row, not two.
+//
+// Rendered only when delegates are configured — the lane always runs here, and
+// stating that on every console would be noise. The one loud case: capacity is
+// configured and NONE of it can currently take work (degraded or disabled),
+// which looks identical to "distributed" from the Start button.
+function WhereItRuns({ runsOn }: { runsOn: LaneWorker[] }) {
+ if (runsOn.length === 0) return null;
+ const noneAvailable = runsOn.every((w) => w.available === 0);
+ return (
+ <p className="text-xs text-muted-foreground">
+ also runs on{" "}
+ {runsOn
+ .map(
+ (w) =>
+ `${w.name} (${w.kind === "llm" ? "LLM endpoint" : "executor"}${
+ w.slots > 1 ? ` ×${w.slots}` : ""
+ })`,
+ )
+ .join(", ")}
+ {noneAvailable && (
+ <span className="text-warning">
+ {" "}
+ — none of it is available right now (degraded or disabled); this
+ machine carries the whole lane until one is re-enabled.
+ </span>
+ )}
+ </p>
+ );
+}
diff --git a/editor/app/operations/components/OperationDetail.tsx b/editor/app/operations/components/OperationDetail.tsx
@@ -3,13 +3,11 @@
import type { ReactNode } from "react";
import type { AutoQueueKind } from "yt-dlp-transcript-common/jobs/autoQueueState";
import type { AutoQueueStatusPayload } from "../status";
-import type { SweepLaneId } from "../lanes";
// Type-only: syncRow.ts is a server module. See OperationRail.
import type { SyncRowView } from "../syncRow";
import { HowPriorityWorks } from "./HowPriorityWorks";
import { OperationRail } from "./OperationRail";
import { RunnerOperationView } from "./RunnerOperationView";
-import { SweepLane } from "./SweepLane";
import { railStates } from "./railStates";
import { useHydrated, useOperationsStatus } from "./useOperationsStatus";
import { RUNNER_TITLE, type Channel } from "./dispatch";
@@ -30,10 +28,8 @@ export function OperationDetail({
channels,
platforms,
bucketsByKind,
- operationsByLane,
activeJobs,
runnerKind,
- laneId,
sync,
cadenceConsole,
dependsOn,
@@ -46,23 +42,15 @@ export function OperationDetail({
channels: Channel[];
platforms: string[];
bucketsByKind: Record<AutoQueueKind, string[]>;
- // The operation ids each lane dispatches, resolved on the SERVER off the
- // registry (it is settings-dependent — `digest.remoteEnabled` moves the digest
- // operation between two queue keys that are both the digest lane). The ladder
- // offers a leaf an operation only for a lane carrying more than one.
- operationsByLane: Record<AutoQueueKind, string[]>;
// Jobs on this operation's queues, SSR-rendered. Empty renders nothing at
// all (RunningJobsList returns null), so a quiet page carries no second
// in-flight list beside the lane's own.
activeJobs: JobRowView[];
- // Set for the operations a RUNNER dispatches; null for everything else.
- // Resolved on the server FROM THE DESCRIPTOR (`op.runner`) so this component
- // never re-decides which is which — and so an operation the registry gains
- // gets the right answer without a branch here.
+ // THE LANE THAT DISPATCHES THIS OPERATION, or null for one nothing here
+ // drives. Resolved on the server by `pauseLaneFor`, which reads the
+ // descriptor rather than the id — so an operation the registry gains gets the
+ // right answer without a branch in this file.
runnerKind: AutoQueueKind | null;
- // The sweep lane that dispatches this, or null for one neither sweep runs.
- // Also resolved on the server, off the descriptor's queue key.
- laneId: SweepLaneId | null;
// The rail's sync row, for the rail at the top of every operation page.
sync: SyncRowView;
// THE CADENCE-TRIGGERED OPERATION'S CONSOLE, built on the server off
@@ -107,15 +95,25 @@ export function OperationDetail({
sync={sync}
/>
- {/* THE RUNNER CONSOLE AND THE SWEEP PANEL, BOTH, for the length of this
- slice. Every lane has a runner now; two of them still have a sweep
- beside it, and an operator mid-migration needs to see which one is
- driving. 1.3 deletes the sweep half. They are distinguishable by
- selector on purpose — the runner section carries `data-lane`, which is
- the contract all four lanes share, and the sweep panel carries
- `data-sweep-lane`, which retires with it. */}
+ {/* ONE CONSOLE, and it is the runner's. Until slice 1.3 this page drew two
+ — the runner's and the sweep panel's — so that an operator
+ mid-migration could see which one was driving. The sweeps are gone;
+ `<section data-lane>` is the contract all four lanes share, and it is
+ the only lane section on the page.
+
+ THE LANE CONTEXT IS ABOVE IT, not inside: this operation's own band,
+ and the sentence saying which other operations share the lane's queue
+ and pause. Outside the runner's <section> because nothing inside it
+ may be a <section> and because those headings would land inside every
+ `section[data-lane]`-scoped lookup in the e2e suite. */}
{runnerKind && (
<>
+ <LaneOperationContext
+ id={id}
+ lane={runnerKind}
+ data={data}
+ activeJobs={activeJobs}
+ />
<HowPriorityWorks />
<RunnerOperationView
kind={runnerKind}
@@ -124,38 +122,33 @@ export function OperationDetail({
channels={channels}
platforms={platforms}
buckets={bucketsByKind[runnerKind]}
- operations={operationsByLane[runnerKind]}
+ operations={data.lanes.operations[runnerKind].map((op) => op.id)}
+ runsOn={data.lanes.runsOn[runnerKind]}
onRefresh={refresh}
/>
+ {/* Operation block first, lane block second — this operation's own
+ switches, then the shared queue's. Both are plain <form>s with
+ their own actions, so saving one cannot read the other's
+ checkboxes as off. */}
+ {operationSettings}
+ {laneSettings}
</>
)}
- {laneId && (
- <SweepOperationView
- id={id}
- laneId={laneId}
- data={data}
- activeJobs={activeJobs}
- onRefresh={refresh}
- operationSettings={operationSettings}
- laneSettings={laneSettings}
- />
- )}
-
- {!runnerKind && !laneId && cadenceConsole && (
+ {!runnerKind && cadenceConsole && (
// The settings slot is drawn here rather than inside the console: the
// console is a moved component with its own contract, and this keeps
- // "console, then this operation's settings" the same order the sweep
- // pages already use.
+ // "console, then this operation's settings" the same order every other
+ // operation page uses.
<>
{cadenceConsole}
{operationSettings}
</>
)}
- {!runnerKind && !laneId && !cadenceConsole && (
+ {!runnerKind && !cadenceConsole && (
// STILL THE FALLTHROUGH, for a registered operation nothing drives.
- // A cadence console is a fourth arm, not a replacement for this one.
+ // A cadence console is a third arm, not a replacement for this one.
<NoConsoleView id={id} data={data} dependsOn={dependsOn} />
)}
@@ -164,59 +157,45 @@ export function OperationDetail({
);
}
-// A REGISTRY OPERATION: digest, or one of the speaker kinds.
+// WHAT THIS PAGE'S OPERATION IS, beside the lane console that dispatches it.
//
-// The lane below is SHARED — every kind on BACKFILL_QUEUE is dispatched by one
-// sweep and held by one gate — and the panel says so rather than letting a
+// The lane below is SHARED — every operation on one queue is dispatched by one
+// runner and held by one gate — and this says so rather than letting a
// per-operation page imply a per-operation switch. What IS this operation's
// alone is the band: its own five populations, its own denominator, its own
-// cost basis. That is the figure the shared panel could not draw before, because a
-// lane holding three operations has no single total that means anything.
-function SweepOperationView({
+// cost basis. That is the figure the shared console cannot draw, because a lane
+// holding three operations has no single total that means anything.
+//
+// A <div>, never a <section>: the runner console below is the page's one
+// `section[data-lane]`, and a second section here would be matched by
+// `locator("section", { has: heading })` and break every scoped lookup on
+// strict mode. `data-operation` names which operation the page is about, which
+// the lane cannot say.
+function LaneOperationContext({
id,
- laneId,
+ lane,
data,
activeJobs,
- onRefresh,
- operationSettings,
- laneSettings,
}: {
id: string;
- laneId: SweepLaneId;
+ lane: AutoQueueKind;
data: AutoQueueStatusPayload;
activeJobs: JobRowView[];
- onRefresh: () => Promise<void>;
- operationSettings?: ReactNode;
- laneSettings?: ReactNode;
}) {
const hydrated = useHydrated();
- const lane = data.lanes[laneId];
const band = data.lanes.bands.find((b) => b.id === id) ?? null;
+ const operations = data.lanes.operations[lane];
// The other operations this lane would dispatch, named. Not a count: "shares
// a lane with 2 others" is exactly the sentence that made "Backfill" mean
// three different costs.
- const siblings = lane.operations.filter((op) => op.id !== id);
+ const siblings = operations.filter((op) => op.id !== id);
// In the CATALOG but not on the lane: the operation is registered and
- // switched off. Worth a sentence, because everything else on this page — an
- // available lane, an armable sweep, a plan — would otherwise read as "ready".
- const switchedOn = lane.operations.some((op) => op.id === id);
+ // switched off. Worth a sentence, because everything else on this page — a
+ // running lane, a work list, a claim ladder — would otherwise read as "ready".
+ const switchedOn = operations.some((op) => op.id === id);
return (
- <section
- // TWO HOOKS, because there are two facts. data-sweep-lane names the LANE
- // this panel controls — what the e2e suite scopes the sweep and pause
- // buttons by, and deliberately the SAME value on every speaker
- // operation's page, since they are one lane. data-operation names which
- // operation the page is about, which the lane cannot say. data-hydrated is
- // the same testing affordance the runner sections carry; see
- // RunnerOperationView.
- //
- // IT WAS `data-lane` UNTIL SLICE 1.2, and it had to move: the runner
- // console now renders beside this panel on the same page, `data-lane` is
- // the contract THAT section carries on all four lanes, and two sections
- // answering one selector fails every scoped lookup on Playwright's strict
- // mode. This attribute is deleted with the panel in 1.3.
- data-sweep-lane={laneId}
+ <div
data-operation={id}
data-hydrated={hydrated ? "true" : undefined}
className="flex flex-col gap-4"
@@ -230,35 +209,23 @@ function SweepOperationView({
{!switchedOn && (
<p className="text-sm text-warning">
This operation is switched off in its settings below, so nothing
- dispatches it — not the sweep, and not the arbiter. That is not the
- same as being finished, which is what an empty lane would otherwise
- imply.
+ dispatches it — its lane will pick every other operation on the queue
+ and skip this one. That is not the same as being finished, which is
+ what an empty work list would otherwise imply.
</p>
)}
{siblings.length > 0 && (
<p className="text-sm text-muted-foreground">
- This operation shares one lane, one sweep and one pause with{" "}
- {joinLabels(siblings.map((s) => s.label))}. Arming the sweep below
- runs whatever is ticked in its scope, not just this one — and the
- counts have different cost bases, so read the scope before arming it.
+ This operation shares one lane, one runner and one pause with{" "}
+ {joinLabels(siblings.map((s) => s.label))}. A rule below that names no
+ operation claims all of them, not just this one — and the counts have
+ different cost bases, so read the rules before starting the lane.
</p>
)}
<RunningJobsList jobs={activeJobs} />
-
- <SweepLane lane={lane} band={band} onRefresh={onRefresh} />
-
- {/* INSIDE the section, deliberately: `data-lane` is what the e2e suite
- scopes this panel's controls by, and a settings form outside it would
- be outside every existing scoped selector. Operation block first, lane
- block second — this operation's own switches, then the shared queue's.
- Both are plain <form>s with their own actions, so saving one cannot
- read the other's checkboxes as off; that is what retired the hidden
- `*FormPresent` markers the single settings form needed. */}
- {operationSettings}
- {laneSettings}
- </section>
+ </div>
);
}
diff --git a/editor/app/operations/components/OperationRail.tsx b/editor/app/operations/components/OperationRail.tsx
@@ -266,7 +266,7 @@ function RailRow({
// `population`, not "unit": the word beside the number names WHICH population
// of the band this is (reachable, blocked, held), and "unit" already means one
-// item of dispatchable work everywhere the arbiter and the worker route use it.
+// item of dispatchable work everywhere the runner and the worker route use it.
function Figure({
n,
population,
diff --git a/editor/app/operations/components/OperationsBoard.tsx b/editor/app/operations/components/OperationsBoard.tsx
@@ -2,14 +2,18 @@
import type { AutoQueueStatusPayload } from "../status";
import type { SyncRowView } from "../syncRow";
-import { ArbiterBar } from "./ArbiterBar";
import { OperationRail } from "./OperationRail";
import { railStates } from "./railStates";
import { useHydrated, useOperationsStatus } from "./useOperationsStatus";
-// THE BOARD: every operation this install runs, one line each, and the
-// dispatcher above them. It is the whole of the page — there is nothing here
-// that is about ONE operation, because that is what /operations/<id> is for.
+// THE BOARD: every operation this install runs, one line each. It is the whole
+// of the page — there is nothing here that is about ONE operation, because that
+// is what /operations/<id> is for.
+//
+// There was a dispatcher bar under the rail until slice 1.3: the arbiter's, a
+// unified dispatcher that never ran a single unit in production. The four lanes
+// each have a runner and a console of their own, and the rail's rows link to
+// them.
//
// This is what the lane <select> used to hide. The console before it stacked
// two identical runner panels and had no room for the other pipelines at all,
@@ -25,12 +29,12 @@ export function OperationsBoard({
initial: AutoQueueStatusPayload;
sync: SyncRowView;
}) {
- const { data, refresh } = useOperationsStatus(initial);
+ const { data } = useOperationsStatus(initial);
const hydrated = useHydrated();
return (
// data-board is the board's addressing hook, and data-hydrated the signal
- // that React is live on it — the arbiter's buttons below are inert until it
+ // that React is live on it — the rail's live figures are stale until it
// is. See useHydrated.
<div
data-board="operations"
@@ -48,8 +52,6 @@ export function OperationsBoard({
selectedId={null}
sync={sync}
/>
-
- <ArbiterBar arbiter={data.lanes.arbiter} onRefresh={refresh} />
</div>
);
}
diff --git a/editor/app/operations/components/RunnerOperationView.tsx b/editor/app/operations/components/RunnerOperationView.tsx
@@ -3,6 +3,7 @@
import { useCallback, useEffect, useState } from "react";
import type { AutoQueueKind } from "yt-dlp-transcript-common/jobs/autoQueueState";
import type { AutoQueueKindStatus, PlatformCooldownView } from "../status";
+import type { LaneWorker } from "../lanes";
import { InFlightList } from "./InFlightList";
import { LaneHeader } from "./LaneHeader";
import { NextUp } from "./NextUp";
@@ -10,7 +11,7 @@ import { PolicyTreeEditor } from "./PolicyTreeEditor";
import { SnoozeControl } from "./SnoozeControl";
import { type Channel, formatClock, formatCooldown, leafOrder } from "./dispatch";
-// ONE RUNNER OPERATION, IN FULL — download or transcription.
+// ONE LANE'S RUNNER, IN FULL — any of the four.
//
// Lifted VERBATIM out of AutoQueueView's KindLane when the console became one
// page per operation. The only thing that changed is what is around it: there
@@ -41,6 +42,7 @@ export function RunnerOperationView({
platforms,
buckets,
operations,
+ runsOn,
onRefresh,
}: {
kind: AutoQueueKind;
@@ -49,9 +51,12 @@ export function RunnerOperationView({
channels: Channel[];
platforms: string[];
buckets: string[];
- // The operation ids this lane dispatches. Empty for the two runner lanes
- // today; the ladder draws a select only when there is more than one.
+ // The operation ids this lane dispatches. Empty for the two bucket lanes; the
+ // ladder draws a select only when there is more than one.
operations: string[];
+ // Delegate workers that can take this lane's work besides this machine.
+ // Read-only; the header draws it, and empty draws nothing.
+ runsOn: LaneWorker[];
onRefresh: () => Promise<void>;
}) {
const [busy, setBusy] = useState(false);
@@ -89,6 +94,7 @@ export function RunnerOperationView({
<LaneHeader
title={title}
status={status}
+ runsOn={runsOn}
now={now}
busy={busy}
onControl={control}
diff --git a/editor/app/operations/components/SweepLane.tsx b/editor/app/operations/components/SweepLane.tsx
@@ -1,422 +0,0 @@
-"use client";
-
-import Link from "next/link";
-import { useState, useTransition } from "react";
-import { Button } from "yt-dlp-transcript-common/components/ui/button";
-import {
- deriveLaneState,
- LANE_DOT,
- LANE_TEXT,
- LANE_WORD,
-} from "../../components/lanes/laneState";
-import {
- startBackfillSweepAction,
- startDigestSweepAction,
- stopBackfillSweepAction,
- stopDigestSweepAction,
-} from "../../jobs/actions";
-import { saveLaneOrderAction } from "../actions";
-import { PauseLaneButton } from "../../components/lanes/pauseControl";
-import type { SweepLaneStatus } from "../lanes";
-import type { OperationBand } from "../../components/pipelines/band";
-import { OrderReach } from "./OrderReach";
-import { SweepPlan } from "./SweepPlan";
-import { SweepScope, WhereItRuns } from "./SweepScope";
-import { formatElapsed } from "./dispatch";
-
-// A sweep-fed lane: digest or backfill.
-//
-// TWO CONTROLS, NOT ONE, and they keep the names they already have everywhere
-// else. The SWEEP is the feed — is there a corpus pass at all. The PAUSE is the
-// gate — is the lane consuming what the feed produces. LaneDeck says why they
-// must stay apart: conflating them is how an operator loses a week of GPU time,
-// because one is cheap to undo and the other is not. Both are here, and the
-// dashboard's cards keep both too.
-//
-// NO CLAIM LADDER. These lanes have no policy tree — settings.autoQueue has
-// exactly two keys — and an empty ladder would read as "no rule matches" where
-// the truth is "rules are not how this lane is dispatched". It gets one when
-// the arbiter does.
-//
-// A <div>, never a nested <section>: the e2e suite scopes with
-// locator("section", { has: heading }), and a section inside a section makes
-// every one of those lookups match two ancestors.
-
-export function SweepLane({
- lane,
- band,
- onRefresh,
-}: {
- lane: SweepLaneStatus;
- // This lane's own band. For backfill that is one of several — the rail shows
- // each kind separately, because summed they are a figure with no cost basis — so the
- // panel names the kinds rather than pretending to a single total.
- band: OperationBand | null;
- onRefresh: () => void | Promise<void>;
-}) {
- const [pending, startTransition] = useTransition();
- const [busy, setBusy] = useState(false);
- const digest = lane.id === "digest";
- // THE SCOPE THE OPERATOR IS COMPOSING, seeded from what is ARMED.
- //
- // Seeded once and then owned locally, deliberately: the payload re-arrives
- // every 3 seconds, and re-seeding on each poll would undo a tick the moment
- // it was made. It is a draft until the arm button writes it — and it is
- // written BY the arm action, never as a separate settings save, so a restart
- // between the two cannot resurrect a bounded run as a corpus-wide one.
- const [scopeKinds, setScopeKinds] = useState<string[]>(lane.scopeKinds);
- const [channelScope, setChannelScope] = useState<ReadonlySet<string> | null>(
- lane.scopeChannels.length > 0 ? new Set(lane.scopeChannels) : null,
- );
- // WHILE A SWEEP IS RUNNING THE CONSOLE SHOWS WHAT IS ARMED, NOT A DRAFT.
- //
- // The scope is read once, at arm time, and persisted there; nothing re-reads
- // it mid-run. So leaving the checkboxes live during a sweep would let an
- // operator untick the expensive lane, watch the plan total fall, and believe
- // they had changed a run that is still doing every operation. The controls go
- // read-only and show the armed scope, and the panel says how to change it.
- const armed = lane.sweeping;
- // A stored scope of `[]` means "every enabled operation" — the rule
- // resolveBackfillLaneOperations applies and the rule an unscoped sweep runs — so it
- // renders as every box ticked rather than none.
- const allKindIds = lane.operations.map((op) => op.id);
- const draftKinds = scopeKinds.length > 0 ? scopeKinds : allKindIds;
- const selectedKinds = armed
- ? lane.scopeKinds.length > 0
- ? lane.scopeKinds
- : allKindIds
- : draftKinds;
- // Ticking every box is not the same as pinning today's three: it re-arms the
- // UNSCOPED sweep, which tracks the registry, so an operation enabled later is
- // picked up by the resumed run instead of being silently excluded forever.
- const armKinds = draftKinds.length === allKindIds.length ? [] : draftKinds;
- const unknownScopeIds = lane.scopeKinds.filter(
- (id) => !lane.operations.some((op) => op.id === id),
- );
- const armChannels = channelScope ? [...channelScope] : undefined;
- const shownChannels = armed
- ? lane.scopeChannels.length > 0
- ? new Set(lane.scopeChannels)
- : null
- : channelScope;
- const canArm =
- lane.available &&
- (digest || draftKinds.length > 0) &&
- (channelScope === null || channelScope.size > 0);
- const state = deriveLaneState({
- available: lane.available,
- gateHeld: lane.gateHeld,
- feedRunning: lane.sweeping,
- activeCount: lane.inFlight.length,
- });
-
- const run = (action: () => Promise<unknown>) => {
- setBusy(true);
- startTransition(async () => {
- try {
- await action();
- await onRefresh();
- } finally {
- setBusy(false);
- }
- });
- };
-
- const working = busy || pending;
-
- return (
- <div className="flex flex-col gap-4">
- <div className="flex flex-wrap items-center gap-x-3 gap-y-2">
- {/* NAMED AFTER WHAT IT HOLDS, then after what this panel does with it.
- "Backfill" was a queue key standing in for three operations with
- different inputs and different costs; lanes.ts derives the group
- name, and "· sweep" says which of the lane's two controls this panel
- is about. */}
- <h2 className="font-display text-lg font-semibold tracking-tight">
- {lane.label} <span className="text-muted-foreground">· sweep</span>
- </h2>
- <span className="flex items-center gap-2 text-sm">
- <span aria-hidden="true" className={`size-2 rounded-full ${LANE_DOT[state]}`} />
- <span className={LANE_TEXT[state]}>{LANE_WORD[state]}</span>
- </span>
- <span className="ml-auto flex flex-wrap gap-2">
- {/* THE FEED, and it now says WHAT IT WILL DO rather than naming the
- mechanism. "Start sweep" and "Pause Backfill" are the same shape
- of phrase for two acts whose costs to undo differ by a week of GPU
- time; different verbs are the cheapest thing that keeps them
- apart. The label is derived from the scope composed below, so the
- button and the plan cannot disagree about what is being armed.
-
- THE SCOPE TRAVELS THROUGH THIS ACTION, never through a separate
- settings save — startBackfillSweep persists the flag and the scope
- in one awaited write, which is what lets the boot hook resume the
- same run rather than a corpus-wide one.
-
- aria-label deliberately UNCHANGED: it is internal addressing that
- confuses nobody, and the e2e suite finds these buttons by it. */}
- <Button
- type="button"
- size="sm"
- variant={lane.sweeping ? "outline" : "default"}
- disabled={working || !lane.available || (!lane.sweeping && !canArm)}
- aria-label={`${lane.sweeping ? "Stop" : "Start"} ${lane.label} sweep`}
- onClick={() =>
- run(
- lane.sweeping
- ? digest
- ? stopDigestSweepAction
- : stopBackfillSweepAction
- : digest
- ? () => startDigestSweepAction(armChannels)
- : () => startBackfillSweepAction(armKinds, armChannels),
- )
- }
- >
- {lane.sweeping
- ? "Stop sweeping"
- : channelScope
- ? `Sweep ${channelScope.size.toLocaleString()} channel${channelScope.size === 1 ? "" : "s"}`
- : "Sweep every channel"}
- </Button>
- {/* THE GATE — the same control the dashboard and the widget draw, so
- there is one definition of what holding a lane looks like.
-
- THREE THINGS CHANGED WHEN IT MOVED HERE, and each is on purpose:
- the aria-label is now the canonical "pause backfill" / "pause
- digests" rather than a composed "Pause Backfill" (Playwright's
- name match is a case-insensitive substring, so every existing
- lookup still finds it); the emphasis is the dashboard's — warning-
- filled when the lane is HELD, where this button used to tint
- itself while the lane was running fine; and the pause no longer
- shares `working` with the sweep button, so starting a sweep does
- not disable the hold mid-flight. `disabled` reaches the pause side
- only: a resume must always be clickable. */}
- <PauseLaneButton
- lane={lane.id}
- held={lane.gateHeld}
- disabled={!lane.available}
- onChange={onRefresh}
- />
- </span>
- </div>
-
- {!lane.available && (
- <p className="text-sm text-muted-foreground">
- No operation on the backfill lane is switched on, so there is no lane
- to run. That is not the same as being finished — turn one on from its
- operation page; this operation's own settings are below.
- </p>
- )}
-
- {/* THE STATE SENTENCE. `holding` is the one an operator has no word for:
- a sweep armed behind a shut gate looks exactly like a wedged runner
- unless something says otherwise.
-
- THE LONG FORM OF sweepLaneNote's `holding` row, deliberately not a
- third caller of it: the rail and the Active Jobs strip need four words
- in a line, and this panel has room for the sentence that says what to
- do about it. Same two facts, same precedence — if that row's wording
- changes, this changes with it. */}
- {state === "holding" && (
- <p className="text-sm text-warning">
- <span className="font-mono text-xs uppercase tracking-[0.14em]">
- Holding
- </span>{" "}
- — {lane.sweeping ? "the sweep is armed but " : ""}the lane is paused, so
- nothing is being consumed. Resume to let it move.
- </p>
- )}
-
- <LaneFigures lane={lane} band={band} />
- {/* A single-operation lane (digest) renders no scope control, so its
- "where it runs" line — the LLM endpoints its calls fan out across —
- lives here instead. Multi-operation lanes carry it per row inside
- SweepScope. */}
- {lane.operations.length < 2 &&
- lane.operations.map((op) => (
- <WhereItRuns key={op.id} op={op} className="block" />
- ))}
- <InFlight lane={lane} />
-
- <OrderReach
- order={lane.order}
- reach={lane.reach}
- tradeoff={
- lane.order === "listed"
- ? null
- : digest
- ? "Digest still runs shortest-first inside a day, so a long VOD can wait behind shorter videos uploaded the same day."
- : "Kinds are still walked in registry order, so the cheapest lane reaches a video first whatever the date order says."
- }
- busy={working}
- onChange={(next) =>
- run(() => saveLaneOrderAction(lane.id, next))
- }
- />
-
- {/* SCOPE, ORDER, PLAN — in the order an operator composes them, and all
- three above the button that commits them.
-
- AN UNAVAILABLE LANE GETS NO CONSOLE. With no operation registered
- there is nothing to scope, and a plan of 68 zeroed rows reads as "all
- caught up" when the truth is "switched off" — which the sentence at
- the top of the panel says instead. */}
- {lane.available && (
- <>
- {armed && (
- <p className="text-xs text-muted-foreground">
- This is the scope the running sweep was armed with. Stop sweeping
- to change it — the scope is read once, when the sweep starts, so
- an edit made now would not reach the run.
- </p>
- )}
- <SweepScope
- operations={lane.operations}
- selected={new Set(selectedKinds)}
- unknownScopeIds={unknownScopeIds}
- busy={working || armed}
- onToggle={(id, next) =>
- setScopeKinds(
- next
- ? [...new Set([...draftKinds, id])]
- : draftKinds.filter((k) => k !== id),
- )
- }
- />
- <SweepPlan
- plan={lane.plan}
- operations={lane.operations}
- scopeKinds={selectedKinds}
- // The order the RUN will use. Only "corpus" reach reaches the
- // CHANNEL order — under "channel" the sweep stays heaviest-first
- // and each per-channel batch applies the order itself — so the plan
- // is drawn the way runSweepLoop resolves it, not the way the
- // dropdown reads.
- order={lane.reach === "corpus" ? lane.order : "listed"}
- runningChannel={lane.inFlight[0]?.channelSlug ?? null}
- channelScope={shownChannels}
- busy={working || armed}
- onChannelScope={setChannelScope}
- />
- </>
- )}
- </div>
- );
-}
-
-function LaneFigures({
- lane,
- band,
-}: {
- lane: SweepLaneStatus;
- band: OperationBand | null;
-}) {
- if (!band) {
- return (
- <p className="text-sm text-muted-foreground">
- This lane covers several operations; see the rail above for each one — a
- single total across them would be a figure with no cost basis.
- </p>
- );
- }
- return (
- <p className="flex flex-wrap items-baseline gap-x-2 gap-y-1 text-sm text-muted-foreground">
- <Figure n={band.reachable} population="reachable now" tone="text-foreground" />
- {band.blocked > 0 && (
- <>
- <Sep />
- <Figure n={band.blocked} population="blocked upstream" />
- </>
- )}
- {band.missingInput > 0 && (
- <>
- <Sep />
- <Figure n={band.missingInput} population="need media back" />
- </>
- )}
- {band.deferred > 0 && (
- <>
- <Sep />
- <Figure n={band.deferred} population="held by a gate" />
- </>
- )}
- <Sep />
- <span>
- {band.present == null || band.eligible == null
- ? "coverage unknown until every channel has been re-reported"
- : `${band.present.toLocaleString()} of ${band.eligible.toLocaleString()} done`}
- </span>
- {lane.sweepJobId && (
- <>
- <Sep />
- <Link
- href={`/jobs/${lane.sweepJobId}`}
- className="underline underline-offset-2 hover:text-foreground"
- >
- sweep log
- </Link>
- </>
- )}
- </p>
- );
-}
-
-function InFlight({ lane }: { lane: SweepLaneStatus }) {
- const now = Date.now();
- return (
- <div className="flex flex-col gap-1">
- <p className="font-mono text-xs uppercase tracking-[0.14em] text-muted-foreground">
- In flight
- </p>
- {lane.inFlight.length === 0 ? (
- <p className="text-sm text-muted-foreground">Nothing running.</p>
- ) : (
- <ul className="flex flex-col gap-0.5 text-sm">
- {lane.inFlight.map((j) => (
- <li key={j.id} className="flex flex-wrap gap-x-3">
- <Link
- href={`/jobs/${j.id}`}
- className="font-mono text-foreground underline underline-offset-2"
- >
- {j.channelSlug ?? j.id}
- </Link>
- {j.startedAt !== null && (
- <span className="tabular-nums text-muted-foreground">
- {formatElapsed(now - j.startedAt)}
- </span>
- )}
- </li>
- ))}
- </ul>
- )}
- </div>
- );
-}
-
-// `population`, not "unit": the word beside the number names WHICH population
-// of the band this is (reachable, blocked, held), and "unit" already means one
-// item of dispatchable work everywhere the arbiter and the worker route use it.
-function Figure({
- n,
- population,
- tone = "",
-}: {
- n: number;
- population: string;
- tone?: string;
-}) {
- return (
- <span>
- <span className={`tabular-nums ${tone}`}>{n.toLocaleString()}</span>{" "}
- {population}
- </span>
- );
-}
-
-function Sep() {
- return (
- <span aria-hidden="true" className="text-border">
- ·
- </span>
- );
-}
diff --git a/editor/app/operations/components/SweepPlan.tsx b/editor/app/operations/components/SweepPlan.tsx
@@ -1,264 +0,0 @@
-"use client";
-
-import { useMemo, useState } from "react";
-import type { AutoQueueOrder } from "yt-dlp-transcript-common/jobs/autoQueuePolicy";
-import {
- foldSweepPlan,
- sweepPlanTotals,
- type SweepChannelCounts,
-} from "yt-dlp-transcript-common/lib/sweepPlan";
-import { bandForScope } from "../../components/pipelines/band";
-import { StateBand } from "../../components/pipelines/StateBand";
-import type { SweepOperation } from "../lanes";
-
-// THE PLAN — the ordered channel itinerary the sweep will actually walk, drawn
-// before it is armed.
-//
-// This is the whole point of the console. "Newest first, across all channels"
-// was a phrase in a dropdown; here it is a consequence you can see, because the
-// list re-sorts under your hand when you change the order, and re-totals when
-// you change the scope. You cannot arm blind, because the thing you are arming
-// is the thing you are reading.
-//
-// SCOPE, PLAN AND PROGRESS ARE ONE VIEW, NOT THREE. The conventional split
-// would put scope in a settings form, a preview behind a modal and progress in
-// a status card. Merging them is justified twice: it is the direct fix for a
-// real bug — a scope that is not recorded at arm time resurrects a bounded run
-// as a corpus-wide one after a restart, so the control that sets the scope must
-// BE the control that arms — and it is the argument OrderReach.tsx already
-// makes for living at the foot of the lane rather than in /settings.
-//
-// NO ANIMATION, and it is a subtraction rather than an oversight: 68 rows
-// easing every 3 seconds is a light show, not an instrument. StateBand's
-// `strip` size is the non-animating one for exactly this reason.
-//
-// NO PER-ROW PERCENTAGE. Digest sits at 0 done on every large channel and
-// diarization is 99.96% media-gone, so "% complete" renders the same number on
-// every row and says nothing; what varies — and what an operator needs — is the
-// SHAPE of the remainder, which is what the band draws.
-
-// How many working rows are drawn before the list is folded. Explicit, with the
-// remainder named and one click away: a silent top-N reads as "this is all of
-// it" when it is not.
-const VISIBLE_ROWS = 12;
-
-function formatPlanDate(key: string): string {
- if (!/^\d{8}$/.test(key)) return "undated";
- return `${key.slice(0, 4)}-${key.slice(4, 6)}-${key.slice(6)}`;
-}
-
-export function SweepPlan({
- plan,
- operations,
- scopeKinds,
- order,
- // The channel this sweep's per-channel job is on right now, for the position
- // marker. Read from the lane's in-flight list rather than stored, so it can
- // never be stale in a way the rest of the lane is not.
- runningChannel,
- // The channel scope, or null for "every channel" — the resting state. Not a
- // separate picker: the plan rows ARE the channel scope, which is why turning
- // it on turns these rows into checkboxes instead of opening a second list.
- channelScope,
- busy,
- onChannelScope,
-}: {
- plan: SweepChannelCounts[];
- operations: SweepOperation[];
- scopeKinds: string[];
- order: AutoQueueOrder;
- runningChannel: string | null;
- channelScope: ReadonlySet<string> | null;
- busy: boolean;
- onChannelScope: (next: ReadonlySet<string> | null) => void;
-}) {
- const [showAll, setShowAll] = useState(false);
-
- const rows = useMemo(() => {
- const byKind = new Map(plan.map((row) => [row.channelSlug, row]));
- const ordered = foldSweepPlan({ counts: plan, kindIds: scopeKinds, order });
- return ordered.map((entry) => {
- const counts = byKind.get(entry.channelSlug);
- // An empty scope is EVERY operation — the same rule foldSweepPlan and
- // resolveBackfillLaneOperations apply, and the same rule an unscoped sweep runs.
- const inScope = scopeKinds;
- const selected = Object.entries(counts?.byKind ?? {}).filter(
- ([id]) => inScope.length === 0 || inScope.includes(id),
- );
- return {
- entry,
- unknown: counts?.unknown === true,
- band: bandForScope({
- id: "sweep-scope",
- // The band's own label and unit are the SCOPE's, not one operation's.
- // With more than one operation selected the unit is "one operation on
- // one video", which is exactly what the sweep dispatches — see
- // bandForScope's header for why that is the one legitimate sum.
- label: operations
- .filter((op) => inScope.length === 0 || inScope.includes(op.id))
- .map((op) => op.label)
- .join(" · "),
- costBasis: "",
- counts: selected.map(([, c]) => c),
- }),
- };
- });
- }, [plan, scopeKinds, order, operations]);
-
- const totals = useMemo(
- () => sweepPlanTotals(rows.map((r) => r.entry)),
- [rows],
- );
- const working = rows.filter((r) => r.entry.reachable > 0);
- const unreported = rows.filter((r) => r.entry.reachable === 0 && r.unknown);
- const idle = rows.length - working.length - unreported.length;
- const visible = showAll ? working : working.slice(0, VISIBLE_ROWS);
- const choosing = channelScope !== null;
-
- const toggleChannel = (slug: string, next: boolean) => {
- const set = new Set(channelScope ?? []);
- if (next) set.add(slug);
- else set.delete(slug);
- onChannelScope(set);
- };
-
- return (
- <div className="flex flex-col gap-2">
- <div className="flex flex-wrap items-baseline justify-between gap-x-4 gap-y-1">
- <p className="font-mono text-xs uppercase tracking-[0.14em] text-muted-foreground">
- The plan
- </p>
- <p className="text-sm text-muted-foreground">
- {/* THE ONE LARGE NUMERAL on this panel, mirroring the transit line's
- one station numeral so the two consoles rhyme. It is the plan
- total — the number the commitment is actually about. */}
- <span className="font-display text-3xl font-semibold tabular-nums text-foreground">
- {totals.reachable.toLocaleString()}
- </span>{" "}
- to do across {totals.channels.toLocaleString()} of{" "}
- {rows.length.toLocaleString()} channels
- {totals.missingInput > 0 && (
- // STATED SEPARATELY, NEVER SUMMED. On this corpus the two are
- // orders of magnitude apart, and one "remaining" figure would say
- // the same thing about a lane that is finished and a lane that
- // cannot start.
- <>
- {" · "}
- <span className="tabular-nums">
- {totals.missingInput.toLocaleString()}
- </span>{" "}
- need their media back first
- </>
- )}
- </p>
- </div>
-
- {working.length === 0 ? (
- <p className="text-sm text-muted-foreground">
- Nothing reachable in this scope. That is not the same as finished —
- widen the operations above, or check what is waiting on media.
- </p>
- ) : (
- <ul className="flex flex-col gap-1">
- {visible.map(({ entry, band }) => {
- const here = entry.channelSlug === runningChannel;
- return (
- <li
- key={entry.channelSlug}
- className="grid grid-cols-[1.25rem_minmax(6rem,1fr)_minmax(4rem,8rem)_auto] items-center gap-x-3 gap-y-1 text-sm"
- >
- <span className="flex items-center justify-center">
- {choosing ? (
- <input
- type="checkbox"
- aria-label={`sweep ${entry.channelSlug}`}
- checked={channelScope?.has(entry.channelSlug) === true}
- disabled={busy}
- onChange={(e) =>
- toggleChannel(entry.channelSlug, e.target.checked)
- }
- />
- ) : (
- // THE POSITION MARKER. Only meaningful while something is
- // running, and deliberately the same arrow the sweep's own
- // log prints for the channel it is entering.
- <span
- aria-hidden="true"
- className={here ? "text-info" : "text-transparent"}
- >
- →
- </span>
- )}
- </span>
- <span
- className={`truncate font-mono text-xs ${
- here ? "text-foreground" : "text-muted-foreground"
- }`}
- title={entry.channelSlug}
- >
- {entry.channelSlug}
- </span>
- <StateBand band={band} size="strip" />
- <span className="flex flex-wrap items-baseline justify-end gap-x-3">
- <span className="tabular-nums">
- {entry.reachable.toLocaleString()} to do
- </span>
- <span className="w-28 text-right text-xs tabular-nums text-muted-foreground">
- {order === "oldest"
- ? `oldest ${formatPlanDate(entry.oldestPending)}`
- : `newest ${formatPlanDate(entry.newestPending)}`}
- </span>
- </span>
- </li>
- );
- })}
- </ul>
- )}
-
- {working.length > visible.length && (
- <button
- type="button"
- className="self-start text-xs underline underline-offset-2 text-muted-foreground hover:text-foreground"
- onClick={() => setShowAll(true)}
- >
- + {(working.length - visible.length).toLocaleString()} more channels
- with work
- </button>
- )}
-
- <div className="flex flex-wrap items-baseline gap-x-4 gap-y-1 text-xs text-muted-foreground">
- {idle > 0 && (
- <span>
- {idle.toLocaleString()} channel{idle === 1 ? "" : "s"} with nothing
- to do
- </span>
- )}
- {unreported.length > 0 && (
- // UNKNOWN IS NOT ZERO. These channels have never been reported on, so
- // 0 here would be a claim nothing has checked. The run's own planner
- // walks them; this list cannot, so it says so.
- <span className="text-warning">
- {unreported.length.toLocaleString()} not reported yet (
- {unreported
- .slice(0, 3)
- .map((r) => r.entry.channelSlug)
- .join(", ")}
- {unreported.length > 3 ? "…" : ""}) — the sweep will still visit them
- </span>
- )}
- <button
- type="button"
- className="ml-auto underline underline-offset-2 hover:text-foreground"
- disabled={busy}
- onClick={() =>
- onChannelScope(
- choosing ? null : new Set(working.map((r) => r.entry.channelSlug)),
- )
- }
- >
- {choosing ? "Use every channel" : "Choose channels"}
- </button>
- </div>
- </div>
- );
-}
diff --git a/editor/app/operations/components/SweepScope.tsx b/editor/app/operations/components/SweepScope.tsx
@@ -1,154 +0,0 @@
-"use client";
-
-import { useId } from "react";
-import type { SweepOperation } from "../lanes";
-
-// WHICH OPERATIONS THIS SWEEP RUNS — the axis that had no control at all.
-//
-// `backfill.sweepKinds` has existed and been honoured by the run since the sweep
-// was written, and no screen has ever set it: arming was one unlabelled button
-// meaning "every enabled operation, whole corpus". On this install that includes
-// speaker-names-from-the-transcript at ~1 model call per transcript CHUNK, on the
-// order of 194,000 calls corpus-wide. "Diarization and names-from-audio only" was
-// a settings.json hand-edit or nothing.
-//
-// EVERY ROW STATES ITS COST BASIS. The three operations here have backlogs of
-// 1, 4 and 11,337 — and the last is not 11,337 times the first in cost, it is
-// far more, because its cost basis is not the video. A checkbox list without the cost basis
-// beside it would be asking for a decision with the deciding fact left out.
-//
-// NO NEW COLOUR. An operation that is out of scope drops to muted text; nothing
-// gains an accent. `can run now` stays the only saturated fill on any pipeline
-// surface — the rule the rail, the /channels strip and the transit line all hold
-// — and a second accent here would break that reading on four surfaces at once.
-//
-// Native <input type=checkbox> like every other policy control on this page; see
-// the note at the foot of dispatch.ts for why that is load-bearing rather than
-// stylistic.
-
-export function SweepScope({
- operations,
- selected,
- // Ids the STORED scope names that no operation answers to. sanitizeBackfill
- // keeps an unknown id and resolveBackfillLaneOperations then matches nothing with it,
- // so a sweep armed on one runs forever doing nothing. Reported here rather
- // than swallowed.
- unknownScopeIds,
- busy,
- onToggle,
-}: {
- operations: SweepOperation[];
- selected: ReadonlySet<string>;
- unknownScopeIds: string[];
- busy: boolean;
- onToggle: (id: string, next: boolean) => void;
-}) {
- const groupId = useId();
- // One operation is not a choice. The digest sweep is exactly this case, and
- // drawing it a single permanently-ticked checkbox would imply a scope that
- // cannot be varied.
- if (operations.length < 2) return null;
-
- return (
- <div className="flex flex-col gap-2 rounded-md border border-border bg-card px-3 py-2">
- <div className="flex flex-wrap items-baseline justify-between gap-x-4 gap-y-1">
- <p
- id={groupId}
- className="font-mono text-xs uppercase tracking-[0.14em] text-muted-foreground"
- >
- Run which operations
- </p>
- <p className="text-xs tabular-nums text-muted-foreground">
- {selected.size} of {operations.length} selected
- </p>
- </div>
- <ul className="flex flex-col gap-1" aria-labelledby={groupId}>
- {operations.map((op) => {
- const on = selected.has(op.id);
- return (
- <li key={op.id}>
- <label
- className={`flex flex-wrap items-baseline gap-x-3 gap-y-0.5 text-sm ${
- on ? "" : "text-muted-foreground"
- }`}
- >
- <input
- type="checkbox"
- className="self-center"
- checked={on}
- disabled={busy}
- onChange={(e) => onToggle(op.id, e.target.checked)}
- />
- <span className={on ? "text-foreground" : ""}>{op.label}</span>
- <span className="tabular-nums">
- {op.reachable.toLocaleString()}
- </span>
- {/* THE UNIT, from the registry. No threshold and no
- editorialising: what is affordable is the operator's call,
- and a "this is a lot" cutoff would be a magic number the next
- operation gets wrong. */}
- <span className="text-xs text-muted-foreground">
- {op.costBasis}
- </span>
- {/* WHERE IT RUNS — read-only, never part of the scope.
- Delegation is decided per item at dispatch by the worker
- pool; this row only reports which configured endpoints and
- executors match, through the same rule the pool grants by. */}
- <WhereItRuns op={op} className="w-full pl-6" />
- </label>
- </li>
- );
- })}
- </ul>
- {selected.size === 0 && (
- <p className="text-xs text-warning">
- Nothing selected — there is no sweep to arm. Tick at least one
- operation.
- </p>
- )}
- {unknownScopeIds.length > 0 && (
- <p className="text-xs text-warning">
- The stored scope also names {unknownScopeIds.join(", ")}, which no
- enabled operation answers to. It counts as nothing and will be dropped
- when you re-arm.
- </p>
- )}
- </div>
- );
-}
-
-// Which delegate workers can take an operation's items, beside this machine.
-// Rendered only when delegates are configured — the lane always runs here, and
-// stating that on every row would be noise. The one loud case: capacity is
-// configured and NONE of it can currently take work (degraded/disabled), which
-// looks identical to "distributed" from the arm button.
-export function WhereItRuns({
- op,
- className = "",
-}: {
- op: SweepOperation;
- className?: string;
-}) {
- if (op.runsOn.length === 0) return null;
- const noneAvailable = op.runsOn.every((w) => w.available === 0);
- return (
- <span className={`text-xs text-muted-foreground ${className}`}>
- also runs on{" "}
- {op.runsOn
- .map(
- (w) =>
- `${w.name} (${w.kind === "llm" ? "LLM endpoint" : "executor"}${
- w.slots > 1 ? ` ×${w.slots}` : ""
- })`,
- )
- .join(", ")}
- {noneAvailable && (
- <span className="text-warning">
- {" "}
- — none of it is available right now (degraded or disabled); this
- machine carries the whole operation until one is re-enabled.
- </span>
- )}
- </span>
- );
-}
diff --git a/editor/app/operations/components/dispatch.ts b/editor/app/operations/components/dispatch.ts
@@ -165,10 +165,6 @@ export function idleReasonText(
return `auto-${what} was switched off`;
case "lane-held":
return "the lane is held — resume it to start dispatching again";
- case "lane-blocked":
- // NAMES THE SWEEP in the runner's own log; here it is the short form,
- // because the operator is looking at the sweep panel beside this one.
- return "this lane's sweep is armed — stop it and the runner resumes";
case "engine-unreachable":
return "the lane's engine did not answer its probe — check the job log";
case "stopped":
diff --git a/editor/app/operations/components/railStates.ts b/editor/app/operations/components/railStates.ts
@@ -1,9 +1,6 @@
import type { AutoQueueStatusPayload } from "../status";
-import type { SweepLaneId, SweepLaneStatus } from "../lanes";
-import {
- deriveLaneState,
- sweepLaneNote,
-} from "../../components/lanes/laneState";
+import { LANES, type AutoQueueKind } from "yt-dlp-transcript-common/lib/autoQueueTypes";
+import { deriveLaneState } from "../../components/lanes/laneState";
import type { RailLaneState } from "./OperationRail";
import { idleReasonText } from "./dispatch";
@@ -19,6 +16,13 @@ import { idleReasonText } from "./dispatch";
// see deriveLaneState: an idle-looking lane claims "all caught up" where the
// truth is "nothing would run this".
//
+// ONE FOLD FOR FOUR LANES since slice 1.3. It used to be two: the runner lanes
+// off `runner.running`, and the two sweep lanes off an armed flag with their own
+// note table. Every lane is a runner lane now, so there is one rule — a stopped
+// runner is "Off", a held one is "Holding", one with units in flight is
+// "Running" — and the note is the runner's own idle reason rather than a second
+// vocabulary for the same states.
+//
// LIVES IN ITS OWN MODULE because the board and every operation page draw the
// same rail, and two copies of this fold would let two pages disagree about
// which lane is holding.
@@ -27,67 +31,47 @@ export function railStates(
): Record<string, RailLaneState> {
const out: Record<string, RailLaneState> = {};
- for (const status of [data.transcription, data.download]) {
+ const laneState = (kind: AutoQueueKind): RailLaneState => {
+ const status = data[kind];
const running = status.runner.running;
- const inFlight = status.runner.inFlight.length;
- out[status.kind] = {
- // A runner has no sweep; its feed is the job queue, so `activeCount` is
- // what tells running from idle. A STOPPED runner is "Off", not "Idle" —
- // it will never pick anything up, which "Idle" does not say.
+ return {
+ // A runner's feed is its policy tree, so `activeCount` is what tells
+ // running from idle. A STOPPED runner is "Off", not "Idle" — it will
+ // never pick anything up, which "Idle" does not say.
state: !running
? "unavailable"
- : deriveLaneState({ gateHeld: status.held, activeCount: inFlight }),
- note: running ? idleReasonText(status.runner.idleReason, status.kind) : null,
- };
- }
-
- // ONE INPUT, TWO READINGS. The word and the note come off the same object
- // through the same precedence, so they cannot disagree about which fact wins
- // — which is exactly what two hand-maintained copies of the table could.
- const sweepState = (sweep: SweepLaneStatus): RailLaneState => {
- const input = {
- available: sweep.available,
- gateHeld: sweep.gateHeld,
- feedRunning: sweep.sweeping,
- activeCount: sweep.inFlight.length,
+ : deriveLaneState({
+ gateHeld: status.held,
+ activeCount: status.runner.inFlight.length,
+ }),
+ note: running ? idleReasonText(status.runner.idleReason, kind) : null,
};
- return { state: deriveLaneState(input), note: sweepLaneNote(input) };
};
- out.digest = sweepState(data.lanes.digest);
+ for (const kind of LANES) out[kind] = laneState(kind);
// Each remaining band takes the state of the lane THAT BAND'S OPERATION
// actually rides — asked of the payload, not assumed from the id.
//
// This loop used to hand every stateless band the backfill lane's state, on
- // the same reasoning the old id-shaped sweepLaneIdFor used: everything that
- // is not digest is backfill. Both were true only because those are the only
- // two lanes registered TODAY. An operation on a third queue would still get a
- // band (the rail is built from allOperations — every switched-on kind,
- // whatever its queue) and would then have advertised a hold belonging to a
- // lane that would never dispatch it.
+ // the reasoning that everything which is not digest is backfill. That was
+ // true only because those are the only two operation lanes registered TODAY.
+ // An operation on a third queue would still get a band (the rail is built
+ // from allOperations — every switched-on kind, whatever its queue) and would
+ // then have advertised a hold belonging to a lane that would never dispatch
+ // it.
//
- // `lane.operations` is the live membership the server already computed, so
- // this cannot drift from it the way a second registry read could. A band in
- // NEITHER lane gets no entry at all, and railStates' contract above supplies
- // the rest: no lane means "Off", never a borrowed "Idle".
- const laneStates: Record<SweepLaneId, RailLaneState> = {
- digest: out.digest,
- backfill: sweepState(data.lanes.backfill),
- };
- const laneIdOf = (operationId: string): SweepLaneId | null => {
- for (const laneId of ["digest", "backfill"] as const) {
- if (data.lanes[laneId].operations.some((op) => op.id === operationId)) {
- return laneId;
- }
- }
- return null;
- };
+ // `lanes.operations` is the live membership the server already computed, so
+ // this cannot drift from it the way a second registry read could. A band on no
+ // lane gets no entry at all, and railStates' contract above supplies the rest:
+ // no lane means "Off", never a borrowed "Idle".
for (const band of data.lanes.bands) {
if (out[band.id]) continue;
- const laneId = laneIdOf(band.id);
- if (!laneId) continue;
- out[band.id] = laneStates[laneId];
+ const lane = LANES.find((k) =>
+ data.lanes.operations[k].some((op) => op.id === band.id),
+ );
+ if (!lane) continue;
+ out[band.id] = out[lane];
}
return out;
}
diff --git a/editor/app/operations/components/settings/LaneSettingsForm.tsx b/editor/app/operations/components/settings/LaneSettingsForm.tsx
@@ -57,11 +57,6 @@ export function LaneSettingsForm({
above, beside the itinerary they draw, which is what lets you read the
commitment before making it.
</p>
- {/* DELIBERATELY NOT THE WORDS "the plan". That phrase is the heading of
- the sweep console this paragraph points at, and this form now
- renders inside the same section — backfill.spec.ts scopes
- getByText("The plan") to section[data-lane], and a second match
- here is a strict-mode violation, not a near miss. */}
<label className="flex items-start gap-2 text-sm">
<input
type="checkbox"
@@ -78,19 +73,11 @@ export function LaneSettingsForm({
</span>
</label>
<Field
- label="Resource share"
- name="backfillWeight"
- defaultValue={String(initial.weight)}
- type="number"
- step="0.05"
- hint="0 (the default) means idle-only: the lane runs only while transcription is quiet, and stands aside the moment it isn't. Above 0 it takes that fraction of its slots as a guaranteed share, floored at 1 — so a small number is a slow lane, not a stopped one."
- />
- <Field
label="Lane concurrency"
name="backfillConcurrency"
defaultValue={String(initial.concurrency)}
type="number"
- hint="Slots the lane may use when it is not standing aside. Default 1 — this is CPU-bound work competing with GPU feeding and the digest sweep for the same threads."
+ hint="Slots the lane may use when it is not standing aside. Default 1 — this is CPU-bound work competing with GPU feeding and the digest lane for the same threads."
/>
<label className="flex items-start gap-2 text-sm">
<input
diff --git a/editor/app/operations/lanes.ts b/editor/app/operations/lanes.ts
@@ -1,157 +1,45 @@
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import { getSettings } from "yt-dlp-transcript-common/lib/settings";
-import {
- isGateHeld,
- type PauseLane,
-} from "yt-dlp-transcript-common/lib/pauseGates";
-import type { AutoQueueOrder, AutoQueueReach } from "yt-dlp-transcript-common/jobs/autoQueuePolicy";
+import { LANES, type AutoQueueKind } from "yt-dlp-transcript-common/lib/autoQueueTypes";
import {
allOperations,
- DIGEST_OPERATION_ID,
- backfillLaneOperations,
- operationCostBasis,
operationLabel,
- operationsActionLabel,
- operationsGroupLabel,
- type OperationDescriptor,
+ operationsForLane,
} from "yt-dlp-transcript-common/lib/operations";
-import { buildSweepChannelCounts } from "yt-dlp-transcript-common/controller/sweepPreview";
-import { loadSweepDates } from "yt-dlp-transcript-common/controller/sweepRecency";
-import type { SweepChannelCounts } from "yt-dlp-transcript-common/lib/sweepPlan";
-import { getDigestSweepJobId } from "yt-dlp-transcript-common/controller/digestSweep";
-import { getBackfillSweepJobId } from "yt-dlp-transcript-common/controller/backfillSweep";
-import { getRegistry } from "yt-dlp-transcript-common/jobs/registry";
-import {
- DIGEST_LOCAL_QUEUE,
- DIGEST_REMOTE_QUEUE,
- BACKFILL_QUEUE,
-} from "yt-dlp-transcript-common/lib/queueKeys";
-import {
- arbiterBlockedReason,
- getArbiterJobId,
-} from "yt-dlp-transcript-common/controller/arbiter";
import { getWorkerPool } from "yt-dlp-transcript-common/jobs/workerPool";
import { workerMatches } from "yt-dlp-transcript-common/lib/workers";
-import { flattenLeaves } from "yt-dlp-transcript-common/jobs/autoQueuePolicy";
import { getChannelBriefs } from "../lib/requestCache";
import {
buildOperationBands,
type OperationBand,
} from "../components/pipelines/buildBands";
-// The two SWEEP-fed lanes, for a console that has to show four pipelines and
-// only has runners for two of them.
+// WHAT THE CONSOLE KNOWS ABOUT A LANE THAT IS NOT ITS RUNNER'S STATE.
//
-// Digest and backfill are dispatched by their own sweeps, not by the policy
-// tree, so they have no rules, no claim ladder and no next-up. What they do
-// have is the same four things every lane has — a state, a reason, work in
-// flight, and an order — and this is where those are read.
-//
-// NO CLAIM LADDER IS RENDERED FOR THEM YET. `settings.autoQueue` carries all
-// four lanes since slice 1.1, so the tree now exists — but nothing dispatches
-// off it until the runner does (slice 1.2), and drawing a ladder over a tree no
-// dispatcher reads would say "these rules decide the order" where the truth is
-// still "the sweep does". The ladder arrives with the runner, and this whole
-// file goes with the sweeps in slice 1.3.
-
-export type SweepLaneId = "digest" | "backfill";
-
-// The sweep lanes are pause lanes too — the other half of the assertion
-// pauseGates.ts makes for AutoQueueKind. One gate definition, two id spaces.
-type _SweepLanesArePauseLanes = SweepLaneId extends PauseLane ? true : never;
-
-// The sweep lane that dispatches an operation — or NULL for one neither sweep
-// runs. Takes the DESCRIPTOR, and reads its queue key.
+// This file used to be the two SWEEP-fed lanes: a state, a reason, a scope, a
+// plan and an arm button apiece, for the two pipelines the auto-queue runner
+// could not dispatch. It can dispatch them now (slice 1.2) and the sweeps are
+// gone (1.3), so a lane's state, its work list and its controls all come from
+// `/api/auto-queue/status` like the other two lanes'.
//
-// The id-shaped version of this ("digest" ? digest : backfill) answered
-// "backfill" for every operation that was not digest, including ones no sweep
-// dispatches at all. It was right only by coincidence of today's registry: an
-// external entry with no runner of its own would have rendered the backfill
-// sweep's whole console, with a live Start button ARMING THE BACKFILL SWEEP,
-// under that operation's heading. The queue key is what the arbiter actually
-// dispatches on, so it is what this asks.
+// TWO THINGS ARE LEFT, and both are properties of the REGISTRY rather than of a
+// runner, which is why they are still built here:
//
-// Null is a real answer, not an error. A caller must render it as "no console
-// here" rather than falling back to a lane.
-export function sweepLaneIdFor(
- op: Pick<OperationDescriptor, "lane"> | null | undefined,
-): SweepLaneId | null {
- const key = op?.lane.queueKey;
- if (key === DIGEST_LOCAL_QUEUE || key === DIGEST_REMOTE_QUEUE) return "digest";
- if (key === BACKFILL_QUEUE) return "backfill";
- return null;
-}
-
-export type SweepLaneStatus = {
- id: SweepLaneId;
- label: string;
- // A corpus-wide sweep is ARMED — the feed. Persisted in settings, so it
- // survives a restart.
- sweeping: boolean;
- // The gate is shut — the lane consumes nothing. SEPARATE from `sweeping` and
- // deliberately so: conflating the two is how an operator loses a week of GPU
- // time, because "stop the sweep" and "hold it at zero throughput" have very
- // different costs to undo.
- gateHeld: boolean;
- // Whether the lane exists at all. False is not "idle": with no backfill
- // feature registered there is nothing to hold, and an idle-looking lane would
- // read as "all caught up" when the truth is "switched off".
- available: boolean;
- // The sweep's own job, for a link to its log. null when nothing is running.
- sweepJobId: string | null;
- // Per-channel jobs this lane has in flight right now, read from the registry
- // rather than from a runner — there is no runner.
- inFlight: { id: string; channelSlug: string | null; startedAt: number | null }[];
- order: AutoQueueOrder;
- reach: AutoQueueReach;
- // ── THE SWEEP'S SCOPE AND ITS PLAN ──────────────────────────────────────
- //
- // A sweep is a commitment, not a toggle: arming the backfill one on this
- // corpus can mean ~194,000 model calls. Everything below exists so the
- // commitment can be READ before it is made, and it is on the same payload as
- // the state above for the reason the rail is: two polls would let the plan
- // and the switch that arms it disagree about the same moment.
- //
- // The operations this sweep can run, in registry order. Exactly one entry for
- // the digest lane, which is why the console offers it no operation list —
- // there is nothing to choose.
- operations: SweepOperation[];
- // What is ARMED right now, read back out of settings rather than held in the
- // client. Empty means unscoped: every operation, every channel — which is
- // what an unscoped sweep has always meant and what the boot hook resumes.
- scopeKinds: string[];
- scopeChannels: string[];
- // The itinerary: one entry per channel, counts per operation, undated where
- // nothing could date them. The client folds this to a plan on every scope
- // change; see common/lib/sweepPlan.ts for why the fold is not done here.
- plan: SweepChannelCounts[];
-};
+// * the BANDS — one per catalogued operation, its five populations off the
+// channel snapshots. The rail draws them, and they are per OPERATION where
+// everything else on this page is per LANE.
+// * WHICH OPERATIONS EACH LANE DISPATCHES, with their labels and the delegate
+// workers that can take them. Settings-dependent (`digest.remoteEnabled`
+// moves the digest operation between two queue keys that are both the
+// digest lane), so it is a server read, and the console is a client
+// component.
-// One operation an operator can put in or out of a sweep's scope.
-//
-// `costBasis` is carried because it is the fact that hid behind a shared lane
-// name: "11,337 reachable" is the same shape of number whether one unit is a
-// single pass over the audio or ~1 model call per transcript CHUNK, a ~17x
-// difference on the same figure. A scope control that did not state it would be
-// asking the operator to choose blind.
-export type SweepOperation = {
+export type LaneOperationView = {
id: string;
label: string;
- costBasis: string;
- // Corpus-wide reachable work for this operation alone, so the checkbox says
- // what ticking it costs before the plan below re-folds.
- reachable: number;
- // WHERE THE WORK CAN RUN besides this machine: the delegate workers (LLM
- // endpoints, tagged unit executors) whose tags match this operation, folded
- // to one row per configured worker with its slot count. READ-ONLY and never
- // persisted into the scope — delegation is a worker-pool concern, decided
- // per item at dispatch, and the console only reports it. Derived through the
- // SAME workerMatches rule the dispatchers grant by, so this row and the
- // actual routing cannot disagree.
- runsOn: SweepOperationWorker[];
};
-export type SweepOperationWorker = {
+export type LaneWorker = {
// The configured worker's id (slot-expansion suffixes folded back together).
id: string;
name: string;
@@ -162,54 +50,27 @@ export type SweepOperationWorker = {
available: number;
};
-// The unified dispatcher's state. Not a lane: it does not do work, it decides
-// which lane does. It is on this payload because the console is where an
-// operator turns it on, and because `blockedReason` has to come from the same
-// function that would refuse the start — an operator told "you can start it"
-// and then refused has been lied to by a race between two copies of one rule.
-export type ArbiterStatus = {
- running: boolean;
- jobId: string | null;
- // Why it will not start, or null when it will. See arbiterBlockedReason.
- blockedReason: string | null;
- // How many rules across both trees name an operation it dispatches. Zero is
- // why it would come straight back up and stop, so the console says so before
- // the click rather than after.
- operationRules: number;
-};
-
export type AutoQueueLanesPayload = {
- digest: SweepLaneStatus;
- backfill: SweepLaneStatus;
// One band per pipeline, in rail order. See components/pipelines/buildBands.ts.
bands: OperationBand[];
- arbiter: ArbiterStatus;
+ // The operations each lane dispatches, in dependency order. Empty for the two
+ // bucket lanes, whose work list is a snapshot bucket rather than an operation.
+ operations: Record<AutoQueueKind, LaneOperationView[]>;
+ // WHERE A LANE'S WORK CAN RUN besides this machine, folded to one row per
+ // configured worker with its slot count. READ-ONLY and never part of any
+ // policy — delegation is a worker-pool concern decided per item at dispatch,
+ // and the console only reports it. Derived through the SAME workerMatches rule
+ // the dispatchers grant by, so this row and the actual routing cannot
+ // disagree.
+ runsOn: Record<AutoQueueKind, LaneWorker[]>;
};
-// Running jobs on a lane's queue keys. The digest lane has two (local and
-// metered) and they are genuinely separate concurrency, so both are read;
-// summing them into one "in flight" is correct here because the lane is one
-// lane to an operator even when it is two queues to the scheduler.
-function inFlightOn(queueKeys: ReadonlyArray<string>): SweepLaneStatus["inFlight"] {
- const keys = new Set(queueKeys);
- return getRegistry()
- .list()
- .filter((j) => j.status === "running" && keys.has(j.queueKey))
- .map((j) => ({
- id: j.id,
- channelSlug: j.channelSlug ?? null,
- startedAt: j.startedAt ?? null,
- }));
-}
-
export async function buildAutoQueueLanes(): Promise<AutoQueueLanesPayload> {
const paths = getPaths();
const settings = getSettings();
const briefs = await getChannelBriefs(paths);
- // allOperations, not backfillLaneOperations: the rail is the CATALOG view —
- // every operation that is switched on, whatever queue it runs on. Filtering
- // to the shared backfill queue here would drop digest, which is the whole
- // reason the rail exists.
+ // allOperations, not one lane's members: the rail is the CATALOG view — every
+ // operation that is switched on, whatever queue it runs on.
const kinds = allOperations(settings);
const bands = buildOperationBands({
@@ -217,73 +78,32 @@ export async function buildAutoQueueLanes(): Promise<AutoQueueLanesPayload> {
operationIds: kinds.map((k) => k.id),
});
- // ── THE PLAN, off the briefs already in hand ────────────────────────────
- //
- // FREE, and that is the third cycle running that this has been true: this
- // function already read every channel's snapshot for the rail and threw the
- // rest away. What it costs now is a fold over data already in memory.
- //
- // NOT THE SWEEP'S OWN PLANNER, which walks the corpus when a snapshot is
- // absent (~474,559 file touches) and is banned from render paths by
- // common/controller/noCorpusWalkInRenderPaths.test.ts. The preview is the
- // same counting off the snapshots, with an unreported channel reported as
- // unknown rather than as zero.
- //
- // (That guard greps for the NAME, in any context — so it is not written here
- // even in prose. Dumb on purpose: a guard that skipped comments would be a
- // guard an offending call could hide behind.)
- const laneKinds = backfillLaneOperations(settings);
- const backfillKindIds = laneKinds.map((k) => k.id);
- // Digest is one operation on its own queue, always catalogued whether or not
- // the feature is switched on — the lane is always available (see below), so
- // its plan must be too.
- const digestKindIds = [DIGEST_OPERATION_ID];
- const planKindIds = [...new Set([...backfillKindIds, ...digestKindIds])];
- // Dates for the ids both plans will order by. Cached for a minute — see
- // controller/sweepRecency.ts for the measured cost and why a stale date
- // cannot mis-plan a sweep that stores no cursor.
- const dates = await loadSweepDates({
- paths,
- briefs,
- kindIds: planKindIds,
- });
- const counts = buildSweepChannelCounts({
- briefs,
- kindIds: planKindIds,
- dates,
- });
- // Each lane sees only its own operations. Handing the backfill console a
- // digest column would offer a scope its arm action cannot express — digest's
- // sweep takes channels and nothing else, because it IS one operation.
- const planFor = (ids: ReadonlyArray<string>): SweepChannelCounts[] =>
- counts.map((row) => ({
- channelSlug: row.channelSlug,
- unknown: row.unknown,
- byKind: Object.fromEntries(
- ids.filter((id) => row.byKind[id]).map((id) => [id, row.byKind[id]]),
- ),
- }));
- // The delegate workers per operation, matched the way the dispatchers match:
- // an llm worker serves an operation its tags (or the llm default set) name;
- // a remote worker takes units only when TAGGED (the unit-dispatch opt-in)
- // and its tags intersect [operation, contended resource]. Local workers are
- // not listed — the lane always runs here, and saying so on every row would
- // be noise.
const workerSummary = getWorkerPool().summary();
const contendsForOf = new Map(
kinds.map((k) => [k.id, (k.laneFor?.(settings) ?? k.lane).contendsFor]),
);
- const runsOnFor = (id: string): SweepOperationWorker[] => {
- const unitRequires = [id, contendsForOf.get(id) ?? "cpu"];
- const byBase = new Map<string, SweepOperationWorker>();
+ // An llm worker serves an operation its tags (or the llm default set) name; a
+ // remote worker takes units only when TAGGED (the unit-dispatch opt-in) and
+ // its tags intersect [operation, contended resource]. Local workers are not
+ // listed — the lane always runs here, and saying so would be noise.
+ // ONE PASS PER WORKER SLOT, not per (operation, slot) pair. A worker serving
+ // two of the lane's operations is one row with one slot count — the slots are
+ // the machine's, not the operation's, and counting them twice would advertise
+ // capacity that does not exist.
+ const runsOnFor = (ids: readonly string[]): LaneWorker[] => {
+ const byBase = new Map<string, LaneWorker>();
for (const w of workerSummary) {
- const takesIt =
+ const takesAny = ids.some((id) =>
w.kind === "llm"
? workerMatches({ kind: "llm", tags: w.tags }, [id])
: w.kind === "remote" &&
(w.tags?.length ?? 0) > 0 &&
- workerMatches({ kind: "remote", tags: w.tags }, unitRequires);
- if (!takesIt) continue;
+ workerMatches({ kind: "remote", tags: w.tags }, [
+ id,
+ contendsForOf.get(id) ?? "cpu",
+ ]),
+ );
+ if (!takesAny) continue;
const base = w.id.split("#")[0];
const entry = byBase.get(base) ?? {
id: base,
@@ -298,73 +118,14 @@ export async function buildAutoQueueLanes(): Promise<AutoQueueLanesPayload> {
}
return [...byBase.values()];
};
- const operationsFor = (ids: ReadonlyArray<string>): SweepOperation[] =>
- ids.map((id) => ({
- id,
- label: operationLabel(id),
- costBasis: operationCostBasis(id),
- reachable: counts.reduce(
- (n, row) => n + (row.byKind[id]?.reachable ?? 0),
- 0,
- ),
- runsOn: runsOnFor(id),
- }));
- return {
- digest: {
- id: "digest",
- label: "Digest",
- sweeping: settings.digest.sweepEnabled,
- gateHeld: isGateHeld(settings, "digest"),
- available: true,
- sweepJobId: getDigestSweepJobId(),
- inFlight: inFlightOn([DIGEST_LOCAL_QUEUE, DIGEST_REMOTE_QUEUE]),
- order: settings.digest.recencyOrder,
- reach: settings.digest.recencyReach,
- // ONE ENTRY, deliberately. The digest sweep runs exactly one operation,
- // so there is no operation scope to offer and the console renders none —
- // the channel itinerary is the whole of its scope.
- operations: operationsFor(digestKindIds),
- scopeKinds: [],
- scopeChannels: settings.digest.sweepChannels,
- plan: planFor(digestKindIds),
- },
- backfill: {
- id: "backfill",
- // NAMED AFTER WHAT IT HOLDS. "Backfill" is this lane's queue key, and on
- // this install it stands for three operations with different inputs and
- // different costs. Derived, so a lane that gains a kind from another
- // group degrades to "Derived data" rather than going stale.
- label: operationsGroupLabel(
- kinds
- .filter((k) => k.lane.queueKey === BACKFILL_QUEUE)
- .map((k) => k.id),
- ),
- sweeping: settings.backfill.sweepEnabled,
- gateHeld: isGateHeld(settings, "backfill"),
- // With no backfill FEATURE switched on there is no lane — see `available`.
- // Digest is always available because the operation is always registered.
- available: kinds.some((k) => k.lane.queueKey === BACKFILL_QUEUE),
- sweepJobId: getBackfillSweepJobId(),
- inFlight: inFlightOn([BACKFILL_QUEUE]),
- order: settings.backfill.order,
- reach: settings.backfill.reach,
- operations: operationsFor(backfillKindIds),
- scopeKinds: settings.backfill.sweepKinds,
- scopeChannels: settings.backfill.sweepChannels,
- plan: planFor(backfillKindIds),
- },
- bands,
- arbiter: {
- running: getArbiterJobId() !== null,
- jobId: getArbiterJobId(),
- blockedReason: arbiterBlockedReason(),
- operationRules: [
- settings.autoQueue.transcription.root,
- settings.autoQueue.download.root,
- ]
- .flatMap((root) => flattenLeaves(root))
- .filter((leaf) => Boolean(leaf.match.operation)).length,
- },
- };
+ const operations = {} as Record<AutoQueueKind, LaneOperationView[]>;
+ const runsOn = {} as Record<AutoQueueKind, LaneWorker[]>;
+ for (const lane of LANES) {
+ const ids = operationsForLane(lane, settings).map((op) => op.id);
+ operations[lane] = ids.map((id) => ({ id, label: operationLabel(id) }));
+ runsOn[lane] = runsOnFor(ids);
+ }
+
+ return { bands, operations, runsOn };
}
diff --git a/editor/app/operations/settingsActions.ts b/editor/app/operations/settingsActions.ts
@@ -212,7 +212,6 @@ export async function saveBackfillLaneSettingsAction(
backfill: {
...dBack,
enabled: formData.get("backfillEnabled") === "on",
- weight: num(formData, "backfillWeight", dBack.weight),
concurrency: num(formData, "backfillConcurrency", dBack.concurrency),
allowRedownload: formData.get("backfillAllowRedownload") === "on",
},
diff --git a/editor/app/widget/components/MonitorWidget.tsx b/editor/app/widget/components/MonitorWidget.tsx
@@ -693,7 +693,7 @@ function BackfillStrip({ data }: { data: WidgetSyncPayload }) {
>
<span
className={`inline-block h-2 w-2 shrink-0 rounded-full ${
- b.sweeping ? "bg-success" : done ? "bg-success/40" : "bg-warning"
+ b.armed ? "bg-success" : done ? "bg-success/40" : "bg-warning"
}`}
/>
<span className="tabular-nums">
@@ -718,14 +718,14 @@ function BackfillStrip({ data }: { data: WidgetSyncPayload }) {
>
<span
className={`inline-block h-2 w-2 shrink-0 rounded-full ${
- b.sweeping ? "bg-success" : done ? "bg-success/40" : "bg-warning"
+ b.armed ? "bg-success" : done ? "bg-success/40" : "bg-warning"
}`}
/>
<span className="tabular-nums">
{b.groupLabel} {b.reachable.toLocaleString()} reachable
{b.needsMedia > 0 && <> · {b.needsMedia.toLocaleString()} need media</>}
{b.held && " · lane off"}
- {!b.held && b.sweeping && " · sweeping"}
+ {!b.held && b.armed && " · running"}
</span>
</section>
);
