commit f7f2f1898713ac7139af4e6a63790529ae2f07f3
parent 45b6eada6508600daf7a3e37d26adb6e660d4fbd
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sat, 29 Aug 2026 17:56:08 -0400
common: sync is a catalogued operation, with a scope and a trigger
Sync had no descriptor, so the board, the rail and /operations/<id> could not
reach it off the same table as everything else. It is in the catalog now, first
— upstream of every entry that depends on the videos it notices.
Two fields carry what makes it different. `scope` says what an operation has a
state FOR (a video, or a channel), and `trigger` says what makes it run (a
backlog, or a cadence). Neither is a workaround for `runner`: that is typed to
the AutoQueueKinds and the sync heartbeat is not one of them, so a console
chosen off `runner` could never be sync's. ExternalOperation is now a projection
of OperationDescriptor rather than a second copy of its field list, which is why
the two new fields arrive there once.
`sync` is a fifth OperationGroup and deliberately NOT in OPERATION_GROUP_ORDER:
that list is the per-video pipeline's stations, and sync draws no /channels
column. GROUP_STAGES.sync is empty for the same reason — its channel surface is
the hand-listed playlist bookend. pauseLaneFor answers null for it: the
scheduler's own `enabled` is its switch, not a lane gate.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Diffstat:
8 files changed, 137 insertions(+), 36 deletions(-)
diff --git a/common/controller/videoOperations.ts b/common/controller/videoOperations.ts
@@ -17,7 +17,9 @@
// transcription already have a per-video surface — the pipeline stage cards
// inside the editor's VideoPanel, which carry their own per-video actions. This
// reader is over OPERATIONS, the backfill/derived-data registry, and that
-// asymmetry is intentional rather than an omission.
+// asymmetry is intentional rather than an omission. Sync is absent for a
+// stronger reason still: it is `scope: "channel"`, so a per-video reader has
+// nothing to ask it.
//
// ONE readVideoFiles PER PAGE. It is a readdir, and the registry's state()
// takes the listing as its probe rather than re-reading it, so the whole page
diff --git a/common/lib/operations.test.ts b/common/lib/operations.test.ts
@@ -1222,6 +1222,7 @@ test("the backfill lane never dispatches digest", () => {
test("the catalog covers every operation, dispatched here or not", () => {
const ids = operationCatalog().map((o) => o.id);
for (const id of [
+ "sync",
"download",
"transcode",
"transcription",
@@ -1257,6 +1258,11 @@ test("`runner` names the auto-queue runner, and only for the two that have one",
if (op.id === "download" || op.id === "transcription") continue;
assert.equal(op.runner, undefined, `${op.id} declares a runner`);
}
+ // Sync included, and it is the interesting one: the sync scheduler's
+ // heartbeat IS a runner in the IA doc's sense, but `runner` is typed to the
+ // AutoQueueKinds and the heartbeat is not one of them. Its console is chosen
+ // off `trigger` instead — see the scope/trigger test below.
+ assert.equal(runners.get("sync"), undefined);
});
test("`appliesTo` is transcode's alone, and it reads the channel config", () => {
@@ -1335,10 +1341,23 @@ test("the three speaker operations share one group; digest does not", () => {
assert.equal(operationGroup("digest"), "digest");
assert.equal(operationGroup("download"), "media");
assert.equal(operationGroup("transcription"), "transcript");
+ // Sync is its own group, not `media`: that group is download + transcode's
+ // /channels column group, and sync draws no column.
+ assert.equal(operationGroup("sync"), "sync");
// An id the catalog does not know is null, NOT filed under the first group.
assert.equal(operationGroup("no-such-operation"), null);
});
+test("`scope` and `trigger`: sync is the one channel-scoped, cadence-triggered operation", () => {
+ // The two fields exist so a per-video surface can EXCLUDE sync by a declared
+ // fact rather than by its id. Every other entry is per-video and backlog-fed.
+ for (const op of operationCatalog()) {
+ assert.equal(op.scope === "channel", op.id === "sync", `${op.id} scope ${op.scope}`);
+ assert.equal(op.trigger === "cadence", op.id === "sync", `${op.id} trigger ${op.trigger}`);
+ }
+ assert.equal(operationCatalog()[0].id, "sync"); // upstream first
+});
+
test("a set's label is derived, so a mixed lane cannot claim one member's name", () => {
// The whole reason this is derived: the backfill lane holds three speaker
// operations today and its station can honestly say "Speakers". Add a kind
diff --git a/common/lib/operations.ts b/common/lib/operations.ts
@@ -242,6 +242,16 @@ export type Lane = {
// on it resolves. See EXTERNAL_OPERATIONS.
export type OperationDispatch = "backfill" | "external";
+// WHAT AN OPERATION ACTS ON, and WHAT MAKES IT RUN. Every media-derived
+// operation is per-video and backlog-driven: it has a state per video, and its
+// work list is whichever videos lack it. Sync is the one exception the IA doc
+// names (plans/editor-operations-ia.md, "Where the noun breaks"): a CHANNEL has
+// a sync state — last synced, next due — and nothing runs it but a cadence. A
+// surface that draws per-video things (a band, a video-page panel, a channel
+// column, a worker tag) filters on `scope`, never on the id.
+export type OperationScope = "video" | "channel";
+export type OperationTrigger = "backlog" | "cadence";
+
export type OperationProbe = {
videoDir: string;
videoId: string;
@@ -319,7 +329,7 @@ export type OperationRunOutcome =
// Deriving the label from the group also means a lane holding a MIX cannot go
// stale: it falls back to "Derived data" rather than naming two of its three
// members.
-export type OperationGroup = "media" | "transcript" | "digest" | "speakers";
+export type OperationGroup = "media" | "transcript" | "digest" | "speakers" | "sync";
// See Operation.settingsBlock. A closed union rather than `string` so the switch
// that picks a settings form is exhaustive: a new block has to be named here and
@@ -329,6 +339,10 @@ export type OperationSettingsBlock = "digest" | "diarization" | "attribution";
// Group order: upstream first. The /channels columns and the transit line both
// lay their pipelines out in this order, so a reader moving between the two
// pages sees the same left-to-right sequence.
+//
+// `sync` is deliberately NOT here: this list is the per-video pipeline's
+// stations, and sync draws no channel column and no station of its own (its
+// channel surface is the hand-listed `playlist` bookend — see GROUP_STAGES).
export const OPERATION_GROUP_ORDER: readonly OperationGroup[] = [
"media",
"transcript",
@@ -346,6 +360,8 @@ export function groupLabel(group: OperationGroup): string {
return "Digest";
case "speakers":
return "Speakers";
+ case "sync":
+ return "Sync";
}
}
@@ -1290,30 +1306,34 @@ export function laneYieldsToTranscription(lane: Lane): boolean {
// - PER-CHANNEL VS CROSS-CHANNEL SCOPE. backfillBatch is one job per channel;
// autoRunner arbitrates across every channel at once, which is the whole
// point of its policy tree.
-export type ExternalOperation = {
- id: string;
- label: string;
- hint: string;
- group: OperationGroup;
- shortLabel: string;
- costBasis: string;
- lane: Lane;
- dependsOn?: readonly string[];
- dispatch: "external";
- // The auto-queue runner that dispatches this, when one does. `dispatch` alone
- // cannot answer it: download and transcription are `external` WITH a runner,
- // and a future `transcode` would be `external` with none. A console that
- // guessed from the id would hand transcode the transcription runner's
- // controls — a live Start button over the wrong lane.
- runner?: AutoQueueKind;
- // Whether this operation applies to a channel AT ALL, off its config. Absent
- // means every channel. The first (and so far only) taker is transcode: a
- // youtube-handling channel or one with no audioFormat never has that work,
- // and the three surfaces that gate the Transcode stage used to each spell
- // `handling === "transcribe" && !!audioFormat` for themselves. Config-scoped
- // on purpose — it is a fact about the channel's configuration, not about any
- // video's state, which is what `state()` on a backfill operation answers.
- appliesTo?: (config: ChannelConfig) => boolean;
+//
+// The SAME shape as a catalog entry, minus the two fields it cannot have: it is
+// `external` by definition, and nothing in settings.json configures a download
+// as an operation. Declared as a projection rather than a second field list so
+// a new descriptor field is declared once and cannot go missing from one of the
+// two — which is how `scope` and `trigger` arrive here for free.
+export type ExternalOperation = Omit<
+ OperationDescriptor,
+ "dispatch" | "settingsBlock"
+> & { dispatch: "external" };
+
+// THE SYNC OPERATION. Catalogued so the board, the rail and /operations/sync
+// come off the same table as everything else, and NOT in EXTERNAL_OPERATIONS —
+// that list is the media-derived pipelines the rail draws bands for (editor
+// buildBands.ts, EXTERNAL_BAND_IDS); sync has no per-video population. The lane
+// is the shape download declares: syncAction runs through runPipelineAction on
+// the per-platform download queue (pipelineActions.ts, downloadQueueKey).
+export const SYNC_OPERATION: OperationDescriptor = {
+ id: "sync",
+ label: "Sync",
+ hint: "Noticing new videos: re-reading each channel's listing on its cadence and fetching what is new. Dispatched by the sync scheduler's heartbeat, and by hand per channel or all at once. A corpus that has stopped noticing new videos is not idle — it is broken.",
+ group: "sync",
+ shortLabel: "Sync",
+ costBasis: "one listing fetch per channel, over the network",
+ lane: { queueKey: "download:<platform>", contendsFor: "network" },
+ dispatch: "external",
+ scope: "channel",
+ trigger: "cadence",
};
export const EXTERNAL_OPERATIONS: readonly ExternalOperation[] = [
@@ -1329,6 +1349,8 @@ export const EXTERNAL_OPERATIONS: readonly ExternalOperation[] = [
// the dependency graph, not dispatch.
lane: { queueKey: "download:<platform>", contendsFor: "network" },
dispatch: "external",
+ scope: "video",
+ trigger: "backlog",
runner: "download",
},
{
@@ -1345,6 +1367,8 @@ export const EXTERNAL_OPERATIONS: readonly ExternalOperation[] = [
lane: { queueKey: TRANSCRIPTION_QUEUE, contendsFor: "cpu" },
dependsOn: ["download"],
dispatch: "external",
+ scope: "video",
+ trigger: "backlog",
// No runner: nothing in the auto-queue feeds this. /operations/transcode
// is the "no console here" panel, on purpose.
appliesTo: (config) =>
@@ -1360,12 +1384,17 @@ export const EXTERNAL_OPERATIONS: readonly ExternalOperation[] = [
lane: { queueKey: TRANSCRIPTION_QUEUE, contendsFor: "gpu" },
dependsOn: ["download"],
dispatch: "external",
+ scope: "video",
+ trigger: "backlog",
runner: "transcription",
},
];
-// Every media-derived operation, dispatched here or not. The catalog — what a
-// dependency id resolves against, and what a future scheduler enumerates.
+// Every operation the console knows, dispatched here or not. The catalog — what
+// a dependency id resolves against, and what the board, the rail and
+// /operations/<id> enumerate. The sync scheduler is IN it now (SYNC_OPERATION),
+// which is why `scope` and `trigger` exist: the entries are no longer all
+// per-video, backlog-fed things.
export type OperationDescriptor = {
id: string;
label: string;
@@ -1376,18 +1405,40 @@ export type OperationDescriptor = {
lane: Lane;
dependsOn?: readonly string[];
dispatch: OperationDispatch;
- // See ExternalOperation.runner. Absent for every backfill kind: those are
- // dispatched by the sweep and the arbiter, not by a runner.
+ // See OperationScope. What this operation has a state FOR — a video for
+ // everything media-derived, a channel for sync.
+ scope: OperationScope;
+ // See OperationTrigger. What makes it run — a backlog for everything the
+ // sweep and the runners feed, a cadence for sync.
+ trigger: OperationTrigger;
+ // The auto-queue runner that dispatches this, when one does. `dispatch` alone
+ // cannot answer it: download and transcription are `external` WITH a runner,
+ // and a future `transcode` would be `external` with none. A console that
+ // guessed from the id would hand transcode the transcription runner's
+ // controls — a live Start button over the wrong lane. Absent for every
+ // backfill kind: those are dispatched by the sweep and the arbiter.
+ //
+ // Typed to the auto-queue kinds on purpose; the sync heartbeat is a runner in
+ // the IA doc's sense but not one of these, which is why /operations/sync is
+ // chosen off `trigger`.
runner?: AutoQueueKind;
- // See ExternalOperation.appliesTo. Absent means every channel.
+ // Whether this operation applies to a channel AT ALL, off its config. Absent
+ // means every channel. The first (and so far only) taker is transcode: a
+ // youtube-handling channel or one with no audioFormat never has that work,
+ // and the three surfaces that gate the Transcode stage used to each spell
+ // `handling === "transcribe" && !!audioFormat` for themselves. Config-scoped
+ // on purpose — it is a fact about the channel's configuration, not about any
+ // video's state, which is what `state()` on a backfill operation answers.
appliesTo?: (config: ChannelConfig) => boolean;
- // See Operation.settingsBlock. Absent for every external operation: nothing in
- // settings.json configures a download or a transcode as an operation.
+ // See Operation.settingsBlock. Absent for every external operation but sync:
+ // nothing in settings.json configures a download or a transcode as an
+ // operation.
settingsBlock?: OperationSettingsBlock;
};
export function operationCatalog(): OperationDescriptor[] {
return [
+ SYNC_OPERATION,
...EXTERNAL_OPERATIONS,
...OPERATIONS.map((k) => ({
id: k.id,
@@ -1399,6 +1450,11 @@ export function operationCatalog(): OperationDescriptor[] {
lane: k.lane,
dependsOn: k.dependsOn,
dispatch: "backfill" as const,
+ // The registry is per-video by construction — `Operation.state(probe)` is
+ // a per-video probe — and every one of its entries is fed by a backlog of
+ // videos that lack the output.
+ scope: "video" as const,
+ trigger: "backlog" as const,
settingsBlock: k.settingsBlock,
})),
];
@@ -1462,6 +1518,8 @@ export function groupActionLabel(group: OperationGroup): string {
return "digests";
case "speakers":
return "speaker work";
+ case "sync":
+ return "syncs";
}
}
diff --git a/common/lib/pauseGates.test.ts b/common/lib/pauseGates.test.ts
@@ -96,9 +96,12 @@ test("pauseLaneFor answers for every catalog id, and transcode is null", () => {
"attribution-diarized": "backfill",
"attribution-text": "backfill",
digest: "digest",
+ // Catalogued, and deliberately gateless: the sync scheduler's own `enabled`
+ // is its switch, and its queue key is neither sweep's.
+ sync: null,
};
const ids = operationCatalog().map((o) => o.id);
- assert.equal(ids.length, 7);
+ assert.equal(ids.length, 8);
for (const id of ids) {
assert.ok(id in expected, `catalog gained ${id} with no expected lane`);
assert.equal(pauseLaneFor(id), expected[id], `pauseLaneFor(${id})`);
diff --git a/common/lib/pauseGates.ts b/common/lib/pauseGates.ts
@@ -57,8 +57,10 @@ type _RunnerLanesArePauseLanes = AutoQueueKind extends PauseLane ? true : never;
// hand it the transcription pause — a live Pause button over a lane that would
// never dispatch it. Its own answer is null.
//
-// Walks operationCatalog() — all seven ids, external ones included — not
-// OPERATION_BY_ID, which knows only the four registry entries.
+// Walks operationCatalog() — all eight ids, external ones included — not
+// OPERATION_BY_ID, which knows only the four registry entries. Sync is in that
+// walk and its answer is null: the scheduler's `enabled` is its own switch, not
+// a lane gate.
export function pauseLaneFor(operationId: string): PauseLane | null {
const op = operationCatalog().find((o) => o.id === operationId);
if (!op) return null;
diff --git a/editor/app/channels/[slug]/lib/stageStatus.ts b/editor/app/channels/[slug]/lib/stageStatus.ts
@@ -87,6 +87,12 @@ export const GROUP_STAGES: Record<OperationGroup, readonly StageId[]> = {
transcript: ["transcribe"],
digest: ["digest"],
speakers: ["speakers"],
+ // Empty on purpose, and this is the paragraph above in practice: sync's
+ // channel surface is the hand-listed `playlist` bookend (JOB_KIND_TO_STAGE
+ // maps the sync job kind to it), which is a channel chore, not a stage this
+ // Record owns. Do not "finish" the derivation by giving sync a stage here —
+ // the channel page would grow a second, duplicate playlist card.
+ sync: [],
};
export type StageTone = "neutral" | "attention" | "danger" | "running" | "ok";
diff --git a/editor/app/channels/page.tsx b/editor/app/channels/page.tsx
@@ -70,6 +70,11 @@ function summariseFreshness(
// stations out in too. A reader moving between the two pages sees the same
// left-to-right sequence, and adding a kind adds a column without touching this
// file.
+//
+// Sync is catalogued but channel-scoped, so no column: `ids` is built from
+// EXTERNAL_BAND_IDS plus the enabled backfill kinds and sync is in neither, and
+// its group is not in OPERATION_GROUP_ORDER either — a column here would have
+// to state a per-video coverage sync does not have.
function pipelineColumns(operationIds: ReadonlyArray<string>): {
ids: string[];
columns: PipelineColumn[];
diff --git a/editor/app/components/pipelines/buildBands.ts b/editor/app/components/pipelines/buildBands.ts
@@ -94,6 +94,12 @@ export type BuildOperationBandsInput = {
// definitions the channel transit line already uses for its Download and
// Transcribe stations, so a corpus figure and a channel figure cannot disagree
// about what "downloaded" means.
+//
+// Sync is catalogued beside them and still gets NO band, here or anywhere: its
+// populations are channels, not videos (`scope: "channel"`), so every one of a
+// band's five numbers would be a category error and the rail would draw
+// "coverage unknown" over a hollow outline. Its rail row is composed instead —
+// see SyncRailRow in OperationRail.tsx.
function addExternalBands(
bands: Map<string, OperationBand>,
snapshot: ChannelSnapshot,
