commit 3b741a45b76fc61f2c24ed57f337e2f2400fe189
parent a9e15e7b204851f9deb7b8794d87d96bce4d68ef
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 28 Aug 2026 15:42:19 -0400
backfill: a channel job can be scoped to video ids
runBackfillBatch already took `ids` and intersected them with disk. The job
around it did not, so there was no way to run the lane over one video: the
only entry points were the stage card's whole-channel run and the sweep's.
`ids` now threads through runBackfillChannelJob to countBackfillWork and the
batch. Counting matters as much as pulling — countBackfillWork walked every
video dir with no filter, so an ids-scoped run would have sized its progress
bar to the whole channel and stalled one short of complete forever.
It is in `spec.params` beside `kindIds`, so a replayed per-video run stays
per-video rather than quietly widening to the channel; jobReplayRegistry parses
it exactly as it parses kindIds. runOperationChannelJob is untouched — the
dispatcher stays channel-scoped.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Diffstat:
5 files changed, 59 insertions(+), 9 deletions(-)
diff --git a/common/controller/backfillBatch.ts b/common/controller/backfillBatch.ts
@@ -870,6 +870,11 @@ export async function countBackfillWork(
paths: Paths,
channelSlug: string,
kindIds?: string[],
+ // Only these videos, intersected with disk — the same scope the batch takes.
+ // The progress target must count what THIS run will pull, and an ids-scoped
+ // run pulls only those; without the filter a per-video run would size its bar
+ // to the whole channel and stall one short of complete forever.
+ ids?: string[],
): Promise<{
reachable: number;
missingInput: number;
@@ -881,7 +886,12 @@ export async function countBackfillWork(
if (kinds.length === 0)
return { reachable: 0, missingInput: 0, deferred: 0, blocked: 0 };
const dataDir = path.join(paths.channelsDir, channelSlug, "data");
- const dirs = await readdir(dataDir).catch(() => [] as string[]);
+ const allDirs = await readdir(dataDir).catch(() => [] as string[]);
+ // Intersected with disk exactly as the batch's own pull does, so an id that
+ // names nothing counts as nothing rather than as one unit of work.
+ const dirs = ids
+ ? ids.filter((id) => allDirs.includes(id))
+ : allDirs;
const targets = new Map<string, unknown>();
for (const kind of kinds) {
targets.set(
diff --git a/common/controller/operationJobs.ts b/common/controller/operationJobs.ts
@@ -64,6 +64,9 @@ export type BackfillChannelJobOptions = {
// card's per-row "run only this" passes one; the group button passes none,
// because that station IS the lane rather than one operation on it.
kindIds?: string[];
+ // Only these videos (intersected with disk by the batch). A stage card passes
+ // none; the video page passes one. In `spec.params` so a replay stays scoped.
+ ids?: string[];
// An operator's queue override from the card's QueueControl. `undefined` is
// "no control wired up" and takes the lane's own key; "" means immediate.
queueKey?: string;
@@ -77,27 +80,30 @@ export type BackfillChannelJobOptions = {
export async function runBackfillChannelJob(
opts: BackfillChannelJobOptions,
): Promise<StreamActionResult> {
- const { paths, channelSlug, kindIds } = opts;
+ const { paths, channelSlug, kindIds, ids } = opts;
const result = await runManagedFunction({
kind: BACKFILL_CHANNEL_KIND,
queueKey: resolveQueueKey(BACKFILL_QUEUE, opts.queueKey),
paths,
channelSlug,
background: opts.background,
- // BOTH fields, because jobReplayRegistry.ts reads `p.kindIds` to replay this
- // job and the queue override has to survive a replay too. Undefined fields
- // drop out of the JSON, so a run with neither stores neither.
+ // ALL THREE fields, because jobReplayRegistry.ts replays this job from
+ // them: the kind scope, the queue override and the id scope all have to
+ // survive a replay — a replayed per-video run that widened to the whole
+ // channel would be a different job wearing the same record. Undefined
+ // fields drop out of the JSON, so a run with none stores none.
spec: {
kind: BACKFILL_CHANNEL_KIND,
slug: channelSlug,
- params: { kindIds, queueKey: opts.queueKey },
+ params: { kindIds, queueKey: opts.queueKey, ids },
},
fn: async (onLog, signal, setProgress, ctx) => {
- const before = await countBackfillWork(paths, channelSlug, kindIds);
+ const before = await countBackfillWork(paths, channelSlug, kindIds, ids);
const batch = await runBackfillBatch({
channelSlug,
paths,
kindIds,
+ ids,
setProgress,
// The bar measures THIS run, from zero. Seeding it with what is already
// on disk cannot represent a regeneration, where no file count moves.
diff --git a/common/controller/videoOperations.test.ts b/common/controller/videoOperations.test.ts
@@ -53,6 +53,7 @@ writeSettings(settingsOn());
const { inspectVideoOperations, orderForVideoPage, shownOnVideoPage } =
await import("./videoOperations");
+const { countBackfillWork } = await import("./backfillBatch");
const { getPaths } = await import("../lib/paths");
const { getSettings } = await import("../lib/settings");
@@ -231,6 +232,30 @@ test("shownOnVideoPage: off with nothing on disk hides; off with a sidecar shows
assert.equal(shownOnVideoPage(withRecord!), true);
});
+// Lives here rather than in backfillBatch.test.ts because countBackfillWork
+// reads settings from disk and this file already owns the settings seam.
+test("countBackfillWork sizes an ids-scoped run to those ids alone", async () => {
+ writeSettings(
+ settingsOn({
+ backfill: { enabled: true, weight: 1, concurrency: 1 },
+ }),
+ );
+ const idA = "vid-count-a";
+ const idB = "vid-count-b";
+ await seed(idA);
+ await seed(idB);
+ const paths = getPaths();
+ // Both are reachable for the text lane — its input is the cue stream, which
+ // every transcribed video has.
+ const whole = await countBackfillWork(paths, SLUG, ["attribution-text"]);
+ assert.ok(whole.reachable >= 2);
+ // Scoped, the progress target must count what THIS run will pull.
+ const scoped = await countBackfillWork(paths, SLUG, ["attribution-text"], [
+ idB,
+ ]);
+ assert.equal(scoped.reachable, 1);
+});
+
test("orderForVideoPage puts the digest group ahead of the speakers group", async () => {
writeSettings(settingsOn());
await seed("vid-sort", { diarization: true });
diff --git a/editor/app/channels/[slug]/backfillActions.ts b/editor/app/channels/[slug]/backfillActions.ts
@@ -25,11 +25,15 @@ export async function backfillChannelAction(
slug: string,
queueKey?: string,
kindIds?: string[],
+ // The video page's per-video Run passes exactly one id; a stage card passes
+ // none and gets the whole channel, as before.
+ ids?: string[],
): Promise<StreamActionResult> {
return runBackfillChannelJob({
paths: getPaths(),
channelSlug: slug,
kindIds,
+ ids,
queueKey,
onDone: () => revalidatePath(`/channels/${slug}`),
});
diff --git a/editor/app/jobs/jobReplayRegistry.ts b/editor/app/jobs/jobReplayRegistry.ts
@@ -275,13 +275,18 @@ export const JOB_REPLAY_HANDLERS: Record<string, ReplayHandler> = {
// Replays correctly with nothing remembered: the batch re-derives its whole
// work-list from disk, so a replay does what is missing NOW rather than what
// was missing when the record was written. The kind scope is carried through
- // so a replayed single-kind run stays single-kind.
+ // so a replayed single-kind run stays single-kind — and the ID scope for the
+ // same reason: a replayed per-video run stays per-video rather than widening
+ // to the whole channel.
"backfill-channel": (spec) => {
const { p, queueKey } = params(spec);
const kindIds = Array.isArray(p.kindIds)
? p.kindIds.filter((k): k is string => typeof k === "string")
: undefined;
- return backfillChannelAction(spec.slug, queueKey, kindIds);
+ const ids = Array.isArray(p.ids)
+ ? p.ids.filter((k): k is string => typeof k === "string")
+ : undefined;
+ return backfillChannelAction(spec.slug, queueKey, kindIds, ids);
},
"check-kept-deleted": (spec) => {
const { queueKey } = params(spec);
