commit 1815abeaae2cfb01b4c69b81a2caa3841b970c78
parent 60fcf2a714547f6760223da0d74de80311c4770f
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sun, 9 Aug 2026 16:48:03 -0400
Tell "waiting on a prerequisite" apart from "the media is gone"
attribution-diarized waits on diarization.json and said so by returning
missing-input. But missing-input means one specific thing to the rest of
the system: the media is gone, re-acquire it. So ~73,000 videos were
counted in the re-acquire population, and with allowRedownload on the
lane would have spent a download each fetching AUDIO -- which can never
satisfy a wait for diarization.json -- and then deleted it again.
`blocked` is now its own state, following the `deferred` precedent
exactly: counted separately, never summed into reachableBackfillWork,
never dispatched, and -- the part that fixes the defect -- entirely
unaffected by allowRedownload. Also immune to `force`, because forcing a
video whose prerequisite does not exist does not make it exist. The
`never` default on candidateAction did its job: omitting the arm was a
compile error, not a silently dispatched video.
dependsOn is declarative and buys two things. resolveBackfillKinds now
topologically sorts by it, so a video diarized during a pass is
attributed in the SAME pass rather than on whatever later pass finds the
sidecar. The sort is stable, which is load-bearing: attribution-text
must stay last so the better lane reaches a diarized video first (one
wasted classification instead of ~30 model calls). A cycle or a
dangling id degrades to declaration order rather than wedging the lane.
Two existing assertions pinned the old behaviour and both now pin the
fix -- backfillKinds.test.ts:562 and attribution.spec.ts's stage card,
where the "needs media re-acquired" line is now absent entirely because
nothing in that fixture needs media.
common 638/638, attribution+backfill+diarization e2e 21/21,
tsc clean both packages.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Diffstat:
9 files changed, 346 insertions(+), 31 deletions(-)
diff --git a/common/controller/backfillBatch.ts b/common/controller/backfillBatch.ts
@@ -127,6 +127,11 @@ export type BackfillBatchResult = {
// same reason missingInput is: it is neither work done nor work failed, and
// folding it into either would make a capped run read as a complete one.
deferred: number;
+ // Videos waiting on a prerequisite kind's output. Separate again, and for a
+ // reason the others do not share: this number is expected to fall by itself
+ // as the prerequisite lane runs, so an operator seeing it should wait rather
+ // than change a setting.
+ blocked: number;
};
// What the candidate pull does with one classification.
@@ -153,7 +158,9 @@ export type CandidateAction =
// Input is gone and re-acquiring is off.
| "missing-input"
// The kind refuses to attempt it under the current configuration.
- | "deferred";
+ | "deferred"
+ // Waiting on a prerequisite kind's output.
+ | "blocked";
export function candidateAction(
state: BackfillClassification,
@@ -173,6 +180,18 @@ export function candidateAction(
// "redo work that looks done", not "ignore the cap". Raising the cap is how
// you ask for a deferred video, and it is one edit in Settings.
return "deferred";
+ case "blocked":
+ // NEVER dispatched, and — like `deferred` — deliberately immune to
+ // `force`. Forcing a video whose prerequisite has not been produced does
+ // not make the prerequisite appear; it just hands the runner an input it
+ // does not have. Running the kind this one dependsOn is how you unblock
+ // it, and the ordering in resolveBackfillKinds tries to do that for you
+ // within the same pass.
+ //
+ // Note what is NOT here: re-acquiring media. That is what separating this
+ // from `missing-input` bought — allowRedownload has no bearing on a
+ // blocked video, so the lane cannot spend a download on one.
+ return "blocked";
case "missing":
case "stale":
return "dispatch";
@@ -204,6 +223,7 @@ export async function runBackfillBatch(
reacquireFailed: 0,
diskFloorHit: false,
deferred: 0,
+ blocked: 0,
};
if (kinds.length === 0) {
@@ -300,6 +320,10 @@ export async function runBackfillBatch(
result.deferred++;
continue;
}
+ if (action === "blocked") {
+ result.blocked++;
+ continue;
+ }
if (action === "missing-input") {
// Counted, not attempted. This is the population the whole
// reachable-vs-needs-re-acquiring split exists to keep visible.
diff --git a/common/lib/backfillKinds.test.ts b/common/lib/backfillKinds.test.ts
@@ -7,11 +7,14 @@ import {
getBackfillKind,
laneBackfillKinds,
resolveBackfillKinds,
+ orderByDependencies,
addBackfillState,
emptyBackfillCounts,
reachableBackfillWork,
type BackfillClassification,
+ type BackfillKind,
} from "./backfillKinds";
+import { candidateAction } from "../controller/backfillBatch";
import {
readVideoFiles,
CUES_JSON_FILENAME,
@@ -559,14 +562,105 @@ test("attribution-text has nothing to do where a diarized record exists", async
);
});
-test("attribution-diarized: no diarization.json is missing-input, not missing", async () => {
- // ~73,000 videos on this corpus, against a handful reachable. The whole reason
- // the four-state split exists — a single "remaining" number here would put
- // every channel at the top of every list forever.
- assert.equal(await classifyAttr(attrDiarized, {}), "missing-input");
+test("attribution-diarized: no diarization.json is BLOCKED, not missing-input", async () => {
+ // ~73,000 videos on this corpus, against a handful reachable — so this state
+ // has to be right or every surface is wrong.
+ //
+ // THIS ASSERTION USED TO SAY missing-input, AND THAT WAS THE DEFECT.
+ // missing-input means one thing to the rest of the system: the media is gone,
+ // re-acquire it. So these videos were counted as needing media re-fetched,
+ // and with allowRedownload on the lane would have spent a download per video
+ // fetching AUDIO — which cannot satisfy a wait for diarization.json — and
+ // then deleted it again. What they are waiting for is the `diarization` kind,
+ // which this table produces.
+ assert.equal(await classifyAttr(attrDiarized, {}), "blocked");
assert.equal(
await classifyAttr(attrDiarized, { attribution: attrSidecar() }),
- "missing-input",
+ "blocked",
+ );
+});
+
+test("blocked is never dispatched, and re-download cannot change that", () => {
+ // The dispatch decision is the consequential one: `blocked` must not reach a
+ // runner whatever the flags say. Note allowRedownload — the flag that DOES
+ // turn missing-input into a dispatch — is deliberately inert here.
+ for (const force of [false, true]) {
+ for (const allowRedownload of [false, true]) {
+ assert.equal(
+ candidateAction("blocked", { force, allowRedownload }),
+ "blocked",
+ `force=${force} allowRedownload=${allowRedownload}`,
+ );
+ }
+ }
+ // The contrast, so this test fails if the two ever get conflated again.
+ assert.equal(
+ candidateAction("missing-input", { force: false, allowRedownload: true }),
+ "dispatch",
+ );
+});
+
+test("blocked is counted, and is NOT reachable work", () => {
+ const counts = emptyBackfillCounts();
+ addBackfillState(counts, "blocked");
+ addBackfillState(counts, "blocked");
+ addBackfillState(counts, "missing");
+ assert.equal(counts.blocked, 2);
+ assert.equal(counts.missing, 1);
+ // The load-bearing line. A corpus with one diarization and 73,000 waiting
+ // attributions must not report 73,000 jobs ready to run.
+ assert.equal(reachableBackfillWork(counts), 1);
+ // And it is its own number, not folded into the re-acquire population.
+ assert.equal(counts.missingInput, 0);
+ assert.equal(counts.deferred, 0);
+});
+
+test("a prerequisite is ordered before the kind that declares it", () => {
+ const base = settingsWithDiarization();
+ const kinds = resolveBackfillKinds(
+ {
+ ...base,
+ attribution: {
+ ...base.attribution,
+ enabled: true,
+ diarizedEnabled: true,
+ textOnlyEnabled: true,
+ },
+ },
+ undefined,
+ );
+ const ids = kinds.map((k) => k.id);
+ const diarizationAt = ids.indexOf("diarization");
+ const diarizedAttrAt = ids.indexOf("attribution-diarized");
+ assert.ok(diarizationAt >= 0 && diarizedAttrAt >= 0, ids.join(","));
+ // Without this, a video diarized during a pass only becomes attributable on
+ // whatever LATER pass happens to find the sidecar on disk.
+ assert.ok(
+ diarizationAt < diarizedAttrAt,
+ `diarization must precede attribution-diarized, got ${ids.join(", ")}`,
+ );
+ // AND the sort is STABLE: attribution-text stays last, which is a separate
+ // deliberate decision (the better lane must reach a diarized video first, or
+ // the text lane spends ~30 model calls to produce a record it must not write).
+ assert.equal(ids[ids.length - 1], "attribution-text", ids.join(","));
+});
+
+test("ordering degrades safely when a prerequisite is absent or cyclic", () => {
+ const a = { id: "a", dependsOn: ["b"] } as unknown as BackfillKind;
+ const b = { id: "b", dependsOn: ["a"] } as unknown as BackfillKind;
+ // A cycle must not wedge the lane or silently drop a kind: every entry comes
+ // back, in declaration order.
+ assert.deepEqual(
+ orderByDependencies([a, b]).map((k) => k.id),
+ ["a", "b"],
+ );
+ // A dependency on something not in the selection imposes no ordering, and
+ // does not remove the dependant from the run.
+ const lonely = { id: "lonely", dependsOn: ["not-here"] } as unknown as
+ BackfillKind;
+ assert.deepEqual(
+ orderByDependencies([lonely]).map((k) => k.id),
+ ["lonely"],
);
});
@@ -751,6 +845,7 @@ test("counts keep reachable work and needs-re-acquiring apart", () => {
stale: 1,
missingInput: 3,
deferred: 0,
+ blocked: 0,
});
// 3, not 6. Measured on the real corpus the difference is 835 vs 77,105, and
// reporting the larger number is what would make every surface useless.
@@ -772,6 +867,7 @@ test("deferred is counted, and is NOT reachable work", async () => {
stale: 1,
missingInput: 0,
deferred: 2,
+ blocked: 0,
});
// 2, not 4. This is the assertion that keeps a capped corpus from ever reading
// as finished, and the one that fails if someone "tidies up" by folding
diff --git a/common/lib/backfillKinds.ts b/common/lib/backfillKinds.ts
@@ -104,16 +104,31 @@ import type { AttributeOneOutcome } from "../controller/attributeOne";
// attempt it under the current configuration. Today that is
// only the diarization duration cap. NEVER summed into
// reachable work, so a capped corpus cannot read as finished.
+// blocked — does not have it, and what it is waiting for is the OUTPUT OF
+// ANOTHER KIND IN THIS TABLE. Not the same thing as
+// missing-input, and conflating them was a real defect: see
+// below.
//
-// `deferred` follows the house rule BackfillRunOutcome = "skipped" already sets
-// on the run side: a deliberate non-action gets its own counter, and is never
-// folded into the work total nor reported as a failure.
+// `deferred` and `blocked` follow the house rule BackfillRunOutcome = "skipped"
+// already sets on the run side: a deliberate non-action gets its own counter,
+// and is never folded into the work total nor reported as a failure.
+//
+// WHY `blocked` IS NOT `missing-input`. `missing-input` means one specific
+// thing to the rest of the system: THE MEDIA IS GONE, RE-ACQUIRE IT. It is the
+// population `allowRedownload` exists for, and backfillReacquire answers it by
+// fetching AUDIO. `attribution-diarized` waits on diarization.json — so
+// reporting that as missing-input told the operator ~73,000 videos needed media
+// re-fetched, and with re-download on, the lane would spend a download per video
+// fetching audio that CANNOT satisfy the wait, then cleaning it up again. The
+// prerequisite is not gone; it has not been produced yet, and this same table
+// knows how to produce it.
export type BackfillState =
| "present"
| "stale"
| "missing"
| "missing-input"
- | "deferred";
+ | "deferred"
+ | "blocked";
// Videos this backfill has no opinion about (not transcribed, marked
// untranscribable). Kept out of BackfillState so it can never be counted.
@@ -177,6 +192,21 @@ export type BackfillKind = {
// One line of UI copy: what this backfill is, in the operator's terms.
hint: string;
tier: BackfillCostTier;
+ // Ids of other kinds in this table whose output this one consumes.
+ //
+ // PURELY DECLARATIVE. It does not gate anything by itself — a kind still
+ // decides for itself, from disk, whether its input is there, and says so by
+ // returning `blocked`. What declaring it buys is two things nothing else
+ // could: resolveBackfillKinds can order a prerequisite before its dependant
+ // within a single pass (so a video diarized this pass can be attributed in
+ // the same one, rather than waiting for whatever LATER pass happens to find
+ // the sidecar on disk), and a surface can say what a blocked video is waiting
+ // FOR rather than just that it is stuck.
+ //
+ // An id naming a kind that is absent or disabled is not an error: the
+ // dependency simply imposes no ordering, and the dependant keeps reporting
+ // `blocked` until something produces its input.
+ dependsOn?: readonly string[];
// The feature's OWN gate. A disabled feature reports no backfill at all —
// otherwise every surface would advertise catch-up work for something the
// operator has switched off.
@@ -401,6 +431,10 @@ const attributionDiarized: BackfillKind = {
label: "Speaker names (from the audio)",
hint: "Names put to the speaker clusters in diarization.json — about one model call per video, and better than the text-only lane. Needs diarization to have run first.",
tier: "lane",
+ // The dependency the hint has always stated in prose. Declaring it is what
+ // turns "needs diarization to have run first" from a sentence an operator
+ // reads into something the scheduler can order by and a counter can name.
+ dependsOn: ["diarization"],
enabled: (settings) =>
settings.attribution.enabled && settings.attribution.diarizedEnabled,
resolveTarget: (settings) => attributionTargetFor(settings, "diarized"),
@@ -414,7 +448,14 @@ const attributionDiarized: BackfillKind = {
// captured during a past run is a perfectly good input after capture is
// switched off again, and refusing to name it would strand exactly the work
// the capture lane exists to protect.
- if (!files.hasDiarization) return "missing-input";
+ //
+ // BLOCKED, NOT MISSING-INPUT. This used to say missing-input, and that was
+ // wrong in a way that cost real work: it put ~73,000 videos into the
+ // "re-acquire the media" population, where allowRedownload would fetch
+ // AUDIO — which can never satisfy a wait for diarization.json — and then
+ // delete it again. What this video is waiting for is the `diarization` kind
+ // declared in dependsOn above, and that is a thing this table produces.
+ if (!files.hasDiarization) return "blocked";
if (!files.entries.includes(ATTRIBUTION_FILENAME)) return "missing";
const record = await loadAttribution(videoDir);
if (!record) return "missing";
@@ -428,7 +469,11 @@ const attributionDiarized: BackfillKind = {
// Only now is the diarization read worth paying for: it is needed solely to
// ask whether the clusters these names point at are still the same clusters.
const diarization = await loadDiarization(videoDir);
- if (!diarization) return "missing-input";
+ // Present in the listing but unreadable — a half-written or corrupt
+ // sidecar. Blocked for the same reason as the branch above, and note that
+ // the `diarization` kind reads a malformed record as ABSENT too, so it will
+ // regenerate this file and unblock the video without anyone intervening.
+ if (!diarization) return "blocked";
return isAttributionFresh(record, {
...(target as AttributionFreshnessTarget),
diarizationGeneratedAt: diarization.generatedAt,
@@ -517,9 +562,71 @@ export function resolveBackfillKinds(
ids: readonly string[] | undefined,
): BackfillKind[] {
const lane = laneBackfillKinds(settings);
- if (!ids || ids.length === 0) return lane;
- const wanted = new Set(ids);
- return lane.filter((k) => wanted.has(k.id));
+ const selected =
+ !ids || ids.length === 0
+ ? lane
+ : lane.filter((k) => new Set(ids).has(k.id));
+ return orderByDependencies(selected);
+}
+
+// Order kinds so a prerequisite is attempted before anything that declares it.
+//
+// WHY THIS IS WORTH DOING AT ALL. backfillBatch walks the kinds in the order it
+// is given, all the way through the video list, before starting the next kind.
+// So with `attribution-diarized` ahead of `diarization`, a video diarized
+// during a pass becomes eligible for attribution only on whatever LATER pass
+// happens to find the sidecar on disk. Ordering by the declaration collapses
+// that into one pass, and costs a topological sort over three entries.
+//
+// STABLE, and that is load-bearing rather than tidiness. BACKFILL_KINDS puts
+// attribution-text LAST on purpose (see the comment there): on a video that has
+// diarization, the better lane must get there first so the text lane finds a
+// record it must not overwrite — one wasted classification instead of ~30 model
+// calls. Kahn's algorithm with a queue seeded and drained in declaration order
+// preserves every ordering the declarations do not contradict, so that decision
+// survives.
+export function orderByDependencies(kinds: BackfillKind[]): BackfillKind[] {
+ const byId = new Map(kinds.map((k) => [k.id, k]));
+ // Only dependencies that are actually IN this selection constrain anything. A
+ // kind that names a disabled or unselected prerequisite is not held back —
+ // it will report `blocked` per video, which is the honest answer, rather than
+ // being silently dropped from the run.
+ const remaining = new Map(
+ kinds.map((k) => [
+ k.id,
+ (k.dependsOn ?? []).filter((d) => byId.has(d) && d !== k.id).length,
+ ]),
+ );
+ const dependants = new Map<string, string[]>();
+ for (const k of kinds) {
+ for (const d of k.dependsOn ?? []) {
+ if (!byId.has(d) || d === k.id) continue;
+ const list = dependants.get(d);
+ if (list) list.push(k.id);
+ else dependants.set(d, [k.id]);
+ }
+ }
+ const out: BackfillKind[] = [];
+ const emitted = new Set<string>();
+ // Repeatedly take the FIRST still-unemitted kind in declaration order whose
+ // prerequisites are all out. Quadratic in the number of kinds, which is three.
+ for (;;) {
+ const next = kinds.find(
+ (k) => !emitted.has(k.id) && (remaining.get(k.id) ?? 0) === 0,
+ );
+ if (!next) break;
+ emitted.add(next.id);
+ out.push(next);
+ for (const id of dependants.get(next.id) ?? []) {
+ remaining.set(id, (remaining.get(id) ?? 1) - 1);
+ }
+ }
+ // A CYCLE leaves entries unemitted. Append them in declaration order rather
+ // than throwing or dropping them: a mis-declared dependency should degrade to
+ // the old behaviour (run in table order), never wedge the lane or silently
+ // stop a backfill from running at all.
+ for (const k of kinds) if (!emitted.has(k.id)) out.push(k);
+ return out;
}
// Per-kind counts, the shape every indicator reads. `missing` and `missingInput`
@@ -533,10 +640,18 @@ export type BackfillCounts = {
// never summed. Snapshots written before this field existed do not carry it,
// so every read site needs `?? 0` — `.toLocaleString()` on undefined throws.
deferred: number;
+ // Waiting on a prerequisite kind's output. A FOURTH number, and the same rule
+ // applies: never summed with the others, and `?? 0` at every read site,
+ // because every snapshot currently on disk predates it.
+ //
+ // This number should FALL on its own as the prerequisite lane works, which is
+ // the whole difference from missingInput — that one only falls if an operator
+ // turns re-download on.
+ blocked: number;
};
export function emptyBackfillCounts(): BackfillCounts {
- return { missing: 0, stale: 0, missingInput: 0, deferred: 0 };
+ return { missing: 0, stale: 0, missingInput: 0, deferred: 0, blocked: 0 };
}
// Fold one classification into a counts record. Central so no surface invents
@@ -550,16 +665,23 @@ export function addBackfillState(
else if (state === "stale") counts.stale++;
else if (state === "missing-input") counts.missingInput++;
else if (state === "deferred") counts.deferred++;
+ else if (state === "blocked") counts.blocked++;
}
// What the lane can act on WITHOUT re-acquiring media. The number every "how
// much is left?" surface should lead with.
//
-// DELIBERATELY UNCHANGED by the addition of `deferred`. This function is the
-// guard: adding a fourth state to the union raises no TypeScript error anywhere
-// (there is no exhaustiveness check over BackfillState in this repo), so the only
-// thing keeping capped videos out of the work total is that they are not added
-// here. If a future state belongs in the total, it goes in on purpose.
+// DELIBERATELY UNCHANGED by the addition of `deferred`, and unchanged again by
+// `blocked`. This function is the guard: adding a state to the union raises no
+// TypeScript error here (the exhaustiveness check lives on the DISPATCH
+// decision, in backfillBatch's candidateAction, which is the branch that can do
+// harm), so the only thing keeping capped and blocked videos out of the work
+// total is that they are not added here. If a future state belongs in the
+// total, it goes in on purpose.
+//
+// A blocked video is emphatically not reachable work: there is nothing this
+// lane can do about it this pass. Counting it would make a corpus with one
+// diarization and 73,000 waiting attributions report 73,000 jobs ready to run.
export function reachableBackfillWork(counts: BackfillCounts): number {
return counts.missing + counts.stale;
}
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,8 @@
# Changelog
## [Unreleased]
+- **"Waiting for an earlier step" is no longer reported as "the media is gone".** Putting names to speakers from the audio needs the speaker-turn capture to have run first. When it hadn't, those videos were reported as *needing their media re-acquired* — which on this corpus is around **73,000 videos**, filed under the one heading that means "fetch the audio again". That was wrong twice over: it was the wrong number in front of the operator, and with re-downloading switched on the system would have spent one download per video fetching audio that **cannot** satisfy the wait, then deleted it again. There is now a distinct **blocked** state for work waiting on an earlier step, counted on its own, never added to the work-to-do total, never handed to a runner, and — crucially — completely unaffected by the re-download setting. The channel card names what each blocked video is waiting for and says plainly that there is nothing to do, because unlike the other numbers **this one falls by itself** as the earlier step runs.
+- **Backfills now declare what they depend on, and run in that order.** A video whose speaker turns were captured during a pass used to become eligible for speaker naming only on some *later* pass that happened to notice the new file on disk. Prerequisites are now declared and ordered, so both happen in the same pass. A mis-declared or circular dependency degrades to the old ordering rather than stalling anything.
- **The automatic downloader now stops when the disk is nearly full — it never did before.** Every download you start by clicking something has checked free space for a long time. The one thing that runs unattended, for days, choosing downloads by itself, did not check at all: there was no mention of disk anywhere in it. That is the process most likely to fill a disk and the least likely to have anyone watching while it does. It now consults the same floor as everything else and simply **goes idle** rather than stopping, so it picks up again on its own once space is free — no restart, nothing to remember. Four more paths that write large files were checked and gated the same way: **re-downloading a single video** (its near-identical sibling, "re-download to archive", already checked — this one had been missed), **persisting kept videos**, which now checks *before each video* instead of once at the start, since it writes full video containers in a loop and the twentieth should not be relying on the first one's headroom, **re-downloading truncated audio**, checked before the old file is deleted rather than after, and the **backfill batch**, whose "disk floor reached" flag was being set and then ignored while it kept asking for more work. Derived files — digests, speaker diarization, attribution — are deliberately **not** gated: they are kilobytes, holding them back frees nothing, and it would throw away days of a multi-week run for no gain.
- **Resuming now needs a little more free space than stopping did, so the pipeline can't flap.** If work restarted at exactly the number that stopped it, the first resumed download would push free space back under the floor, stop again, and oscillate — logging a stop each time, which is precisely when the log needs to stay readable. There is a new **Resume margin** setting (default **2 GB**) that a stopped pipeline has to clear before it starts writing again, so "resumed" means you actually freed something rather than a temporary file being tidied away. Set it to 0 for the old behaviour.
- **A disk stop now says it is a disk stop.** The dashboard had one red "downloads paused" marker, and it could only ever mean the manual pause button — during a real disk stop it said nothing at all, and if both were true you would un-pause and watch nothing happen. Manual pause and disk stop are now **two separate readouts** with their own wording, and the disk one shows how much is free, what the floor is, and — when it is holding for the resume margin — the number it is waiting for. A live free-space readout is now always visible whenever the floor is switched on, not only once something has gone wrong.
diff --git a/editor/app/channels/[slug]/backfillActions.ts b/editor/app/channels/[slug]/backfillActions.ts
@@ -55,6 +55,9 @@ export async function backfillChannelAction(
(batch.deferred > 0
? `; ${batch.deferred} deferred (over the diarization length limit)`
: "") +
+ (batch.blocked > 0
+ ? `; ${batch.blocked} waiting on a prerequisite backfill`
+ : "") +
(batch.reacquired > 0
? `; ${batch.reacquired} re-acquired, ${batch.reacquireCleaned} cleaned up`
: "") +
diff --git a/editor/app/channels/[slug]/components/stages/BackfillStage.tsx b/editor/app/channels/[slug]/components/stages/BackfillStage.tsx
@@ -31,6 +31,13 @@ export type BackfillKindView = {
// diarization duration cap). A THIRD number, never added to the other two:
// summing it would let a capped corpus report as finished.
deferred: number;
+ // Videos waiting on another kind's output. A FOURTH number, never added to
+ // the others either — but unlike the three above it needs nothing from the
+ // operator, because the prerequisite lane brings it down on its own.
+ blocked: number;
+ // Labels of the kinds this one waits on, resolved from the registry's
+ // dependsOn on the server so the copy can name them.
+ dependsOnLabels: string[];
stale: number;
};
@@ -60,6 +67,15 @@ export function BackfillStage({
const reachable = kinds.reduce((n, k) => n + k.reachableIds.length, 0);
const missingInput = kinds.reduce((n, k) => n + k.missingInput, 0);
const deferred = kinds.reduce((n, k) => n + k.deferred, 0);
+ const blocked = kinds.reduce((n, k) => n + k.blocked, 0);
+ // What the blocked videos are waiting for, named. Only kinds that actually
+ // have blocked videos contribute, so the sentence never lists a prerequisite
+ // that is not holding anything up.
+ const blockedOn = [
+ ...new Set(
+ kinds.filter((k) => k.blocked > 0).flatMap((k) => k.dependsOnLabels),
+ ),
+ ];
const allReachableIds = [
...new Set(kinds.flatMap((k) => k.reachableIds)),
].sort();
@@ -100,6 +116,21 @@ export function BackfillStage({
include {deferred === 1 ? "it" : "them"}.
</p>
)}
+ {blocked > 0 && (
+ <p
+ aria-label="backfill blocked"
+ className="mt-1 text-sm text-muted-foreground"
+ >
+ {blocked.toLocaleString()}{" "}
+ {blocked === 1 ? "video is" : "videos are"} waiting on{" "}
+ {blockedOn.length > 0
+ ? blockedOn.join(" and ")
+ : "an earlier backfill"}{" "}
+ and will become available as{" "}
+ {blockedOn.length === 1 ? "it runs" : "those run"} — nothing to do
+ here.
+ </p>
+ )}
</div>
{kinds.length > 1 &&
@@ -114,6 +145,7 @@ export function BackfillStage({
{k.stale > 0 && ` (${k.stale} stale)`} ·{" "}
{k.missingInput.toLocaleString()} needing media
{k.deferred > 0 && ` · ${k.deferred.toLocaleString()} deferred`}
+ {k.blocked > 0 && ` · ${k.blocked.toLocaleString()} blocked`}
</p>
))}
diff --git a/editor/app/channels/[slug]/lib/stageStatus.ts b/editor/app/channels/[slug]/lib/stageStatus.ts
@@ -397,6 +397,12 @@ export function computeStageStatuses(
(n, e) => n + (e.deferred ?? 0),
0,
);
+ // Same treatment again, and the wording matters: this number falls on its own
+ // as the prerequisite lane runs, so it must not read as something to fix.
+ const backfillBlocked = backfillEntries.reduce(
+ (n, e) => n + (e.blocked ?? 0),
+ 0,
+ );
const backfillParts: string[] = [];
if (backfillPending > 0) {
backfillParts.push(
@@ -417,6 +423,11 @@ export function computeStageStatuses(
`${backfillDeferred.toLocaleString()} deferred (too long to diarize)`,
);
}
+ if (backfillBlocked > 0) {
+ backfillParts.push(
+ `${backfillBlocked.toLocaleString()} waiting on an earlier backfill`,
+ );
+ }
const backfill: StageStatus = {
id: "backfill",
title: "Backfill",
diff --git a/editor/app/channels/[slug]/page.tsx b/editor/app/channels/[slug]/page.tsx
@@ -76,7 +76,10 @@ import { TranscodeStage } from "./components/stages/TranscodeStage";
import { TranscribeStage } from "./components/stages/TranscribeStage";
import { DigestStage } from "./components/stages/DigestStage";
import { BackfillStage } from "./components/stages/BackfillStage";
-import { laneBackfillKinds } from "yt-dlp-transcript-common/lib/backfillKinds";
+import {
+ getBackfillKind,
+ laneBackfillKinds,
+} from "yt-dlp-transcript-common/lib/backfillKinds";
import { VideoListPane } from "./components/VideoListPane";
import { VideoListPaneSection } from "./components/VideoListPaneSection";
import { VideoPanel, type VideoFile } from "./videos/[id]/components/VideoPanel";
@@ -429,6 +432,13 @@ export default async function ChannelDetailPage({
// cap existed have no `deferred` field, and .toLocaleString() on
// undefined throws in the render path.
deferred: entry?.deferred ?? 0,
+ // Same reason: every snapshot on disk predates this field.
+ blocked: entry?.blocked ?? 0,
+ // Resolved here, on the server, because BACKFILL_KIND_BY_ID reads
+ // the filesystem and must never reach a client component.
+ dependsOnLabels: (kind.dependsOn ?? [])
+ .map((id) => getBackfillKind(id)?.label)
+ .filter((l): l is string => Boolean(l)),
stale: entry?.stale ?? 0,
};
})}
diff --git a/editor/e2e/attribution.spec.ts b/editor/e2e/attribution.spec.ts
@@ -306,14 +306,25 @@ test("the stage card and /actionable show attribution beside diarization, with t
// component, unmodified, driven by a bigger registry.
//
// The diarized lane can reach ONE video (the one with diarization.json) and
- // reports the other as needing its input re-acquired. That 1-vs-1 split is the
+ // reports the other as BLOCKED on diarization. That 1-vs-1 split is the
// corpus's handful-vs-73,000 in miniature, and the card must never show "2".
+ //
+ // This used to assert "1 needing media", and that was the defect: the second
+ // video is not waiting for media, it is waiting for the diarization lane —
+ // and re-acquiring audio for it could never have helped.
await expect(
section.getByLabel("backfill kind attribution-diarized"),
).toContainText("1 reachable");
await expect(
section.getByLabel("backfill kind attribution-diarized"),
- ).toContainText("1 needing media");
+ ).toContainText("0 needing media");
+ await expect(
+ section.getByLabel("backfill kind attribution-diarized"),
+ ).toContainText("1 blocked");
+ // And the card names what it is waiting for, rather than just saying stuck.
+ await expect(section.getByLabel("backfill blocked")).toContainText(
+ "Speaker diarization",
+ );
// The text lane reaches BOTH: its input is the cue stream, which every
// transcribed video has. That is exactly why running it corpus-wide is the
// expensive option.
@@ -328,16 +339,20 @@ test("the stage card and /actionable show attribution beside diarization, with t
await expect(section.getByRole("heading")).toContainText(
"Backfill derived data (3)",
);
- await expect(section.getByLabel("backfill needs re-acquiring")).toContainText(
- "1",
- );
+ // AND THE RE-ACQUIRE LINE IS GONE, which is the point of the whole change.
+ // Nothing in this fixture needs media fetched: the one video that cannot be
+ // attributed from audio is waiting for the diarization lane, and no download
+ // would have helped it. Before this, that video was reported here and would
+ // have cost a download-and-delete under allowRedownload.
+ await expect(section.getByLabel("backfill needs re-acquiring")).toHaveCount(0);
+ await expect(section.getByLabel("backfill blocked")).toContainText("1");
- // /actionable, which sums across kinds but keeps the two populations in
- // separate columns.
+ // /actionable, which sums across kinds but keeps the populations in separate
+ // columns.
await page.goto("/actionable");
const row = page.getByLabel(`backfill row ${CHANNEL}`);
await expect(row).toBeVisible();
- // 3 reachable and 1 needing media, side by side and never added.
+ // 3 reachable, and still never added to anything else.
await expect(row).toContainText("3");
await expect(page.getByLabel("backfill", { exact: true })).toContainText(
"Needs media",
