commit 318c9d86d2643d89cdc65f4cbab2fe4ce8431ec4
parent 1b92723509047969a1d7a8982f7e1b2d48cfd694
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 25 Sep 2026 11:44:51 -0400
Merge main bfcd2fcf (release 7 slice Y) into one-core/r7-cli
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
20 files changed, 960 insertions(+), 59 deletions(-)
diff --git a/common/controller/autoRunner.ts b/common/controller/autoRunner.ts
@@ -68,11 +68,12 @@ import {
writeAutoQueueState,
} from "../jobs/autoQueueState";
import {
- clearBackoff,
isCoolingDown,
- nextBackoff,
+ isVideoDeferred,
+ pruneDeferred,
pruneExpired,
} from "../jobs/platformBackoff";
+import { applyUnitOutcome } from "../jobs/unitOutcome";
import { type DownloadFailureClass } from "../lib/availability";
import { resolveCookiePolicy } from "../lib/cookiePolicy";
import { isAutoSubsOnly } from "../lib/subtitleProvenance";
@@ -208,6 +209,9 @@ export type AutoRunnerIdleReason =
// Every pending video belongs to a platform inside a rate-limit/network
// cooldown window. Download only.
| "cooldown"
+ // Every pending video left after the platform gates was rate-limited
+ // recently and is deferred (videoDeferrals). Download only.
+ | "deferred"
// No enabled, non-degraded worker slot exists. Transcription only.
| "no-workers"
// The worker pool is pause-all'd. Transcription only, and DISTINCT from
@@ -1152,6 +1156,9 @@ async function runLoop(
// Drop long-lapsed platform cooldowns on boot; entries still in (or recently
// out of) their window are kept so an Odysee 429 cooldown survives a restart.
pruneExpired(kindState.platformBackoff, Date.now());
+ // And lapsed per-video deferrals (live ones survive a restart by design — a
+ // restart must not re-hit the rate-limited video at fails+1).
+ pruneDeferred(kindState.videoDeferrals, Date.now());
// The session's "already done, do not re-pick" set, KEYED BY (operation,
// video) rather than by video.
//
@@ -1617,6 +1624,9 @@ async function runLoop(
// download (busy) OR is in a rate-limit/network backoff window (cooling
// down), so a busy/throttled platform yields to the next-priority free one.
let anyCooling = false;
+ // Download only: some pending video was dropped because it is deferred
+ // after a recent rate limit (see unitOutcome.ts).
+ let anyDeferred = false;
platformSkip.clear();
if (kind === "download") {
// Merge in any cooldown a manual sync/import wrote to the shared state
@@ -1653,6 +1663,17 @@ async function runLoop(
});
}
}
+ // A video rate-limited recently is skipped until its deferral lapses, so
+ // after a cooldown the runner moves on rather than re-picking it.
+ if (Object.keys(kindState.videoDeferrals).length > 0) {
+ for (const leafId of Object.keys(pending)) {
+ pending[leafId] = pending[leafId].filter((id) => {
+ if (!isVideoDeferred(kindState.videoDeferrals, id, now)) return true;
+ anyDeferred = true;
+ return false;
+ });
+ }
+ }
}
// ── THE SCAN COMES BEFORE THE DOWNLOADS ────────────────────────────────
@@ -1738,7 +1759,11 @@ async function runLoop(
// platform gates just took away — the page says which.
if (pendingBeforeGates === 0) live.idleReason = "no-pending";
else if (countPending(pending) === 0) {
- live.idleReason = anyCooling ? "cooldown" : "capped";
+ live.idleReason = anyCooling
+ ? "cooldown"
+ : anyDeferred
+ ? "deferred"
+ : "capped";
} else live.idleReason = "capped";
return null;
}
@@ -1933,33 +1958,27 @@ async function runLoop(
reportRunnerProgress(Math.max(0, lastPending - 1));
}
- // Per-platform backoff: a rate-limit / network failure pauses the whole
- // platform with an exponential cooldown, and the video is deliberately NOT
- // marked completed so it's retried once the cooldown lapses. Any other
- // outcome marks the video done-for-session; a success also clears the
- // platform's cooldown.
- const backoffHit =
- unitPlatform !== null &&
- (result.failureClass === "rate_limit" ||
- result.failureClass === "network");
- if (backoffHit && unitPlatform) {
- const entry = nextBackoff(
- kindState.platformBackoff[unitPlatform],
- Date.now(),
- );
- kindState.platformBackoff[unitPlatform] = entry;
- const secs = Math.round((entry.until - Date.now()) / 1000);
- onLog(
- `Auto-download: ${unitPlatform} ${result.failureClass} — backing off ${secs}s (attempt ${entry.fails}). ${pick.videoId} will retry after cooldown.`,
- );
- } else {
- if (unitPlatform && result.outcome === "transcribed") {
- clearBackoff(kindState.platformBackoff, unitPlatform);
- }
- // An operation lane retires (operation, video) pairs inside
- // runOperationPick — it is the only thing that knows WHICH operation
- // ran — so a bare id here would be a key nothing ever reads.
- if (!isOperationLane(kind)) markCompleted(pick.videoId);
+ // Pacing: a rate-limit / network failure backs the whole platform off,
+ // and a rate-limited video is also deferred so the next pick after the
+ // cooldown is a DIFFERENT video — see unitOutcome.ts. Any other outcome
+ // retires the video for the session; a success clears the cooldown.
+ const effect = applyUnitOutcome(
+ kindState,
+ {
+ platform: unitPlatform,
+ videoId: pick.videoId,
+ channelSlug,
+ outcome: result.outcome,
+ ...(result.failureClass ? { failureClass: result.failureClass } : {}),
+ },
+ Date.now(),
+ );
+ if (effect.line) onLog(effect.line);
+ // An operation lane retires (operation, video) pairs inside
+ // runOperationPick — it is the only thing that knows WHICH operation
+ // ran — so a bare id here would be a key nothing ever reads.
+ if (effect.markCompleted && !isOperationLane(kind)) {
+ markCompleted(pick.videoId);
}
// Persist so the cooldown (and reset) survive a restart.
persist();
diff --git a/common/controller/metadataScanStore.test.ts b/common/controller/metadataScanStore.test.ts
@@ -132,6 +132,43 @@ test("an unchanged upsert skips the write entirely", async () => {
});
});
+test("an identical error a cooldown old refreshes its `at` (25 h later rewrites)", async () => {
+ await withPaths(async (paths) => {
+ const err = { class: "members_only", message: "join", at: T1 };
+ await upsertMetadataScan(paths, SLUG, { errors: { a: err } }, T1);
+ const later = new Date(Date.parse(T1) + 25 * 3_600_000).toISOString();
+ await upsertMetadataScan(
+ paths,
+ SLUG,
+ { errors: { a: { ...err, at: later } } },
+ later,
+ );
+ const scan = await loadMetadataScan(paths, SLUG);
+ assert.equal(scan.errors.a.at, later);
+ assert.equal(scan.updatedAt, later);
+ });
+});
+
+test("an identical error inside the cooldown leaves the file alone (1 h later)", async () => {
+ await withPaths(async (paths) => {
+ const err = { class: "members_only", message: "join", at: T1 };
+ await upsertMetadataScan(paths, SLUG, { errors: { a: err } }, T1);
+ const file = metadataScanPath(paths, SLUG);
+ const before = (await stat(file)).mtimeMs;
+ const later = new Date(Date.parse(T1) + 3_600_000).toISOString();
+ await upsertMetadataScan(
+ paths,
+ SLUG,
+ { errors: { a: { ...err, at: later } } },
+ later,
+ );
+ assert.equal((await stat(file)).mtimeMs, before);
+ const scan = await loadMetadataScan(paths, SLUG);
+ assert.equal(scan.errors.a.at, T1);
+ assert.equal(scan.updatedAt, T1);
+ });
+});
+
test("normalize-on-read drops unknown fields and malformed records", async () => {
await withPaths(async (paths) => {
await writeFile(
diff --git a/common/controller/metadataScanStore.ts b/common/controller/metadataScanStore.ts
@@ -247,7 +247,16 @@ export async function upsertMetadataScan(
// a later pass must not un-scan it.
if (scan.entries[id]) continue;
const prev = scan.errors[id];
- if (!prev || prev.class !== err.class || prev.message !== err.message) {
+ // An IDENTICAL error still rewrites once its `at` is a cooldown old: `at`
+ // is what metadataScanWanted reads, so leaving it stale re-queues the id
+ // at every runner start forever (142 members-only videos re-scanned,
+ // cookie-authed, at each start). Refreshed, the id rests for another day.
+ if (
+ !prev ||
+ prev.class !== err.class ||
+ prev.message !== err.message ||
+ Date.parse(now) - Date.parse(prev.at) >= METADATA_SCAN_ERROR_COOLDOWN_MS
+ ) {
scan.errors[id] = err;
changed = true;
}
diff --git a/common/jobs/autoQueueState.test.ts b/common/jobs/autoQueueState.test.ts
@@ -101,6 +101,7 @@ test("every lane is keyed, and a state file written before a lane existed coerce
runtime: { currentWeights: {} },
picks: [],
platformBackoff: {},
+ videoDeferrals: {},
});
assert.deepEqual(back.backfill.picks, []);
// And the lanes it does name are untouched.
@@ -127,6 +128,46 @@ test("a lane missing from an in-memory state object still writes", async () => {
});
});
+test("videoDeferrals round-trips through write/read; other lanes stay {}", async () => {
+ await withPaths(async (paths) => {
+ const state = emptyAutoQueueState();
+ state.download.videoDeferrals = {
+ a1: { until: 123456, channelSlug: "alpha" },
+ };
+ await writeAutoQueueState(paths, state);
+
+ const back = await readAutoQueueState(paths);
+ assert.deepEqual(back.download.videoDeferrals, {
+ a1: { until: 123456, channelSlug: "alpha" },
+ });
+ assert.deepEqual(back.transcription.videoDeferrals, {});
+ });
+});
+
+test("a state file written before videoDeferrals existed coerces it to {}; corrupt entries drop", async () => {
+ await withPaths(async (paths) => {
+ await writeAutoQueueState(paths, emptyAutoQueueState());
+ await writeFile(
+ paths.autoQueueStateFile,
+ JSON.stringify({
+ download: {
+ runtime: { currentWeights: {} },
+ picks: [],
+ platformBackoff: { youtube: { until: 5, fails: 2 } },
+ },
+ transcription: { videoDeferrals: { x: { until: "nope" }, y: 7 } },
+ }),
+ );
+
+ const back = await readAutoQueueState(paths);
+ assert.deepEqual(back.download.videoDeferrals, {});
+ assert.deepEqual(back.download.platformBackoff, {
+ youtube: { until: 5, fails: 2 },
+ });
+ assert.deepEqual(back.transcription.videoDeferrals, {});
+ });
+});
+
// --- write safety ------------------------------------------------------------
test("overlapping writes do not collide on the tmp file", async () => {
diff --git a/common/jobs/autoQueueState.ts b/common/jobs/autoQueueState.ts
@@ -7,7 +7,9 @@ import {
} from "./autoQueuePolicy";
import {
type PlatformBackoffState,
+ type VideoDeferralState,
coercePlatformBackoff,
+ coerceVideoDeferrals,
} from "./platformBackoff";
// Persistent fairness state for the auto-queue runners. Unlike in-flight worker
@@ -41,6 +43,11 @@ export type AutoQueueKindState = {
// the "download" kind; empty for transcription. Persisted so an Odysee 429
// cooldown survives a server restart instead of re-storming on boot.
platformBackoff: PlatformBackoffState;
+ // Per-video deferrals (a rate-limited video is skipped by the auto-download
+ // pick for VIDEO_RATE_LIMIT_DEFER_MS). Download kind only; `{}` elsewhere.
+ // Persisted so a restart does not re-hit the same video at fails+1. A file
+ // written before the key existed coerces to `{}`; an older build ignores it.
+ videoDeferrals: VideoDeferralState;
};
// KEYED BY EVERY LANE, including the two with no executor yet. A lane whose
@@ -53,7 +60,12 @@ export type AutoQueueState = Record<AutoQueueKind, AutoQueueKindState>;
export const AUTO_QUEUE_PICK_LOG_LIMIT = 50;
export function emptyAutoQueueKindState(): AutoQueueKindState {
- return { runtime: emptyAutoQueueRuntime(), picks: [], platformBackoff: {} };
+ return {
+ runtime: emptyAutoQueueRuntime(),
+ picks: [],
+ platformBackoff: {},
+ videoDeferrals: {},
+ };
}
export function emptyAutoQueueState(): AutoQueueState {
@@ -109,6 +121,7 @@ function coerceKindState(value: unknown): AutoQueueKindState {
runtime: coerceRuntime(r.runtime),
picks,
platformBackoff: coercePlatformBackoff(r.platformBackoff),
+ videoDeferrals: coerceVideoDeferrals(r.videoDeferrals),
};
}
@@ -148,6 +161,7 @@ export async function writeAutoQueueState(
runtime: k.runtime,
picks: k.picks.slice(0, AUTO_QUEUE_PICK_LOG_LIMIT),
platformBackoff: k.platformBackoff ?? {},
+ videoDeferrals: k.videoDeferrals ?? {},
});
const out = Object.fromEntries(
LANES.map((lane) => [lane, trim(state[lane] ?? emptyAutoQueueKindState())]),
diff --git a/common/jobs/platformBackoff.test.ts b/common/jobs/platformBackoff.test.ts
@@ -9,6 +9,12 @@ import {
isCoolingDown,
nextBackoff,
pruneExpired,
+ VIDEO_RATE_LIMIT_DEFER_MS,
+ type VideoDeferralState,
+ coerceVideoDeferrals,
+ deferVideo,
+ isVideoDeferred,
+ pruneDeferred,
} from "./platformBackoff";
// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test common/jobs/platformBackoff.test.ts
@@ -91,3 +97,51 @@ test("coercePlatformBackoff tolerates corrupt/missing shapes", () => {
{ odysee: { until: 5, fails: 2 }, youtube: { until: 7, fails: 1 } },
);
});
+
+test("deferVideo defers a video for 6 h by default, keyed by id", () => {
+ const state: VideoDeferralState = {};
+ deferVideo(state, "v1", "alpha", 1_000);
+ assert.deepEqual(state, {
+ v1: { until: 1_000 + VIDEO_RATE_LIMIT_DEFER_MS, channelSlug: "alpha" },
+ });
+ assert.equal(VIDEO_RATE_LIMIT_DEFER_MS, 6 * 60 * 60_000);
+});
+
+test("deferVideo re-defers from now (a later 429 extends the window)", () => {
+ const state: VideoDeferralState = {};
+ deferVideo(state, "v1", "alpha", 1_000, 10);
+ deferVideo(state, "v1", "alpha", 5_000, 10);
+ assert.equal(state.v1.until, 5_010);
+});
+
+test("isVideoDeferred reflects the until window, per video", () => {
+ const state: VideoDeferralState = { v1: { until: 2_000, channelSlug: "a" } };
+ assert.equal(isVideoDeferred(state, "v1", 1_999), true);
+ assert.equal(isVideoDeferred(state, "v1", 2_000), false);
+ assert.equal(isVideoDeferred(state, "v2", 1_000), false);
+});
+
+test("pruneDeferred drops lapsed deferrals only", () => {
+ const state: VideoDeferralState = {
+ old: { until: 1_000, channelSlug: "a" },
+ edge: { until: 2_000, channelSlug: "a" },
+ live: { until: 3_000, channelSlug: "b" },
+ };
+ pruneDeferred(state, 2_000);
+ assert.deepEqual(Object.keys(state), ["live"]);
+});
+
+test("coerceVideoDeferrals tolerates corrupt/missing shapes", () => {
+ assert.deepEqual(coerceVideoDeferrals(undefined), {});
+ assert.deepEqual(coerceVideoDeferrals("nope"), {});
+ assert.deepEqual(
+ coerceVideoDeferrals({
+ ok: { until: 5, channelSlug: "a" },
+ noSlug: { until: 5 },
+ badUntil: { until: "5", channelSlug: "a" },
+ inf: { until: Infinity, channelSlug: "a" },
+ nul: null,
+ }),
+ { ok: { until: 5, channelSlug: "a" } },
+ );
+});
diff --git a/common/jobs/platformBackoff.ts b/common/jobs/platformBackoff.ts
@@ -92,3 +92,75 @@ export function coercePlatformBackoff(value: unknown): PlatformBackoffState {
}
return out;
}
+
+// ---------------------------------------------------------------------------
+// Per-video deferral (release 7, YouTube lane pacing).
+//
+// A platform cooldown alone does not pace a lane whose queue is ordered: with
+// `order: "listed"` the runner re-picks the SAME video the moment the cooldown
+// lapses, so one video whose subtitle fetch keeps answering 429 climbs `fails`
+// to the 30-minute cap and holds the whole platform (2026-09-24: one Short
+// retried 12x). A rate-limited video is therefore also DEFERRED for
+// VIDEO_RATE_LIMIT_DEFER_MS: the auto-download pick skips it, so after the
+// cooldown the runner moves on to the next video. Persisted beside
+// `platformBackoff` (a restart must not re-hit the same video at fails+1).
+// Manual Sync / download-missing do not consult it.
+
+export type VideoDeferral = {
+ // Epoch ms: the auto-download pick skips this video until this instant.
+ until: number;
+ // The channel the video was picked from — for the status view's link.
+ channelSlug: string;
+};
+
+// Keyed by video id.
+export type VideoDeferralState = Record<string, VideoDeferral>;
+
+export const VIDEO_RATE_LIMIT_DEFER_MS = 6 * 60 * 60_000; // 6 hours
+
+// Defer a video from `now` for `ms`. Mutates in place.
+export function deferVideo(
+ state: VideoDeferralState,
+ videoId: string,
+ channelSlug: string,
+ now: number,
+ ms: number = VIDEO_RATE_LIMIT_DEFER_MS,
+): void {
+ state[videoId] = { until: now + ms, channelSlug };
+}
+
+// True while the video's deferral window is open.
+export function isVideoDeferred(
+ state: VideoDeferralState,
+ videoId: string,
+ now: number,
+): boolean {
+ const entry = state[videoId];
+ return entry !== undefined && entry.until > now;
+}
+
+// Drop deferrals whose window has lapsed. Mutates in place — keeps the map
+// bounded (unlike platform backoff there is no escalation memory to retain).
+export function pruneDeferred(state: VideoDeferralState, now: number): void {
+ for (const [videoId, entry] of Object.entries(state)) {
+ if (entry.until <= now) delete state[videoId];
+ }
+}
+
+// Defensive coercion for the persisted shape; mirrors coercePlatformBackoff.
+export function coerceVideoDeferrals(value: unknown): VideoDeferralState {
+ const out: VideoDeferralState = {};
+ if (!value || typeof value !== "object") return out;
+ for (const [videoId, raw] of Object.entries(value as Record<string, unknown>)) {
+ if (!raw || typeof raw !== "object") continue;
+ const r = raw as Record<string, unknown>;
+ if (
+ typeof r.until === "number" &&
+ Number.isFinite(r.until) &&
+ typeof r.channelSlug === "string"
+ ) {
+ out[videoId] = { until: r.until, channelSlug: r.channelSlug };
+ }
+ }
+ return out;
+}
diff --git a/common/jobs/unitOutcome.test.ts b/common/jobs/unitOutcome.test.ts
@@ -0,0 +1,105 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ BACKOFF_BASE_MS,
+ VIDEO_RATE_LIMIT_DEFER_MS,
+ isVideoDeferred,
+} from "./platformBackoff";
+import { type UnitOutcomeState, applyUnitOutcome } from "./unitOutcome";
+
+// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test jobs/unitOutcome.test.ts
+
+const noJitter = () => 0.5;
+const empty = (): UnitOutcomeState => ({ platformBackoff: {}, videoDeferrals: {} });
+const unit = (
+ videoId: string,
+ over: Partial<Parameters<typeof applyUnitOutcome>[1]> = {},
+) => ({
+ platform: "youtube",
+ videoId,
+ channelSlug: "alpha",
+ outcome: "failed" as const,
+ ...over,
+});
+
+test("rate_limit backs the platform off AND defers the video 6h", () => {
+ const s = empty();
+ const now = 1_000_000;
+ const r = applyUnitOutcome(s, unit("v1", { failureClass: "rate_limit" }), now, noJitter);
+ assert.equal(r.markCompleted, false);
+ assert.deepEqual(s.platformBackoff.youtube, { until: now + BACKOFF_BASE_MS, fails: 1 });
+ assert.deepEqual(s.videoDeferrals.v1, {
+ until: now + VIDEO_RATE_LIMIT_DEFER_MS,
+ channelSlug: "alpha",
+ });
+ assert.equal(
+ r.line,
+ "Auto-download: youtube rate_limit — backing off 60s (attempt 1). v1 deferred 6h; next video after cooldown.",
+ );
+});
+
+test("network backs off only — no deferral, today's line", () => {
+ const s = empty();
+ const r = applyUnitOutcome(s, unit("v1", { failureClass: "network" }), 0, noJitter);
+ assert.equal(r.markCompleted, false);
+ assert.equal(s.platformBackoff.youtube.fails, 1);
+ assert.deepEqual(s.videoDeferrals, {});
+ assert.equal(
+ r.line,
+ "Auto-download: youtube network — backing off 60s (attempt 1). v1 will retry after cooldown.",
+ );
+});
+
+test("transcribed clears the platform cooldown and retires the video", () => {
+ const s = empty();
+ s.platformBackoff.youtube = { until: 10, fails: 3 };
+ const r = applyUnitOutcome(s, unit("v1", { outcome: "transcribed" }), 0, noJitter);
+ assert.deepEqual(r, { markCompleted: true, line: null });
+ assert.deepEqual(s.platformBackoff, {});
+});
+
+test("any other failure retires the video and leaves the cooldown alone", () => {
+ const s = empty();
+ s.platformBackoff.youtube = { until: 10, fails: 3 };
+ const r = applyUnitOutcome(s, unit("v1", { failureClass: "per_video" }), 0, noJitter);
+ assert.deepEqual(r, { markCompleted: true, line: null });
+ assert.deepEqual(s.platformBackoff.youtube, { until: 10, fails: 3 });
+});
+
+test("a non-download unit (no platform) touches neither map", () => {
+ const s = empty();
+ const r = applyUnitOutcome(
+ s,
+ unit("v1", { platform: null, failureClass: "rate_limit" }),
+ 0,
+ noJitter,
+ );
+ assert.deepEqual(r, { markCompleted: true, line: null });
+ assert.deepEqual(s, empty());
+});
+
+test("every branch prunes lapsed deferrals", () => {
+ const s = empty();
+ s.videoDeferrals = {
+ old: { until: 5, channelSlug: "a" },
+ live: { until: 50, channelSlug: "a" },
+ };
+ applyUnitOutcome(s, unit("v1", { outcome: "skipped" }), 10, noJitter);
+ assert.deepEqual(Object.keys(s.videoDeferrals), ["live"]);
+});
+
+test("fails climbs only across DISTINCT ids: the deferred video is not re-picked", () => {
+ // The runner's pick skips a deferred id, so after a cooldown the next 429 is
+ // necessarily a different video — the escalation now measures the platform,
+ // not one stubborn video.
+ const s = empty();
+ let now = 0;
+ const ids = ["v1", "v2", "v3"];
+ for (const id of ids) {
+ assert.equal(isVideoDeferred(s.videoDeferrals, id, now), false);
+ applyUnitOutcome(s, unit(id, { failureClass: "rate_limit" }), now, noJitter);
+ now = s.platformBackoff.youtube.until; // the cooldown lapses
+ }
+ assert.equal(s.platformBackoff.youtube.fails, 3);
+ for (const id of ids) assert.equal(isVideoDeferred(s.videoDeferrals, id, now), true);
+});
diff --git a/common/jobs/unitOutcome.ts b/common/jobs/unitOutcome.ts
@@ -0,0 +1,85 @@
+// What one finished auto-queue unit does to the lane's persisted pacing state —
+// the per-platform backoff and the per-video deferral. Pure: no I/O, `now` and
+// `rand` injected, so the whole decision is unit-testable (the runner's loop
+// that calls it is not exported). autoRunner.ts calls it once per unit, logs the
+// line, retires the video when told to, and persists.
+//
+// - `rate_limit` (HTTP 429 / throttle): the platform backs off exponentially
+// (nextBackoff) AND the video is deferred for VIDEO_RATE_LIMIT_DEFER_MS, so
+// after the cooldown the runner moves on to the NEXT video instead of
+// re-picking the same one. A 429 is per video on YouTube (the subtitle
+// fetch), and with `order: "listed"` the same video sits at the head of the
+// queue: without the deferral one video climbs `fails` to the 30-minute cap
+// and holds the whole platform (2026-09-24, one Short retried 12x).
+// - `network`: backoff only — the video is retried after the cooldown (today's
+// behaviour; a network error says nothing about the video).
+// - either: the video is NOT retired for the session (`markCompleted: false`).
+// - anything else retires it; a success (`transcribed`) also clears the
+// platform's cooldown.
+// - every branch prunes lapsed deferrals, keeping the map bounded.
+
+import type { DownloadFailureClass } from "../lib/availability";
+import {
+ type PlatformBackoffState,
+ type VideoDeferralState,
+ clearBackoff,
+ deferVideo,
+ nextBackoff,
+ pruneDeferred,
+} from "./platformBackoff";
+
+export type UnitOutcomeState = {
+ platformBackoff: PlatformBackoffState;
+ videoDeferrals: VideoDeferralState;
+};
+
+export type FinishedUnit = {
+ // The download platform the unit ran against; null on a non-download lane.
+ platform: string | null;
+ videoId: string;
+ channelSlug: string;
+ outcome: "transcribed" | "skipped" | "failed";
+ failureClass?: DownloadFailureClass;
+};
+
+export type UnitOutcomeEffect = {
+ // Retire the video for the session (the caller still skips this on an
+ // operation lane, which retires (operation, video) pairs itself).
+ markCompleted: boolean;
+ // The line to log, or null when there is nothing to say.
+ line: string | null;
+};
+
+export function applyUnitOutcome(
+ state: UnitOutcomeState,
+ unit: FinishedUnit,
+ now: number,
+ rand: () => number = Math.random,
+): UnitOutcomeEffect {
+ pruneDeferred(state.videoDeferrals, now);
+ const pf = unit.platform;
+ if (
+ pf !== null &&
+ (unit.failureClass === "rate_limit" || unit.failureClass === "network")
+ ) {
+ const entry = nextBackoff(state.platformBackoff[pf], now, rand);
+ state.platformBackoff[pf] = entry;
+ const secs = Math.round((entry.until - now) / 1000);
+ const head = `Auto-download: ${pf} ${unit.failureClass} — backing off ${secs}s (attempt ${entry.fails}).`;
+ if (unit.failureClass === "rate_limit") {
+ deferVideo(state.videoDeferrals, unit.videoId, unit.channelSlug, now);
+ return {
+ markCompleted: false,
+ line: `${head} ${unit.videoId} deferred 6h; next video after cooldown.`,
+ };
+ }
+ return {
+ markCompleted: false,
+ line: `${head} ${unit.videoId} will retry after cooldown.`,
+ };
+ }
+ if (pf !== null && unit.outcome === "transcribed") {
+ clearBackoff(state.platformBackoff, pf);
+ }
+ return { markCompleted: true, line: null };
+}
diff --git a/common/views/activeJobs.test.ts b/common/views/activeJobs.test.ts
@@ -322,6 +322,21 @@ test("a stopped runner reads unavailable, not idle", async () => {
assert.equal(payload.lanes.length, 4);
});
+test("a download runner idling on deferred videos says so on its lane", async () => {
+ const h = harness({
+ runner: (kind) =>
+ kind === "download"
+ ? runnerStatus({ kind, idleReason: "deferred" })
+ : runnerStatus({ kind, idleReason: "no-pending" }),
+ });
+ const payload = await buildActiveJobsPayload(h.inputs);
+ const download = payload.lanes.find((l) => l.label === "Auto-download");
+ assert.equal(
+ download?.note,
+ "every pending video was rate-limited recently and is deferred",
+ );
+});
+
test("the disk block is the gate the shell sampled, with `low` derived", async () => {
const h = harness({
disk: { ...OK_DISK, ok: false, freeBytes: 1, reason: "below-floor", message: "full" },
diff --git a/common/views/activeJobs.ts b/common/views/activeJobs.ts
@@ -13,7 +13,10 @@ import { deriveLaneState } from "./laneState";
import { autoRunnerJobKind } from "../controller/autoRunner";
import { isGateHeld } from "../lib/pauseGates";
import { LANES, type AutoQueueKind } from "../lib/autoQueueTypes";
-import type { AutoRunnerStatus } from "../controller/autoRunner";
+import type {
+ AutoRunnerIdleReason,
+ AutoRunnerStatus,
+} from "../controller/autoRunner";
import type { JobRecord } from "../jobs/registry";
import type { QueueView } from "../jobs/scheduler";
import type { JobMeta } from "../jobs/jobMeta";
@@ -385,7 +388,7 @@ const LANE_LABEL: Record<AutoQueueKind, string> = {
// kept server-side because this payload is consumed by three clients and the
// sentence must be the same in all three.
function autoIdleNote(
- reason: string | null,
+ reason: AutoRunnerIdleReason | null,
kind: AutoQueueKind,
): string | null {
switch (reason) {
@@ -395,6 +398,8 @@ function autoIdleNote(
return "every route to the work is at a worker cap";
case "cooldown":
return "every pending platform is in a rate-limit cooldown";
+ case "deferred":
+ return "every pending video was rate-limited recently and is deferred";
case "no-workers":
return "no enabled worker";
case "workers-paused":
@@ -411,7 +416,10 @@ function autoIdleNote(
return "snoozed";
case "disabled":
return `the ${LANE_LABEL[kind].toLowerCase()} lane is switched off`;
- default:
+ // Exhaustive (no default), so a new idle reason is a compile error here
+ // as it is in the console's idleReasonText.
+ case "stopped":
+ case null:
return null;
}
}
diff --git a/common/views/autoQueueStatus.test.ts b/common/views/autoQueueStatus.test.ts
@@ -98,6 +98,32 @@ test("a cooldown is filtered by the injected clock, newest first", () => {
);
});
+test("deferred videos: live only, soonest first, ties by id, clock-driven", () => {
+ const state = emptyAutoQueueState();
+ // Inserted out of order on purpose; the strip must not depend on key order.
+ state.download.videoDeferrals = {
+ late: { until: NOW + 6 * 3_600_000, channelSlug: "beta" },
+ tieB: { until: NOW + 60_000, channelSlug: "alpha" },
+ lapsed: { until: NOW - 1_000, channelSlug: "alpha" },
+ // Strictly greater-than, like the cooldowns: lapsing exactly now has lapsed.
+ boundary: { until: NOW, channelSlug: "alpha" },
+ tieA: { until: NOW + 60_000, channelSlug: "gamma" },
+ };
+ const payload = buildAutoQueueStatusPayload(inputs({ state }));
+ assert.deepEqual(payload.download.deferred, [
+ { videoId: "tieA", channelSlug: "gamma", untilMs: NOW + 60_000 },
+ { videoId: "tieB", channelSlug: "alpha", untilMs: NOW + 60_000 },
+ { videoId: "late", channelSlug: "beta", untilMs: NOW + 6 * 3_600_000 },
+ ]);
+ assert.deepEqual(payload.transcription.deferred, []);
+ // Past the tied pair, only the 6 h one is left.
+ assert.deepEqual(
+ buildAutoQueueStatusPayload(inputs({ state, now: () => NOW + 60_000 }))
+ .download.deferred,
+ [{ videoId: "late", channelSlug: "beta", untilMs: NOW + 6 * 3_600_000 }],
+ );
+});
+
test("transcription's hold is the pool's, every other lane's is the gate", () => {
// The asymmetry is deliberate and is the reason `pool` is on these inputs at
// all: transcription's hold is LIVE on the worker pool, while every other
diff --git a/common/views/autoQueueStatus.ts b/common/views/autoQueueStatus.ts
@@ -33,6 +33,14 @@ export type PlatformCooldownView = {
fails: number;
};
+// A video the auto-download pick skips until `untilMs` because it was
+// rate-limited recently (see jobs/unitOutcome.ts). Download kind only.
+export type VideoDeferralView = {
+ videoId: string;
+ channelSlug: string;
+ untilMs: number;
+};
+
export type AutoQueueKindStatus = {
kind: AutoQueueKind;
// THE POLICY AS THE LANE DISPATCHES IT, not as it is stored. Every field is
@@ -58,6 +66,9 @@ export type AutoQueueKindStatus = {
picks: AutoQueuePick[];
// Platforms paused by a rate-limit/network backoff (download kind only).
cooldowns: PlatformCooldownView[];
+ // Videos deferred after a rate limit, soonest to return first (download
+ // kind only; empty elsewhere and whenever none is live).
+ deferred: VideoDeferralView[];
// This runner's lane gate — what the page's pause button draws, and what lets
// the operations rail show a runner HOLDING for the first time.
//
@@ -143,6 +154,22 @@ function buildKind(
.filter(([, e]) => e.until > now)
.map(([platform, e]) => ({ platform, untilMs: e.until, fails: e.fails }))
.sort((a, b) => b.untilMs - a.untilMs);
+ const deferred: VideoDeferralView[] = Object.entries(
+ inputs.state[kind].videoDeferrals ?? {},
+ )
+ .filter(([, d]) => d.until > now)
+ .map(([videoId, d]) => ({
+ videoId,
+ channelSlug: d.channelSlug,
+ untilMs: d.until,
+ }))
+ // Ties by id in CODE-POINT order, not localeCompare: YouTube ids are
+ // mixed-case and the strip's order must not depend on the server's locale.
+ .sort(
+ (a, b) =>
+ a.untilMs - b.untilMs ||
+ (a.videoId < b.videoId ? -1 : a.videoId > b.videoId ? 1 : 0),
+ );
return {
kind,
policy,
@@ -154,6 +181,7 @@ function buildKind(
nextUp: pending.nextUp,
picks: inputs.state[kind].picks,
cooldowns,
+ deferred,
// THE ASYMMETRY IS TRANSCRIPTION'S ALONE, and it is not a special case for
// "the first lane": its hold is LIVE on the worker pool, while every other
// lane's flag IS its gate. See lib/pauseGates.ts.
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,9 @@
# Changelog
## [Unreleased]
+- **Auto-download no longer retries the same rate-limited video over and over; it moves on to the next one.** When a download answered HTTP 429, the runner paused the whole platform for a while and then picked the same video again, because it was still first in the queue. Each retry doubled the pause, up to 30 minutes. On 2026-09-24 one YouTube Short was retried 12 times this way and kept YouTube paused all evening. A YouTube 429 comes from the subtitle fetch for one video, not from the whole site. Now a rate-limited video is also **deferred for 6 hours**: auto-download skips it, so when the pause ends the runner takes the next video. The pause still grows only when *different* videos keep hitting the limit. Deferrals are kept in `.auto-queue/state.json` beside the platform cooldowns, so a restart does not retry the video early. The log line reads `… (attempt 1). <id> deferred 6h; next video after cooldown.` A manual Sync or *download missing* ignores deferrals and still fetches the video. When every video left is deferred, the runner reports that it is idle for that reason: "every pending video was rate-limited recently and is deferred".
+- **The cooldown strip on `/operations/download` also lists deferred videos.** It is now a region named *Rate-limit cooldown*, with a *Platforms in cooldown* list (unchanged) and a *Deferred videos* list. Each deferred video links to its page and shows how long it has left (`alpha/a1 — 5h 59m left`). The strip appears when either list has something in it. Times over an hour now read `5h 59m` instead of `359m 58s`.
+- **A video that keeps failing its metadata scan is no longer rescanned at every runner start.** When a scan hit the same error again (members-only, for example), the error's timestamp was not updated. The one-day rest that timestamp controls therefore ran out once and never started again, and each runner start rescanned all of them: 142 members-only videos on one channel, with cookies. The same error seen again after a day now updates the timestamp, so the video waits another day.
- **A video the server answers with HTTP 410 Gone is recorded as removed, not as an error.** Rumble answers a taken-down video with `HTTP Error 410: Gone`; the availability check read that as a generic error (one Rekieta Law Rumble video has said "error" since 2026-08-21), and a download that hit it could stop the batch. It now reads as removed, like "Video unavailable" does, so the check says so and a download skips that one video and carries on. Existing records change the next time the video is checked.
- **umtool's report videos can fetch Rumble clips again.** The clip fetch and the source availability check in `umtool/report-to-video` ran yt-dlp without the browser fingerprint Rumble now requires, so every Rumble clip failed with 403 and every Rumble source looked missing. They now pass the same Rumble arguments as the editor, from the same single table.
- **A transcript pulled back from a remote worker is written safely.** It used to be written straight onto `transcript.json`, so a crash part-way through left a truncated transcript; it now goes through the editor's one atomic write (temp file, then rename), like every other file the editor writes.
diff --git a/editor/app/operations/components/RunnerOperationView.tsx b/editor/app/operations/components/RunnerOperationView.tsx
@@ -1,10 +1,12 @@
"use client";
+import Link from "next/link";
import { useCallback, useEffect, useState } from "react";
import type { AutoQueueKind } from "yt-dlp-transcript-common/jobs/autoQueueState";
import type {
AutoQueueKindStatus,
PlatformCooldownView,
+ VideoDeferralView,
} from "yt-dlp-transcript-common/views/autoQueueStatus";
import type { LaneWorker } from "yt-dlp-transcript-common/views/autoQueueLanes";
import { InFlightList } from "./InFlightList";
@@ -113,8 +115,12 @@ export function RunnerOperationView({
</p>
)}
- {status.cooldowns.length > 0 && (
- <CooldownStrip cooldowns={status.cooldowns} now={now} />
+ {(status.cooldowns.length > 0 || status.deferred.length > 0) && (
+ <CooldownStrip
+ cooldowns={status.cooldowns}
+ deferred={status.deferred}
+ now={now}
+ />
)}
<SnoozeControl
@@ -191,36 +197,80 @@ export function useNow(): number | null {
return now;
}
-// Platforms paused by a rate-limit/network backoff. A manual Sync on one of
-// these is refused until it lapses; the runner skips it meanwhile.
+// Platforms paused by a rate-limit/network backoff, and the videos deferred
+// after a rate limit. A manual Sync on a cooling platform is refused until it
+// lapses; the runner skips it meanwhile. A deferred video is skipped by
+// auto-download only — a manual Sync / download-missing still fetches it.
+//
+// A <div role="region">, NOT a <section>: this sits inside the lane's own
+// <section>, and the structural contract above forbids a nested one. The
+// accessible role and name are the same either way.
function CooldownStrip({
cooldowns,
+ deferred,
now,
}: {
cooldowns: PlatformCooldownView[];
+ deferred: VideoDeferralView[];
now: number | null;
}) {
+ const left = (untilMs: number) =>
+ now === null ? null : Math.max(0, Math.ceil((untilMs - now) / 1000));
return (
- <div className="flex flex-col gap-1 rounded-md border border-warning/30 bg-warning-soft px-3 py-2 text-sm">
- <span className="font-medium text-warning">
- Rate-limit cooldown — auto-download and manual sync are paused on:
- </span>
- <ul className="flex flex-wrap gap-x-4 gap-y-1 text-warning">
- {cooldowns.map((c) => {
- const secs =
- now === null
- ? null
- : Math.max(0, Math.ceil((c.untilMs - now) / 1000));
- return (
- <li key={c.platform} className="tabular-nums">
- <span className="font-mono">{c.platform}</span>
- {secs !== null && (
- <> — {formatCooldown(secs)} left (attempt {c.fails})</>
- )}
- </li>
- );
- })}
- </ul>
+ <div
+ role="region"
+ aria-label="Rate-limit cooldown"
+ className="flex flex-col gap-1 rounded-md border border-warning/30 bg-warning-soft px-3 py-2 text-sm"
+ >
+ {cooldowns.length > 0 && (
+ <>
+ <span className="font-medium text-warning">
+ Rate-limit cooldown — auto-download and manual sync are paused on:
+ </span>
+ <ul
+ aria-label="Platforms in cooldown"
+ className="flex flex-wrap gap-x-4 gap-y-1 text-warning"
+ >
+ {cooldowns.map((c) => {
+ const secs = left(c.untilMs);
+ return (
+ <li key={c.platform} className="tabular-nums">
+ <span className="font-mono">{c.platform}</span>
+ {secs !== null && (
+ <> — {formatCooldown(secs)} left (attempt {c.fails})</>
+ )}
+ </li>
+ );
+ })}
+ </ul>
+ </>
+ )}
+ {deferred.length > 0 && (
+ <>
+ <span className="font-medium text-warning">
+ Deferred videos — skipped by auto-download until:
+ </span>
+ <ul
+ aria-label="Deferred videos"
+ className="flex flex-wrap gap-x-4 gap-y-1 text-warning"
+ >
+ {deferred.map((d) => {
+ const secs = left(d.untilMs);
+ return (
+ <li key={d.videoId} className="tabular-nums">
+ <Link
+ href={`/channels/${d.channelSlug}/videos/${d.videoId}`}
+ className="font-mono underline underline-offset-2 hover:text-brand"
+ >
+ {d.channelSlug}/{d.videoId}
+ </Link>
+ {secs !== null && <> — {formatCooldown(secs)} left</>}
+ </li>
+ );
+ })}
+ </ul>
+ </>
+ )}
</div>
);
}
diff --git a/editor/app/operations/components/dispatch.ts b/editor/app/operations/components/dispatch.ts
@@ -151,6 +151,8 @@ export function idleReasonText(
return "work exists, but every route to it is at a worker cap";
case "cooldown":
return "every pending platform is in a rate-limit cooldown";
+ case "deferred":
+ return "every pending video was rate-limited recently and is deferred";
case "no-workers":
return "no enabled worker to run it";
case "workers-paused":
@@ -199,8 +201,16 @@ export function formatRecency(
return key.estimated ? `≈${iso}` : iso;
}
+// A remaining cooldown: 42s, 3m 21s, 5h 59m. The hours arm is for a 6 h video
+// deferral, which would otherwise read "359m 58s"; past an hour the seconds
+// are noise and are dropped.
export function formatCooldown(secs: number): string {
if (secs < 60) return `${secs}s`;
+ if (secs >= 3600) {
+ const h = Math.floor(secs / 3600);
+ const m = Math.floor((secs % 3600) / 60);
+ return m === 0 ? `${h}h` : `${h}h ${m}m`;
+ }
const m = Math.floor(secs / 60);
const s = secs % 60;
return s === 0 ? `${m}m` : `${m}m ${s}s`;
diff --git a/editor/e2e/fixtures/bin/fake-ytdlp.mjs b/editor/e2e/fixtures/bin/fake-ytdlp.mjs
@@ -444,6 +444,17 @@ async function modeYoutubeSingleUrlManaged(url) {
`youtube-single:${url} cookies=${cookieArg()}\n`,
);
if (cookieGateBlocked(url)) failCookieGate(url);
+ // `dl429`: the 2026-09-24 YouTube shape — the metadata prefetch succeeded
+ // (the prefetch branch has no such sentinel), then the real download's
+ // SUBTITLE fetch answers 429. Per video, not IP-wide: availability.ts
+ // classifies it rate_limit, so the runner backs youtube off AND defers this
+ // one video (pacing.spec.ts).
+ if (url.toLowerCase().includes("dl429")) {
+ process.stderr.write(
+ `ERROR: [youtube] ${url}: Unable to download video subtitles for 'en': HTTP Error 429: Too Many Requests\n`,
+ );
+ process.exit(1);
+ }
process.stdout.write(`[fake-ytdlp] managed single-URL ${id}\n`);
// URL sentinels for no-subs-fallback tests. The sentinels live in the
diff --git a/editor/e2e/pacing.spec.ts b/editor/e2e/pacing.spec.ts
@@ -0,0 +1,227 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { test, expect } from "@playwright/test";
+import { readJson, resetData, resolvePath, writeSettings } from "./helpers";
+import { baseUrl } from "./baseUrl";
+
+// YouTube lane pacing (release 7): a rate-limited video is DEFERRED for 6 h as
+// well as backing the platform off, so after the cooldown the auto-download
+// runner moves on to the NEXT video instead of re-picking the same one. The
+// pure decision is unit-tested (common/jobs/unitOutcome.test.ts); this proves
+// the runner's pick honours a persisted deferral, the lane says why it idles,
+// the page shows it, and a live 429 (the fake's `dl429` sentinel) defers.
+
+type Leaf = { id: string; match: { type: string; value?: string } };
+type Group = { id: string; mode: string; children: (Group | Leaf)[] };
+
+const ONE_WORKER = [
+ {
+ id: "w1",
+ name: "W1",
+ kind: "local",
+ enabled: true,
+ priority: 0,
+ appId: "whisper-cpp",
+ config: {},
+ },
+];
+
+const STATE_FILE = "test-transcripts/.auto-queue/state.json";
+
+// Copied from auto-queue.spec.ts: a YouTube channel with undownloaded videos.
+async function makeDownloadChannel(slug: string, ids: string[]) {
+ const root = resolvePath(`test-transcripts/channels/${slug}`);
+ await mkdir(`${root}/data`, { recursive: true });
+ await writeFile(
+ `${root}/config.json`,
+ JSON.stringify({
+ handling: "youtube",
+ name: slug,
+ url: `https://www.youtube.com/@${slug}/videos`,
+ }),
+ );
+ await writeFile(
+ `${root}/playlist`,
+ ids.map((id) => `https://www.youtube.com/watch?v=${id}`).join("\n") + "\n",
+ );
+ await writeFile(
+ `${root}/snapshot.json`,
+ JSON.stringify({
+ generatedAt: "2026-06-01T00:00:00.000Z",
+ totals: { videos: ids.length, transcribed: 0, downloaded: 0 },
+ buckets: {
+ noTranscript: [],
+ downloadedNoTranscript: [],
+ untranscoded: [],
+ multipleAudioFormats: [],
+ transcribedWithAudio: [],
+ untranscribable: [],
+ noMetadata: [],
+ failedListed: [],
+ missingFromArchive: [],
+ duplicateDirs: [],
+ partialDownloads: [],
+ corruptSource: [],
+ nonStandardVtt: [],
+ skippedByFilter: [],
+ },
+ undownloadedIds: ids,
+ }),
+ );
+}
+
+async function setup(ids: string[]) {
+ await resetData(null);
+ await makeDownloadChannel("alpha", ids);
+ const root: Group = {
+ id: "root",
+ mode: "strict",
+ children: [{ id: "leaf-alpha", match: { type: "channel", value: "alpha" } }],
+ };
+ await writeSettings({
+ adminTitle: "Test Admin",
+ maxTranscriptPageBytes: 8388608,
+ sleepBetweenDownloadsSeconds: 0,
+ minFreeDiskGB: 0,
+ workers: ONE_WORKER,
+ autoQueue: {
+ transcription: {},
+ download: { enabled: true, maxWorkers: null, root },
+ },
+ });
+}
+
+type KindStatus = {
+ runner: { running: boolean; idleReason: string | null };
+ picks: { videoId: string }[];
+ deferred: { videoId: string; channelSlug: string; untilMs: number }[];
+};
+type Status = { download: KindStatus };
+
+async function getStatus(
+ request: import("@playwright/test").APIRequestContext,
+): Promise<Status> {
+ const res = await request.get(`${baseUrl}/api/auto-queue/status`);
+ expect(res.ok()).toBeTruthy();
+ return res.json();
+}
+
+// Chronological pick order (the status log is newest-first).
+function pickOrder(status: Status): string[] {
+ return status.download.picks.map((p) => p.videoId).reverse();
+}
+
+async function control(
+ request: import("@playwright/test").APIRequestContext,
+ action: "start" | "stop",
+) {
+ const res = await request.post(`${baseUrl}/api/auto-queue/control`, {
+ data: { kind: "download", action },
+ });
+ if (action === "start") expect(res.ok()).toBeTruthy();
+}
+
+type PersistedDownload = {
+ platformBackoff: Record<string, { until: number; fails: number }>;
+ videoDeferrals?: Record<string, { until: number; channelSlug: string }>;
+};
+
+async function persistedDownload(): Promise<PersistedDownload | null> {
+ try {
+ return (await readJson<{ download: PersistedDownload }>(STATE_FILE))
+ .download;
+ } catch {
+ return null;
+ }
+}
+
+test.afterEach(async ({ request }) => {
+ await control(request, "stop");
+});
+
+test("a deferred video is skipped by auto-download, and the lane says so", async ({
+ page,
+ request,
+}) => {
+ await setup(["a1", "a2"]);
+ // Seed a live deferral on a1 — as if its download had just answered 429.
+ await mkdir(resolvePath("test-transcripts/.auto-queue"), { recursive: true });
+ await writeFile(
+ resolvePath(STATE_FILE),
+ JSON.stringify({
+ transcription: { runtime: {}, picks: [], platformBackoff: {} },
+ download: {
+ runtime: {},
+ picks: [],
+ platformBackoff: {},
+ videoDeferrals: {
+ a1: { until: Date.now() + 60 * 60_000, channelSlug: "alpha" },
+ },
+ },
+ }),
+ );
+
+ await control(request, "start");
+
+ await expect
+ .poll(async () => (await getStatus(request)).download.runner.idleReason, {
+ timeout: 60_000,
+ })
+ .toBe("deferred");
+ const status = await getStatus(request);
+ // a1 is listed first, and was never picked.
+ expect(pickOrder(status)).toEqual(["a2"]);
+ expect(status.download.deferred.map((d) => d.videoId)).toEqual(["a1"]);
+
+ await page.goto("/operations/download");
+ const region = page.getByRole("region", { name: "Rate-limit cooldown" });
+ await expect(region).toBeVisible({ timeout: 20_000 });
+ const list = region.getByRole("list", { name: "Deferred videos" });
+ await expect(list).toContainText("alpha/a1");
+ // Nothing is cooling, so the platform list is not drawn at all.
+ await expect(
+ region.getByRole("list", { name: "Platforms in cooldown" }),
+ ).toHaveCount(0);
+ // The idle sentence (dispatch.ts) — drawn by the rail and by the lane, so
+ // both say it; one visible is the assertion.
+ await expect(
+ page.getByText(/every pending video was rate-limited recently and is deferred/).first(),
+ ).toBeVisible();
+});
+
+test("a live 429 backs youtube off, defers the video, and the next pick is the next video", async ({
+ request,
+}) => {
+ // A REAL cooldown: base 60 s ±10 %. The whole point is what happens when it
+ // lapses — the runner must move on to a2, not re-pick dl429vid1.
+ test.setTimeout(150_000);
+ await setup(["dl429vid1", "a2"]);
+
+ await control(request, "start");
+
+ await expect
+ .poll(
+ async () => {
+ const d = await persistedDownload();
+ return {
+ deferred: Boolean(d?.videoDeferrals?.dl429vid1),
+ fails: d?.platformBackoff.youtube?.fails ?? 0,
+ };
+ },
+ { timeout: 30_000 },
+ )
+ .toEqual({ deferred: true, fails: 1 });
+ const d = await persistedDownload();
+ expect(d?.videoDeferrals?.dl429vid1?.channelSlug).toBe("alpha");
+ // Deferred for ~6 h, not for the cooldown's minute.
+ expect(d!.videoDeferrals!.dl429vid1.until - Date.now()).toBeGreaterThan(
+ 5 * 60 * 60_000,
+ );
+
+ await expect
+ .poll(async () => pickOrder(await getStatus(request)), {
+ timeout: 100_000,
+ intervals: [2_000],
+ })
+ .toContain("a2");
+ expect(pickOrder(await getStatus(request))).toEqual(["dl429vid1", "a2"]);
+});
diff --git a/plans/release-7.md b/plans/release-7.md
@@ -314,6 +314,83 @@ incomplete; the LM chat-only tier (operator config).
## Record
+### Slice Y, as shipped — YouTube lane pacing (2026-09-25)
+
+Branch `one-core/r7-pacing` off `main` `6ee1d336`. `main` did not move during the slice. Every
+spec anchor was checked on `6ee1d336` before its edit, and all of them held at `0032ed8a` line
+numbers. Items 1 to 8 were built as written. Item 9 (`sleepBetweenDownloadsSeconds`) stays out.
+Item 10: no numbers were taken. The slice fixes two things. First, a rate-limited video is
+**deferred for 6 h** as well as backing its platform off, so when the cooldown lapses the runner
+picks the next video instead of re-picking the same one. The deferral is persisted beside
+`platformBackoff` so a restart honours it. Second, an identical metadata-scan error now refreshes
+its `at` once it is a cooldown old, so members-only ids stop being re-scanned at every runner start.
+
+| sha | what |
+|---|---|
+| `65dca073` | items 1 + 2. `platformBackoff.ts` appends `VideoDeferral {until, channelSlug}`, `VideoDeferralState`, `VIDEO_RATE_LIMIT_DEFER_MS` (6 h), `deferVideo`, `isVideoDeferred`, `pruneDeferred` (drops `until <= now`; there is no retention, since a deferral has no escalation memory) and `coerceVideoDeferrals` (mirrors `coercePlatformBackoff`). `nextBackoff` is untouched. `AutoQueueKindState.videoDeferrals` is added to the empty state, to `coerceKindState` (a missing or corrupt key becomes `{}`) and to the write trim. `platformBackoff.test` +5, `autoQueueState.test` +2; the existing "lane coerces to empty" deepEqual gained the key |
+| `71c755c8` | item 3. New `common/jobs/unitOutcome.ts` `applyUnitOutcome(state, unit, now, rand?) → {markCompleted, line}`. `rate_limit` runs `nextBackoff` + `deferVideo` and logs `Auto-download: <pf> rate_limit — backing off <s>s (attempt <n>). <id> deferred 6h; next video after cooldown.` `network` backs off with today's line and no deferral. `transcribed` calls `clearBackoff`. A unit with no platform (a non-download lane) touches neither map, as before. Every branch prunes. The block at `autoRunner.ts:1941-1963` is now the call + `onLog(line)` + `markCompleted` (still skipped on an operation lane), and `persist()` is unchanged. `unitOutcome.test.ts` has 7 cases, including "`fails` climbs only across distinct ids" |
+| `a9af6a0a` | item 4. The boot prune adds `pruneDeferred`. After the platform gate, the download branch drops pending ids with a live deferral and sets `anyDeferred` only when it actually dropped one. The idle reason is `anyCooling ? "cooldown" : anyDeferred ? "deferred" : "capped"`. `AutoRunnerIdleReason` gains `"deferred"`. `dispatch.ts` `idleReasonText` gains "every pending video was rate-limited recently and is deferred" |
+| `cb041841` | item 5. `metadataScanStore.ts` `upsertMetadataScan` also rewrites an identical error when `Date.parse(now) - Date.parse(prev.at) >= METADATA_SCAN_ERROR_COOLDOWN_MS`. The flush passes `new Date().toISOString()` as `now` (`ytdlp/metadataScan.ts:374-379`). `metadataScanStore.test` +2: 25 h later rewrites `at` and `updatedAt`; 1 h later leaves the file's mtime alone |
+| `04f5c9be` | items 6 + 7. `autoQueueStatus.ts` adds `VideoDeferralView {videoId, channelSlug, untilMs}` and `AutoQueueKindStatus.deferred` (live deferrals only, soonest first). `RunnerOperationView.tsx` draws the strip when there are cooldowns OR deferrals. The strip is `role="region"` `aria-label="Rate-limit cooldown"`. Under the existing heading is `<ul aria-label="Platforms in cooldown">` (items unchanged, drawn only when a platform cools). Then, when there are deferrals, the heading "Deferred videos — skipped by auto-download until:" and `<ul aria-label="Deferred videos">`, whose items read `alpha/a1 — 5h 59m left` with the id a `Link` to the video page, as `NextUp` does. **The region is a `<div role="region">`, not the `<section>` the spec wrote**: the strip sits inside the lane's own `<section>`, and the file's structural contract (`:30-32`) forbids a nested one because every `locator("section", {has})` in the suite would match two ancestors. The accessible role and name are the same. `formatCooldown` gains an hours arm (`5h 59m`, or `6h` when the minutes are zero). Its only caller is this strip, and platform cooldowns cap at ~33 min, so their text does not change |
+| `17dc70c0` | item 8. The fake's `dl429` sentinel is in `modeYoutubeSingleUrlManaged` only: stderr `ERROR: [youtube] <url>: Unable to download video subtitles for 'en': HTTP Error 429: Too Many Requests`, exit 1. The prefetch branch still succeeds, which is the real two-spawn shape. New `editor/e2e/pacing.spec.ts`. T1 seeds `download.videoDeferrals.a1` and checks: picks `["a2"]`, idle `deferred`, `deferred == [a1]`, and the region + `Deferred videos` list containing `alpha/a1` with no `Platforms in cooldown` list. T2 checks: `state.json` shows `videoDeferrals.dl429vid1` (slug `alpha`, > 5 h left) and `platformBackoff.youtube.fails === 1`; then, after the real ~60 s cooldown, picks become `["dl429vid1","a2"]` (`setTimeout(150_000)`) |
+| `ffe71f42` | e2e fix. T1's idle-sentence assertion hit two elements (the rail and the lane both draw the sentence, a strict-mode violation), so it now asserts `.first()` |
+| `5be6ceb5` | this record, `[Unreleased]` bullets, and the correction note at the top of `plans/youtube-lane-pacing.md` |
+| `8094615d` | (review fix) `common/views/activeJobs.ts` `autoIdleNote` gains `case "deferred"` with `dispatch.ts`'s exact sentence. It is now typed `AutoRunnerIdleReason \| null` with no `default` (`"stopped"` and `null` return null), so a new idle reason fails to compile here as it does in `idleReasonText`. `activeJobs.test` +1: a download runner idling `deferred` shows that sentence as the `Auto-download` lane's note |
+| `74c24b1f` | (review fix) `autoQueueStatus.test` +1 for `deferred`. Lapsed and boundary (`until === NOW`) entries drop. Out-of-order input comes out soonest first, and entries tied on `until` are ordered by `videoId`. The shape is `{videoId, channelSlug, untilMs}`, transcription gets `[]`, and a later injected clock drops the tied pair. The builder's tie-break changed from `localeCompare` to code-point order, so the strip's order does not depend on the server locale |
+| *(this commit)* | record: the two review-fix rows and the re-gate |
+
+**Gates** (worktree root, on `ffe71f42`). tsc (`pnpm -r --no-bail --workspace-concurrency=1 exec
+tsc --noEmit`) was clean before every commit. common **1770/1770** = 1754 + 5 (`platformBackoff`)
++ 2 (`autoQueueState`) + 7 (`unitOutcome`) + 2 (`metadataScanStore`). Editor unit **72/72**.
+test:scripts **159 pass + 1 skip**. mcp **219/219**. `pnpm --filter editor exec next build` ok
+(compiled in 27.6 s). `pnpm --filter export exec next build` ok (10.5 s; no dangling links under
+`export/public`). EDITOR e2e, `$T/y-specs.txt` =
+`pacing.spec fetch-window.spec queues.spec rumble-sweep.spec auto-queue.spec lane-runner.spec`
+(all six exist):
+- run 0 (`y-e2e0-misscoped.log`): **aborted at test 6 of 642**. The brief's bare names
+ (`pacing …`) are Playwright path regexes, and `pacing` matches the worktree path
+ `one-core-r7-pacing/`, so every spec was selected. Killed along with its orphan servers on
+ :3711/:3710 and its ollama stub; the list was rewritten with `.spec` suffixes. **A worktree whose
+ path contains a spec's name needs the suffixed form.**
+- run 1 (`y-e2e1.log`): **44 passed, 1 failed, 3.9 min**. T1 failed on its last assertion, the
+ strict-mode double match fixed in `ffe71f42`. T2 passed (1.1 min).
+- run 2 (`y-e2e2.log`, `pacing.spec` alone): **2 passed, 0 failed, 1.4 min**.
+- run 3 (`y-e2e3.log`, the full list on `ffe71f42`): **45 passed, 0 failed, 4.3 min**. No run
+ waited in the queue.
+
+**Numbers: none.** `.auto-queue/state.json` is outside both numbers tools (settings: 1,353 paths;
+files: config/site/sidecars), and no `settings.json`, `site.json` or `config.json` key changed.
+**`videoDeferrals: {}` will appear on all four lanes of `state.json` at the first persist after
+the rollout boot.** That is the expected change outside the md5 baseline (rollout step 4 records
+`.download|keys` before). An older build drops the key on its next write, so rollback is safe.
+
+**Re-gate after the review fixes** (tip `74c24b1f`). `editor/.next/dev` was removed first:
+e2e run 3 had left a truncated generated `validator.ts` there, and it was the only file tsc
+failed on. tsc clean. common **1772/1772** (1770 + 1 `activeJobs` + 1 `autoQueueStatus`). Editor
+unit **72/72**. EDITOR e2e `pacing.spec` alone (`y-e2e4.log`): **2 passed, 0 failed, 1.8 min**.
+The fixes change no runner behaviour, only the `/jobs` note, a test, and a same-`until` tie-break,
+so the other five specs were not rerun. test:scripts, mcp and the builds were not rerun either,
+because the fixes touch no file they cover beyond `common/views`, which tsc checks.
+
+**Found and left.**
+- **Manual Sync and *download missing* do not consult deferrals**, by design. A manual retry of a
+ deferred video still runs, and if it succeeds it clears the platform cooldown. It does not
+ clear the video's deferral: the runner retires a fetched id anyway, because it leaves
+ `undownloadedIds` at the next snapshot. A manual 429 goes through `recordDownloadBackoff` (a
+ platform cooldown only, no deferral). That read-modify-write now carries `videoDeferrals`
+ through, since `readAutoQueueState` coerces it.
+- **The runner does not merge deferrals from disk mid-run** the way it merges `platformBackoff`.
+ Only the runner writes deferrals, so there is nothing outside it to merge. A deferral
+ hand-seeded into `state.json` takes effect at the next runner start, which is what T1 does.
+- *(Fixed on review, `8094615d` / `74c24b1f`:)* `activeJobs.ts` `autoIdleNote` had no `deferred`
+ case, and `autoQueueStatus.test.ts` had no `deferred` case. Ownership of both files was
+ extended to Y for these fixes.
+- The plan file's step 3 (a `plans/FACTS.md` entry: the lane defers a rate-limited video; the
+ cooldown escalates across distinct videos; YouTube 429s are per-video timedtext) is the
+ parent's to write.
+- **Commit trailers** name `Claude Opus 5.5 (1M context)`, as the release-6 implementers did.
+ `implementer-rules.md` still names Fable 5.1.
+
### Slice C, as shipped — the archilyzer CLI, hub deploy path, posts-only fix (2026-09-25)
Branch `one-core/r7-cli` off `main` `6ee1d336`. There is now one command line, `common/bin/archilyzer.ts`,
@@ -420,4 +497,5 @@ project, no hub project and no homepage build.
the ownership list by name.
- **Commit trailers** name `Claude Opus 5.5 (1M context)`, as in releases 5 and 6.
+
## Rollout
diff --git a/plans/youtube-lane-pacing.md b/plans/youtube-lane-pacing.md
@@ -1,5 +1,14 @@
# Plan: one YouTube video must not keep the whole platform in a 429 cooldown
+> **Corrected by `plans/release-7.md` "## Slice Y" and shipped as "### Slice Y, as shipped"
+> there (2026-09-25).** The deferral is PERSISTED as `videoDeferrals` beside `platformBackoff`
+> in `.auto-queue/state.json`, not in memory (a rollout restart would otherwise re-hit the video
+> at `fails+1`). The outcome logic moved to a pure seam, `common/jobs/unitOutcome.ts`, because
+> `runLoop` is not exported. An all-deferred lane idles with its own reason, `deferred`, not
+> `cooldown`. The page lists deferred videos. The identical-scan-error `at` refresh is also in
+> the slice. `sleepBetweenDownloadsSeconds` stays out. The steps and tests below are the
+> original plan; read the release-7 record for what shipped.
+
**Found 2026-09-25**, from STATE.md: "YouTube held a 429 cooldown across three attempts (18:43,
19:13, 19:41)", and the owed md5-sweep sync was refused all three times. Verified read-only on
`main` `93dcb532`, the live `settings.json` and the retained `transcripts/.jobs/*` (meta
