commit 6d65595ff74f1c219f9162f844a008d757728533
parent 69129b97d545fb250ea0e867b723f4134f3dfcfe
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 2 Oct 2026 01:40:22 -0400
Merge main (9ee7f1ee: RL) into r17/media-tier-mover
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
# Conflicts:
# editor/CHANGELOG.md
# plans/release-17.md
Diffstat:
50 files changed, 3624 insertions(+), 97 deletions(-)
diff --git a/SETTINGS.md b/SETTINGS.md
@@ -21,6 +21,7 @@ A copied example PINS every default it spells — including each lane's `autoQue
| [`cookieMode`](#cookiemode) | `"when-required"` |
| [`social`](#social) | object — see below |
| [`sleepBetweenDownloadsSeconds`](#sleepbetweendownloadsseconds) | `10` |
+| [`pacing`](#pacing) | object — see below |
| [`downloadFormat`](#downloadformat) | `"auto"` |
| [`minFreeDiskGB`](#minfreediskgb) | `5` |
| [`resumeMarginGB`](#resumemargingb) | `2` |
@@ -185,10 +186,34 @@ Default:
## `sleepBetweenDownloadsSeconds`
-Pause (seconds) inserted between per-video yt-dlp invocations in managed batch downloads. yt-dlp's own `-t sleep` only paces requests within one invocation, so without this the managed loop hammers the source IP back-to-back. 0 disables. Per-channel override available.
+Pause (seconds) inserted between per-video yt-dlp invocations in managed batch downloads, and between two auto-download units on one platform (release 17; the lane ignored it before). yt-dlp's own `-t sleep` only paces requests within one invocation, so without this the managed loop hammers the source IP back-to-back. The adaptive pace above its base (see `pacing`) is added to it. 0 disables. Per-channel override available for batch downloads.
Default: `10`
+## `pacing`
+
+How the download pace adapts to rate limits, per platform (release 17). Every yt-dlp spawn against a platform paces its requests (`--sleep-requests`) at the platform's adaptive pace, and the download lane waits sleepBetweenDownloadsSeconds plus the pace above its fixed value between units. A rate limit doubles the pace; clean units ease it back; a rate limit that outlasts the cooldown cap holds the platform to one probe at a time. The live pace, cooldowns and holds are in `.auto-queue/state.json`, shown on /operations/download. See common/jobs/platformBackoff.ts.
+
+#### `pacing`
+
+| Key | Default | Description |
+|---|---|---|
+| `sleepRequestsCapSeconds` | `16` | Ceiling (seconds) on a platform's adaptive `--sleep-requests`. The pace starts at the platform's fixed value (1 s for YouTube and Rumble) and doubles on every platform-level rate limit until it reaches this. A subtitle-only 429 never moves it. Clamped 1–120; default 16. |
+| `decayAfterCleanUnits` | `5` | How many clean auto-download units on a platform halve its pace one step back toward the fixed value. Clamped 1–1000; default 5. |
+| `holdAfterFailsAtCap` | `3` | How many consecutive failures AT the 30-minute cooldown cap put a platform in a hold: the lane then runs one probe unit per holdProbeMinutes instead of one per cooldown, and a manual Sync or download on it is refused with the next probe's time. A clean probe clears the hold and the backoff. Clamped 1–100; default 3. |
+| `holdProbeMinutes` | `60` | Minutes between probes while a platform is held. Clamped 1–1440; default 60. |
+
+Default:
+
+```json
+{
+ "sleepRequestsCapSeconds": 16,
+ "decayAfterCleanUnits": 5,
+ "holdAfterFailsAtCap": 3,
+ "holdProbeMinutes": 60
+}
+```
+
## `downloadFormat`
Default yt-dlp `-f` download format for every channel that doesn't set its own (ChannelConfig.downloadFormat). "auto" picks per-source: `original` for Odysee (whose HLS rungs are CDN-truncated), `bestaudio/worst` elsewhere. See common/ytdlp/downloadFormat.ts.
diff --git a/common/bin/doctor.test.ts b/common/bin/doctor.test.ts
@@ -590,3 +590,40 @@ test("the drive-health timings: applied from settings.storage.health before the
assert.equal(healthTimings().budgetMs, 5000, "applied to this process");
applyHealthTimings();
});
+
+test("download pacing: a held platform and a raised pace warn, and nothing is written (release 17, RL)", async () => {
+ const c = checkout();
+ const stateFile = path.join(c.paths.transcriptsDir, ".auto-queue", "state.json");
+ (c.paths as { autoQueueStateFile: string }).autoQueueStateFile = stateFile;
+ mkdirSync(path.dirname(stateFile), { recursive: true });
+ const now = Date.UTC(2026, 9, 1, 12, 0);
+ writeFileSync(
+ stateFile,
+ JSON.stringify({
+ download: {
+ platformBackoff: {
+ youtube: { until: now + 42 * 60_000, fails: 8 },
+ rumble: { until: now + 90_000, fails: 2 },
+ },
+ platformHolds: { youtube: { since: now - 60_000, probeAt: now + 42 * 60_000, rateLimited: true } },
+ platformPace: { youtube: { sleepRequestsSeconds: 4, baseSeconds: 1, cleanUnits: 1, steppedAt: now } },
+ },
+ }),
+ );
+ const before = tree(c.root);
+ const r = await run(c, {}, { now: new Date(now) });
+ assert.equal(r.ok, true, renderDoctorReport(r));
+ const dp = r.checks.filter((x) => x.section === "download pacing");
+ assert.deepEqual(
+ dp.map((x) => [x.id, x.status]),
+ [
+ ["youtube", "warn"],
+ ["rumble", "warn"],
+ ["youtube pace", "warn"],
+ ],
+ );
+ assert.match(dp[0].detail, /^held since .* after 8 rate-limited\/network failures in a row; next probe in 42 min/);
+ assert.match(dp[1].detail, /^in a rate-limit cooldown for 2 min more \(attempt 2\)$/);
+ assert.match(dp[2].detail, /^4s between requests \(base 1s\)/);
+ assert.deepEqual(tree(c.root), before);
+});
diff --git a/common/bin/doctor.ts b/common/bin/doctor.ts
@@ -183,6 +183,51 @@ export async function collectDoctorReport(deps: DoctorDeps): Promise<DoctorRepor
}
}
+ // ── download pacing ──────────────────────────────────────────────────────
+ // The auto-download lane's persisted rate-limit state (release 17, slice
+ // RL): a platform in a cooldown or a HOLD, and a request pace a rate limit
+ // raised. A warning, never a failure — the lane is doing what it should, and
+ // the operator may want to know why YouTube has been quiet all evening.
+ // Read-only: the state reader never writes.
+ const DP = "download pacing";
+ if (paths.autoQueueStateFile && existsSync(paths.autoQueueStateFile)) {
+ const { readAutoQueueState } = await import("../jobs/autoQueueState");
+ const st = (await readAutoQueueState(paths)).download;
+ const nowMs = (deps.now ?? new Date()).getTime();
+ const mins = (ms: number) => `${Math.max(1, Math.ceil(ms / 60_000))} min`;
+ let quiet = true;
+ for (const [pf, e] of Object.entries(st.platformBackoff)) {
+ const hold = st.platformHolds[pf];
+ if (hold) {
+ quiet = false;
+ add(DP, pf, "warn",
+ `held since ${stamp(new Date(hold.since))} after ${e.fails} ${hold.rateLimited ? "rate-limited/network" : "network"} failures in a row; ` +
+ (e.until > nowMs
+ ? `next probe in ${mins(e.until - nowMs)} — manual fetches on it are refused until then`
+ : "probe overdue — a clean manual Sync lifts it, or Clear hold on /operations/download"));
+ } else if (e.until > nowMs) {
+ quiet = false;
+ add(DP, pf, "warn", `in a rate-limit cooldown for ${mins(e.until - nowMs)} more (attempt ${e.fails})`);
+ }
+ }
+ const { effectivePaceSeconds } = await import("../jobs/platformBackoff");
+ for (const [pf, p] of Object.entries(st.platformPace)) {
+ const v = effectivePaceSeconds(p, nowMs);
+ if (v <= p.baseSeconds) continue;
+ quiet = false;
+ add(DP, `${pf} pace`, "warn",
+ `${v}s between requests (base ${p.baseSeconds}s) — a rate limit raised it; it eases one step per hour with no rate limit, and one step per run of clean units (${p.cleanUnits} so far)`);
+ }
+ const subs = Object.values(st.subtitleDeferrals).filter((d) => d.until > nowMs);
+ if (subs.length > 0) {
+ add(DP, "subtitles", "info",
+ `${subs.length} video(s) with subtitles deferred after a subtitle 429 (${subs.filter((d) => d.count >= 3).length} left alone for 7 days) — /operations/download lists them`);
+ }
+ if (quiet) add(DP, "platforms", "ok", "no platform in a cooldown or a hold; every pace at its base");
+ } else {
+ add(DP, "platforms", "info", "no auto-queue state yet — nothing has been rate-limited");
+ }
+
// ── settings ─────────────────────────────────────────────────────────────
const S = "settings";
const settingsText = readOrNull(paths.settingsFile);
diff --git a/common/controller/autoRunner.ts b/common/controller/autoRunner.ts
@@ -70,12 +70,17 @@ import {
writeAutoQueueState,
} from "../jobs/autoQueueState";
import {
+ currentPaceSeconds,
+ downloadGapMs,
isCoolingDown,
+ isPlatformHeld,
isVideoDeferred,
mergeBackoffEntry,
pruneDeferred,
- pruneExpired,
+ prunePlatformPacing,
+ pruneSubtitleDeferrals,
} from "../jobs/platformBackoff";
+import { staticSleepRequestsSeconds } from "../ytdlp/platformArgs.mjs";
import { applyUnitOutcome } from "../jobs/unitOutcome";
import { type DownloadFailureClass } from "../lib/availability";
import { resolveCookiePolicy } from "../lib/cookiePolicy";
@@ -218,6 +223,14 @@ export type AutoRunnerIdleReason =
// Every pending video left after the platform gates was rate-limited
// recently and is deferred (videoDeferrals). Download only.
| "deferred"
+ // Every pending platform is HELD: its rate limit outlasted the cooldown cap,
+ // and the lane runs one probe per `pacing.holdProbeMinutes` until one comes
+ // back clean (release 17, slice RL). Download only.
+ | "held"
+ // Every pending platform is inside the gap between two units:
+ // `sleepBetweenDownloadsSeconds` plus the adaptive pace above its base
+ // (release 17, slice RL). Download only.
+ | "paced"
// No enabled, non-degraded worker slot exists. Transcription only.
| "no-workers"
// The worker pool is pause-all'd. Transcription only, and DISTINCT from
@@ -1170,7 +1183,10 @@ async function runLoop(
const runtime = kindState.runtime;
// Drop long-lapsed platform cooldowns on boot; entries still in (or recently
// out of) their window are kept so an Odysee 429 cooldown survives a restart.
- pruneExpired(kindState.platformBackoff, Date.now());
+ // A HELD platform's entry is kept however old: a hold ends with a clean
+ // probe, not with time (release 17, slice RL).
+ prunePlatformPacing(kindState, Date.now());
+ pruneSubtitleDeferrals(kindState.subtitleDeferrals, Date.now());
// And lapsed per-video deferrals (live ones survive a restart by design — a
// restart must not re-hit the rate-limited video at fails+1).
pruneDeferred(kindState.videoDeferrals, Date.now());
@@ -1251,6 +1267,13 @@ async function runLoop(
// "unknown" for unrecognized hosts). Unused for transcription.
const platformInFlight = new Map<string, number>();
const PER_PLATFORM_CAP = 1;
+ // THE GAP BETWEEN TWO UNITS ON ONE PLATFORM (release 17, slice RL): the epoch
+ // ms before which the platform takes no new unit. Set when a unit settles to
+ // `sleepBetweenDownloadsSeconds` plus the adaptive pace above its base
+ // (downloadGapMs) — the lane used to start the next unit the moment one
+ // settled, ignoring the setting every batch download honours. In memory: a
+ // restart is itself a gap.
+ const platformNextStartAt = new Map<string, number>();
// Channels this runner has already scanned (or tried to). See the pre-pick in
// next() for why a scan is never retried inside one runner's lifetime.
const scannedThisRun = new Set<string>();
@@ -1642,6 +1665,10 @@ async function runLoop(
// Download only: some pending video was dropped because it is deferred
// after a recent rate limit (see unitOutcome.ts).
let anyDeferred = false;
+ // Download only: a platform skipped for its hold (one probe at a time) or
+ // for the gap between units — each its own answer to "why idle?".
+ let anyHeld = false;
+ let anyPaced = false;
platformSkip.clear();
if (kind === "download") {
// Merge in any cooldown a manual sync/import wrote to the shared state
@@ -1669,8 +1696,16 @@ async function runLoop(
platformSkip.add(pf);
// A platform skipped for a cooldown is a different answer to "why is
// it idle?" than one skipped for being busy, so the two are tracked
- // apart rather than both reading as "capped".
- anyCooling = true;
+ // apart rather than both reading as "capped". A held platform's
+ // cooldown is its next probe.
+ if (isPlatformHeld(kindState.platformHolds, pf)) anyHeld = true;
+ else anyCooling = true;
+ }
+ }
+ for (const [pf, at] of platformNextStartAt) {
+ if (at > now && !platformSkip.has(pf)) {
+ platformSkip.add(pf);
+ anyPaced = true;
}
}
if (platformSkip.size > 0) {
@@ -1779,9 +1814,13 @@ async function runLoop(
else if (countPending(pending) === 0) {
live.idleReason = anyCooling
? "cooldown"
- : anyDeferred
- ? "deferred"
- : "capped";
+ : anyHeld
+ ? "held"
+ : anyPaced
+ ? "paced"
+ : anyDeferred
+ ? "deferred"
+ : "capped";
} else live.idleReason = "capped";
return null;
}
@@ -1985,6 +2024,8 @@ async function runLoop(
// and a rate-limited video is also deferred so the next pick after the
// cooldown is a DIFFERENT video — see unitOutcome.ts. Any other outcome
// retires the video for the session; a success clears the cooldown.
+ const settledAt = Date.now();
+ const unitSettings = getSettings();
const effect = applyUnitOutcome(
kindState,
{
@@ -1994,9 +2035,23 @@ async function runLoop(
outcome: result.outcome,
...(result.failureClass ? { failureClass: result.failureClass } : {}),
},
- Date.now(),
+ settledAt,
+ Math.random,
+ unitSettings.pacing,
);
if (effect.line) onLog(effect.line);
+ // The gap before this platform's next unit, at the pace this outcome
+ // left it (release 17, slice RL).
+ if (unitPlatform) {
+ const base = staticSleepRequestsSeconds(unitPlatform);
+ const gap = downloadGapMs(
+ unitSettings.sleepBetweenDownloadsSeconds,
+ currentPaceSeconds(kindState.platformPace, unitPlatform, base),
+ base,
+ );
+ if (gap > 0) platformNextStartAt.set(unitPlatform, settledAt + gap);
+ else platformNextStartAt.delete(unitPlatform);
+ }
// An operation lane retires (operation, video) pairs inside
// runOperationPick — it is the only thing that knows WHICH operation
// ran — so a bare id here would be a key nothing ever reads.
@@ -2292,7 +2347,13 @@ async function launchUnit(args: LaunchArgs): Promise<UnitResult> {
// Complete-but-malformed source: terminal and kept on disk. Treat as skipped
// (not failed) so it doesn't drive backoff and isn't re-picked for download.
if (unitStatus === "corrupt-full-source") return { outcome: "skipped" };
- if (unitStatus && unitStatus.startsWith("ok")) return { outcome: "transcribed" };
+ // A success whose SUBTITLE fetch alone answered 429 says so: the runner
+ // defers the video's subtitles and leaves the platform alone (release 17).
+ if (unitStatus && unitStatus.startsWith("ok")) {
+ return unitFailureClass === "subs_rate_limit"
+ ? { outcome: "transcribed", failureClass: "subs_rate_limit" }
+ : { outcome: "transcribed" };
+ }
// failureClass drives the runner's per-platform backoff (rate_limit/network).
return { outcome: "failed", failureClass: unitFailureClass };
}
diff --git a/common/controller/checkAvailability.ts b/common/controller/checkAvailability.ts
@@ -27,7 +27,7 @@ import { getSettings } from "../lib/settings";
// digest registry's claude-code lane, which has the same problem.
import { parseStdoutJson } from "../lib/parseStdoutJson";
import { readChannelConfig } from "./channels";
-import { channelExtraArgs, platformArgs } from "../ytdlp/channelArgs";
+import { channelExtraArgs, pacedPlatformArgs } from "../ytdlp/channelArgs";
import { detectPlatform } from "../lib/platform";
import { resolveShardItems } from "./shard";
import type { Paths } from "../lib/paths";
@@ -250,7 +250,8 @@ export async function runAvailabilityCheck({
? channelExtraArgs(config, probeCookies)
: [
...cookieArgs(probeCookies),
- ...platformArgs(detectPlatform(url)),
+ // At the platform's current pace (release 17, slice RL).
+ ...pacedPlatformArgs(detectPlatform(url)),
]),
"--",
url,
diff --git a/common/jobs/autoQueueState.test.ts b/common/jobs/autoQueueState.test.ts
@@ -102,6 +102,9 @@ test("every lane is keyed, and a state file written before a lane existed coerce
picks: [],
platformBackoff: {},
videoDeferrals: {},
+ platformPace: {},
+ platformHolds: {},
+ subtitleDeferrals: {},
});
assert.deepEqual(back.backfill.picks, []);
// And the lanes it does name are untouched.
@@ -168,6 +171,42 @@ test("a state file written before videoDeferrals existed coerces it to {}; corru
});
});
+test("the pacing memory round-trips; a file written before it coerces to {} (release 17, RL)", async () => {
+ await withPaths(async (paths) => {
+ const state = emptyAutoQueueState();
+ state.download.platformPace = {
+ youtube: { sleepRequestsSeconds: 4, baseSeconds: 1, cleanUnits: 2, steppedAt: 7 },
+ };
+ state.download.platformHolds = { youtube: { since: 10, probeAt: 20, rateLimited: false } };
+ state.download.subtitleDeferrals = {
+ v1: { count: 2, lastAt: 5, until: 6, channelSlug: "alpha" },
+ };
+ await writeAutoQueueState(paths, state);
+ const back = await readAutoQueueState(paths);
+ assert.deepEqual(back.download.platformPace, state.download.platformPace);
+ assert.deepEqual(back.download.platformHolds, state.download.platformHolds);
+ assert.deepEqual(back.download.subtitleDeferrals, state.download.subtitleDeferrals);
+ assert.deepEqual(back.transcription.platformPace, {});
+
+ await writeFile(
+ paths.autoQueueStateFile,
+ JSON.stringify({
+ download: {
+ platformPace: { youtube: { sleepRequestsSeconds: "x" }, rumble: { sleepRequestsSeconds: 2 } },
+ platformHolds: { youtube: { since: 1 } },
+ subtitleDeferrals: { v1: { count: 0, lastAt: 1, until: 2, channelSlug: "a" } },
+ },
+ }),
+ );
+ const old = await readAutoQueueState(paths);
+ assert.deepEqual(old.download.platformPace, {
+ rumble: { sleepRequestsSeconds: 2, baseSeconds: 0, cleanUnits: 0, steppedAt: 0 },
+ });
+ assert.deepEqual(old.download.platformHolds, {});
+ assert.deepEqual(old.download.subtitleDeferrals, {});
+ });
+});
+
// --- write safety ------------------------------------------------------------
test("overlapping writes do not collide on the tmp file", async () => {
diff --git a/common/jobs/autoQueueState.ts b/common/jobs/autoQueueState.ts
@@ -7,9 +7,16 @@ import {
} from "./autoQueuePolicy";
import {
type PlatformBackoffState,
+ type PlatformHoldState,
+ type PlatformPaceState,
+ type SubtitleDeferralState,
type VideoDeferralState,
coercePlatformBackoff,
+ coercePlatformHolds,
+ coercePlatformPace,
+ coerceSubtitleDeferrals,
coerceVideoDeferrals,
+ effectivePaceSeconds,
} from "./platformBackoff";
// Persistent fairness state for the auto-queue runners. Unlike in-flight worker
@@ -48,6 +55,17 @@ export type AutoQueueKindState = {
// Persisted so a restart does not re-hit the same video at fails+1. A file
// written before the key existed coerces to `{}`; an older build ignores it.
videoDeferrals: VideoDeferralState;
+ // THE PACING MEMORY (release 17, slice RL) — download kind only, `{}`
+ // elsewhere; a file written before a key existed coerces it to `{}` and an
+ // older build drops it on its next write. See platformBackoff.ts.
+ // Per-platform `--sleep-requests` above the static value, while it is above.
+ platformPace: PlatformPaceState;
+ // Platforms whose rate limit outlasted the cooldown cap: one probe per
+ // `pacing.holdProbeMinutes` until a clean one.
+ platformHolds: PlatformHoldState;
+ // Videos whose SUBTITLE fetch answered 429 (their media did not): the count,
+ // and until when the batch subtitle fetch leaves them alone.
+ subtitleDeferrals: SubtitleDeferralState;
};
// KEYED BY EVERY LANE, including the two with no executor yet. A lane whose
@@ -65,6 +83,9 @@ export function emptyAutoQueueKindState(): AutoQueueKindState {
picks: [],
platformBackoff: {},
videoDeferrals: {},
+ platformPace: {},
+ platformHolds: {},
+ subtitleDeferrals: {},
};
}
@@ -122,6 +143,9 @@ function coerceKindState(value: unknown): AutoQueueKindState {
picks,
platformBackoff: coercePlatformBackoff(r.platformBackoff),
videoDeferrals: coerceVideoDeferrals(r.videoDeferrals),
+ platformPace: coercePlatformPace(r.platformPace),
+ platformHolds: coercePlatformHolds(r.platformHolds),
+ subtitleDeferrals: coerceSubtitleDeferrals(r.subtitleDeferrals),
};
}
@@ -137,9 +161,11 @@ export async function readAutoQueueState(paths: Paths): Promise<AutoQueueState>
}
if (!raw || typeof raw !== "object") return emptyAutoQueueState();
const r = raw as Record<string, unknown>;
- return Object.fromEntries(
+ const state = Object.fromEntries(
LANES.map((lane) => [lane, coerceKindState(r[lane])]),
) as AutoQueueState;
+ notePace(state);
+ return state;
}
// Write the state atomically (tmp + rename), creating the .auto-queue dir on
@@ -162,10 +188,14 @@ export async function writeAutoQueueState(
picks: k.picks.slice(0, AUTO_QUEUE_PICK_LOG_LIMIT),
platformBackoff: k.platformBackoff ?? {},
videoDeferrals: k.videoDeferrals ?? {},
+ platformPace: k.platformPace ?? {},
+ platformHolds: k.platformHolds ?? {},
+ subtitleDeferrals: k.subtitleDeferrals ?? {},
});
const out = Object.fromEntries(
LANES.map((lane) => [lane, trim(state[lane] ?? emptyAutoQueueKindState())]),
) as AutoQueueState;
+ notePace(out);
await writeJsonAtomic(paths.autoQueueStateFile, out, { mkdir: true });
}
@@ -181,6 +211,9 @@ type AutoQueueStateHolder = {
// receives ONE object.
loading: Promise<AutoQueueState> | null;
loadingFile: string | null;
+ // The download lane's pace as last read or written, by ANY reader of the
+ // file — for `livePlatformPaceSeconds` when no runner holds the object.
+ lastPace: PlatformPaceState | null;
};
declare global {
@@ -195,11 +228,32 @@ function getHolder(): AutoQueueStateHolder {
stateFile: null,
loading: null,
loadingFile: null,
+ lastPace: null,
};
}
return globalThis.__yttAutoQueueState__;
}
+function notePace(state: AutoQueueState): void {
+ getHolder().lastPace = state.download?.platformPace ?? {};
+}
+
+// THE PACE A SPAWN USES, SYNCHRONOUSLY (release 17, slice RL). Every yt-dlp
+// argv in the repo is built by `channelExtraArgs`, which is synchronous and is
+// called from a dozen places; threading an awaited state read through each
+// would be a dozen chances to forget one. So this answers from what the process
+// already holds: the shared object when a runner holds it (the one every lane
+// and `recordDownloadBackoff` write through), else the pace the last read or
+// write of the file saw (the status poll reads it every few seconds), else
+// undefined — the caller then uses the static pace. Never reads the disk.
+export function livePlatformPaceSeconds(platform: string): number | undefined {
+ const holder = getHolder();
+ const pace = holder.state?.download?.platformPace ?? holder.lastPace;
+ const entry = pace?.[platform];
+ // With the hourly easing applied (platformBackoff.ts, PACE_TIME_DECAY_MS).
+ return entry ? effectivePaceSeconds(entry, Date.now()) : undefined;
+}
+
// THE PERSISTED STATE IS ONE OBJECT FOR EVERY LANE'S RUNNER, and it has to be.
//
// `writeAutoQueueState` serializes the WHOLE file — all four lanes — so two
diff --git a/common/jobs/downloadBackoff.test.ts b/common/jobs/downloadBackoff.test.ts
@@ -5,9 +5,15 @@ import { tmpdir } from "node:os";
import path from "node:path";
import type { Paths } from "../lib/paths";
import {
+ clearSubtitleDeferral,
+ heldPlatformRefusal,
platformCooldownRemainingMs,
+ readSubtitleDeferrals,
recordDownloadBackoff,
+ recordSubtitleDeferral,
} from "./downloadBackoff";
+import { PACING_DEFAULTS } from "./platformBackoff";
+import { clearPlatformHold, recordPlatformClean } from "./downloadBackoff";
import { BACKOFF_BASE_MS } from "./platformBackoff";
import {
emptyAutoQueueState,
@@ -151,3 +157,95 @@ test("with no runner live, recordDownloadBackoff stays a disk round trip and loa
assert.equal(onDisk.download.platformBackoff.youtube?.fails, 1);
});
});
+
+// ── release 17, slice RL ─────────────────────────────────────────────────────
+
+test("a manual rate limit doubles the pace; a network failure does not", async () => {
+ await withTempPaths(async (paths) => {
+ await recordDownloadBackoff("youtube", paths, "rate_limit", PACING_DEFAULTS);
+ await recordDownloadBackoff("youtube", paths, "network", PACING_DEFAULTS);
+ const onDisk = await readAutoQueueState(paths);
+ assert.equal(onDisk.download.platformBackoff.youtube.fails, 2);
+ const { steppedAt, ...pace } = onDisk.download.platformPace.youtube;
+ assert.deepEqual(pace, { sleepRequestsSeconds: 2, baseSeconds: 1, cleanUnits: 0 });
+ assert.ok(Math.abs(steppedAt - Date.now()) < 60_000);
+ });
+});
+
+test("a manual 429 at the cap holds the platform, and the refusal names the next probe", async () => {
+ await withTempPaths(async (paths) => {
+ const state = emptyAutoQueueState();
+ state.download.platformBackoff.youtube = { until: Date.now() - 1, fails: 7 };
+ await writeAutoQueueState(paths, state);
+ const pacing = { ...PACING_DEFAULTS, holdAfterFailsAtCap: 3, holdProbeMinutes: 60 };
+ assert.equal(await heldPlatformRefusal("youtube", "Sync", paths), null);
+ await recordDownloadBackoff("youtube", paths, "rate_limit", pacing);
+ const onDisk = await readAutoQueueState(paths);
+ assert.ok(onDisk.download.platformHolds.youtube);
+ assert.equal(
+ onDisk.download.platformBackoff.youtube.until,
+ onDisk.download.platformHolds.youtube.probeAt,
+ );
+ const text = await heldPlatformRefusal("youtube", "Sync", paths);
+ assert.match(text ?? "", /^youtube is held: .*\(8 failures in a row, held since \d\d:\d\d UTC\)\..* the next probe is in 60 min\. Sync will run once a probe comes back clean\.$/);
+ });
+});
+
+test("subtitle deferrals count up through the file and clear on success", async () => {
+ await withTempPaths(async (paths) => {
+ const a = await recordSubtitleDeferral("v1", "alpha", paths, 1000);
+ const b = await recordSubtitleDeferral("v1", "alpha", paths, 2000);
+ assert.equal(a.count, 1);
+ assert.equal(b.count, 2);
+ const onDisk = await readAutoQueueState(paths);
+ assert.equal(onDisk.download.subtitleDeferrals.v1.count, 2);
+ // Nothing platform-wide moved.
+ assert.deepEqual(onDisk.download.platformBackoff, {});
+ assert.deepEqual(onDisk.download.platformPace, {});
+ await clearSubtitleDeferral("v1", paths);
+ assert.deepEqual(await readSubtitleDeferrals(paths), {});
+ });
+});
+
+test("with a runner live, a subtitle deferral is written through the shared object", async () => {
+ await withTempPaths(async (paths) => {
+ const live = await sharedAutoQueueState(paths);
+ await recordSubtitleDeferral("v9", "beta", paths);
+ assert.equal(live.download.subtitleDeferrals.v9.count, 1);
+ assert.equal((await readAutoQueueState(paths)).download.subtitleDeferrals.v9.count, 1);
+ });
+});
+
+test("an overdue hold no longer refuses; a clean manual run lifts it (review H2)", async () => {
+ await withTempPaths(async (paths) => {
+ const state = emptyAutoQueueState();
+ const now = Date.now();
+ state.download.platformBackoff.youtube = { until: now - 1_000, fails: 9 };
+ state.download.platformHolds.youtube = { since: now - 3_600_000, probeAt: now - 1_000, rateLimited: true };
+ await writeAutoQueueState(paths, state);
+ assert.equal(await heldPlatformRefusal("youtube", "Sync", paths), null);
+ const line = await recordPlatformClean("youtube", paths, PACING_DEFAULTS);
+ assert.match(line ?? "", /^youtube answered cleanly: its hold and backoff are cleared/);
+ const after = (await readAutoQueueState(paths)).download;
+ assert.deepEqual(after.platformHolds, {});
+ assert.deepEqual(after.platformBackoff, {});
+ // Nothing to settle: no line and no write.
+ assert.equal(await recordPlatformClean("youtube", paths, PACING_DEFAULTS), null);
+ });
+});
+
+test("Clear hold drops the hold, the backoff and the pace, and says so", async () => {
+ await withTempPaths(async (paths) => {
+ const state = emptyAutoQueueState();
+ const now = Date.now();
+ state.download.platformBackoff.youtube = { until: now + 3_600_000, fails: 9 };
+ state.download.platformHolds.youtube = { since: now - 60_000, probeAt: now + 3_600_000, rateLimited: true };
+ state.download.platformPace.youtube = { sleepRequestsSeconds: 8, baseSeconds: 1, cleanUnits: 0, steppedAt: now };
+ await writeAutoQueueState(paths, state);
+ const line = await clearPlatformHold("youtube", paths);
+ assert.match(line ?? "", /^Cleared by hand for youtube: the hold \(since .* UTC\), the backoff \(9 failures\), the pace \(8s → base 1s\)\./);
+ const after = (await readAutoQueueState(paths)).download;
+ assert.deepEqual([after.platformHolds, after.platformBackoff, after.platformPace], [{}, {}, {}]);
+ assert.equal(await clearPlatformHold("youtube", paths), null);
+ });
+});
diff --git a/common/jobs/downloadBackoff.ts b/common/jobs/downloadBackoff.ts
@@ -19,13 +19,29 @@
// politeness, never concurrency.
import { getPaths, type Paths } from "../lib/paths";
+import { getSettings } from "../lib/settings";
import {
type AutoQueueState,
liveAutoQueueState,
readAutoQueueState,
writeAutoQueueState,
} from "./autoQueueState";
-import { mergeBackoffEntry, nextBackoff, pruneExpired } from "./platformBackoff";
+import {
+ type PacingSettings,
+ type PlatformHoldEntry,
+ type SubtitleDeferral,
+ type SubtitleDeferralState,
+ clearPlatformPacing,
+ deferSubtitles,
+ effectivePaceSeconds,
+ escalatePlatform,
+ heldPlatformSentence,
+ mergeBackoffEntry,
+ prunePlatformPacing,
+ pruneSubtitleDeferrals,
+ settlePlatformClean,
+} from "./platformBackoff";
+import { staticSleepRequestsSeconds } from "../ytdlp/platformArgs.mjs";
// Milliseconds remaining in the platform's current cooldown window, or 0 if it
// is not cooling down. Prefers the live shared object; else reads the file.
@@ -40,13 +56,23 @@ export async function platformCooldownRemainingMs(
return entry && entry.until > now ? entry.until - now : 0;
}
-// Record a rate-limit/network failure against a platform, escalating its
-// exponential cooldown — through the live shared object when a runner holds
-// one, else as a read-modify-write of the file.
-export async function recordDownloadBackoff(
- platform: string,
- paths: Paths = getPaths(),
-): Promise<void> {
+// The `pacing` settings block, read defensively: a settings read that throws
+// (a CLI with no corpus, a test's stub paths) paces on the defaults.
+function pacingSettings(): PacingSettings | undefined {
+ try {
+ return getSettings().pacing;
+ } catch {
+ return undefined;
+ }
+}
+
+// THE ONE WRITE-THROUGH: mutate the live shared object when a runner holds
+// one (after folding in the backoff on disk, as the runner's own merge does),
+// else read-modify-write the file. Every writer below goes through it.
+async function mutateDownloadState<T>(
+ paths: Paths,
+ mutate: (state: AutoQueueState) => T,
+): Promise<T> {
const live = await liveAutoQueueState(paths);
let state: AutoQueueState;
if (live) {
@@ -64,11 +90,160 @@ export async function recordDownloadBackoff(
} else {
state = await readAutoQueueState(paths);
}
- const now = Date.now();
- pruneExpired(state.download.platformBackoff, now);
- state.download.platformBackoff[platform] = nextBackoff(
- state.download.platformBackoff[platform],
- now,
- );
+ const out = mutate(state);
await writeAutoQueueState(paths, state);
+ return out;
+}
+
+// Record a rate-limit/network failure against a platform, escalating its
+// exponential cooldown — through the live shared object when a runner holds
+// one, else as a read-modify-write of the file. A rate limit also doubles the
+// platform's pace, and a backoff failing at the cap often enough holds it
+// (release 17, slice RL) — the same escalation the lane applies to its units.
+export async function recordDownloadBackoff(
+ platform: string,
+ paths: Paths = getPaths(),
+ failureClass: "rate_limit" | "network" = "rate_limit",
+ pacing: PacingSettings | undefined = pacingSettings(),
+): Promise<void> {
+ await mutateDownloadState(paths, (state) => {
+ const now = Date.now();
+ prunePlatformPacing(state.download, now);
+ escalatePlatform(state.download, platform, failureClass, now, {
+ baseSeconds: staticSleepRequestsSeconds(platform),
+ ...(pacing ? { pacing } : {}),
+ });
+ });
+}
+
+// Record a SUBTITLE 429 against one video (release 17, slice RL): the count
+// climbs and the batch subtitle fetch leaves it alone for 6 h, or for 7 days
+// from the third. The platform's backoff, hold and pace are not touched.
+export async function recordSubtitleDeferral(
+ videoId: string,
+ channelSlug: string,
+ paths: Paths = getPaths(),
+ now: number = Date.now(),
+): Promise<SubtitleDeferral> {
+ return mutateDownloadState(paths, (state) => {
+ pruneSubtitleDeferrals(state.download.subtitleDeferrals, now);
+ return deferSubtitles(state.download.subtitleDeferrals, videoId, channelSlug, now);
+ });
+}
+
+// A video's subtitles came down: its deferral (and its count) is forgotten.
+// A no-op — no write — when it had none.
+export async function clearSubtitleDeferral(
+ videoId: string,
+ paths: Paths = getPaths(),
+): Promise<void> {
+ const state =
+ (await liveAutoQueueState(paths)) ?? (await readAutoQueueState(paths));
+ if (!state.download.subtitleDeferrals[videoId]) return;
+ await mutateDownloadState(paths, (s) => {
+ delete s.download.subtitleDeferrals[videoId];
+ });
+}
+
+// Every recorded subtitle deferral (live object first, else the file).
+export async function readSubtitleDeferrals(
+ paths: Paths = getPaths(),
+): Promise<SubtitleDeferralState> {
+ const state =
+ (await liveAutoQueueState(paths)) ?? (await readAutoQueueState(paths));
+ return state.download.subtitleDeferrals;
+}
+
+// A platform's hold, when it is held, with the backoff's failure count.
+export async function platformHold(
+ platform: string,
+ paths: Paths = getPaths(),
+): Promise<{ hold: PlatformHoldEntry; fails: number } | null> {
+ const state =
+ (await liveAutoQueueState(paths)) ?? (await readAutoQueueState(paths));
+ const hold = state.download.platformHolds[platform];
+ if (!hold) return null;
+ return { hold, fails: state.download.platformBackoff[platform]?.fails ?? 0 };
+}
+
+// The refusal for a manual Sync / download on a HELD platform, or null. Only
+// while it is held AND its probe is still in the future (review H2): once the
+// probe is due, a manual run IS the probe — refusing it too would leave a
+// platform with the lane off, or nothing pending on it, held for ever. `what`
+// names the refused thing ("Sync", "This download").
+export async function heldPlatformRefusal(
+ platform: string,
+ what: string,
+ paths: Paths = getPaths(),
+ now: number = Date.now(),
+): Promise<string | null> {
+ const state =
+ (await liveAutoQueueState(paths)) ?? (await readAutoQueueState(paths));
+ const hold = state.download.platformHolds[platform];
+ const entry = state.download.platformBackoff[platform];
+ if (!hold || !entry || entry.until <= now) return null;
+ const probeMinutes =
+ pacingSettings()?.holdProbeMinutes ?? Math.round((hold.probeAt - hold.since) / 60_000);
+ return heldPlatformSentence(platform, hold, entry.fails, now, {
+ probeMinutes,
+ what,
+ });
+}
+
+// A MANUAL RUN THAT CAME BACK CLEAN (review H2) — a Sync, a download, a
+// metadata scan that the source answered without a rate limit or a network
+// failure — settles the platform exactly as a clean lane unit does: the
+// backoff and any hold clear, and it counts toward the pace's easing. Returns
+// the line for the job's log, or null when there was nothing to settle (and
+// then writes nothing).
+export async function recordPlatformClean(
+ platform: string,
+ paths: Paths = getPaths(),
+ pacing: PacingSettings | undefined = pacingSettings(),
+): Promise<string | null> {
+ const peek =
+ (await liveAutoQueueState(paths)) ?? (await readAutoQueueState(paths));
+ const d = peek.download;
+ if (!d.platformBackoff[platform] && !d.platformHolds[platform] && !d.platformPace[platform]) {
+ return null;
+ }
+ return mutateDownloadState(paths, (state) => {
+ const now = Date.now();
+ prunePlatformPacing(state.download, now);
+ const hadBackoff = Boolean(state.download.platformBackoff[platform]);
+ const fx = settlePlatformClean(state.download, platform, {
+ baseSeconds: staticSleepRequestsSeconds(platform),
+ now,
+ ...(pacing ? { pacing } : {}),
+ });
+ if (fx.releasedHold) {
+ return `${platform} answered cleanly: its hold and backoff are cleared (as a clean probe would).\n`;
+ }
+ if (hadBackoff) return `${platform} answered cleanly: its rate-limit backoff is cleared.\n`;
+ if (fx.decayed) return `${platform} pace eased to ${fx.paceSeconds}s between requests.\n`;
+ return null;
+ });
+}
+
+// "Clear hold" (review H2): the operator's word beats the machine, as with an
+// auto-pause. The hold, the backoff and the raised pace all go. Returns the
+// sentence for the job log, or null when the platform had none of them.
+export async function clearPlatformHold(
+ platform: string,
+ paths: Paths = getPaths(),
+): Promise<string | null> {
+ return mutateDownloadState(paths, (state) => {
+ const now = Date.now();
+ const pace = state.download.platformPace[platform];
+ const was = clearPlatformPacing(state.download, platform);
+ if (!was.hold && was.fails === 0 && was.paceSeconds === null) return null;
+ const parts = [
+ was.hold
+ ? `the hold (since ${new Date(was.hold.since).toISOString().slice(0, 16).replace("T", " ")} UTC)`
+ : null,
+ was.fails > 0 ? `the backoff (${was.fails} failures)` : null,
+ pace ? `the pace (${effectivePaceSeconds(pace, now)}s → base ${pace.baseSeconds}s)` : null,
+ ].filter(Boolean);
+ return `Cleared by hand for ${platform}: ${parts.join(", ")}. The next failure starts from the bottom of the backoff.\n`;
+ });
}
diff --git a/common/jobs/jobKinds.ts b/common/jobs/jobKinds.ts
@@ -100,6 +100,18 @@ const JOB_KINDS: Record<string, JobKindMeta> = {
queueKeyStrategy: "platform",
needsMedia: true,
},
+ // THE OPERATOR'S "Clear hold" on /operations/download (release 17 slice
+ // RL): one step that drops a held platform's hold, backoff and raised pace,
+ // and says so in its log. Its own queue (`pacing:<platform>`), so a Sync
+ // running on the platform's queue never makes the click wait. Touches no
+ // channel and no file but the auto-queue state.
+ "clear-platform-hold": {
+ kind: "clear-platform-hold",
+ label: "Clear rate-limit hold",
+ drainable: false,
+ replayable: false,
+ queueKeyStrategy: "custom",
+ },
// The two OPERATION lanes' runners. Same shape as the two above — one
// long-lived job per lane on queueKey "", drainable, never replayable — with
// one difference worth stating: their units make NO job record. A digest or a
diff --git a/common/jobs/platformBackoff.test.ts b/common/jobs/platformBackoff.test.ts
@@ -16,7 +16,14 @@ import {
deferVideo,
isVideoDeferred,
pruneDeferred,
+ PACING_DEFAULTS,
+ downloadGapMs,
+ failsAtCap,
+ FAILS_TO_REACH_CAP,
+ heldPlatformSentence,
+ prunePlatformPacing,
} from "./platformBackoff";
+import { sanitizePacing } from "../lib/settingsSchema";
// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test common/jobs/platformBackoff.test.ts
@@ -164,3 +171,59 @@ test("mergeBackoffEntry takes the max of until and of fails separately", () => {
{ until: 300, fails: 7 },
);
});
+
+// ── release 17, slice RL ─────────────────────────────────────────────────────
+
+test("PACING_DEFAULTS are the settings schema's defaults", () => {
+ assert.deepEqual({ ...PACING_DEFAULTS }, sanitizePacing({}));
+ assert.deepEqual(sanitizePacing({ sleepRequestsCapSeconds: 999, holdProbeMinutes: 0 }), {
+ ...PACING_DEFAULTS,
+ sleepRequestsCapSeconds: 120,
+ holdProbeMinutes: 1,
+ });
+});
+
+test("the cap is reached at the sixth failure; failsAtCap counts from there", () => {
+ assert.equal(FAILS_TO_REACH_CAP, 6);
+ assert.equal(BACKOFF_BASE_MS * 2 ** (FAILS_TO_REACH_CAP - 2) < BACKOFF_MAX_MS, true);
+ assert.deepEqual([5, 6, 7, 8].map(failsAtCap), [0, 1, 2, 3]);
+});
+
+test("the lane gap is the operator's sleep plus the pace above its base", () => {
+ assert.equal(downloadGapMs(0, 1, 1), 0);
+ assert.equal(downloadGapMs(30, 1, 1), 30_000);
+ assert.equal(downloadGapMs(30, 4, 1), 33_000);
+ assert.equal(downloadGapMs(0, 8, 1), 7_000);
+ assert.equal(downloadGapMs(10, 2, 0), 12_000);
+ assert.equal(downloadGapMs(-5, 0.5, 1), 0);
+});
+
+test("a held platform's backoff survives the prune; a hold with no backoff is dropped", () => {
+ const now = 10 * BACKOFF_MAX_MS;
+ const state = {
+ platformBackoff: {
+ youtube: { until: 0, fails: 9 },
+ rumble: { until: 0, fails: 2 },
+ } as PlatformBackoffState,
+ platformPace: {},
+ platformHolds: {
+ youtube: { since: 0, probeAt: 0, rateLimited: true },
+ odysee: { since: 0, probeAt: 0, rateLimited: true },
+ },
+ };
+ prunePlatformPacing(state, now);
+ assert.deepEqual(Object.keys(state.platformBackoff), ["youtube"]);
+ assert.deepEqual(Object.keys(state.platformHolds), ["youtube"]);
+});
+
+test("the held sentence names the hold and the next probe", () => {
+ const since = Date.UTC(2026, 9, 1, 14, 2);
+ assert.equal(
+ heldPlatformSentence("youtube", { since, probeAt: since + 42 * 60_000, rateLimited: true }, 8, since, {
+ probeMinutes: 60,
+ what: "Sync",
+ }),
+ "youtube is held: its rate limit outlasted the cooldown cap (8 failures in a row, held since 14:02 UTC). " +
+ "Auto-download probes it once every 60 min — the next probe is in 42 min. Sync will run once a probe comes back clean.",
+ );
+});
diff --git a/common/jobs/platformBackoff.ts b/common/jobs/platformBackoff.ts
@@ -177,3 +177,435 @@ export function coerceVideoDeferrals(value: unknown): VideoDeferralState {
}
return out;
}
+
+// ---------------------------------------------------------------------------
+// The adaptive pace, the hold and the subtitle deferral (release 17, slice RL).
+//
+// Measured 2026-10-01: every one of the day's 78 HTTP 429s was YouTube's
+// subtitle (timedtext) fetch, per video, after the media formats had already
+// resolved — and the platform backoff read `fails: 80`, retrying one unit every
+// 30 minutes into the same refusal. Three answers, each pure here and persisted
+// beside `platformBackoff` in `.auto-queue/state.json` (download lane only):
+//
+// - A SUBTITLE 429 IS NOT A PLATFORM SIGNAL. `classifyDownloadFailure` says
+// `subs_rate_limit` when the subtitle fetch is the only failure; the download
+// goes on to the media, and the video's subtitles are deferred here — the
+// platform backoff and the pace are not touched. Three deferrals and the
+// subtitles are left alone for a week.
+// - THE PACE ADAPTS. `--sleep-requests` for a platform starts at its static
+// value (PLATFORM_ARGS), doubles on every platform-level `rate_limit` up to
+// `pacing.sleepRequestsCapSeconds`, and halves back toward the base after
+// every `pacing.decayAfterCleanUnits` clean units.
+// - A BLOCK IS NOT A BURST. A platform whose backoff has failed at the
+// 30-minute cap `pacing.holdAfterFailsAtCap` times in a row is HELD: its
+// cooldown becomes one probe per `pacing.holdProbeMinutes`, and a clean probe
+// clears the hold and the backoff together.
+
+// The `pacing` settings block (settingsSchema.ts), restated here so jobs/ need
+// not import the schema module.
+export type PacingSettings = {
+ sleepRequestsCapSeconds: number;
+ decayAfterCleanUnits: number;
+ holdAfterFailsAtCap: number;
+ holdProbeMinutes: number;
+};
+
+export const PACING_DEFAULTS: Readonly<PacingSettings> = Object.freeze({
+ sleepRequestsCapSeconds: 16,
+ decayAfterCleanUnits: 5,
+ holdAfterFailsAtCap: 3,
+ holdProbeMinutes: 60,
+});
+
+// The current pace of one platform, when it differs from the static one. An
+// entry back at its base is deleted, so `{}` means "every platform at its base".
+export type PlatformPaceEntry = {
+ // Seconds between the HTTP requests one yt-dlp process makes
+ // (`--sleep-requests`), for every spawn against this platform.
+ sleepRequestsSeconds: number;
+ // The platform's static value the pace decays back to (PLATFORM_ARGS; 0 for
+ // a platform with no entry). Stored so a reader with no access to the args
+ // table (the status view) can say "base 1 s".
+ baseSeconds: number;
+ // Clean units since the last doubling or decay step.
+ cleanUnits: number;
+ // Epoch ms of the last doubling or decay step. THE ONE TIME-BASED RULE
+ // (review L2): the pace also eases one step per PACE_TIME_DECAY_MS with no
+ // rate limit, so a raised pace cannot outlive its cause while the lane is
+ // off or has nothing pending on the platform. 0 for an entry written before
+ // the field existed (it eases at once).
+ steppedAt: number;
+};
+
+// One easing step per hour with no rate limit (see `steppedAt`).
+export const PACE_TIME_DECAY_MS = 60 * 60_000;
+
+export type PlatformPaceState = Record<string, PlatformPaceEntry>;
+
+// A platform in a HOLD. Its backoff entry's `until` is the next probe.
+export type PlatformHoldEntry = {
+ // Epoch ms the hold began.
+ since: number;
+ // Epoch ms of the next probe (mirrors the backoff entry's `until`).
+ probeAt: number;
+ // Whether a RATE LIMIT is among the failures that held it — false for a
+ // hold reached through network failures alone, which is worded "failing",
+ // not "rate-limited" (review L5).
+ rateLimited: boolean;
+};
+
+export type PlatformHoldState = Record<string, PlatformHoldEntry>;
+
+// A video whose subtitles answered 429 while its media did not.
+export type SubtitleDeferral = {
+ // How many times its subtitle fetch was rate-limited.
+ count: number;
+ // Epoch ms of the most recent one.
+ lastAt: number;
+ // Epoch ms until which the batch subtitle fetch (download-missing-subs)
+ // leaves it alone: VIDEO_RATE_LIMIT_DEFER_MS after a strike, and
+ // SUBTITLE_HOLD_MS once `count` reaches SUBTITLE_HOLD_AFTER.
+ until: number;
+ channelSlug: string;
+};
+
+export type SubtitleDeferralState = Record<string, SubtitleDeferral>;
+
+export const SUBTITLE_HOLD_AFTER = 3;
+export const SUBTITLE_HOLD_MS = 7 * 24 * 60 * 60_000; // 7 days
+
+// The smallest `fails` whose cooldown is the cap (60 s doubling → 32 min ≥ 30).
+export const FAILS_TO_REACH_CAP = (() => {
+ let f = 1;
+ while (BACKOFF_BASE_MS * 2 ** (f - 1) < BACKOFF_MAX_MS) f++;
+ return f;
+})();
+
+// How many consecutive failures were spent AT the cap.
+export function failsAtCap(fails: number): number {
+ return Math.max(0, fails - FAILS_TO_REACH_CAP + 1);
+}
+
+export function isPlatformHeld(holds: PlatformHoldState, platform: string): boolean {
+ return holds[platform] !== undefined;
+}
+
+// The pace a platform's spawns use now: its entry, else its base.
+export function currentPaceSeconds(
+ pace: PlatformPaceState,
+ platform: string,
+ baseSeconds: number,
+): number {
+ return pace[platform]?.sleepRequestsSeconds ?? baseSeconds;
+}
+
+// What the lane waits between two units on one platform, and what a batch
+// sleeps between two videos: the operator's `sleepBetweenDownloadsSeconds`
+// plus the pace ABOVE ITS BASE. The base pace is already paid inside every
+// spawn (`--sleep-requests`); what the gap adds is the part a rate limit added.
+export function downloadGapMs(
+ sleepBetweenDownloadsSeconds: number,
+ paceSeconds: number,
+ baseSeconds: number,
+): number {
+ const sleep = Math.max(0, sleepBetweenDownloadsSeconds);
+ const extra = Math.max(0, paceSeconds - baseSeconds);
+ return Math.round((sleep + extra) * 1000);
+}
+
+export type PlatformPacingState = {
+ platformBackoff: PlatformBackoffState;
+ platformPace: PlatformPaceState;
+ platformHolds: PlatformHoldState;
+};
+
+export type EscalationEffect = {
+ entry: PlatformBackoffEntry;
+ // The pace after this failure (unchanged on a network failure).
+ paceSeconds: number;
+ // True when this failure put the platform into its hold.
+ enteredHold: boolean;
+ held: boolean;
+};
+
+// One platform-level failure (`rate_limit` or `network`): the backoff
+// escalates exactly as before, a rate limit doubles the pace, and a backoff
+// that has now failed at the cap `holdAfterFailsAtCap` times holds the
+// platform — its next try becomes a probe `holdProbeMinutes` away. Mutates.
+export function escalatePlatform(
+ state: PlatformPacingState,
+ platform: string,
+ failureClass: "rate_limit" | "network",
+ now: number,
+ opts: { pacing?: PacingSettings; baseSeconds: number; rand?: () => number },
+): EscalationEffect {
+ const pacing = opts.pacing ?? PACING_DEFAULTS;
+ const entry = nextBackoff(state.platformBackoff[platform], now, opts.rand);
+ if (failureClass === "rate_limit") {
+ const cur = currentPaceSeconds(state.platformPace, platform, opts.baseSeconds);
+ const cap = Math.max(pacing.sleepRequestsCapSeconds, opts.baseSeconds);
+ state.platformPace[platform] = {
+ sleepRequestsSeconds: Math.min(cap, Math.max(1, cur * 2)),
+ baseSeconds: opts.baseSeconds,
+ cleanUnits: 0,
+ steppedAt: now,
+ };
+ }
+ const wasHeld = isPlatformHeld(state.platformHolds, platform);
+ const held = wasHeld || failsAtCap(entry.fails) >= pacing.holdAfterFailsAtCap;
+ if (held) {
+ entry.until = now + pacing.holdProbeMinutes * 60_000;
+ state.platformHolds[platform] = {
+ since: state.platformHolds[platform]?.since ?? now,
+ probeAt: entry.until,
+ rateLimited:
+ (state.platformHolds[platform]?.rateLimited ?? false) ||
+ failureClass === "rate_limit",
+ };
+ }
+ state.platformBackoff[platform] = entry;
+ return {
+ entry,
+ paceSeconds: currentPaceSeconds(state.platformPace, platform, opts.baseSeconds),
+ enteredHold: held && !wasHeld,
+ held,
+ };
+}
+
+export type CleanEffect = {
+ // True when this clean unit was the probe that lifted a hold.
+ releasedHold: boolean;
+ // The pace after this unit, and whether it stepped down.
+ paceSeconds: number;
+ decayed: boolean;
+};
+
+// One clean unit on a platform: the backoff and any hold clear (a clean probe
+// is what lifts a hold), and every `decayAfterCleanUnits` of them halves the
+// pace back toward its base — an entry that reaches the base is deleted.
+export function settlePlatformClean(
+ state: PlatformPacingState,
+ platform: string,
+ opts: {
+ pacing?: PacingSettings;
+ baseSeconds: number;
+ // False for a probe whose media came down but whose subtitles 429'd: it
+ // lifts a hold, and does not count toward the pace's easing (review L1).
+ countForDecay?: boolean;
+ now?: number;
+ },
+): CleanEffect {
+ const pacing = opts.pacing ?? PACING_DEFAULTS;
+ const releasedHold = isPlatformHeld(state.platformHolds, platform);
+ clearBackoff(state.platformBackoff, platform);
+ delete state.platformHolds[platform];
+ const pace = state.platformPace[platform];
+ let decayed = false;
+ if (pace && opts.countForDecay !== false) {
+ pace.cleanUnits += 1;
+ if (pace.cleanUnits >= Math.max(1, pacing.decayAfterCleanUnits)) {
+ decayed = true;
+ const next = pace.sleepRequestsSeconds / 2;
+ if (next <= pace.baseSeconds) delete state.platformPace[platform];
+ else
+ state.platformPace[platform] = {
+ ...pace,
+ sleepRequestsSeconds: next,
+ cleanUnits: 0,
+ steppedAt: opts.now ?? pace.steppedAt,
+ };
+ }
+ }
+ return {
+ releasedHold,
+ paceSeconds: currentPaceSeconds(state.platformPace, platform, opts.baseSeconds),
+ decayed,
+ };
+}
+
+// Prune the backoff map without dropping a HELD platform's entry: a hold lasts
+// until a clean probe, however long the lane was stopped, and its `fails` is
+// the escalation memory that keeps it a hold. A hold whose backoff entry is
+// gone (a hand-edited file) is dropped with it. Mutates.
+export function prunePlatformPacing(state: PlatformPacingState, now: number): void {
+ decayPaceByTime(state.platformPace, now);
+ const before = { ...state.platformBackoff };
+ pruneExpired(state.platformBackoff, now);
+ for (const pf of Object.keys(state.platformHolds)) {
+ if (before[pf] && !state.platformBackoff[pf]) state.platformBackoff[pf] = before[pf];
+ if (!state.platformBackoff[pf]) delete state.platformHolds[pf];
+ }
+}
+
+// The pace an entry stands at NOW once the hourly easing is applied: one
+// halving per whole PACE_TIME_DECAY_MS since `steppedAt`, never below its base.
+// Pure — the readers (the args builder, the view, the doctor) use it so a
+// stale entry on disk never paces a spawn above what the rule allows.
+export function effectivePaceSeconds(entry: PlatformPaceEntry, now: number): number {
+ const steps = Math.max(0, Math.floor((now - entry.steppedAt) / PACE_TIME_DECAY_MS));
+ let v = entry.sleepRequestsSeconds;
+ for (let i = 0; i < steps && v > entry.baseSeconds; i++) v = v / 2;
+ return Math.max(v, entry.baseSeconds);
+}
+
+// Apply the hourly easing to the stored map (an entry at its base is
+// deleted). Mutates. Called where the state is pruned and on every unit.
+export function decayPaceByTime(pace: PlatformPaceState, now: number): void {
+ for (const [pf, e] of Object.entries(pace)) {
+ const steps = Math.max(0, Math.floor((now - e.steppedAt) / PACE_TIME_DECAY_MS));
+ if (steps === 0) continue;
+ const v = effectivePaceSeconds(e, now);
+ if (v <= e.baseSeconds) delete pace[pf];
+ else pace[pf] = { ...e, sleepRequestsSeconds: v, steppedAt: e.steppedAt + steps * PACE_TIME_DECAY_MS };
+ }
+}
+
+// THE OPERATOR'S WORD (review H2): "Clear hold" on the lane page. The hold,
+// the backoff and the raised pace all go, as if the platform had never been
+// refused; the next failure starts the escalation from the bottom. Mutates;
+// returns what there was, for the job log.
+export function clearPlatformPacing(
+ state: PlatformPacingState,
+ platform: string,
+): { hold: PlatformHoldEntry | null; fails: number; paceSeconds: number | null } {
+ const out = {
+ hold: state.platformHolds[platform] ?? null,
+ fails: state.platformBackoff[platform]?.fails ?? 0,
+ paceSeconds: state.platformPace[platform]?.sleepRequestsSeconds ?? null,
+ };
+ delete state.platformHolds[platform];
+ clearBackoff(state.platformBackoff, platform);
+ delete state.platformPace[platform];
+ return out;
+}
+
+// Record one subtitle 429 against a video: the count climbs, and the batch
+// subtitle fetch leaves it alone for 6 h — or for 7 days from the third. A
+// manual fetch never reads this. Mutates; returns the entry.
+export function deferSubtitles(
+ state: SubtitleDeferralState,
+ videoId: string,
+ channelSlug: string,
+ now: number,
+): SubtitleDeferral {
+ const count = (state[videoId]?.count ?? 0) + 1;
+ const entry: SubtitleDeferral = {
+ count,
+ lastAt: now,
+ until: now + (count >= SUBTITLE_HOLD_AFTER ? SUBTITLE_HOLD_MS : VIDEO_RATE_LIMIT_DEFER_MS),
+ channelSlug,
+ };
+ state[videoId] = entry;
+ return entry;
+}
+
+export function isSubtitleDeferred(
+ state: SubtitleDeferralState,
+ videoId: string,
+ now: number,
+): boolean {
+ const e = state[videoId];
+ return e !== undefined && e.until > now;
+}
+
+// Drop a deferral a week after its window lapsed: the count is what makes the
+// third strike a week, so it outlives the 6 h window it opened. Mutates.
+export function pruneSubtitleDeferrals(state: SubtitleDeferralState, now: number): void {
+ for (const [id, e] of Object.entries(state)) {
+ if (e.until + SUBTITLE_HOLD_MS <= now) delete state[id];
+ }
+}
+
+// ---- coercion (the persisted shapes; mirrors coercePlatformBackoff) ----
+
+const finite = (v: unknown): v is number => typeof v === "number" && Number.isFinite(v);
+
+export function coercePlatformPace(value: unknown): PlatformPaceState {
+ const out: PlatformPaceState = {};
+ if (!value || typeof value !== "object") return out;
+ for (const [pf, raw] of Object.entries(value as Record<string, unknown>)) {
+ if (!raw || typeof raw !== "object") continue;
+ const r = raw as Record<string, unknown>;
+ if (finite(r.sleepRequestsSeconds) && r.sleepRequestsSeconds > 0) {
+ out[pf] = {
+ sleepRequestsSeconds: r.sleepRequestsSeconds,
+ baseSeconds: finite(r.baseSeconds) && r.baseSeconds >= 0 ? r.baseSeconds : 0,
+ cleanUnits: finite(r.cleanUnits) && r.cleanUnits >= 0 ? Math.floor(r.cleanUnits) : 0,
+ steppedAt: finite(r.steppedAt) ? r.steppedAt : 0,
+ };
+ }
+ }
+ return out;
+}
+
+export function coercePlatformHolds(value: unknown): PlatformHoldState {
+ const out: PlatformHoldState = {};
+ if (!value || typeof value !== "object") return out;
+ for (const [pf, raw] of Object.entries(value as Record<string, unknown>)) {
+ if (!raw || typeof raw !== "object") continue;
+ const r = raw as Record<string, unknown>;
+ if (finite(r.since) && finite(r.probeAt)) {
+ out[pf] = {
+ since: r.since,
+ probeAt: r.probeAt,
+ rateLimited: typeof r.rateLimited === "boolean" ? r.rateLimited : true,
+ };
+ }
+ }
+ return out;
+}
+
+export function coerceSubtitleDeferrals(value: unknown): SubtitleDeferralState {
+ const out: SubtitleDeferralState = {};
+ if (!value || typeof value !== "object") return out;
+ for (const [id, raw] of Object.entries(value as Record<string, unknown>)) {
+ if (!raw || typeof raw !== "object") continue;
+ const r = raw as Record<string, unknown>;
+ if (
+ finite(r.count) &&
+ r.count >= 1 &&
+ finite(r.lastAt) &&
+ finite(r.until) &&
+ typeof r.channelSlug === "string"
+ ) {
+ out[id] = {
+ count: Math.floor(r.count),
+ lastAt: r.lastAt,
+ until: r.until,
+ channelSlug: r.channelSlug,
+ };
+ }
+ }
+ return out;
+}
+
+// ---- words (shared by the refusals, the status view and the doctor) ----
+
+function clockOf(ms: number): string {
+ return new Date(ms).toISOString().slice(11, 16) + " UTC";
+}
+
+function minutesText(ms: number): string {
+ const m = Math.max(1, Math.ceil(ms / 60_000));
+ return m >= 120 ? `${Math.floor(m / 60)}h ${m % 60}m` : `${m} min`;
+}
+
+// The sentence a manual Sync / download on a held platform is refused with.
+export function heldPlatformSentence(
+ platform: string,
+ hold: PlatformHoldEntry,
+ fails: number,
+ now: number,
+ opts: { probeMinutes: number; what?: string },
+): string {
+ const what = opts.what ?? "This";
+ const why = hold.rateLimited
+ ? "its rate limit outlasted the cooldown cap"
+ : "it kept failing (network errors) past the cooldown cap";
+ return (
+ `${platform} is held: ${why} ` +
+ `(${fails} failures in a row, held since ${clockOf(hold.since)}). ` +
+ `Auto-download probes it once every ${opts.probeMinutes} min — the next probe ` +
+ `is in ${minutesText(hold.probeAt - now)}. ${what} will run once a probe comes back clean.`
+ );
+}
diff --git a/common/jobs/unitOutcome.test.ts b/common/jobs/unitOutcome.test.ts
@@ -2,7 +2,13 @@ import { test } from "node:test";
import assert from "node:assert/strict";
import {
BACKOFF_BASE_MS,
+ FAILS_TO_REACH_CAP,
+ PACING_DEFAULTS,
+ PACE_TIME_DECAY_MS,
+ SUBTITLE_HOLD_MS,
VIDEO_RATE_LIMIT_DEFER_MS,
+ effectivePaceSeconds,
+ heldPlatformSentence,
isVideoDeferred,
} from "./platformBackoff";
import { type UnitOutcomeState, applyUnitOutcome } from "./unitOutcome";
@@ -10,7 +16,13 @@ import { type UnitOutcomeState, applyUnitOutcome } from "./unitOutcome";
// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test jobs/unitOutcome.test.ts
const noJitter = () => 0.5;
-const empty = (): UnitOutcomeState => ({ platformBackoff: {}, videoDeferrals: {} });
+const empty = (): UnitOutcomeState => ({
+ platformBackoff: {},
+ videoDeferrals: {},
+ platformPace: {},
+ platformHolds: {},
+ subtitleDeferrals: {},
+});
const unit = (
videoId: string,
over: Partial<Parameters<typeof applyUnitOutcome>[1]> = {},
@@ -34,7 +46,7 @@ test("rate_limit backs the platform off AND defers the video 6h", () => {
});
assert.equal(
r.line,
- "Auto-download: youtube rate_limit — backing off 60s (attempt 1). v1 deferred 6h; next video after cooldown.",
+ "Auto-download: youtube rate_limit — backing off 60s (attempt 1). v1 deferred 6h; next video after cooldown. Pace now 2s between requests.",
);
});
@@ -103,3 +115,188 @@ test("fails climbs only across DISTINCT ids: the deferred video is not re-picked
assert.equal(s.platformBackoff.youtube.fails, 3);
for (const id of ids) assert.equal(isVideoDeferred(s.videoDeferrals, id, now), true);
});
+
+// ── release 17, slice RL: the subtitle deferral, the pace, the hold ─────────
+
+test("subs_rate_limit retires the video, defers its subtitles, and touches no platform state", () => {
+ const s = empty();
+ s.platformBackoff.youtube = { until: 10, fails: 3 };
+ s.platformPace.youtube = { sleepRequestsSeconds: 4, baseSeconds: 1, cleanUnits: 2, steppedAt: 1_000_000 };
+ const before = structuredClone({
+ platformBackoff: s.platformBackoff,
+ platformPace: s.platformPace,
+ platformHolds: s.platformHolds,
+ });
+ const now = 1_000_000;
+ const r = applyUnitOutcome(
+ s,
+ unit("v1", { outcome: "transcribed", failureClass: "subs_rate_limit" }),
+ now,
+ noJitter,
+ );
+ assert.equal(r.markCompleted, true);
+ assert.equal(
+ r.line,
+ "Auto-download: v1 downloaded; its subtitles were rate-limited (1×) — deferred 6h for download-missing-subs. youtube is not backed off.",
+ );
+ assert.deepEqual(s.subtitleDeferrals.v1, {
+ count: 1,
+ lastAt: now,
+ until: now + VIDEO_RATE_LIMIT_DEFER_MS,
+ channelSlug: "alpha",
+ });
+ // Not cleared like a success, not escalated like a failure, no video deferral.
+ assert.deepEqual(
+ { platformBackoff: s.platformBackoff, platformPace: s.platformPace, platformHolds: s.platformHolds },
+ before,
+ );
+ assert.deepEqual(s.videoDeferrals, {});
+});
+
+test("the third subtitle deferral leaves the subtitles alone for 7 days", () => {
+ const s = empty();
+ let now = 0;
+ for (let i = 1; i <= 3; i++) {
+ const r = applyUnitOutcome(
+ s,
+ unit("v1", { outcome: "transcribed", failureClass: "subs_rate_limit" }),
+ now,
+ noJitter,
+ );
+ assert.equal(s.subtitleDeferrals.v1.count, i);
+ if (i < 3) {
+ assert.equal(s.subtitleDeferrals.v1.until, now + VIDEO_RATE_LIMIT_DEFER_MS);
+ now += VIDEO_RATE_LIMIT_DEFER_MS + 1; // the 6 h window lapses; the count stays
+ } else {
+ assert.equal(s.subtitleDeferrals.v1.until, now + SUBTITLE_HOLD_MS);
+ assert.match(r.line ?? "", /\(3×\) — left alone for 7 days/);
+ }
+ }
+ // A week after the hold lapsed the record is pruned, count and all.
+ applyUnitOutcome(s, unit("v2", { outcome: "skipped" }), now + 2 * SUBTITLE_HOLD_MS + 1, noJitter);
+ assert.deepEqual(s.subtitleDeferrals, {});
+});
+
+test("the pace doubles on each rate limit, never on network, and stops at the cap", () => {
+ const s = empty();
+ const pacing = { ...PACING_DEFAULTS, sleepRequestsCapSeconds: 8 };
+ let now = 0;
+ const paces: number[] = [];
+ for (let i = 0; i < 5; i++) {
+ applyUnitOutcome(s, unit(`v${i}`, { failureClass: "rate_limit" }), now, noJitter, pacing);
+ paces.push(s.platformPace.youtube.sleepRequestsSeconds);
+ now = s.platformBackoff.youtube.until;
+ }
+ assert.deepEqual(paces, [2, 4, 8, 8, 8]);
+ applyUnitOutcome(s, unit("vn", { failureClass: "network" }), now, noJitter, pacing);
+ assert.equal(s.platformPace.youtube.sleepRequestsSeconds, 8);
+ // A platform with no static pace starts from nothing and goes to 1 s.
+ applyUnitOutcome(s, unit("o1", { platform: "odysee", failureClass: "rate_limit" }), now, noJitter, pacing);
+ assert.deepEqual(s.platformPace.odysee, { sleepRequestsSeconds: 1, baseSeconds: 0, cleanUnits: 0, steppedAt: now });
+});
+
+test("the pace eases one step per N clean units, and an entry back at its base is gone", () => {
+ const s = empty();
+ const pacing = { ...PACING_DEFAULTS, decayAfterCleanUnits: 2 };
+ s.platformPace.youtube = { sleepRequestsSeconds: 4, baseSeconds: 1, cleanUnits: 0, steppedAt: 0 };
+ const lines: (string | null)[] = [];
+ for (let i = 0; i < 4; i++) {
+ lines.push(applyUnitOutcome(s, unit(`c${i}`, { outcome: "transcribed" }), 0, noJitter, pacing).line);
+ }
+ assert.deepEqual(lines, [
+ null,
+ "Auto-download: youtube pace eased to 2s between requests.",
+ null,
+ "Auto-download: youtube pace eased to 1s between requests.",
+ ]);
+ assert.deepEqual(s.platformPace, {});
+ // A subtitle-429 unit is not clean: it does not count toward the decay.
+ s.platformPace.youtube = { sleepRequestsSeconds: 4, baseSeconds: 1, cleanUnits: 1, steppedAt: 0 };
+ applyUnitOutcome(s, unit("x", { outcome: "transcribed", failureClass: "subs_rate_limit" }), 0, noJitter, pacing);
+ assert.equal(s.platformPace.youtube.cleanUnits, 1);
+});
+
+test("a backoff failing at the cap N times holds the platform: one probe per holdProbeMinutes", () => {
+ const s = empty();
+ const pacing = { ...PACING_DEFAULTS, holdAfterFailsAtCap: 3, holdProbeMinutes: 60 };
+ let now = 0;
+ const lines: string[] = [];
+ // Climb to the cap, then two failures AT it: no hold yet.
+ for (let i = 0; i < FAILS_TO_REACH_CAP + 1; i++) {
+ lines.push(applyUnitOutcome(s, unit(`v${i}`, { failureClass: "rate_limit" }), now, noJitter, pacing).line ?? "");
+ now = s.platformBackoff.youtube.until;
+ }
+ assert.deepEqual(s.platformHolds, {});
+ assert.equal(s.platformBackoff.youtube.fails, FAILS_TO_REACH_CAP + 1);
+ // The third at the cap: held, and the next try is a probe an hour away.
+ const r = applyUnitOutcome(s, unit("h1", { failureClass: "rate_limit" }), now, noJitter, pacing);
+ assert.deepEqual(s.platformHolds.youtube, { since: now, probeAt: now + 60 * 60_000, rateLimited: true });
+ assert.equal(s.platformBackoff.youtube.until, now + 60 * 60_000);
+ assert.match(r.line ?? "", /^Auto-download: youtube rate_limit — held after 8 failures in a row; next probe in 3600s\./);
+ // A failed probe keeps the hold (and its `since`) and re-arms the hour.
+ const since = now;
+ now = s.platformBackoff.youtube.until;
+ const r2 = applyUnitOutcome(s, unit("h2", { failureClass: "network" }), now, noJitter, pacing);
+ assert.deepEqual(s.platformHolds.youtube, { since, probeAt: now + 60 * 60_000, rateLimited: true });
+ assert.match(r2.line ?? "", /still held/);
+ // A clean probe clears the hold AND the backoff.
+ const r3 = applyUnitOutcome(s, unit("h3", { outcome: "transcribed" }), now + 60 * 60_000, noJitter, pacing);
+ assert.deepEqual(s.platformHolds, {});
+ assert.equal(s.platformBackoff.youtube, undefined);
+ assert.equal(
+ r3.line,
+ "Auto-download: youtube probe came back clean — the hold and the backoff are cleared.",
+ );
+});
+
+test("a probe whose media came down but whose subtitles 429'd lifts the hold, without easing the pace (review L1)", () => {
+ const s = empty();
+ s.platformBackoff.youtube = { until: 100, fails: 9 };
+ s.platformHolds.youtube = { since: 0, probeAt: 100, rateLimited: true };
+ s.platformPace.youtube = { sleepRequestsSeconds: 4, baseSeconds: 1, cleanUnits: 0, steppedAt: 100 };
+ const r = applyUnitOutcome(
+ s,
+ unit("p", { outcome: "transcribed", failureClass: "subs_rate_limit" }),
+ 100,
+ noJitter,
+ );
+ assert.deepEqual(s.platformHolds, {});
+ assert.equal(s.platformBackoff.youtube, undefined);
+ assert.equal(s.platformPace.youtube.cleanUnits, 0);
+ assert.equal(s.subtitleDeferrals.p.count, 1);
+ assert.match(r.line ?? "", /The youtube probe's media came down: the hold and the backoff are cleared\.$/);
+});
+
+test("the pace eases one step per hour with no rate limit (the one time-based rule, review L2)", () => {
+ const s = empty();
+ const t0 = 10 * PACE_TIME_DECAY_MS;
+ s.platformPace.youtube = { sleepRequestsSeconds: 8, baseSeconds: 1, cleanUnits: 0, steppedAt: t0 };
+ applyUnitOutcome(s, unit("x", { outcome: "skipped" }), t0 + PACE_TIME_DECAY_MS - 1, noJitter);
+ assert.equal(s.platformPace.youtube.sleepRequestsSeconds, 8);
+ applyUnitOutcome(s, unit("x", { outcome: "skipped" }), t0 + 2 * PACE_TIME_DECAY_MS, noJitter);
+ assert.deepEqual(s.platformPace.youtube, {
+ sleepRequestsSeconds: 2,
+ baseSeconds: 1,
+ cleanUnits: 0,
+ steppedAt: t0 + 2 * PACE_TIME_DECAY_MS,
+ });
+ // The readers see the same easing without a write.
+ assert.equal(effectivePaceSeconds(s.platformPace.youtube, t0 + 3 * PACE_TIME_DECAY_MS), 1);
+ applyUnitOutcome(s, unit("x", { outcome: "skipped" }), t0 + 9 * PACE_TIME_DECAY_MS, noJitter);
+ assert.deepEqual(s.platformPace, {});
+});
+
+test("a hold reached through network failures alone is worded as failing, not rate-limited (review L5)", () => {
+ const s = empty();
+ const pacing = { ...PACING_DEFAULTS, holdAfterFailsAtCap: 1 };
+ s.platformBackoff.youtube = { until: 0, fails: 5 };
+ applyUnitOutcome(s, unit("n", { failureClass: "network" }), 10, noJitter, pacing);
+ assert.equal(s.platformHolds.youtube.rateLimited, false);
+ assert.match(
+ heldPlatformSentence("youtube", s.platformHolds.youtube, 6, 10, { probeMinutes: 60 }),
+ /^youtube is held: it kept failing \(network errors\) past the cooldown cap/,
+ );
+ // A rate limit while held makes it a rate-limited hold.
+ applyUnitOutcome(s, unit("r", { failureClass: "rate_limit" }), 20, noJitter, pacing);
+ assert.equal(s.platformHolds.youtube.rateLimited, true);
+});
diff --git a/common/jobs/unitOutcome.ts b/common/jobs/unitOutcome.ts
@@ -17,20 +17,46 @@
// - anything else retires it; a success (`transcribed`) also clears the
// platform's cooldown.
// - every branch prunes lapsed deferrals, keeping the map bounded.
+//
+// Release 17, slice RL (see platformBackoff.ts, "The adaptive pace"):
+// - `rate_limit` also doubles the platform's pace, and a backoff that has
+// failed at the cap `pacing.holdAfterFailsAtCap` times HOLDS the platform:
+// its next try is a probe `pacing.holdProbeMinutes` away.
+// - `subs_rate_limit` (the subtitle fetch alone answered 429 and the media came
+// down) retires the video like any success and defers its SUBTITLES — the
+// platform's backoff and pace are not touched; on a HELD platform it is the
+// clean probe that lifts the hold (review L1).
+// - the pace eases one step per hour with no rate limit (decayPaceByTime).
+// - a clean unit (`transcribed` with no subtitle 429) clears the backoff and
+// any hold — a clean probe is what lifts one — and counts toward the pace's
+// decay.
import type { DownloadFailureClass } from "../lib/availability";
import {
+ type PacingSettings,
type PlatformBackoffState,
+ type PlatformHoldState,
+ type PlatformPaceState,
+ type SubtitleDeferralState,
type VideoDeferralState,
- clearBackoff,
+ SUBTITLE_HOLD_AFTER,
+ decayPaceByTime,
+ deferSubtitles,
deferVideo,
- nextBackoff,
+ isPlatformHeld,
+ escalatePlatform,
pruneDeferred,
+ pruneSubtitleDeferrals,
+ settlePlatformClean,
} from "./platformBackoff";
+import { staticSleepRequestsSeconds } from "../ytdlp/platformArgs.mjs";
export type UnitOutcomeState = {
platformBackoff: PlatformBackoffState;
videoDeferrals: VideoDeferralState;
+ platformPace: PlatformPaceState;
+ platformHolds: PlatformHoldState;
+ subtitleDeferrals: SubtitleDeferralState;
};
export type FinishedUnit = {
@@ -55,22 +81,33 @@ export function applyUnitOutcome(
unit: FinishedUnit,
now: number,
rand: () => number = Math.random,
+ pacing?: PacingSettings,
): UnitOutcomeEffect {
pruneDeferred(state.videoDeferrals, now);
+ pruneSubtitleDeferrals(state.subtitleDeferrals, now);
+ decayPaceByTime(state.platformPace, now);
const pf = unit.platform;
if (
pf !== null &&
(unit.failureClass === "rate_limit" || unit.failureClass === "network")
) {
- const entry = nextBackoff(state.platformBackoff[pf], now, rand);
- state.platformBackoff[pf] = entry;
+ const fx = escalatePlatform(state, pf, unit.failureClass, now, {
+ baseSeconds: staticSleepRequestsSeconds(pf),
+ rand,
+ ...(pacing ? { pacing } : {}),
+ });
+ const entry = fx.entry;
const secs = Math.round((entry.until - now) / 1000);
- const head = `Auto-download: ${pf} ${unit.failureClass} — backing off ${secs}s (attempt ${entry.fails}).`;
+ const head = fx.held
+ ? `Auto-download: ${pf} ${unit.failureClass} — ${fx.enteredHold ? "held" : "still held"} after ${entry.fails} failures in a row; next probe in ${secs}s.`
+ : `Auto-download: ${pf} ${unit.failureClass} — backing off ${secs}s (attempt ${entry.fails}).`;
+ const pace =
+ unit.failureClass === "rate_limit" ? ` Pace now ${fx.paceSeconds}s between requests.` : "";
if (unit.failureClass === "rate_limit") {
deferVideo(state.videoDeferrals, unit.videoId, unit.channelSlug, now);
return {
markCompleted: false,
- line: `${head} ${unit.videoId} deferred 6h; next video after cooldown.`,
+ line: `${head} ${unit.videoId} deferred 6h; next video after cooldown.${pace}`,
};
}
return {
@@ -78,8 +115,52 @@ export function applyUnitOutcome(
line: `${head} ${unit.videoId} will retry after cooldown.`,
};
}
+ if (unit.failureClass === "subs_rate_limit") {
+ // The media came down; only the subtitle fetch was refused. Not a platform
+ // signal (the timedtext 429 is per video): nothing platform-wide moves.
+ const d = deferSubtitles(state.subtitleDeferrals, unit.videoId, unit.channelSlug, now);
+ const left =
+ d.count >= SUBTITLE_HOLD_AFTER
+ ? "left alone for 7 days"
+ : "deferred 6h for download-missing-subs";
+ // A HELD platform's probe whose media came down answered on every
+ // platform-level request: that is the clean probe the hold waits for
+ // (review L1). It does not count toward the pace's easing.
+ if (pf !== null && isPlatformHeld(state.platformHolds, pf)) {
+ settlePlatformClean(state, pf, {
+ baseSeconds: staticSleepRequestsSeconds(pf),
+ countForDecay: false,
+ now,
+ ...(pacing ? { pacing } : {}),
+ });
+ return {
+ markCompleted: true,
+ line: `Auto-download: ${unit.videoId} downloaded; its subtitles were rate-limited (${d.count}×) — ${left}. The ${pf} probe's media came down: the hold and the backoff are cleared.`,
+ };
+ }
+ return {
+ markCompleted: true,
+ line: `Auto-download: ${unit.videoId} downloaded; its subtitles were rate-limited (${d.count}×) — ${left}. ${pf ?? "the platform"} is not backed off.`,
+ };
+ }
if (pf !== null && unit.outcome === "transcribed") {
- clearBackoff(state.platformBackoff, pf);
+ const fx = settlePlatformClean(state, pf, {
+ baseSeconds: staticSleepRequestsSeconds(pf),
+ now,
+ ...(pacing ? { pacing } : {}),
+ });
+ if (fx.releasedHold) {
+ return {
+ markCompleted: true,
+ line: `Auto-download: ${pf} probe came back clean — the hold and the backoff are cleared.`,
+ };
+ }
+ if (fx.decayed) {
+ return {
+ markCompleted: true,
+ line: `Auto-download: ${pf} pace eased to ${fx.paceSeconds}s between requests.`,
+ };
+ }
}
return { markCompleted: true, line: null };
}
diff --git a/common/lib/availability.test.ts b/common/lib/availability.test.ts
@@ -2,7 +2,10 @@ import { test } from "node:test";
import assert from "node:assert/strict";
import {
classifyDownloadFailure,
+ hasNonRateLimitSubtitleFailure,
+ hasSubtitleRateLimit,
isSoftBlock,
+ isSubtitleRateLimitOnly,
parseUnavailableFromStderr,
} from "./availability";
@@ -126,3 +129,89 @@ test("a genuinely removed or unavailable video is still deleted / per_video", ()
assert.equal(parseUnavailableFromStderr(members), "members_only");
assert.equal(classifyDownloadFailure(members, "members_only"), "per_video");
});
+
+// ── release 17, slice RL: a subtitle 429 is not a platform failure ──────────
+
+const SUB_429 =
+ "ERROR: [youtube] abc: Unable to download video subtitles for 'en': HTTP Error 429: Too Many Requests";
+
+test("a subtitle 429 alone is subs_rate_limit — as an ERROR or a WARNING", () => {
+ assert.equal(classifyDownloadFailure(SUB_429, "error"), "subs_rate_limit");
+ assert.equal(
+ classifyDownloadFailure(
+ "WARNING: Unable to download video subtitles for 'en-orig': HTTP Error 429: Too Many Requests",
+ undefined,
+ ),
+ "subs_rate_limit",
+ );
+ // yt-dlp's own re-extraction after a --load-info-json subtitle failure is a
+ // WARNING; a second subtitle 429 from the URL is still subtitle-only.
+ assert.equal(
+ classifyDownloadFailure(
+ [
+ SUB_429,
+ "WARNING: The info failed to download: ERROR: Unable to download video subtitles for 'en': HTTP Error 429: Too Many Requests; trying with URL https://www.youtube.com/watch?v=abc",
+ SUB_429,
+ ].join("\n"),
+ "error",
+ ),
+ "subs_rate_limit",
+ );
+ assert.equal(isSubtitleRateLimitOnly(SUB_429), true);
+});
+
+test("a subtitle 429 beside any other ERROR is a platform rate limit", () => {
+ const webpage429 =
+ "ERROR: [youtube] abc: Unable to download webpage: HTTP Error 429: Too Many Requests";
+ assert.equal(classifyDownloadFailure(`${SUB_429}\n${webpage429}`, "error"), "rate_limit");
+ assert.equal(classifyDownloadFailure(webpage429, "error"), "rate_limit");
+ // The soft block and the bot check are platform signals, whatever else failed.
+ assert.equal(
+ classifyDownloadFailure(
+ `${SUB_429}\nWARNING: This content isn't available, try again later.`,
+ "error",
+ ),
+ "rate_limit",
+ );
+ assert.equal(
+ isSubtitleRateLimitOnly(`${SUB_429}\nSign in to confirm you’re not a bot`),
+ false,
+ );
+});
+
+test("a subtitle failure that is not a rate limit is not subs_rate_limit", () => {
+ const sub404 =
+ "ERROR: [youtube] abc: Unable to download video subtitles for 'en': HTTP Error 404: Not Found";
+ assert.equal(hasSubtitleRateLimit(sub404), false);
+ assert.equal(classifyDownloadFailure(sub404, "error"), "unknown");
+});
+
+test("a subtitle failure that is not a 429 is told apart; a WARNING webpage 429 is a platform signal (review H1, N9)", () => {
+ const w403 = "WARNING: Unable to download video subtitles for 'en': HTTP Error 403: Forbidden";
+ const chat = "WARNING: Unable to download video subtitles for 'live_chat': HTTP Error 404: Not Found";
+ const w429 = "WARNING: Unable to download video subtitles for 'en': HTTP Error 429: Too Many Requests";
+ assert.equal(hasNonRateLimitSubtitleFailure(w403), true);
+ assert.equal(hasNonRateLimitSubtitleFailure(chat), true);
+ assert.equal(hasNonRateLimitSubtitleFailure(w429), false);
+ assert.equal(isSubtitleRateLimitOnly(`${w429}\n${chat}`), false);
+ assert.equal(
+ isSubtitleRateLimitOnly(`${w429}\nWARNING: [youtube] abc: Unable to download webpage: HTTP Error 429: Too Many Requests (retrying)`),
+ false,
+ );
+});
+
+test("a subtitle 429 beside another subtitle failure classes as the OTHER failure, never rate_limit (re-review R1)", () => {
+ const w429 = "WARNING: Unable to download video subtitles for 'en': HTTP Error 429: Too Many Requests";
+ const chat404 = "WARNING: Unable to download video subtitles for 'live_chat': HTTP Error 404: Not Found";
+ const sub403 = "WARNING: Unable to download video subtitles for 'en-orig': HTTP Error 403: Forbidden";
+ assert.equal(classifyDownloadFailure(`${w429}\n${chat404}`, "error"), "unknown");
+ assert.equal(classifyDownloadFailure(`${w429}\n${sub403}`, "error"), "network");
+ // A real platform 429 beside them still wins.
+ assert.equal(
+ classifyDownloadFailure(
+ `${w429}\n${chat404}\nERROR: [youtube] abc: Unable to download webpage: HTTP Error 429: Too Many Requests`,
+ "error",
+ ),
+ "rate_limit",
+ );
+});
diff --git a/common/lib/availability.ts b/common/lib/availability.ts
@@ -231,12 +231,64 @@ export function parseUnavailableFromStderr(stderr: string): Availability {
// `per_video` — the rest of the batch can continue. Rate-limit and network
// errors are batch-level signals: continuing would just hammer the source or
// waste cycles. Everything else is `unknown` and treated as fatal to be safe.
+//
+// `subs_rate_limit` (release 17, slice RL) is the one class that is not a
+// failure of the download: the SUBTITLE fetch answered 429 and nothing else
+// failed. YouTube's timedtext endpoint refuses per video while the webpage,
+// player and media requests in the same spawn succeed (measured 2026-09-25 and
+// 2026-10-01), so it says nothing about the platform: the downloader goes on to
+// the media, the record is a success carrying this class, and only the video's
+// subtitles are deferred (jobs/platformBackoff.ts).
export type DownloadFailureClass =
| "per_video"
| "rate_limit"
+ | "subs_rate_limit"
| "network"
| "unknown";
+// yt-dlp's subtitle-fetch failure, as an ERROR (the default) or as a WARNING
+// (under --ignore-errors, which is how the managed downloader asks it to carry
+// on to the media): `Unable to download video subtitles for 'en': HTTP Error
+// 429: Too Many Requests`.
+const SUBTITLE_RATE_LIMIT_RE =
+ /unable to download video subtitles for [^\n]*?(?:http error 429|too many requests|rate[- ]?limit)/i;
+
+export function hasSubtitleRateLimit(stderr: string): boolean {
+ return SUBTITLE_RATE_LIMIT_RE.test(stderr);
+}
+
+const SUBTITLE_FAILURE_RE = /unable to download video subtitles/i;
+const RATE_LIMIT_WORDS_RE = /http error 429|too many requests|rate[- ]?limit/i;
+
+// A subtitle failure that is NOT a rate limit — a 403/404/5xx on the subtitle
+// URL, a failed live_chat replay, an OSError writing the file. Under
+// --ignore-errors yt-dlp reports EVERY subtitle failure as a WARNING and exits
+// 0, so the managed downloader asks this to keep such a failure a failure:
+// only a 429 takes the "download the media anyway" path (review H1).
+export function hasNonRateLimitSubtitleFailure(stderr: string): boolean {
+ return stderr
+ .split("\n")
+ .some((l) => SUBTITLE_FAILURE_RE.test(l) && !RATE_LIMIT_WORDS_RE.test(l));
+}
+
+// True when a subtitle 429 is the ONLY failure in the tail: every ERROR line
+// is a subtitle-download line (yt-dlp's own "The info failed to download …
+// trying with URL" retry is a WARNING and is allowed), every subtitle failure
+// is a rate limit, every 429 / too-many-requests line is a subtitle line (a
+// retried webpage 429 logged as a WARNING is a platform signal), and neither
+// the soft block nor the bot check — both platform signals — is anywhere in it.
+export function isSubtitleRateLimitOnly(stderr: string): boolean {
+ if (!hasSubtitleRateLimit(stderr)) return false;
+ if (isSoftBlock(stderr) || isBotCheck(stderr)) return false;
+ if (hasNonRateLimitSubtitleFailure(stderr)) return false;
+ for (const line of stderr.split("\n")) {
+ const subtitleLine = SUBTITLE_FAILURE_RE.test(line);
+ if (/^\s*ERROR:/i.test(line) && !subtitleLine) return false;
+ if (RATE_LIMIT_WORDS_RE.test(line) && !subtitleLine) return false;
+ }
+ return true;
+}
+
const PER_VIDEO_CLASSES: ReadonlyArray<Availability> = [
"private",
"members_only",
@@ -269,13 +321,23 @@ export function classifyDownloadFailure(
// that also names a removed video) must still back off. Erring this way costs
// one cooldown; erring the other way keeps requesting into the block.
if (isSoftBlock(stderrTail)) return "rate_limit";
+ // The subtitle fetch alone was refused: the media is unaffected, and the
+ // platform is not to back off for it (release 17, slice RL).
+ if (isSubtitleRateLimitOnly(stderrTail)) return "subs_rate_limit";
if (
availabilityClass !== undefined &&
PER_VIDEO_CLASSES.includes(availabilityClass)
) {
return "per_video";
}
- const s = stderrTail.toLowerCase();
+ // A subtitle 429 beside another subtitle failure is still not a platform
+ // signal (release 17 re-review R1): its lines are dropped before the
+ // generic rate-limit test, so the class is the OTHER failure's.
+ const s = stderrTail
+ .split("\n")
+ .filter((l) => !(SUBTITLE_FAILURE_RE.test(l) && RATE_LIMIT_WORDS_RE.test(l)))
+ .join("\n")
+ .toLowerCase();
if (
/http error 429/.test(s) ||
/too many requests/.test(s) ||
diff --git a/common/lib/downloadOutcome.ts b/common/lib/downloadOutcome.ts
@@ -66,7 +66,12 @@ export type DownloadAttemptKind =
// The live-chat pass a "chat-only" filter verdict runs INSTEAD of a download:
// --skip-download --write-subs --sub-langs live_chat, no media, no archive
// line. Recorded with n: 1, since it is the only real attempt there is.
- | "live-chat-only";
+ | "live-chat-only"
+ // The primary again, with every subtitle refused, after a primary that
+ // failed ONLY on its subtitle fetch (a 429): the media is what the download
+ // is for, and the subtitles are deferred (release 17, slice RL). Recorded
+ // with n: 1 beside the primary it re-runs.
+ | "primary-without-subs";
export type AudioCheckProbeVerdict = "clean" | "partial" | "malformed";
@@ -101,7 +106,9 @@ export type DownloadOutcomeRecord = {
// rate-limit that yt-dlp logs as a WARNING before failing with a different
// final line (e.g. Odysee "HTTP Error 429" → "No video formats found") is
// still recognised. Drives the auto-runner's per-platform backoff. Undefined
- // on success / skipped-filtered.
+ // on success / skipped-filtered — except `subs_rate_limit`, which is set on a
+ // SUCCESS whose subtitle fetch alone answered 429: the media came down, and
+ // the video's subtitles are deferred (release 17, slice RL).
failureClass?: DownloadFailureClass;
fellBackToTranscribe?: boolean;
// Set when status is "failed-short-audio": the measured shortfall, so the UI
diff --git a/common/lib/settingsDocs.ts b/common/lib/settingsDocs.ts
@@ -16,6 +16,7 @@ import {
BACKFILL_SETTINGS_FIELD_DOCS,
BUILD_PIPELINE_SETTINGS_FIELD_DOCS,
DIARIZATION_SETTINGS_FIELD_DOCS,
+ PACING_SETTINGS_FIELD_DOCS,
DIGEST_SETTINGS_FIELD_DOCS,
SAVED_VIDEO_BACKUP_SETTINGS_FIELD_DOCS,
SOCIAL_LINK_FIELD_DOCS,
@@ -126,6 +127,13 @@ export function blockTables(d: SiteSettings): Partial<Record<keyof SiteSettings,
{ path: "workers[].remote", docs: REMOTE_WORKER_CONFIG_FIELD_DOCS },
{ path: "workers[].llm", docs: LLM_WORKER_CONFIG_FIELD_DOCS },
],
+ pacing: [
+ {
+ path: "pacing",
+ docs: PACING_SETTINGS_FIELD_DOCS,
+ defaults: fromObject(d.pacing),
+ },
+ ],
archiveStorage: [
{
path: "archiveStorage",
diff --git a/common/lib/settingsSchema.test.ts b/common/lib/settingsSchema.test.ts
@@ -32,6 +32,7 @@ import type {
BuildPipelineSettings,
DiarizationSettings,
DigestSettings,
+ PacingSettingsBlock,
ReportDebouncePreset,
SavedVideoBackupSettings,
SocialLink,
@@ -66,6 +67,8 @@ type PreSchemaSiteSettings = {
// Release 16 slice XL — the one key it adds.
social: SocialSettings;
sleepBetweenDownloadsSeconds: number;
+ // Release 17 slice RL — the adaptive pace and the hold.
+ pacing: PacingSettingsBlock;
downloadFormat: DownloadFormatPreset;
minFreeDiskGB: number;
resumeMarginGB: number;
@@ -95,7 +98,7 @@ type PreSchemaSiteSettings = {
type Same<A, B> = [A] extends [B] ? ([B] extends [A] ? true : false) : false;
const shapeUnchanged: Same<SiteSettings, PreSchemaSiteSettings> = true;
-test("SiteSettings keeps its 32 fields, in file order", () => {
+test("SiteSettings keeps its 33 fields, in file order", () => {
assert.equal(shapeUnchanged, true);
assert.deepEqual(Object.keys(siteSettingsSchema.shape), [
"adminTitle",
@@ -107,6 +110,7 @@ test("SiteSettings keeps its 32 fields, in file order", () => {
"cookieMode",
"social",
"sleepBetweenDownloadsSeconds",
+ "pacing",
"downloadFormat",
"minFreeDiskGB",
"resumeMarginGB",
diff --git a/common/lib/settingsSchema.ts b/common/lib/settingsSchema.ts
@@ -645,6 +645,74 @@ export const ARCHIVE_STORAGE_SETTINGS_FIELD_DOCS: FieldDocs<ArchiveStorageSettin
"happen.",
};
+// Each field is documented in PACING_SETTINGS_FIELD_DOCS below (rendered into SETTINGS.md).
+export type PacingSettingsBlock = {
+ sleepRequestsCapSeconds: number;
+ decayAfterCleanUnits: number;
+ holdAfterFailsAtCap: number;
+ holdProbeMinutes: number;
+};
+
+export const PACING_SETTINGS_FIELD_DOCS: FieldDocs<PacingSettingsBlock> = {
+ sleepRequestsCapSeconds:
+ "Ceiling (seconds) on a platform's adaptive `--sleep-requests`. The pace " +
+ "starts at the platform's fixed value (1 s for YouTube and Rumble) and " +
+ "doubles on every platform-level rate limit until it reaches this. A " +
+ "subtitle-only 429 never moves it. Clamped 1–120; default 16.",
+ decayAfterCleanUnits:
+ "How many clean auto-download units on a platform halve its pace one step " +
+ "back toward the fixed value. Clamped 1–1000; default 5.",
+ holdAfterFailsAtCap:
+ "How many consecutive failures AT the 30-minute cooldown cap put a " +
+ "platform in a hold: the lane then runs one probe unit per " +
+ "holdProbeMinutes instead of one per cooldown, and a manual Sync or " +
+ "download on it is refused with the next probe's time. A clean probe " +
+ "clears the hold and the backoff. Clamped 1–100; default 3.",
+ holdProbeMinutes:
+ "Minutes between probes while a platform is held. Clamped 1–1440; " +
+ "default 60.",
+};
+
+export const PACING_DEFAULT_SLEEP_REQUESTS_CAP_SECONDS = 16;
+export const PACING_DEFAULT_DECAY_AFTER_CLEAN_UNITS = 5;
+export const PACING_DEFAULT_HOLD_AFTER_FAILS_AT_CAP = 3;
+export const PACING_DEFAULT_HOLD_PROBE_MINUTES = 60;
+
+function clampInt(v: unknown, min: number, max: number, dflt: number): number {
+ if (typeof v !== "number" || !Number.isFinite(v)) return dflt;
+ return Math.min(max, Math.max(min, Math.floor(v)));
+}
+
+export function sanitizePacing(v: unknown): PacingSettingsBlock {
+ const r = (v && typeof v === "object" ? v : {}) as Record<string, unknown>;
+ return {
+ sleepRequestsCapSeconds: clampInt(
+ r.sleepRequestsCapSeconds,
+ 1,
+ 120,
+ PACING_DEFAULT_SLEEP_REQUESTS_CAP_SECONDS,
+ ),
+ decayAfterCleanUnits: clampInt(
+ r.decayAfterCleanUnits,
+ 1,
+ 1000,
+ PACING_DEFAULT_DECAY_AFTER_CLEAN_UNITS,
+ ),
+ holdAfterFailsAtCap: clampInt(
+ r.holdAfterFailsAtCap,
+ 1,
+ 100,
+ PACING_DEFAULT_HOLD_AFTER_FAILS_AT_CAP,
+ ),
+ holdProbeMinutes: clampInt(
+ r.holdProbeMinutes,
+ 1,
+ 1440,
+ PACING_DEFAULT_HOLD_PROBE_MINUTES,
+ ),
+ };
+}
+
export const SLEEP_BETWEEN_DOWNLOADS_MAX_SECONDS = 600;
export const SLEEP_BETWEEN_DOWNLOADS_DEFAULT_SECONDS = 10;
@@ -1489,7 +1557,10 @@ export const siteSettingsSchema = z.object({
"Per-platform settings of the social posts. Today two keys, both X's, both chosen in the X account session section of /settings: where the X fetchers' login comes from (`social.x.cookieSource`) and where X posts may appear (`social.x.visibility`). See common/social/xCookieSource.ts.",
),
sleepBetweenDownloadsSeconds: settingsField((v): number => clampSleepBetweenDownloadsSeconds(v)).describe(
- "Pause (seconds) inserted between per-video yt-dlp invocations in managed batch downloads. yt-dlp's own `-t sleep` only paces requests within one invocation, so without this the managed loop hammers the source IP back-to-back. 0 disables. Per-channel override available.",
+ "Pause (seconds) inserted between per-video yt-dlp invocations in managed batch downloads, and between two auto-download units on one platform (release 17; the lane ignored it before). yt-dlp's own `-t sleep` only paces requests within one invocation, so without this the managed loop hammers the source IP back-to-back. The adaptive pace above its base (see `pacing`) is added to it. 0 disables. Per-channel override available for batch downloads.",
+ ),
+ pacing: settingsField((v): PacingSettingsBlock => sanitizePacing(v)).describe(
+ "How the download pace adapts to rate limits, per platform (release 17). Every yt-dlp spawn against a platform paces its requests (`--sleep-requests`) at the platform's adaptive pace, and the download lane waits sleepBetweenDownloadsSeconds plus the pace above its fixed value between units. A rate limit doubles the pace; clean units ease it back; a rate limit that outlasts the cooldown cap holds the platform to one probe at a time. The live pace, cooldowns and holds are in `.auto-queue/state.json`, shown on /operations/download. See common/jobs/platformBackoff.ts.",
),
downloadFormat: settingsField((v): DownloadFormatPreset =>
isDownloadFormatPreset(v) ? v : "auto").describe(
diff --git a/common/views/activeJobs.ts b/common/views/activeJobs.ts
@@ -400,6 +400,10 @@ function autoIdleNote(
return "every pending platform is in a rate-limit cooldown";
case "deferred":
return "every pending video was rate-limited recently and is deferred";
+ case "held":
+ return "every pending platform is held after repeated rate limits — one probe at a time";
+ case "paced":
+ return "every pending platform is pausing between downloads";
case "no-workers":
return "no enabled worker";
case "workers-paused":
diff --git a/common/views/autoQueueStatus.test.ts b/common/views/autoQueueStatus.test.ts
@@ -89,14 +89,40 @@ test("a cooldown is filtered by the injected clock, newest first", () => {
};
const payload = buildAutoQueueStatusPayload(inputs({ state }));
assert.deepEqual(payload.download.cooldowns, [
- { platform: "twitch", untilMs: NOW + 600_000, fails: 5 },
- { platform: "rumble", untilMs: NOW + 60_000, fails: 2 },
+ { platform: "twitch", untilMs: NOW + 600_000, fails: 5, hold: null },
+ { platform: "rumble", untilMs: NOW + 60_000, fails: 2, hold: null },
]);
// The clock is a function, so the same state read later says something else.
assert.deepEqual(
buildAutoQueueStatusPayload(inputs({ state, now: () => NOW + 120_000 }))
.download.cooldowns,
- [{ platform: "twitch", untilMs: NOW + 600_000, fails: 5 }],
+ [{ platform: "twitch", untilMs: NOW + 600_000, fails: 5, hold: null }],
+ );
+});
+
+test("a hold, the pace and the subtitle deferrals reach the download lane (release 17, RL)", () => {
+ const state = emptyAutoQueueState();
+ state.download.platformBackoff = { youtube: { until: NOW + 3_600_000, fails: 8 } };
+ state.download.platformHolds = { youtube: { since: NOW - 60_000, probeAt: NOW + 3_600_000, rateLimited: true } };
+ state.download.platformPace = {
+ youtube: { sleepRequestsSeconds: 4, baseSeconds: 1, cleanUnits: 0, steppedAt: NOW },
+ };
+ state.download.subtitleDeferrals = {
+ b: { count: 3, lastAt: NOW - 1, until: NOW + 7 * 86_400_000, channelSlug: "alpha" },
+ a: { count: 1, lastAt: NOW - 1, until: NOW + 3_600_000, channelSlug: "alpha" },
+ gone: { count: 1, lastAt: NOW - 10, until: NOW - 1, channelSlug: "alpha" },
+ };
+ const d = buildAutoQueueStatusPayload(inputs({ state })).download;
+ assert.deepEqual(d.cooldowns, [
+ { platform: "youtube", untilMs: NOW + 3_600_000, fails: 8, hold: { sinceMs: NOW - 60_000, rateLimited: true } },
+ ]);
+ assert.deepEqual(d.pace, [{ platform: "youtube", sleepRequestsSeconds: 4, baseSeconds: 1 }]);
+ assert.deepEqual(
+ d.subtitleDeferred.map((x) => [x.videoId, x.count, x.held]),
+ [
+ ["a", 1, false],
+ ["b", 3, true],
+ ],
);
});
@@ -303,3 +329,13 @@ test("memo: a different key misses the memo and the computation in flight", asyn
assert.equal(await a, "late A2");
assert.equal(await memo.get("tree C", async () => "unused"), "C");
});
+
+test("a held platform stays listed after its probe time passed (review H2)", () => {
+ const state = emptyAutoQueueState();
+ state.download.platformBackoff = { youtube: { until: NOW - 5_000, fails: 9 }, rumble: { until: NOW - 5_000, fails: 2 } };
+ state.download.platformHolds = { youtube: { since: NOW - 7_200_000, probeAt: NOW - 5_000, rateLimited: false } };
+ const d = buildAutoQueueStatusPayload(inputs({ state })).download;
+ assert.deepEqual(d.cooldowns, [
+ { platform: "youtube", untilMs: NOW - 5_000, fails: 9, hold: { sinceMs: NOW - 7_200_000, rateLimited: false } },
+ ]);
+});
diff --git a/common/views/autoQueueStatus.ts b/common/views/autoQueueStatus.ts
@@ -12,6 +12,7 @@ import type {
AutoQueueState,
} from "../jobs/autoQueueState";
import { LANES } from "../lib/autoQueueTypes";
+import { effectivePaceSeconds } from "../jobs/platformBackoff";
import type { AutoQueuePolicy } from "../jobs/autoQueuePolicy";
import { isGateHeld } from "../lib/pauseGates";
import type { FocusSummary } from "../lib/channelPriority";
@@ -31,6 +32,32 @@ export type PlatformCooldownView = {
platform: string;
untilMs: number;
fails: number;
+ // The platform's HOLD (release 17, slice RL): its failures outlasted the
+ // cooldown cap, and `untilMs` is the next probe. Null for a plain cooldown.
+ // A held platform is listed WHATEVER its probe time (review H2): once the
+ // probe is overdue (`untilMs` passed) it stays held until a clean probe, a
+ // clean manual run or Clear hold. `rateLimited` is false for a hold reached
+ // through network failures alone.
+ hold: { sinceMs: number; rateLimited: boolean } | null;
+};
+
+// A platform whose request pace is above its static value (release 17, slice
+// RL): what every yt-dlp spawn against it now waits between requests.
+export type PlatformPaceView = {
+ platform: string;
+ sleepRequestsSeconds: number;
+ baseSeconds: number;
+};
+
+// A video whose subtitle fetch answered 429 while its media came down.
+// `held` once the count reached three (left alone for 7 days).
+export type SubtitleDeferralView = {
+ videoId: string;
+ channelSlug: string;
+ count: number;
+ lastAtMs: number;
+ untilMs: number;
+ held: boolean;
};
// A video the auto-download pick skips until `untilMs` because it was
@@ -69,6 +96,11 @@ export type AutoQueueKindStatus = {
// Videos deferred after a rate limit, soonest to return first (download
// kind only; empty elsewhere and whenever none is live).
deferred: VideoDeferralView[];
+ // Platforms whose pace a rate limit raised, and the videos whose subtitles
+ // are deferred (download kind only; release 17, slice RL). The deferrals
+ // are listed while their window is open, soonest first.
+ pace: PlatformPaceView[];
+ subtitleDeferred: SubtitleDeferralView[];
// This runner's lane gate — what the page's pause button draws, and what lets
// the operations rail show a runner HOLDING for the first time.
//
@@ -151,9 +183,46 @@ function buildKind(
const cooldowns: PlatformCooldownView[] = Object.entries(
inputs.state[kind].platformBackoff,
)
- .filter(([, e]) => e.until > now)
- .map(([platform, e]) => ({ platform, untilMs: e.until, fails: e.fails }))
+ .filter(([platform, e]) => e.until > now || Boolean(inputs.state[kind].platformHolds?.[platform]))
+ .map(([platform, e]) => {
+ const hold = inputs.state[kind].platformHolds?.[platform];
+ return {
+ platform,
+ untilMs: e.until,
+ fails: e.fails,
+ hold: hold ? { sinceMs: hold.since, rateLimited: hold.rateLimited } : null,
+ };
+ })
.sort((a, b) => b.untilMs - a.untilMs);
+ const pace: PlatformPaceView[] = Object.entries(
+ inputs.state[kind].platformPace ?? {},
+ )
+ // At the pace the hourly easing leaves it (effectivePaceSeconds); an
+ // entry already back at its base is not listed.
+ .map(([platform, p]) => ({
+ platform,
+ sleepRequestsSeconds: effectivePaceSeconds(p, now),
+ baseSeconds: p.baseSeconds,
+ }))
+ .filter((p) => p.sleepRequestsSeconds > p.baseSeconds)
+ .sort((a, b) => (a.platform < b.platform ? -1 : a.platform > b.platform ? 1 : 0));
+ const subtitleDeferred: SubtitleDeferralView[] = Object.entries(
+ inputs.state[kind].subtitleDeferrals ?? {},
+ )
+ .filter(([, d]) => d.until > now)
+ .map(([videoId, d]) => ({
+ videoId,
+ channelSlug: d.channelSlug,
+ count: d.count,
+ lastAtMs: d.lastAt,
+ untilMs: d.until,
+ held: d.count >= 3,
+ }))
+ .sort(
+ (a, b) =>
+ a.untilMs - b.untilMs ||
+ (a.videoId < b.videoId ? -1 : a.videoId > b.videoId ? 1 : 0),
+ );
const deferred: VideoDeferralView[] = Object.entries(
inputs.state[kind].videoDeferrals ?? {},
)
@@ -182,6 +251,8 @@ function buildKind(
picks: inputs.state[kind].picks,
cooldowns,
deferred,
+ pace,
+ subtitleDeferred,
// THE ASYMMETRY IS TRANSCRIPTION'S ALONE, and it is not a special case for
// "the first lane": its hold is LIVE on the worker pool, while every other
// lane's flag IS its gate. See lib/pauseGates.ts.
diff --git a/common/ytdlp/channelArgs.test.ts b/common/ytdlp/channelArgs.test.ts
@@ -1,6 +1,14 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { channelExtraArgs, platformArgs, PLATFORM_ARGS } from "./channelArgs";
+import {
+ channelExtraArgs,
+ channelPaceSeconds,
+ pacedPlatformArgs,
+ platformArgs,
+ PLATFORM_ARGS,
+ staticSleepRequestsSeconds,
+ withSleepRequests,
+} from "./channelArgs";
import type { ChannelConfig } from "../lib/channelConfig";
// Run with:
@@ -80,3 +88,62 @@ test("platformArgs returns a copy, not the table's array", () => {
a.push("--mutated");
assert.deepEqual(PLATFORM_ARGS.rumble, RUMBLE);
});
+
+// ── release 17, slice RL: the adaptive pace ─────────────────────────────────
+
+test("the pace raises --sleep-requests in place, and a channel's own still wins", () => {
+ assert.deepEqual(
+ channelExtraArgs(cfg({ url: "https://www.youtube.com/@x" }), undefined, 4),
+ ["--sleep-requests", "4"],
+ );
+ assert.deepEqual(
+ channelExtraArgs(cfg({ url: "https://rumble.com/c/x" }), undefined, 8),
+ ["--impersonate", "chrome", "--sleep-requests", "8"],
+ );
+ const own = channelExtraArgs(
+ cfg({ url: "https://www.youtube.com/@x", ytdlpExtraArgs: ["--sleep-requests", "3"] }),
+ undefined,
+ 8,
+ );
+ assert.deepEqual(own, ["--sleep-requests", "8", "--sleep-requests", "3"]);
+});
+
+test("a platform with no static pace gets one appended; a pace never lowers one", () => {
+ assert.deepEqual(withSleepRequests([], 2), ["--sleep-requests", "2"]);
+ assert.deepEqual(withSleepRequests(["--sleep-requests", "1"], 0.5), ["--sleep-requests", "1"]);
+ assert.deepEqual(withSleepRequests(["--sleep-requests", "1"], undefined), ["--sleep-requests", "1"]);
+ assert.deepEqual(pacedPlatformArgs("odysee", 2), ["--sleep-requests", "2"]);
+ assert.equal(staticSleepRequestsSeconds("youtube"), 1);
+ assert.equal(staticSleepRequestsSeconds("odysee"), 0);
+ assert.equal(staticSleepRequestsSeconds(null), 0);
+});
+
+test("with no pace recorded, the static args are unchanged", () => {
+ globalThis.__yttAutoQueueState__ = undefined;
+ assert.equal(channelPaceSeconds(cfg({ url: "https://www.youtube.com/@x" })), 1);
+ assert.deepEqual(
+ channelExtraArgs(cfg({ url: "https://www.youtube.com/@x" })),
+ ["--sleep-requests", "1"],
+ );
+});
+
+test("the live pace (the shared state) reaches every channelExtraArgs call", () => {
+ globalThis.__yttAutoQueueState__ = {
+ state: null,
+ stateFile: null,
+ loading: null,
+ loadingFile: null,
+ lastPace: { youtube: { sleepRequestsSeconds: 4, baseSeconds: 1, cleanUnits: 0, steppedAt: Date.now() } },
+ };
+ try {
+ assert.deepEqual(
+ channelExtraArgs(cfg({ url: "https://www.youtube.com/@x" })),
+ ["--sleep-requests", "4"],
+ );
+ assert.equal(channelPaceSeconds(cfg({ url: "https://www.youtube.com/@x" })), 4);
+ // Another platform is untouched.
+ assert.deepEqual(channelExtraArgs(cfg({ url: "https://rumble.com/c/x" })), RUMBLE);
+ } finally {
+ globalThis.__yttAutoQueueState__ = undefined;
+ }
+});
diff --git a/common/ytdlp/channelArgs.ts b/common/ytdlp/channelArgs.ts
@@ -1,6 +1,7 @@
// The per-channel argv every yt-dlp invocation in this repo appends: the cookie
// source when the resolved policy calls for one, then the platform's fixed
-// args (PLATFORM_ARGS), then the channel's own `ytdlpExtraArgs` verbatim.
+// args (PLATFORM_ARGS) at the platform's CURRENT pace, then the channel's own
+// `ytdlpExtraArgs` verbatim.
//
// Lifted out of ytdlp/downloadOneManaged.ts (where it was `channelConfigArgs`,
// which still delegates here) because the clip-window fetch has to honour the
@@ -10,18 +11,37 @@
//
// This is the ONE builder. `configArgs` (runYtdlp.ts), the metadata scan, the
// quick availability check and the new-channel probe all go through it (the
-// probe has no config yet, so it calls `platformArgs` directly). Before
+// probe has no config yet, so it calls `pacedPlatformArgs` directly). Before
// release 5 there were four copies and the probe had none, so a Rumble
// channel could not even be created.
+//
+// THE PACE ADAPTS (release 17, slice RL). `--sleep-requests` is the platform's
+// static value until a platform-level rate limit doubles it; the current value
+// lives in the download lane's persisted state (`platformPace`, see
+// jobs/platformBackoff.ts) and is read here synchronously through
+// `livePlatformPaceSeconds`, so EVERY spawn against the platform — listing,
+// prefetch, primary, availability, metadata scan, clip — slows down together.
+// The channel's own args still come last and win.
import type { ChannelConfig } from "../lib/channelConfig";
import { cookieArgs } from "../lib/cookiePolicy";
import { detectPlatform, type Platform } from "../lib/platform";
+import { livePlatformPaceSeconds } from "../jobs/autoQueueState";
// The platform args table lives in `platformArgs.mjs` (plain JS so umtool's
// `.mjs` scripts can import the same copy); re-exported here for TS callers.
-import { platformArgs } from "./platformArgs.mjs";
-export { PLATFORM_ARGS, platformArgs, platformArgsForUrl } from "./platformArgs.mjs";
+import {
+ platformArgs,
+ staticSleepRequestsSeconds,
+ withSleepRequests,
+} from "./platformArgs.mjs";
+export {
+ PLATFORM_ARGS,
+ platformArgs,
+ platformArgsForUrl,
+ staticSleepRequestsSeconds,
+ withSleepRequests,
+} from "./platformArgs.mjs";
export function channelPlatform(
config: Pick<ChannelConfig, "platform" | "url">,
@@ -29,13 +49,46 @@ export function channelPlatform(
return config.platform ?? detectPlatform(config.url);
}
+// The key the pacing state is kept under — the same one the rate-limit
+// cooldown uses (`detectPlatform(url) ?? "unknown"`, autoRunner/runYtdlp), so a
+// 429 recorded by any path paces every path.
+export function pacingPlatformKey(config: Pick<ChannelConfig, "url">): string {
+ return detectPlatform(config.url) ?? "unknown";
+}
+
+// The platform's current `--sleep-requests`: the adaptive pace while one is
+// recorded, else the static value.
+export function channelPaceSeconds(
+ config: Pick<ChannelConfig, "platform" | "url">,
+): number {
+ return (
+ livePlatformPaceSeconds(pacingPlatformKey(config)) ??
+ staticSleepRequestsSeconds(channelPlatform(config))
+ );
+}
+
+// PLATFORM_ARGS for a platform at a given pace (default: its current one) —
+// for the spawns that have a URL and no channel config.
+export function pacedPlatformArgs(
+ platform: Platform | null | undefined,
+ paceSeconds: number | undefined = platform
+ ? livePlatformPaceSeconds(platform)
+ : undefined,
+): string[] {
+ return withSleepRequests(platformArgs(platform), paceSeconds);
+}
+
export function channelExtraArgs(
config: ChannelConfig,
cookies?: string,
+ // The pace to apply; default: the platform's current one. Tests pass it.
+ paceSeconds: number | undefined = livePlatformPaceSeconds(
+ pacingPlatformKey(config),
+ ),
): string[] {
const args: string[] = [];
args.push(...cookieArgs(cookies));
- args.push(...platformArgs(channelPlatform(config)));
+ args.push(...withSleepRequests(platformArgs(channelPlatform(config)), paceSeconds));
if (config.ytdlpExtraArgs?.length) args.push(...config.ytdlpExtraArgs);
return args;
}
diff --git a/common/ytdlp/downloadOneManaged.ts b/common/ytdlp/downloadOneManaged.ts
@@ -7,6 +7,8 @@ import { execa } from "execa";
import {
AUTH_RETRY_CLASSES,
classifyDownloadFailure,
+ hasNonRateLimitSubtitleFailure,
+ hasSubtitleRateLimit,
parseUnavailableFromStderr,
} from "../lib/availability";
import {
@@ -65,7 +67,7 @@ import {
import type { Paths } from "../lib/paths";
import { transcribeWithWorker } from "../controller/transcribeOne";
import { extractVideoId, outputArgsForUrl } from "./runYtdlp";
-import { channelExtraArgs } from "./channelArgs";
+import { channelExtraArgs, channelPaceSeconds } from "./channelArgs";
import {
ARCHIVE_MARKER,
runOneYtdlp as runOneYtdlpRaw,
@@ -168,6 +170,16 @@ export type ManagedDownloadOpts = {
// prefetch pass already wrote (fed in via sourceArgs as --load-info-json), so
// we drop --write-info-json (the file is already on disk). When false, the
// legacy single-call behavior: write the info json during this download.
+//
+// A SUBTITLE 429 IS A WARNING HERE, NOT THE END (release 17, slice RL). Under
+// yt-dlp's default, a subtitle file it cannot fetch raises: the video fails,
+// and — reading from --load-info-json — yt-dlp re-extracts from the URL once and
+// asks the throttled endpoint again. `--ignore-errors` makes it report the
+// failure as `WARNING: Unable to download video subtitles for …` and carry on
+// (exit 0, the line still in the log for the classifier); the caller then sees
+// "no transcript" and fetches the media (attempt 3). Any other error is still
+// an ERROR and a non-zero exit. `--sleep-subtitles` is the pace before each
+// subtitle request — the request YouTube throttles — never below `-t sleep`'s 5.
function youtubeHandlingArgs(
config: ChannelConfig,
reuseInfoJson = false,
@@ -179,10 +191,20 @@ function youtubeHandlingArgs(
config.subLangs ?? "en.*,live_chat",
];
if (!reuseInfoJson) args.push("--write-info-json");
- args.push("--skip-download", "-t", "sleep");
+ args.push(
+ "--skip-download",
+ "-t",
+ "sleep",
+ "--sleep-subtitles",
+ String(Math.max(YOUTUBE_SLEEP_SUBTITLES_FLOOR, channelPaceSeconds(config))),
+ "--ignore-errors",
+ );
return args;
}
+// `-t sleep`'s own `--sleep-subtitles`: the adaptive pace only ever raises it.
+const YOUTUBE_SLEEP_SUBTITLES_FLOOR = 5;
+
// Format/extract args for a transcribe-handling download under a resolved
// persistence plan. "ytdlp" mode is the legacy path: yt-dlp extracts the audio
// itself (-x) and optionally keeps its bestaudio source via -k. "app" mode omits
@@ -1192,9 +1214,96 @@ async function runManagedDownload(
});
if (primaryRes.archiveLine) lastArchiveLine = primaryRes.archiveLine;
+ // ---------- A subtitle 429 is not a failed download (release 17, RL) ----------
+ // YouTube's timedtext endpoint refuses per video while the media requests
+ // succeed. When the subtitle fetch is all that failed, the download goes on
+ // to the media and the record carries `subs_rate_limit` (a success): the
+ // caller defers the video's SUBTITLES, never the platform.
+ // - youtube handling runs the primary with --ignore-errors, so yt-dlp
+ // exits 0 with the failure as a WARNING; the media pass below fetches
+ // the audio when no transcript track arrived.
+ // - any other primary that died on its subtitles alone (a channel whose
+ // own args ask for subtitles) is run ONCE more with every subtitle
+ // refused: a second spawn, the media, no timedtext request.
+ // - ONLY a rate limit (review H1). --ignore-errors turns EVERY subtitle
+ // failure into a WARNING and exit 0 — a 403/404/5xx, a failed live_chat
+ // replay, a file it could not write. Any of those keeps today's meaning:
+ // the attempt failed (not archived, classified from the tail), exactly
+ // as the exit 1 it used to be.
+ let subsRateLimited = false;
+ let subtitleFailedOtherwise = false;
+ if (
+ !audioCheckEnabled &&
+ !audioCheckCorruptSource &&
+ !audioCheckCorruptFullSource
+ ) {
+ if (
+ attemptSucceeded(primaryRes.exitCode) &&
+ hasNonRateLimitSubtitleFailure(primaryRes.stderrTail)
+ ) {
+ subtitleFailedOtherwise = true;
+ const last = attempts.at(-1);
+ if (last) {
+ last.error = trimError(primaryRes.stderrTail);
+ last.availabilityClass = parseUnavailableFromStderr(primaryRes.stderrTail);
+ }
+ opts.onLog(
+ `The subtitle fetch for ${canonicalId ?? opts.videoUrl} failed (not a rate limit); ` +
+ `the download is a failed attempt, as before.\n`,
+ );
+ } else if (
+ attemptSucceeded(primaryRes.exitCode) &&
+ hasSubtitleRateLimit(primaryRes.stderrTail)
+ ) {
+ subsRateLimited = true;
+ } else if (
+ !attemptSucceeded(primaryRes.exitCode) &&
+ !opts.signal.aborted &&
+ classifyDownloadFailure(primaryRes.stderrTail, primaryAvail) ===
+ "subs_rate_limit"
+ ) {
+ subsRateLimited = true;
+ opts.onLog(
+ `The subtitle fetch for ${canonicalId ?? opts.videoUrl} was rate-limited (HTTP 429) and nothing else failed; ` +
+ `downloading without subtitles — they are deferred, the platform is not backed off.\n`,
+ );
+ const noSubsRes = await runOneYtdlp(opts, channelDir, [
+ "--ignore-config",
+ "--restrict-filenames",
+ ...FULL_LOG_PROGRESS_ARGS,
+ ...mediaArgs,
+ "--print",
+ `after_video:${ARCHIVE_MARKER} %(extractor)s %(id)s`,
+ ...channelConfigArgs(opts.channelConfig, primaryCookies),
+ // After the channel's own args: yt-dlp keeps the last occurrence.
+ "--no-write-subs",
+ "--no-write-auto-subs",
+ ...sourceArgs(opts.videoUrl, infoJsonPath),
+ ]);
+ lastFullTail = noSubsRes.stderrTail;
+ attempts.push({
+ n: 1,
+ kind: "primary-without-subs",
+ handling: opts.channelConfig.handling,
+ usedCookies: Boolean(primaryCookies),
+ ytdlpExitCode: noSubsRes.exitCode,
+ availabilityClass: attemptSucceeded(noSubsRes.exitCode)
+ ? undefined
+ : parseUnavailableFromStderr(noSubsRes.stderrTail),
+ error: attemptSucceeded(noSubsRes.exitCode)
+ ? undefined
+ : trimError(noSubsRes.stderrTail),
+ });
+ if (noSubsRes.archiveLine) lastArchiveLine = noSubsRes.archiveLine;
+ primaryRes = noSubsRes;
+ if (!attemptSucceeded(noSubsRes.exitCode)) subsRateLimited = false;
+ }
+ }
+
let lastSucceeded =
!audioCheckCorruptSource &&
!audioCheckCorruptFullSource &&
+ !subtitleFailedOtherwise &&
attemptSucceeded(primaryRes.exitCode);
if (lastSucceeded) {
// "ok-with-cookies" keeps meaning "cookies were NEEDED" (the prefetch only
@@ -1344,7 +1453,10 @@ async function runManagedDownload(
) {
const hasTranscript = await hasAnyTranscriptOnDisk(videoDir);
const noCaptions = await metadataReportsNoCaptions(videoDir);
- const noSubsFallback = !hasTranscript && noCaptions;
+ // A subtitle 429 left no transcript: the media is fetched exactly as for a
+ // video with no captions, and the subtitles wait (release 17, slice RL).
+ const subsDeferredFallback = subsRateLimited && !hasTranscript;
+ const noSubsFallback = !hasTranscript && (noCaptions || subsDeferredFallback);
// Forced only when the fallback would NOT have run on its own: a video with
// no transcript and no captions takes today's path whoever asked.
const forced = !noSubsFallback && opts.forceMedia === true;
@@ -1373,6 +1485,11 @@ async function runManagedDownload(
hasTranscript ? "a transcript is on disk" : "captions exist"
}\n`,
);
+ } else if (subsDeferredFallback) {
+ opts.onLog(
+ `The subtitles for ${videoId} were rate-limited (HTTP 429); downloading the media anyway — ` +
+ `the subtitles are deferred for download-missing-subs, the platform is not backed off${plan.persist ? " (keeping source video)" : ""}.\n`,
+ );
} else {
opts.onLog(
`No subs available for ${videoId}; falling back to audio download + whisper${plan.persist ? " (keeping source video)" : ""}.\n`,
@@ -1411,7 +1528,10 @@ async function runManagedDownload(
...channelConfigArgs(fallbackConfig, fallbackCookieOverride),
// yt-dlp takes the LAST occurrence of an option, so the refusals go
// after the channel's own args (the chat-only pass's rule).
- ...(keepTranscript ? ["--no-write-subs", "--no-write-auto-subs"] : []),
+ // A deferred subtitle fetch is not asked again in the same minute.
+ ...(keepTranscript || subsDeferredFallback
+ ? ["--no-write-subs", "--no-write-auto-subs"]
+ : []),
...sourceArgs(
opts.videoUrl,
path.join(videoDir, "metadata.info.json"),
@@ -1586,6 +1706,7 @@ async function runManagedDownload(
lastFullTail,
fellBackToTranscribe,
shortAudio: shortAudioInfo,
+ subsRateLimited,
});
}
@@ -1604,17 +1725,23 @@ async function writeOutcome(
lastFullTail: string;
fellBackToTranscribe?: boolean;
shortAudio?: DownloadOutcomeRecord["shortAudio"];
+ // The subtitle fetch alone answered 429 and the download went on.
+ subsRateLimited?: boolean;
},
): Promise<DownloadOutcomeRecord> {
const { status, attempts } = o;
const finishedAt = new Date().toISOString();
// Classify a failed download against the FULL stderr tail of the last attempt
// so a rate-limit logged as a WARNING (then masked by a different final error)
- // is still caught. Skipped for successes and filter-skips.
+ // is still caught. Skipped for successes and filter-skips — except that a
+ // success whose subtitles were rate-limited says so (`subs_rate_limit`), so
+ // its caller defers the subtitles (release 17, slice RL).
const isFailure = status === "failed" || status === "failed-corrupt-source";
const failureClass = isFailure
? classifyDownloadFailure(o.lastFullTail, attempts.at(-1)?.availabilityClass)
- : undefined;
+ : o.subsRateLimited && status.startsWith("ok")
+ ? ("subs_rate_limit" as const)
+ : undefined;
const record: DownloadOutcomeRecord = {
videoId: o.videoId,
webpageUrl: opts.videoUrl,
diff --git a/common/ytdlp/managedDownloadsSleep.test.ts b/common/ytdlp/managedDownloadsSleep.test.ts
@@ -246,3 +246,61 @@ test("declinedWithoutMediaFetch: a chat-only pass is a fetch", () => {
false,
);
});
+
+// ── release 17, slice RL ─────────────────────────────────────────────────────
+
+// Runs two fetched videos with an injected pace and returns the slept ms.
+async function gapFor(
+ paceSeconds: number,
+ outcomes: DownloadOutcomeRecord[] = [FETCHED, FETCHED],
+): Promise<{ slept: number[]; deferred: string[] }> {
+ const slept: number[] = [];
+ const deferred: string[] = [];
+ let i = 0;
+ const urls = outcomes.map(
+ (_, n) => `https://www.youtube.com/watch?v=vid${String(n).padStart(8, "0")}`,
+ );
+ const channelConfig = {
+ handling: "youtube",
+ url: "https://www.youtube.com/@c/videos",
+ } as never;
+ await runManagedDownloads(
+ {
+ channelSlug: "c",
+ mode: "download-missing" as never,
+ channelConfig,
+ paths: getPaths(),
+ onLog: () => {},
+ signal: new AbortController().signal,
+ abortOnError: false,
+ },
+ urls,
+ channelConfig,
+ undefined,
+ {
+ downloadOne: async () => outcomes[i++],
+ sleep: async (ms) => {
+ slept.push(ms);
+ },
+ paceSeconds: () => paceSeconds,
+ recordSubtitleDeferral: async (id, slug) => {
+ deferred.push(`${slug}/${id}`);
+ },
+ },
+ );
+ return { slept, deferred };
+}
+
+test("the gap is sleepBetweenDownloadsSeconds plus the pace above its base", async () => {
+ // youtube's base pace is 1 s: at the base the gap is the setting alone.
+ assert.deepEqual((await gapFor(1)).slept, [30_000]);
+ // After two rate limits (pace 4 s) the batch waits 3 s more per video.
+ assert.deepEqual((await gapFor(4)).slept, [33_000]);
+});
+
+test("a subtitle 429 on a success is recorded as a subtitle deferral, and the batch goes on", async () => {
+ const subs = { ...outcome("ok", [PREFETCH, PRIMARY], "subs_rate_limit"), videoId: "s1" };
+ const r = await gapFor(1, [subs, FETCHED]);
+ assert.deepEqual(r.deferred, ["c/s1"]);
+ assert.deepEqual(r.slept, [30_000]);
+});
diff --git a/common/ytdlp/metadataScan.ts b/common/ytdlp/metadataScan.ts
@@ -31,7 +31,8 @@ import {
authRetryCookies,
resolveCookiePolicy,
} from "../lib/cookiePolicy";
-import { channelExtraArgs } from "./channelArgs";
+import { channelExtraArgs, pacingPlatformKey } from "./channelArgs";
+import { recordPlatformClean } from "../jobs/downloadBackoff";
import type { Paths } from "../lib/paths";
import { getSettings } from "../lib/settings";
import { extractVideoId } from "../lib/videoId";
@@ -683,6 +684,15 @@ export async function runMetadataScan(
`${scannedTotal} video(s) were read and saved; the rest stay unscanned. A per-platform cooldown has been recorded — run the scan again once it lapses.\n`,
);
await opts.onPlatformBackoff?.("rate_limit");
+ } else if (stopped === undefined && scannedTotal > 0 && !opts.signal.aborted) {
+ // A scan the source answered cleanly settles the platform like a clean
+ // probe — its backoff and any hold clear (release 17 review H2).
+ try {
+ const line = await recordPlatformClean(pacingPlatformKey(channelConfig), paths);
+ if (line) opts.onLog(line);
+ } catch {
+ /* shared-state write is best-effort */
+ }
}
const errorCount = Object.keys(errors).length;
diff --git a/common/ytdlp/platformArgs.mjs b/common/ytdlp/platformArgs.mjs
@@ -58,3 +58,43 @@ export function platformArgs(platform) {
export function platformArgsForUrl(url) {
return platformArgs(detectPlatform(url));
}
+
+// THE STATIC PACE a platform's spawns carry (`--sleep-requests` in its
+// PLATFORM_ARGS entry), or 0 for a platform with none. The adaptive pace
+// (common/jobs/platformBackoff.ts, release 17 slice RL) starts here, doubles on
+// a rate limit and decays back to it.
+/**
+ * @param {string | null | undefined} platform
+ * @returns {number}
+ */
+export function staticSleepRequestsSeconds(platform) {
+ if (!platform) return 0;
+ /** @type {Record<string, readonly string[] | undefined>} */
+ const table = PLATFORM_ARGS;
+ const args = table[platform];
+ if (!args) return 0;
+ const i = args.indexOf("--sleep-requests");
+ const v = i >= 0 ? Number(args[i + 1]) : 0;
+ return Number.isFinite(v) && v > 0 ? v : 0;
+}
+
+// `args` with its `--sleep-requests` raised to `seconds` — replaced when the
+// args carry a lower one, appended when they carry none. Never LOWERS a pace:
+// a value below what the args already say is ignored. Returns a copy.
+/**
+ * @param {readonly string[]} args
+ * @param {number | undefined | null} seconds
+ * @returns {string[]}
+ */
+export function withSleepRequests(args, seconds) {
+ const out = [...args];
+ if (seconds == null || !Number.isFinite(seconds) || seconds <= 0) return out;
+ const i = out.indexOf("--sleep-requests");
+ if (i < 0) {
+ out.push("--sleep-requests", String(seconds));
+ return out;
+ }
+ const cur = Number(out[i + 1]);
+ if (!Number.isFinite(cur) || seconds > cur) out[i + 1] = String(seconds);
+ return out;
+}
diff --git a/common/ytdlp/platformClean.test.ts b/common/ytdlp/platformClean.test.ts
@@ -0,0 +1,58 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { chmodSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test ytdlp/platformClean.test.ts
+//
+// A CLEAN MANUAL RUN SETTLES THE PLATFORM — ONLY IF IT ASKED THE SOURCE
+// SOMETHING (release 17 slice RL, re-review R3). runYtdlp calls
+// `onPlatformClean` after a run that neither threw nor recorded a backoff; a
+// run that made no request (download-missing with nothing to fetch) must not,
+// or it would lift a due hold on no evidence. getPaths() memoizes, so the env
+// is set before anything imports it.
+const ROOT = mkdtempSync(path.join(os.tmpdir(), "platform-clean-"));
+process.env.TRANSCRIPTS_DIR = ROOT;
+process.env.SETTINGS_FILE = path.join(ROOT, "settings.json");
+writeFileSync(
+ process.env.SETTINGS_FILE,
+ JSON.stringify({ minFreeDiskGB: 0, sleepBetweenDownloadsSeconds: 0 }) + "\n",
+);
+const BIN = path.join(ROOT, "fake-ytdlp.sh");
+writeFileSync(BIN, "#!/bin/sh\nexit 0\n");
+chmodSync(BIN, 0o755);
+process.env.YTDLP_BIN = BIN;
+mkdirSync(path.join(ROOT, "channels", "c", "data"), { recursive: true });
+// An empty saved playlist: download-missing has nothing to fetch.
+writeFileSync(path.join(ROOT, "channels", "c", "playlist"), "");
+
+const { runYtdlp } = await import("./runYtdlp");
+const { getPaths } = await import("../lib/paths");
+
+async function cleanCalls(mode: "download-missing" | "download-one-audio"): Promise<number> {
+ let calls = 0;
+ await runYtdlp({
+ channelSlug: "c",
+ mode,
+ channelConfig: { handling: "transcribe", url: "https://www.youtube.com/@c/videos" } as never,
+ paths: getPaths(),
+ onLog: () => {},
+ signal: new AbortController().signal,
+ singleVideoUrl: "https://www.youtube.com/watch?v=abcdefghijk",
+ onPlatformClean: async () => {
+ calls++;
+ return null;
+ },
+ });
+ return calls;
+}
+
+test("a run that made no request is not a clean run", async () => {
+ assert.equal(await cleanCalls("download-missing"), 0);
+});
+
+test("a run that made a request and recorded no backoff is", async () => {
+ assert.equal(await cleanCalls("download-one-audio"), 1);
+});
diff --git a/common/ytdlp/runYtdlp.ts b/common/ytdlp/runYtdlp.ts
@@ -14,7 +14,13 @@ import { getSettings } from "../lib/settings";
import { patchChannelConfig } from "../controller/channels";
import { diskGate } from "../lib/diskSpace";
import { detectPlatform } from "../lib/platform";
-import { channelExtraArgs, platformArgs } from "./channelArgs";
+import {
+ channelExtraArgs,
+ channelPaceSeconds,
+ channelPlatform,
+ pacedPlatformArgs,
+ staticSleepRequestsSeconds,
+} from "./channelArgs";
import { isRealAudioFile } from "../lib/videoStatus";
import { readVttProvenance } from "../lib/subtitleProvenance";
import { extractVideoId } from "../lib/videoId";
@@ -56,9 +62,21 @@ import {
isFullSweepDue,
resolveFullSweepIntervalMinutes,
} from "../jobs/deepSync";
-import { platformCooldownRemainingMs } from "../jobs/downloadBackoff";
+import {
+ clearSubtitleDeferral,
+ platformCooldownRemainingMs,
+ readSubtitleDeferrals,
+ recordSubtitleDeferral,
+} from "../jobs/downloadBackoff";
+import {
+ SUBTITLE_HOLD_AFTER,
+ downloadGapMs,
+ isSubtitleDeferred,
+} from "../jobs/platformBackoff";
+import { hasSubtitleRateLimit } from "../lib/availability";
import { computeKeepWindow, type KeepWindow } from "../controller/keptVideos";
import { downloadOneManaged } from "./downloadOneManaged";
+import { STDERR_TAIL_BYTES } from "./runOneYtdlp";
import {
resolveDownloadFormatPreset,
resolveDownloadFormatSelector,
@@ -161,6 +179,17 @@ export type RunYtdlpOpts = {
onPlatformBackoff?: (
failureClass: "rate_limit" | "network",
) => void | Promise<void>;
+ // Called once when the run finished — not thrown, not cancelled — with no
+ // platform backoff recorded during it: the source answered cleanly, so a
+ // manual run settles the platform the way a clean lane probe does (its
+ // backoff and any hold clear; release 17 review H2). Returns a line for the
+ // log, or null.
+ onPlatformClean?: () => Promise<string | null>;
+ // Internal: how many requests this run made of the source (a listing, a
+ // per-video download, a raw spawn). Set by runYtdlpMode and shared through
+ // every `{...opts}` copy by reference; a run that asked nothing is not clean
+ // (release 17 re-review R3).
+ requestCounter?: { n: number };
};
// The batch modes whose child writes media into `data/<id>/` — yt-dlp's own
@@ -204,7 +233,35 @@ export async function runYtdlp(opts: RunYtdlpOpts): Promise<void> {
}
}
+// A CLEAN RUN SETTLES THE PLATFORM (release 17 review H2): with no rate limit
+// or network failure recorded while it ran, the run is the probe a held
+// platform waits for — the lane may be off, or have nothing pending on it.
async function runYtdlpMode(opts: RunYtdlpOpts): Promise<void> {
+ if (!opts.onPlatformClean) return runYtdlpModeInner(opts);
+ let backedOff = false;
+ const inner = opts.onPlatformBackoff;
+ const requestCounter = { n: 0 };
+ await runYtdlpModeInner({
+ ...opts,
+ requestCounter,
+ onPlatformBackoff: async (failureClass) => {
+ backedOff = true;
+ await inner?.(failureClass);
+ },
+ });
+ // A run that asked the source nothing (download-missing with nothing to
+ // fetch, download-missing-subs with every video deferred) says nothing about
+ // the platform, and must not lift a due hold.
+ if (backedOff || opts.signal.aborted || requestCounter.n === 0) return;
+ try {
+ const line = await opts.onPlatformClean();
+ if (line) opts.onLog(line);
+ } catch {
+ /* shared-state write is best-effort */
+ }
+}
+
+async function runYtdlpModeInner(opts: RunYtdlpOpts): Promise<void> {
switch (opts.mode) {
case "store-playlist":
await storePlaylist(opts);
@@ -441,6 +498,7 @@ async function enumeratePlaylistUrls(
opts.channelConfig.url!,
);
opts.onLog(`$ ${opts.paths.ytdlpBin} ${args.join(" ")}\n`);
+ if (opts.requestCounter) opts.requestCounter.n++;
const child = execa(opts.paths.ytdlpBin, args, {
cwd: root,
@@ -530,8 +588,8 @@ export async function probeChannelMeta(opts: {
"--print",
"%(channel,uploader,playlist_title,playlist,uploader_id,title)s",
// No channel config exists yet; the platform's fixed args still apply
- // (a Rumble probe 403s without `--impersonate`).
- ...platformArgs(detectPlatform(url)),
+ // (a Rumble probe 403s without `--impersonate`), at its current pace.
+ ...pacedPlatformArgs(detectPlatform(url)),
url,
];
log(`$ ${paths.ytdlpBin} ${args.join(" ")}\n`);
@@ -940,6 +998,10 @@ async function downloadPlaylistManaged(
export type ManagedDownloadsDeps = {
downloadOne?: typeof downloadOneManaged;
sleep?: (ms: number, signal: AbortSignal) => Promise<void>;
+ // Where a video's subtitle 429 is recorded (release 17, slice RL).
+ recordSubtitleDeferral?: (videoId: string, channelSlug: string) => Promise<unknown>;
+ // The platform's current pace (default: channelPaceSeconds).
+ paceSeconds?: () => number;
};
// True when the download filter declined the video before any media request:
@@ -1011,6 +1073,14 @@ export async function runManagedDownloads(
const sleepSeconds =
opts.channelConfig.sleepBetweenDownloadsSeconds ??
settings.sleepBetweenDownloadsSeconds;
+ // THE PACE ABOVE ITS BASE joins the gap (release 17, slice RL): after a
+ // rate limit doubled the platform's pace, a batch spaces its videos further
+ // apart too. Read per gap, so a 429 in this batch slows the rest of it.
+ const basePace = staticSleepRequestsSeconds(channelPlatform(effectiveChannelConfig));
+ const paceNow = deps.paceSeconds ?? (() => channelPaceSeconds(effectiveChannelConfig));
+ const recordSubs =
+ deps.recordSubtitleDeferral ??
+ ((id: string, slug: string) => recordSubtitleDeferral(id, slug, opts.paths));
// Title-filter rejections from this batch, written to the channel-level
// metadata-scan store ONCE at the end. Per-video writes would rewrite the
// whole file for every non-matching video — the exact shape of work a
@@ -1066,6 +1136,7 @@ export async function runManagedDownloads(
});
let outcome;
try {
+ if (opts.requestCounter) opts.requestCounter.n++;
outcome = await downloadOne({
channelSlug: opts.channelSlug,
channelConfig: effectiveChannelConfig,
@@ -1089,6 +1160,16 @@ export async function runManagedDownloads(
} finally {
task?.end();
}
+ // A subtitle 429 on a download that otherwise succeeded: the video's
+ // subtitles are deferred, the platform is not backed off, the batch
+ // goes on (release 17, slice RL). Best-effort, like the backoff.
+ if (outcome.failureClass === "subs_rate_limit") {
+ try {
+ await recordSubs(outcome.videoId, opts.channelSlug);
+ } catch {
+ /* shared-state write is best-effort */
+ }
+ }
if (outcome.status === "skipped-filtered") {
skippedCount++;
} else if (outcome.status === "chat-only") {
@@ -1142,15 +1223,16 @@ export async function runManagedDownloads(
// declined slept 30 s after nothing but its metadata prefetch. Every
// other outcome still sleeps — a real fetch, success or failure, and
// every failure, per-video ones included (see declinedWithoutMediaFetch).
+ const gapMs = downloadGapMs(sleepSeconds, paceNow(), basePace);
if (
- sleepSeconds > 0 &&
+ gapMs > 0 &&
!isLast &&
!willAbortLoop &&
!opts.signal.aborted &&
!declinedWithoutMediaFetch(outcome)
) {
- opts.onLog(`Sleeping ${sleepSeconds}s before next download...\n`);
- await sleep(sleepSeconds * 1000, opts.signal);
+ opts.onLog(`Sleeping ${gapMs / 1000}s before next download...\n`);
+ await sleep(gapMs, opts.signal);
}
}),
),
@@ -1228,12 +1310,15 @@ function trackOnDisk(entries: string[], track: string): boolean {
// Fetch only the missing subtitle tracks for one already-downloaded video.
// Uses the literal canonical output path (outputArgsForUrl) so the .vtt files
// land in the same data/<canonicalId>/ dir as the rest of the video.
+// THE BATCH SUBTITLE FETCH, per video. Returns the stderr it printed so the
+// caller can tell a subtitle 429 (deferred, release 17 slice RL) from any
+// other failure; throws on any non-zero exit as before.
async function downloadSubsForUrl(
opts: RunYtdlpOpts,
root: string,
url: string,
subLangs: string,
-): Promise<void> {
+): Promise<string> {
const args: string[] = [
"--ignore-config",
"--restrict-filenames",
@@ -1252,7 +1337,7 @@ async function downloadSubsForUrl(
"--",
url,
];
- await runChildAndStream(opts, root, args);
+ return runChildAndStream(opts, root, args);
}
async function downloadMissingSubs(opts: RunYtdlpOpts): Promise<void> {
@@ -1281,6 +1366,13 @@ async function downloadMissingSubs(opts: RunYtdlpOpts): Promise<void> {
let unidentifiable = 0;
let noMetadata = 0;
let noExpectedTracks = 0;
+ // Videos whose subtitles answered 429 recently (release 17, slice RL): left
+ // alone until their deferral lapses — 6 h after a strike, 7 days from the
+ // third. The video page's own download still fetches them.
+ const subDeferrals = await readSubtitleDeferrals(opts.paths).catch(
+ () => ({}) as Awaited<ReturnType<typeof readSubtitleDeferrals>>,
+ );
+ const deferredNow: string[] = [];
for (const url of urls) {
// Post-reconcile a video lives in data/<canonicalId>/, so the URL's
@@ -1290,6 +1382,10 @@ async function downloadMissingSubs(opts: RunYtdlpOpts): Promise<void> {
unidentifiable++;
continue;
}
+ if (isSubtitleDeferred(subDeferrals, dirId, Date.now())) {
+ deferredNow.push(dirId);
+ continue;
+ }
const videoDir = path.join(dataDir, dirId);
let metaRaw: string;
try {
@@ -1339,6 +1435,14 @@ async function downloadMissingSubs(opts: RunYtdlpOpts): Promise<void> {
: "") +
(noMetadata ? `, ${noMetadata} without metadata (skipped)` : "") +
(unidentifiable ? `, ${unidentifiable} unidentifiable URLs` : "") +
+ (deferredNow.length
+ ? `, ${deferredNow.length} deferred after a subtitle rate limit (${deferredNow
+ .map((id) => {
+ const d = (subDeferrals as Record<string, { count: number; until: number }>)[id];
+ return `${id} ${d.count}×${d.count >= SUBTITLE_HOLD_AFTER ? " — left alone" : ""} until ${new Date(d.until).toISOString().slice(0, 16).replace("T", " ")} UTC`;
+ })
+ .join(", ")})`
+ : "") +
`.\n`,
);
@@ -1353,26 +1457,68 @@ async function downloadMissingSubs(opts: RunYtdlpOpts): Promise<void> {
let processed = 0;
let failed = 0;
let firstFailure: Error | null = null;
+ // THE GAP BETWEEN TWO VIDEOS (release 17 review L3): the same
+ // sleepBetweenDownloadsSeconds plus the pace above its base the lane and the
+ // batch downloads wait, so a channel whose subtitles are being refused is
+ // not asked once per video back to back.
+ const subsSleepSeconds =
+ opts.channelConfig.sleepBetweenDownloadsSeconds ??
+ getSettings().sleepBetweenDownloadsSeconds;
+ const subsBasePace = staticSleepRequestsSeconds(channelPlatform(opts.channelConfig));
await Promise.all(
- tofetch.map((url) =>
+ tofetch.map((url, index) =>
limit(async () => {
if (opts.signal.aborted || opts.drainSignal?.aborted) return;
if (firstFailure) return;
+ if (index > 0) {
+ const gapMs = downloadGapMs(
+ subsSleepSeconds,
+ channelPaceSeconds(opts.channelConfig),
+ subsBasePace,
+ );
+ if (gapMs > 0) {
+ opts.onLog(`Sleeping ${gapMs / 1000}s before the next video...\n`);
+ await abortableSleep(gapMs, opts.signal);
+ if (opts.signal.aborted || opts.drainSignal?.aborted) return;
+ }
+ }
const task = opts.tracker?.start({
id: extractVideoId(url) ?? url,
label: extractVideoId(url) ?? url,
kind: "download",
});
+ const id = extractVideoId(url);
try {
- await downloadSubsForUrl(
+ const stderr = await downloadSubsForUrl(
task ? { ...opts, onLog: task.onLog } : opts,
root,
url,
subLangs,
);
+ // yt-dlp exited clean. A subtitle 429 it reported anyway (a WARNING)
+ // is still a deferral; otherwise the subtitles came down and the
+ // video's deferral, if it had one, is forgotten.
+ if (id) {
+ if (hasSubtitleRateLimit(stderr)) {
+ await recordSubtitleDeferral(id, opts.channelSlug, opts.paths).catch(() => {});
+ } else {
+ await clearSubtitleDeferral(id, opts.paths).catch(() => {});
+ }
+ }
} catch (err) {
failed++;
- if (abortOnError && !firstFailure) firstFailure = err as Error;
+ // A SUBTITLE 429 IS PER VIDEO (release 17, slice RL): the video is
+ // deferred and the batch goes on to the next one, abortOnError or
+ // not. Any other failure keeps today's abort.
+ const tail = (err as { stderrTail?: string }).stderrTail ?? "";
+ if (id && classifyDownloadFailure(tail, undefined) === "subs_rate_limit") {
+ const d = await recordSubtitleDeferral(id, opts.channelSlug, opts.paths).catch(() => null);
+ (task?.onLog ?? opts.onLog)(
+ `Subtitles for ${id} were rate-limited (HTTP 429)${d ? ` — ${d.count}×, deferred` : ""}; continuing with the next video.\n`,
+ );
+ } else if (abortOnError && !firstFailure) {
+ firstFailure = err as Error;
+ }
} finally {
task?.end();
}
@@ -1978,12 +2124,15 @@ async function safeBackfillAvailability(opts: RunYtdlpOpts): Promise<void> {
}
}
+// Returns the last STDERR_TAIL_BYTES of stderr (and attaches it to the thrown
+// error as `stderrTail`), so a caller can classify what yt-dlp said.
async function runChildAndStream(
opts: RunYtdlpOpts,
cwd: string,
args: string[],
-): Promise<void> {
+): Promise<string> {
opts.onLog(`$ ${opts.paths.ytdlpBin} ${args.join(" ")}\n`);
+ if (opts.requestCounter) opts.requestCounter.n++;
const child = execa(opts.paths.ytdlpBin, args, {
cwd,
cancelSignal: opts.signal,
@@ -1992,6 +2141,10 @@ async function runChildAndStream(
reject: false,
});
child.all?.on("data", (c: Buffer) => opts.onLog(c.toString("utf8")));
+ let stderrTail = "";
+ child.stderr?.on("data", (c: Buffer) => {
+ stderrTail = (stderrTail + c.toString("utf8")).slice(-STDERR_TAIL_BYTES);
+ });
const result = await child;
// yt-dlp exit code convention: 101 = "break-on-existing" / "max-downloads"
// (clean stop, not an error). Treat it the same as 0.
@@ -2000,11 +2153,14 @@ async function runChildAndStream(
result.exitCode !== 101 &&
!opts.signal.aborted
) {
- throw new Error(`yt-dlp exited with code ${result.exitCode}`);
+ throw Object.assign(new Error(`yt-dlp exited with code ${result.exitCode}`), {
+ stderrTail,
+ });
}
if (result.exitCode === 101) {
opts.onLog(`yt-dlp stopped on existing entry (exit 101).\n`);
}
+ return stderrTail;
}
// The three sync-state stamps (CHANNEL_SYNC_STATE_KEYS). Each re-reads the
diff --git a/common/ytdlp/subtitleRateLimit.test.ts b/common/ytdlp/subtitleRateLimit.test.ts
@@ -0,0 +1,175 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import type { ChannelConfig } from "../lib/channelConfig";
+import { downloadOneManaged } from "./downloadOneManaged";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test ytdlp/subtitleRateLimit.test.ts
+//
+// A SUBTITLE 429 NEVER FAILS A DOWNLOAD (release 17, slice RL). The yt-dlp here
+// is a temp node script that plays the 2026-10-01 shape: the prefetch and the
+// media requests succeed, the subtitle fetch answers 429 — as a WARNING when
+// asked to --ignore-errors (yt-dlp's own semantics), as an ERROR otherwise. It
+// records every argv so the spawns can be counted and read.
+
+const ID = "H64QQZuw-aA";
+const VIDEO = `https://www.youtube.com/watch?v=${ID}`;
+const SUB_429 = `Unable to download video subtitles for 'en': HTTP Error 429: Too Many Requests`;
+
+const FAKE = `#!/usr/bin/env node
+const fs = require("node:fs");
+const path = require("node:path");
+const argv = process.argv.slice(2);
+fs.appendFileSync(process.env.FAKE_LOG, JSON.stringify(argv) + "\\n");
+const has = (f) => argv.includes(f);
+const dir = path.join("data", ${JSON.stringify(ID)});
+fs.mkdirSync(dir, { recursive: true });
+if (has("--no-write-auto-subs") && has("--write-info-json") && !has("-x") && !has("-f")) {
+ // the metadata prefetch: captions are listed
+ fs.writeFileSync(path.join(dir, "metadata.info.json"), JSON.stringify({
+ id: ${JSON.stringify(ID)}, title: "t", automatic_captions: { en: [{}] }, subtitles: {},
+ }));
+ process.exit(0);
+}
+const subsOn = (has("--write-subs") || has("--write-auto-subs")) &&
+ !(argv.lastIndexOf("--no-write-subs") > argv.lastIndexOf("--write-subs"));
+const SUB_FAIL = process.env.FAKE_SUB_FAIL || ${JSON.stringify(SUB_429)};
+if (subsOn) {
+ if (has("--ignore-errors")) {
+ process.stderr.write("WARNING: [youtube] ${ID}: " + SUB_FAIL + "\\n");
+ if (has("--skip-download")) { process.stdout.write("DLOM_ARCHIVE youtube ${ID}\\n"); process.exit(0); }
+ } else {
+ process.stderr.write("ERROR: [youtube] ${ID}: " + SUB_FAIL + "\\n");
+ process.exit(1);
+ }
+}
+// the media
+fs.writeFileSync(path.join(dir, "audio.mp3"), "audio");
+process.stdout.write("DLOM_ARCHIVE youtube ${ID}\\n");
+process.exit(0);
+`;
+
+async function run(config: Partial<ChannelConfig>, subFail?: string): Promise<{
+ record: Awaited<ReturnType<typeof downloadOneManaged>>;
+ spawns: string[][];
+ audio: boolean;
+ archived: boolean;
+ transcript: boolean;
+ log: string;
+}> {
+ const root = await mkdtemp(path.join(tmpdir(), "subs-rl-"));
+ try {
+ const bin = path.join(root, "fake-ytdlp.cjs");
+ await writeFile(bin, FAKE);
+ await chmod(bin, 0o755);
+ const logFile = path.join(root, "argv.log");
+ process.env.FAKE_LOG = logFile;
+ if (subFail) process.env.FAKE_SUB_FAIL = subFail;
+ else delete process.env.FAKE_SUB_FAIL;
+ const paths = {
+ channelsDir: path.join(root, "channels"),
+ ytdlpBin: bin,
+ } as Paths;
+ let log = "";
+ const record = await downloadOneManaged({
+ channelSlug: "c",
+ channelConfig: {
+ handling: "youtube",
+ url: "https://www.youtube.com/@c/videos",
+ ...config,
+ } as ChannelConfig,
+ paths,
+ videoUrl: VIDEO,
+ onLog: (s) => {
+ log += s;
+ },
+ signal: new AbortController().signal,
+ });
+ const spawns = (await readFile(logFile, "utf8"))
+ .split("\n")
+ .filter(Boolean)
+ .map((l) => JSON.parse(l) as string[]);
+ const dir = path.join(paths.channelsDir, "c", "data", ID);
+ return {
+ record,
+ spawns,
+ audio: existsSync(path.join(dir, "audio.mp3")),
+ archived: existsSync(path.join(paths.channelsDir, "c", "archive")),
+ transcript: existsSync(path.join(dir, "transcript.en.vtt")),
+ log,
+ };
+ } finally {
+ delete process.env.FAKE_LOG;
+ delete process.env.FAKE_SUB_FAIL;
+ await rm(root, { recursive: true, force: true });
+ }
+}
+
+test("youtube handling: the subtitle 429 is a warning, the media comes down, the record says subs_rate_limit", async () => {
+ const r = await run({});
+ assert.equal(r.record.status, "ok");
+ assert.equal(r.record.failureClass, "subs_rate_limit");
+ assert.equal(r.record.fellBackToTranscribe, true);
+ assert.equal(r.audio, true);
+ assert.equal(r.transcript, false);
+ assert.deepEqual(
+ r.record.attempts.map((a) => a.kind),
+ ["metadata-prefetch", "primary", "no-subs-fallback"],
+ );
+ // The primary asked yt-dlp to carry on past a subtitle failure, and paced
+ // the subtitle request at no less than `-t sleep`'s 5 s.
+ const primary = r.spawns[1];
+ assert.ok(primary.includes("--ignore-errors"));
+ const ss = primary.indexOf("--sleep-subtitles");
+ assert.ok(ss > primary.indexOf("sleep"), "--sleep-subtitles comes after -t sleep");
+ assert.equal(primary[ss + 1], "5");
+ // The media pass does not ask the throttled endpoint again.
+ const media = r.spawns[2];
+ assert.ok(media.lastIndexOf("--no-write-subs") >= 0);
+ assert.ok(media.lastIndexOf("--no-write-auto-subs") >= 0);
+ assert.match(r.log, /subtitles for H64QQZuw-aA were rate-limited \(HTTP 429\); downloading the media anyway/);
+ // Three spawns: prefetch, subtitles, media — the same as a video with no captions.
+ assert.equal(r.spawns.length, 3);
+});
+
+test("a primary that died on its subtitles alone is run once more without them", async () => {
+ // A transcribe-handling channel whose own args ask for subtitles.
+ const r = await run({ handling: "transcribe", ytdlpExtraArgs: ["--write-subs"] });
+ assert.equal(r.record.status, "ok");
+ assert.equal(r.record.failureClass, "subs_rate_limit");
+ assert.deepEqual(
+ r.record.attempts.map((a) => a.kind),
+ ["metadata-prefetch", "primary", "primary-without-subs"],
+ );
+ assert.equal(r.audio, true);
+ const again = r.spawns[2];
+ // The refusals come after the channel's own --write-subs (last flag wins).
+ assert.ok(again.lastIndexOf("--no-write-subs") > again.lastIndexOf("--write-subs"));
+});
+
+// Review H1: --ignore-errors turns EVERY subtitle failure into a WARNING and
+// exit 0. Only a 429 takes the new path; anything else is a failed attempt,
+// exactly as before — never `ok` with no transcript, no media and an archive line.
+for (const [name, fail, cls] of [
+ ["a subtitle 403", "Unable to download video subtitles for 'en': HTTP Error 403: Forbidden", "network"],
+ ["a failed live_chat replay", "Unable to download video subtitles for 'live_chat': HTTP Error 404: Not Found", "unknown"],
+] as const) {
+ test(`youtube handling: ${name} under --ignore-errors is still a failed attempt`, async () => {
+ const r = await run({}, fail);
+ assert.equal(r.record.status, "failed");
+ assert.equal(r.record.failureClass, cls);
+ assert.equal(r.audio, false);
+ assert.equal(r.archived, false);
+ assert.deepEqual(
+ r.record.attempts.map((a) => a.kind),
+ ["metadata-prefetch", "primary"],
+ );
+ assert.match(r.record.attempts[1].error ?? "", /Unable to download video subtitles/);
+ assert.equal(r.spawns.length, 2);
+ });
+}
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -35,6 +35,8 @@
- **A channel's text stays on the fast disk when its media moves, so a slow or unplugged media drive no longer holds its transcripts.** A channel's big files — the audio and the raw live-chat replay — can now live in the channel's own `media` folder, on this disk or another, while its transcripts, cues, metadata and every other small file stay in `data/` where they always were; each big file that moves leaves a small link behind, so everything that opens it by name still finds it. New downloads, transcodes and live-chat normalizes put their big files there as they finish, and every cleanup that deletes audio removes the file the link points to, not just the link. What that changes when a media drive is stalled, unplugged or mid-move: **the index and stats builds never wait on it or are held by it** (a live chat whose transcript cues are out of date keeps the cues the last build read until the drive answers), the channel's report still refreshes (its media size reads as unknown until the drive answers), **digests keep running — even during a move of that channel's media** — and so do normalize, the availability checks, the metadata scan and clip eviction. Transcription, downloads, the backfill lane and anything else that opens the audio are held as before. **A channel moved the old way — its whole `data/` on the other drive — is now shown as "Media layout retired" and held by everything, the builds and digests included, until `archilyzer storage migrate-tier <channel>` brings its text home;** every refusal says so. Deleting a video from its page is refused while its channel's media drive is not reachable, so its audio is never left behind on the drive. Needs a rebuild and restart of the editor.
- **Moving a channel's media now moves only its big files.** The Storage panel's **Move media** copies the channel's audio and raw live-chat replays to `<root>/<channel>/media` on the destination and leaves its transcripts, metadata and every other small file in `data/` on this disk; a channel whose audio is still in `data/` has it put into the channel's `media` folder on this disk first, and the preview says how many files ("2 file(s) tiered first"). **Move back in place** brings the `media` folder home; the links in `data/` are not touched either way. The panel shows the media path and the text path, each with its size. While a channel's media is held — moving, or on a drive that is unplugged or not answering — its text stays readable: the video page and the videos list open as usual and list the audio with its size unknown, the transcript opens, the audio answers "not reachable, try again" (503) instead of "not found", and a digest may run, during a move too; only the jobs and lanes that open the audio wait. A move, its preview and **Resume move** refuse a channel still on the retired whole-directory layout and name `archilyzer storage migrate-tier`; `/storage` counts such channels as unreachable "(n to migrate)". On `/storage` a location's figure is now the media on it, and the corpus volume's row adds every channel's text and clip windows ("text … + clips … on the corpus volume, plus … media of in-place channels"). Re-pointing a location, renaming a channel and deleting one follow the `media` folder. Needs a rebuild and restart of the editor.
- **An export build no longer reads a raw live-chat replay from a media drive that is unplugged, not answering or mid-move.** It publishes the live-chat cues already on disk for that channel, and its log says how many are older than their replay.
+- **A video whose YouTube subtitles answer "Too Many Requests" (HTTP 429) is downloaded anyway, and YouTube is not put in a cooldown for it.** YouTube refuses a subtitle file per video while the video itself downloads fine; every one of the day's 429s on 2026-10-01 was a subtitle fetch, and each failed its download, put all of YouTube in a cooldown that reached 30 minutes, and deferred the video for 6 hours to fail the same way again. Now that refusal is noted and the download goes on to the audio, as for a video with no captions, so the transcription lane transcribes it; nothing platform-wide is backed off. The video's subtitles are deferred: **Download missing subs** skips them for 6 hours, and from the third time they are refused, for 7 days; a download from the video's own page still fetches them. The download lane's page lists them under **Deferred subtitles**, and the video page says how many times and when. **Download missing subs** also goes on to the next video when one video's subtitles are refused, instead of stopping. Any other subtitle failure (a 403, a missing file, a chat replay that fails) still fails the download, as before. Needs a rebuild and restart of the editor.
+- **The download pace adapts to rate limits, a rate limit that outlasts the cooldown holds the platform, and the auto-download lane waits between downloads.** Every yt-dlp run against a platform now waits its platform's current pace between requests: 1 second for YouTube and Rumble, doubled by each real rate limit (up to 16 seconds) and eased back one step after every 5 clean downloads, and one step for every hour with no rate limit; a subtitle-only refusal never raises it. When a platform has failed three times in a row at the 30-minute cooldown, it is **held**: auto-download tries it once an hour instead of every 30 minutes, and until that try is due a manual **Sync**, download or metadata scan on it is refused with a sentence giving its time. A clean try lifts the hold — the lane's, or a manual Sync, download or scan once the try is due, which is how a hold ends while auto-download is off or has nothing to fetch on that platform — and so does a try whose video came down although its subtitles were refused. The lane page keeps a held platform listed until then (saying when the lane is off), with a **Clear hold** button that drops the hold, the cooldown and the raised pace at once and says so in a job log. The auto-download lane now waits **Sleep between downloads** between two downloads on one platform, as a channel's batch downloads always did, plus whatever the pace was raised by; batch downloads add that too. The lane page's **Rate-limit cooldown** box shows held platforms, the raised paces and the deferred subtitles, the lane says when it is idle because a platform is held or it is pausing between downloads, and `archilyzer doctor` warns about a platform in a cooldown or held, and about a raised pace. **Download missing subs** also waits that gap between videos. The four numbers are the new `pacing` block in `settings.json` (SETTINGS.md). **After the restart, YouTube may be held at its first real failure:** its cooldown count from before the update (the subtitle refusals) still stands, so one failure puts it straight past the cap — **Clear hold** on the download lane's page resets it, and a clean download does too. Needs a rebuild and restart of the editor.
## [0.11.0] - 2026-09-30
- **Transcripts that arrived after a video was first seen are counted.** The stats behind the homepage, the hub and every site's charts were cached per video and refreshed only when the video's metadata changed, so a transcript that came later — a Whisper run days after the download, or a video downloaded after the last index build — never reached them, and a video with YouTube captions alone had no transcription date. Counts and charts were low; the homepage could show a site with 0 transcripts, 0 channels and 0 hours while it served its videos. A stat is now also redone whenever the index re-reads the video, every transcript has a date, and a captioned video is dated by when its captions arrived rather than by a later Normalize run, so its place on "Transcribed over time" can move. **After updating, rebuild and restart the editor before anything else:** until then, **Build stats dataset** runs the old code and would undo the new stats, while a site, hub or homepage build already runs the new code — and the first stats build of any kind re-reads every video once (about 10–30 minutes on a large archive; it can be stopped and picks up where it stopped). Then build the index, the stats, the homepage, the hub, and the sites.
diff --git a/editor/app/api/channels/[slug]/videos/[id]/subtitle-deferral/route.ts b/editor/app/api/channels/[slug]/videos/[id]/subtitle-deferral/route.ts
@@ -0,0 +1,26 @@
+import { NextResponse } from "next/server";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { readSubtitleDeferrals } from "yt-dlp-transcript-common/jobs/downloadBackoff";
+
+export const dynamic = "force-dynamic";
+
+// GET → { count, lastAt, until } | null — the video's SUBTITLE deferral
+// (release 17, slice RL): how many times its subtitle fetch answered 429, when
+// last, and until when Download missing subs leaves it alone. Read by the video
+// page's one-line note.
+//
+// A ROUTE, NOT A SERVER ACTION: the note asks on mount, and Next runs a page's
+// server actions one at a time — a mount-time action would sit in front of the
+// operator's first click (Mark untranscribable, Download, …) while it ran.
+export async function GET(
+ _req: Request,
+ ctx: { params: Promise<{ slug: string; id: string }> },
+): Promise<NextResponse> {
+ const { slug, id } = await ctx.params;
+ const all = await readSubtitleDeferrals(getPaths()).catch(() => ({}));
+ const d = (all as Record<string, { count: number; lastAt: number; until: number; channelSlug: string }>)[id];
+ return NextResponse.json(
+ d && d.channelSlug === slug ? { count: d.count, lastAt: d.lastAt, until: d.until } : null,
+ { headers: { "cache-control": "no-store" } },
+ );
+}
diff --git a/editor/app/channels/[slug]/pipelineActions.ts b/editor/app/channels/[slug]/pipelineActions.ts
@@ -14,8 +14,10 @@ import {
} from "yt-dlp-transcript-common/lib/queueKeys";
import { detectPlatform } from "yt-dlp-transcript-common/lib/platform";
import {
+ heldPlatformRefusal,
platformCooldownRemainingMs,
recordDownloadBackoff,
+ recordPlatformClean,
} from "yt-dlp-transcript-common/jobs/downloadBackoff";
import {
countNotYetDownloaded,
@@ -126,6 +128,20 @@ async function runPipelineAction(
// Platform key shared with the auto-download runner (detectPlatform(url) ??
// "unknown") — drives the 429 cooldown both honor.
const platform = detectPlatform(channelConfig.url) ?? "unknown";
+ // A HELD platform refuses every manual fetch, not only a Sync (release 17,
+ // slice RL), while its next probe is still ahead: the lane is probing it
+ // once an hour. Once the probe is due, a manual run IS the probe, and a
+ // clean one lifts the hold (onPlatformClean below). The sentence names the
+ // hold and the next probe. store-playlist stays allowed, as it is under the
+ // downloads pause.
+ if (mode !== "store-playlist") {
+ const held = await heldPlatformRefusal(
+ platform,
+ mode === "sync" ? "Sync" : "This download",
+ paths,
+ );
+ if (held) return { ok: false, info: true, error: held };
+ }
// A manually clicked Sync RESPECTS the per-platform rate-limit cooldown: if
// auto-download (or a prior sync) hit a 429, refuse rather than re-storm the
// source. Neutral (info) result so the UI shows it as a notice, not an error.
@@ -211,8 +227,13 @@ async function runPipelineAction(
setProgress,
progressBaseline,
// On a 429/network failure, record the shared per-platform cooldown so
- // the auto-download runner (and a later manual sync) back off too.
- onPlatformBackoff: () => recordDownloadBackoff(platform, paths),
+ // the auto-download runner (and a later manual sync) back off too. The
+ // class matters since release 17: only a rate limit doubles the pace.
+ onPlatformBackoff: (failureClass) =>
+ recordDownloadBackoff(platform, paths, failureClass),
+ // A run the source answered cleanly settles the platform — a held
+ // one's hold included — as a clean lane probe does (release 17).
+ onPlatformClean: () => recordPlatformClean(platform, paths),
});
// The channel report (snapshot) is regenerated automatically after this
// job finishes, via the global debounced scheduler hooked into
@@ -342,6 +363,8 @@ export async function runMetadataScanAction(
return { ok: false, error: "Channel has no `url` configured" };
}
const platform = detectPlatform(channelConfig.url) ?? "unknown";
+ const held = await heldPlatformRefusal(platform, "The metadata scan", paths);
+ if (held) return { ok: false, info: true, error: held };
const remainingMs = await platformCooldownRemainingMs(platform, paths);
if (remainingMs > 0) {
const secs = Math.ceil(remainingMs / 1000);
diff --git a/editor/app/channels/[slug]/videos/[id]/components/SubtitleDeferralLine.tsx b/editor/app/channels/[slug]/videos/[id]/components/SubtitleDeferralLine.tsx
@@ -0,0 +1,56 @@
+"use client";
+
+import { useEffect, useState } from "react";
+
+// THE SUBTITLE DEFERRAL, IN ONE LINE (release 17, slice RL). A video whose
+// YouTube subtitles answered HTTP 429 while its media came down: how many
+// times, when last, and until when the batch subtitle fetch (Download missing
+// subs) leaves it alone — 6 h after a strike, 7 days from the third. A
+// download from this page still fetches them. Draws nothing when there is no
+// deferral, which is every video but a handful.
+export function SubtitleDeferralLine({
+ slug,
+ videoId,
+}: {
+ slug: string;
+ videoId: string;
+}) {
+ const [d, setD] = useState<{ count: number; lastAt: number; until: number } | null>(null);
+ useEffect(() => {
+ let live = true;
+ // A plain GET, not a server action: Next queues a page's server actions,
+ // and this one would run ahead of the operator's first click.
+ fetch(
+ `/api/channels/${encodeURIComponent(slug)}/videos/${encodeURIComponent(videoId)}/subtitle-deferral`,
+ { cache: "no-store" },
+ )
+ .then((r) => (r.ok ? r.json() : null))
+ .then((r: { count: number; lastAt: number; until: number } | null) => {
+ if (live) setD(r);
+ })
+ .catch(() => {});
+ return () => {
+ live = false;
+ };
+ }, [slug, videoId]);
+ if (!d) return null;
+ const day = (ms: number) => new Date(ms).toISOString().slice(0, 16).replace("T", " ") + " UTC";
+ const held = d.count >= 3;
+ const active = d.until > Date.now();
+ return (
+ <p
+ role="note"
+ aria-label="subtitle deferral"
+ className="rounded-md border border-warning/30 bg-warning-soft px-3 py-2 text-sm text-warning"
+ >
+ The subtitles for this video were rate-limited (HTTP 429){" "}
+ {d.count}× — last {day(d.lastAt)}.{" "}
+ {active
+ ? held
+ ? `Left alone by Download missing subs until ${day(d.until)}.`
+ : `Download missing subs retries them after ${day(d.until)}.`
+ : "Download missing subs will retry them."}{" "}
+ A download from this page still fetches them.
+ </p>
+ );
+}
diff --git a/editor/app/channels/[slug]/videos/[id]/components/VideoPanel.tsx b/editor/app/channels/[slug]/videos/[id]/components/VideoPanel.tsx
@@ -35,6 +35,7 @@ import { RedownloadSection } from "./cards/RedownloadSection";
import { ShortAudioBanner } from "./cards/ShortAudioBanner";
import { SourceVideoSection } from "./cards/SourceVideoSection";
import { TranscriptSourceSection } from "./cards/TranscriptSourceSection";
+import { SubtitleDeferralLine } from "./SubtitleDeferralLine";
import {
CANONICAL_VTT,
WHISPER_FILENAME,
@@ -314,6 +315,8 @@ export function VideoPanel({
</div>
</PipelineStageCard>
+ <SubtitleDeferralLine slug={slug} videoId={videoId} />
+
{vttTracks.length > 0 && (
<PipelineStageCard
id="transcript-source"
diff --git a/editor/app/channels/[slug]/videos/[id]/videoActions.ts b/editor/app/channels/[slug]/videos/[id]/videoActions.ts
@@ -74,6 +74,7 @@ import {
import { detectPlatform } from "yt-dlp-transcript-common/lib/platform";
import { applyTagAssignmentsAction } from "../../../../tags/actions";
import {
+ heldPlatformRefusal,
platformCooldownRemainingMs,
recordDownloadBackoff,
} from "yt-dlp-transcript-common/jobs/downloadBackoff";
@@ -1011,7 +1012,12 @@ export async function fetchWindowAction(req: {
// handful of seconds, but a 429 is a fact about the SOURCE, not about the
// size of the request.
const platform = detectPlatform(url) ?? "unknown";
+ // A held platform's refusal names the hold and the next probe (release 17).
+ const held = await heldPlatformRefusal(platform, "This fetch", paths);
const cooldownMs = await platformCooldownRemainingMs(platform, paths);
+ if (held && cooldownMs > 0) {
+ return { ok: false, status: 409, cooldownMs, platform, error: held };
+ }
if (cooldownMs > 0) {
return {
ok: false,
@@ -1163,7 +1169,12 @@ export async function fetchFullSourceAction(req: {
};
}
const platform = detectPlatform(url) ?? "unknown";
+ // A held platform's refusal names the hold and the next probe (release 17).
+ const held = await heldPlatformRefusal(platform, "This fetch", paths);
const cooldownMs = await platformCooldownRemainingMs(platform, paths);
+ if (held && cooldownMs > 0) {
+ return { ok: false, status: 409, cooldownMs, platform, error: held };
+ }
if (cooldownMs > 0) {
return {
ok: false,
diff --git a/editor/app/operations/components/RunnerOperationView.tsx b/editor/app/operations/components/RunnerOperationView.tsx
@@ -6,6 +6,8 @@ import type { AutoQueueKind } from "yt-dlp-transcript-common/jobs/autoQueueState
import type {
AutoQueueKindStatus,
PlatformCooldownView,
+ PlatformPaceView,
+ SubtitleDeferralView,
VideoDeferralView,
} from "yt-dlp-transcript-common/views/autoQueueStatus";
import type { LaneWorker } from "yt-dlp-transcript-common/views/autoQueueLanes";
@@ -15,6 +17,15 @@ import { NextUp } from "./NextUp";
import { PolicyTreeEditor } from "./PolicyTreeEditor";
import { SnoozeControl } from "./SnoozeControl";
import { type Channel, formatClock, formatCooldown, leafOrder } from "./dispatch";
+import { clearPlatformHoldAction } from "../pacingActions";
+
+// The idle reasons that mean "paused or gated", not "waiting for work".
+const LANE_PAUSE_REASONS: ReadonlySet<string> = new Set([
+ "lane-held",
+ "downloads-paused",
+ "snoozed",
+ "disk-gate",
+]);
// ONE LANE'S RUNNER, IN FULL — any of the four.
//
@@ -65,6 +76,9 @@ export function RunnerOperationView({
onRefresh: () => Promise<void>;
}) {
const [busy, setBusy] = useState(false);
+ // What the last Clear hold did. Held HERE, not in the strip: a clear that
+ // leaves nothing to list unmounts the strip, and the sentence must outlive it.
+ const [holdNote, setHoldNote] = useState<string | null>(null);
const now = useNow();
const control = useCallback(
@@ -115,13 +129,27 @@ export function RunnerOperationView({
</p>
)}
- {(status.cooldowns.length > 0 || status.deferred.length > 0) && (
+ {(status.cooldowns.length > 0 ||
+ status.deferred.length > 0 ||
+ (status.pace?.length ?? 0) > 0 ||
+ (status.subtitleDeferred?.length ?? 0) > 0) && (
<CooldownStrip
cooldowns={status.cooldowns}
deferred={status.deferred}
+ pace={status.pace ?? []}
+ subtitleDeferred={status.subtitleDeferred ?? []}
+ laneRunning={status.runner.running}
+ laneHeld={status.held || (status.runner.idleReason !== null && LANE_PAUSE_REASONS.has(status.runner.idleReason))}
+ onRefresh={onRefresh}
+ onCleared={setHoldNote}
now={now}
/>
)}
+ {holdNote && (
+ <span aria-label="clear hold result" className="text-xs text-muted-foreground">
+ {holdNote}
+ </span>
+ )}
<SnoozeControl
kind={kind}
@@ -202,27 +230,123 @@ export function useNow(): number | null {
// lapses; the runner skips it meanwhile. A deferred video is skipped by
// auto-download only — a manual Sync / download-missing still fetches it.
//
+// RELEASE 17, SLICE RL adds three lists to the same region: the platforms
+// HELD (a rate limit that outlasted the cooldown cap — one probe at a time,
+// and every manual fetch on them refused), the request PACE of every platform
+// a rate limit slowed, and the videos whose SUBTITLES are deferred (their
+// media came down; the platform was not backed off for them).
+//
// A <div role="region">, NOT a <section>: this sits inside the lane's own
// <section>, and the structural contract above forbids a nested one. The
// accessible role and name are the same either way.
function CooldownStrip({
cooldowns,
deferred,
+ pace,
+ subtitleDeferred,
+ laneRunning,
+ laneHeld,
+ onRefresh,
+ onCleared,
now,
}: {
cooldowns: PlatformCooldownView[];
deferred: VideoDeferralView[];
+ pace: PlatformPaceView[];
+ subtitleDeferred: SubtitleDeferralView[];
+ laneRunning: boolean;
+ // The lane is up but paused or gated (its own hold, the downloads pause,
+ // a snooze, the disk gate): it is not waiting for a video.
+ laneHeld: boolean;
+ onRefresh: () => Promise<void>;
+ onCleared: (message: string) => void;
now: number | null;
}) {
+ const [clearing, setClearing] = useState<string | null>(null);
+ const clearHold = async (platform: string) => {
+ setClearing(platform);
+ try {
+ const res = await clearPlatformHoldAction(platform);
+ onCleared(res.ok ? res.message : res.error);
+ await onRefresh();
+ } finally {
+ setClearing(null);
+ }
+ };
const left = (untilMs: number) =>
now === null ? null : Math.max(0, Math.ceil((untilMs - now) / 1000));
+ const paceOf = (platform: string) =>
+ pace.find((p) => p.platform === platform)?.sleepRequestsSeconds;
+ const cooling = cooldowns.filter(
+ (c) => !c.hold && (now === null || c.untilMs > now),
+ );
+ const held = cooldowns.filter((c) => c.hold);
return (
<div
role="region"
aria-label="Rate-limit cooldown"
className="flex flex-col gap-1 rounded-md border border-warning/30 bg-warning-soft px-3 py-2 text-sm"
>
- {cooldowns.length > 0 && (
+ {held.length > 0 && (
+ <>
+ <span className="font-medium text-warning">
+ Held — failures outlasted the cooldown cap; auto-download probes
+ once at a time, and manual fetches wait for the next probe:
+ </span>
+ <ul
+ aria-label="Platforms held"
+ className="flex flex-col gap-1 text-warning"
+ >
+ {held.map((c) => {
+ const secs = left(c.untilMs);
+ const p = paceOf(c.platform);
+ const overdue = secs !== null && secs <= 0;
+ return (
+ <li key={c.platform} className="flex flex-wrap items-center gap-x-2 tabular-nums">
+ <span>
+ <span className="font-mono">{c.platform}</span> —{" "}
+ {c.hold!.rateLimited ? "rate-limited" : "failing (network)"}
+ , held since {formatClock(c.hold!.sinceMs)} ({c.fails}{" "}
+ failures in a row)
+ {secs !== null &&
+ (overdue ? (
+ laneRunning && !laneHeld ? (
+ <>
+ , probe due — it waits for a pending video on this
+ platform; a clean manual Sync also lifts it
+ </>
+ ) : laneRunning ? (
+ <>
+ , probe overdue — the lane is paused; a clean
+ manual Sync also lifts it
+ </>
+ ) : (
+ <>
+ , probe overdue — the lane is off; a clean manual
+ Sync also lifts it
+ </>
+ )
+ ) : (
+ <>, next probe in {formatCooldown(secs)}</>
+ ))}
+ {p !== undefined && <> · pace {p}s</>}
+ </span>
+ <button
+ type="button"
+ aria-label={`Clear hold on ${c.platform}`}
+ disabled={clearing !== null}
+ onClick={() => void clearHold(c.platform)}
+ className="rounded border border-warning/40 px-2 py-0.5 text-xs hover:bg-warning/10 disabled:opacity-50"
+ >
+ {clearing === c.platform ? "Clearing…" : "Clear hold"}
+ </button>
+ </li>
+ );
+ })}
+ </ul>
+ </>
+ )}
+ {cooling.length > 0 && (
<>
<span className="font-medium text-warning">
Rate-limit cooldown — auto-download and manual sync are paused on:
@@ -231,20 +355,41 @@ function CooldownStrip({
aria-label="Platforms in cooldown"
className="flex flex-wrap gap-x-4 gap-y-1 text-warning"
>
- {cooldowns.map((c) => {
+ {cooling.map((c) => {
const secs = left(c.untilMs);
+ const p = paceOf(c.platform);
return (
<li key={c.platform} className="tabular-nums">
<span className="font-mono">{c.platform}</span>
{secs !== null && (
<> — {formatCooldown(secs)} left (attempt {c.fails})</>
)}
+ {p !== undefined && <> · pace {p}s</>}
</li>
);
})}
</ul>
</>
)}
+ {pace.length > 0 && (
+ <>
+ <span className="font-medium text-warning">
+ Request pace — seconds between requests, raised by a rate limit:
+ </span>
+ <ul
+ aria-label="Request pace"
+ className="flex flex-wrap gap-x-4 gap-y-1 text-warning"
+ >
+ {pace.map((p) => (
+ <li key={p.platform} className="tabular-nums">
+ <span className="font-mono">{p.platform}</span> —{" "}
+ {p.sleepRequestsSeconds}s between requests (base{" "}
+ {p.baseSeconds}s)
+ </li>
+ ))}
+ </ul>
+ </>
+ )}
{deferred.length > 0 && (
<>
<span className="font-medium text-warning">
@@ -271,6 +416,40 @@ function CooldownStrip({
</ul>
</>
)}
+ {subtitleDeferred.length > 0 && (
+ <>
+ <span className="font-medium text-warning">
+ Deferred subtitles — downloaded, but the subtitles answered HTTP
+ 429; Download missing subs retries them after:
+ </span>
+ <ul
+ aria-label="Deferred subtitles"
+ className="flex flex-wrap gap-x-4 gap-y-1 text-warning"
+ >
+ {subtitleDeferred.map((d) => {
+ const secs = left(d.untilMs);
+ return (
+ <li key={d.videoId} className="tabular-nums">
+ <Link
+ href={`/channels/${d.channelSlug}/videos/${d.videoId}`}
+ className="font-mono underline underline-offset-2 hover:text-brand"
+ >
+ {d.channelSlug}/{d.videoId}
+ </Link>{" "}
+ — {d.count}×
+ {secs !== null && (
+ <>
+ , {d.held ? "left alone " : ""}
+ {formatCooldown(secs)}
+ {d.held ? "" : " left"}
+ </>
+ )}
+ </li>
+ );
+ })}
+ </ul>
+ </>
+ )}
</div>
);
}
diff --git a/editor/app/operations/components/dispatch.ts b/editor/app/operations/components/dispatch.ts
@@ -153,6 +153,10 @@ export function idleReasonText(
return "every pending platform is in a rate-limit cooldown";
case "deferred":
return "every pending video was rate-limited recently and is deferred";
+ case "held":
+ return "every pending platform is held after repeated rate limits — one probe at a time";
+ case "paced":
+ return "every pending platform is pausing between downloads";
case "no-workers":
return "no enabled worker to run it";
case "workers-paused":
@@ -206,6 +210,12 @@ export function formatRecency(
// are noise and are dropped.
export function formatCooldown(secs: number): string {
if (secs < 60) return `${secs}s`;
+ // The days arm is for a subtitle hold (7 days), release 17.
+ if (secs >= 86400) {
+ const d = Math.floor(secs / 86400);
+ const h = Math.floor((secs % 86400) / 3600);
+ return h === 0 ? `${d}d` : `${d}d ${h}h`;
+ }
if (secs >= 3600) {
const h = Math.floor(secs / 3600);
const m = Math.floor((secs % 3600) / 60);
diff --git a/editor/app/operations/pacingActions.ts b/editor/app/operations/pacingActions.ts
@@ -0,0 +1,41 @@
+"use server";
+
+import { revalidatePath } from "next/cache";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { runManagedFunction } from "yt-dlp-transcript-common/jobs/streamCommand";
+import { clearPlatformHold } from "yt-dlp-transcript-common/jobs/downloadBackoff";
+
+// "CLEAR HOLD" (release 17 slice RL, review H2). The operator's word beats the
+// machine, as with an auto-pause: a held platform's hold, its backoff and its
+// raised pace all go, and the next failure starts the escalation from the
+// bottom. Run as a one-step job so the clearing is in a job log (/jobs, kind
+// `clear-platform-hold`), on its own queue — a Sync running on the platform's
+// queue must not make the click wait.
+export async function clearPlatformHoldAction(
+ platform: string,
+): Promise<{ ok: true; message: string } | { ok: false; error: string }> {
+ if (!/^[a-z0-9.-]{1,64}$/i.test(platform)) {
+ return { ok: false, error: `Not a platform: ${platform}` };
+ }
+ const paths = getPaths();
+ let message = `${platform} had no hold, backoff or raised pace to clear.`;
+ const res = await runManagedFunction({
+ kind: "clear-platform-hold",
+ queueKey: `pacing:${platform}`,
+ paths,
+ fn: async (onLog) => {
+ const line = await clearPlatformHold(platform, paths);
+ if (line) message = line.trim();
+ onLog(`${message}\n`);
+ },
+ });
+ if (!res.ok) return { ok: false, error: res.error };
+ void res.stream.cancel();
+ const done = await res.done;
+ if (done.status !== "done") {
+ return { ok: false, error: `Clearing the ${platform} hold ended ${done.status}.` };
+ }
+ revalidatePath("/operations");
+ revalidatePath("/operations/[id]", "page");
+ return { ok: true, message };
+}
diff --git a/editor/e2e/fixtures/bin/fake-ytdlp.mjs b/editor/e2e/fixtures/bin/fake-ytdlp.mjs
@@ -462,15 +462,26 @@ async function modeYoutubeSingleUrlManaged(url) {
`youtube-single:${url} cookies=${cookieArg()}\n`,
);
if (cookieGateBlocked(url)) failCookieGate(url);
- // `dl429`: the 2026-09-24 YouTube shape — the metadata prefetch succeeded
- // (the prefetch branch has no such sentinel), then the real download's
- // SUBTITLE fetch answers 429. Per video, not IP-wide: availability.ts
- // classifies it rate_limit, so the runner backs youtube off AND defers this
- // one video (pacing.spec.ts).
+ // `dl429`: the 2026-09-24 / 2026-10-01 YouTube shape — the metadata
+ // prefetch succeeded (the prefetch branch has no such sentinel), then the
+ // real download's SUBTITLE fetch answers 429. Per video, not IP-wide. Like
+ // the real yt-dlp: under --ignore-errors (which the managed primary passes
+ // since release 17, slice RL) the failure is a WARNING, nothing is written
+ // and the exit is clean; without it, an ERROR and exit 1. availability.ts
+ // classifies either as `subs_rate_limit`: the downloader fetches the media
+ // and the runner defers only the subtitles (rate-limit.spec.ts).
if (url.toLowerCase().includes("dl429")) {
- process.stderr.write(
- `ERROR: [youtube] ${url}: Unable to download video subtitles for 'en': HTTP Error 429: Too Many Requests\n`,
- );
+ const line = `[youtube] ${url}: Unable to download video subtitles for 'en': HTTP Error 429: Too Many Requests`;
+ if (has("--ignore-errors")) {
+ process.stderr.write(`WARNING: ${line}\n`);
+ const id429 = urlIdYouTube(url);
+ if (!existsSync(path.join(videoDir, "metadata.info.json"))) {
+ await writeMetadata(videoDir, id429, urlSentinels(url));
+ }
+ process.stdout.write(`DLOM_ARCHIVE youtube ${id429}\n`);
+ return;
+ }
+ process.stderr.write(`ERROR: ${line}\n`);
process.exit(1);
}
process.stdout.write(`[fake-ytdlp] managed single-URL ${id}\n`);
@@ -821,6 +832,16 @@ async function main() {
`prefetch:${url} cookies=${cookieArg()}\n`,
);
if (cookieGateBlocked(url)) failCookieGate(url);
+ // `wp429`: a PLATFORM-level 429 — the watch page itself is refused. The
+ // prefetch is the first request a managed download makes, so the video
+ // stops there classified `rate_limit`: the platform backs off, its pace
+ // doubles, and the video is deferred (pacing.spec.ts, rate-limit.spec.ts).
+ if (url.toLowerCase().includes("wp429")) {
+ process.stderr.write(
+ `ERROR: [youtube] ${id}: Unable to download webpage: HTTP Error 429: Too Many Requests\n`,
+ );
+ process.exit(1);
+ }
await writeMetadata(videoDir, id, urlSentinels(url));
process.stdout.write(`[fake-ytdlp] metadata prefetch ${id}\n`);
return;
diff --git a/editor/e2e/pacing.spec.ts b/editor/e2e/pacing.spec.ts
@@ -8,7 +8,9 @@ import { baseUrl } from "./baseUrl";
// runner moves on to the NEXT video instead of re-picking the same one. The
// pure decision is unit-tested (common/jobs/unitOutcome.test.ts); this proves
// the runner's pick honours a persisted deferral, the lane says why it idles,
-// the page shows it, and a live 429 (the fake's `dl429` sentinel) defers.
+// the page shows it, and a live 429 (the fake's `wp429` sentinel — a refused
+// watch page; a SUBTITLE 429, `dl429`, no longer backs the platform off since
+// release 17 slice RL, see rate-limit.spec.ts) defers.
type Leaf = { id: string; match: { type: string; value?: string } };
type Group = { id: string; mode: string; children: (Group | Leaf)[] };
@@ -192,9 +194,9 @@ test("a live 429 backs youtube off, defers the video, and the next pick is the n
request,
}) => {
// A REAL cooldown: base 60 s ±10 %. The whole point is what happens when it
- // lapses — the runner must move on to a2, not re-pick dl429vid1.
+ // lapses — the runner must move on to a2, not re-pick wp429vid1.
test.setTimeout(150_000);
- await setup(["dl429vid1", "a2"]);
+ await setup(["wp429vid1", "a2"]);
await control(request, "start");
@@ -203,7 +205,7 @@ test("a live 429 backs youtube off, defers the video, and the next pick is the n
async () => {
const d = await persistedDownload();
return {
- deferred: Boolean(d?.videoDeferrals?.dl429vid1),
+ deferred: Boolean(d?.videoDeferrals?.wp429vid1),
fails: d?.platformBackoff.youtube?.fails ?? 0,
};
},
@@ -211,9 +213,9 @@ test("a live 429 backs youtube off, defers the video, and the next pick is the n
)
.toEqual({ deferred: true, fails: 1 });
const d = await persistedDownload();
- expect(d?.videoDeferrals?.dl429vid1?.channelSlug).toBe("alpha");
+ expect(d?.videoDeferrals?.wp429vid1?.channelSlug).toBe("alpha");
// Deferred for ~6 h, not for the cooldown's minute.
- expect(d!.videoDeferrals!.dl429vid1.until - Date.now()).toBeGreaterThan(
+ expect(d!.videoDeferrals!.wp429vid1.until - Date.now()).toBeGreaterThan(
5 * 60 * 60_000,
);
@@ -223,5 +225,5 @@ test("a live 429 backs youtube off, defers the video, and the next pick is the n
intervals: [2_000],
})
.toContain("a2");
- expect(pickOrder(await getStatus(request))).toEqual(["dl429vid1", "a2"]);
+ expect(pickOrder(await getStatus(request))).toEqual(["wp429vid1", "a2"]);
});
diff --git a/editor/e2e/rate-limit.spec.ts b/editor/e2e/rate-limit.spec.ts
@@ -0,0 +1,375 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { test, expect } from "@playwright/test";
+import {
+ pathExists,
+ readJson,
+ resetData,
+ resolvePath,
+ writeSettings,
+} from "./helpers";
+import { baseUrl } from "./baseUrl";
+
+// A SUBTITLE 429 DOES NOT FAIL A DOWNLOAD, AND THE PACE ADAPTS (release 17,
+// slice RL). The pure decisions are unit-tested (common/jobs/unitOutcome.test.ts,
+// common/ytdlp/subtitleRateLimit.test.ts); this proves them through the real
+// download lane and the fake yt-dlp:
+// - `dl429`: the 2026-10-01 shape — the prefetch succeeds, the subtitle fetch
+// answers 429. The unit succeeds with the media and no subtitles, the
+// video's subtitles are deferred, youtube is NOT backed off, and the lane
+// page and the video page say so.
+// - `wp429`: a platform-level 429 (the watch page refused) on a backoff
+// already at the cap: youtube is HELD, its next try is a probe
+// `pacing.holdProbeMinutes` away, the pace doubled, the page shows the
+// hold and the pace, and a manual Sync is refused with the hold sentence.
+// - a probe that comes back clean clears the hold and the backoff.
+
+type Leaf = { id: string; match: { type: string; value?: string } };
+type Group = { id: string; mode: string; children: (Group | Leaf)[] };
+
+const STATE_FILE = "test-transcripts/.auto-queue/state.json";
+const TOKEN = "test-worker-token";
+const AUTH = { authorization: `Bearer ${TOKEN}` };
+
+// Copied from pacing.spec.ts: a YouTube channel with undownloaded videos.
+async function makeDownloadChannel(slug: string, ids: string[]) {
+ const root = resolvePath(`test-transcripts/channels/${slug}`);
+ await mkdir(`${root}/data`, { recursive: true });
+ await writeFile(
+ `${root}/config.json`,
+ JSON.stringify({
+ handling: "youtube",
+ name: slug,
+ url: `https://www.youtube.com/@${slug}/videos`,
+ }),
+ );
+ await writeFile(
+ `${root}/playlist`,
+ ids.map((id) => `https://www.youtube.com/watch?v=${id}`).join("\n") + "\n",
+ );
+ await writeFile(
+ `${root}/snapshot.json`,
+ JSON.stringify({
+ generatedAt: "2026-06-01T00:00:00.000Z",
+ totals: { videos: ids.length, transcribed: 0, downloaded: 0 },
+ buckets: {
+ noTranscript: [],
+ downloadedNoTranscript: [],
+ untranscoded: [],
+ multipleAudioFormats: [],
+ transcribedWithAudio: [],
+ untranscribable: [],
+ noMetadata: [],
+ failedListed: [],
+ missingFromArchive: [],
+ duplicateDirs: [],
+ partialDownloads: [],
+ corruptSource: [],
+ nonStandardVtt: [],
+ skippedByFilter: [],
+ },
+ undownloadedIds: ids,
+ }),
+ );
+}
+
+async function setup(
+ ids: string[],
+ pacing: Record<string, number> = {},
+ seed?: Record<string, unknown>,
+ laneEnabled = true,
+) {
+ await resetData(null);
+ await makeDownloadChannel("alpha", ids);
+ const root: Group = {
+ id: "root",
+ mode: "strict",
+ children: [{ id: "leaf-alpha", match: { type: "channel", value: "alpha" } }],
+ };
+ // Settings first: writing them clears the server's shared auto-queue state,
+ // so the seed below is what the runner reads when it starts.
+ await writeSettings({
+ adminTitle: "Test Admin",
+ maxTranscriptPageBytes: 8388608,
+ sleepBetweenDownloadsSeconds: 0,
+ minFreeDiskGB: 0,
+ pacing,
+ autoQueue: {
+ transcription: {},
+ download: { enabled: laneEnabled, maxWorkers: null, root },
+ },
+ });
+ if (seed) {
+ await mkdir(resolvePath("test-transcripts/.auto-queue"), { recursive: true });
+ await writeFile(
+ resolvePath(STATE_FILE),
+ JSON.stringify({
+ transcription: { runtime: {}, picks: [], platformBackoff: {} },
+ download: { runtime: {}, picks: [], platformBackoff: {}, ...seed },
+ }),
+ );
+ }
+}
+
+type Persisted = {
+ platformBackoff: Record<string, { until: number; fails: number }>;
+ videoDeferrals?: Record<string, { until: number; channelSlug: string }>;
+ platformPace?: Record<string, { sleepRequestsSeconds: number; baseSeconds: number }>;
+ platformHolds?: Record<string, { since: number; probeAt: number }>;
+ subtitleDeferrals?: Record<string, { count: number; until: number; channelSlug: string }>;
+};
+
+async function persisted(): Promise<Persisted | null> {
+ try {
+ return (await readJson<{ download: Persisted }>(STATE_FILE)).download;
+ } catch {
+ return null;
+ }
+}
+
+type Status = {
+ download: {
+ runner: { running: boolean; idleReason: string | null };
+ picks: { videoId: string }[];
+ };
+};
+
+async function getStatus(
+ request: import("@playwright/test").APIRequestContext,
+): Promise<Status> {
+ const res = await request.get(`${baseUrl}/api/auto-queue/status`);
+ expect(res.ok()).toBeTruthy();
+ return res.json();
+}
+
+async function control(
+ request: import("@playwright/test").APIRequestContext,
+ action: "start" | "stop",
+) {
+ const res = await request.post(`${baseUrl}/api/auto-queue/control`, {
+ data: { kind: "download", action },
+ });
+ if (action === "start") expect(res.ok()).toBeTruthy();
+}
+
+test.afterEach(async ({ request }) => {
+ await control(request, "stop");
+});
+
+test("a subtitle 429 downloads the media, defers only the subtitles, and backs nothing off", async ({
+ page,
+ request,
+}) => {
+ await setup(["dl429sub1"]);
+ await control(request, "start");
+
+ await expect
+ .poll(async () => (await persisted())?.subtitleDeferrals?.dl429sub1?.count ?? 0, {
+ timeout: 60_000,
+ })
+ .toBe(1);
+ const d = await persisted();
+ // The platform was not backed off, its pace was not raised, and the video
+ // itself is not deferred — its media is done.
+ expect(d?.platformBackoff.youtube).toBeUndefined();
+ expect(d?.platformPace ?? {}).toEqual({});
+ expect(d?.videoDeferrals ?? {}).toEqual({});
+ expect(d?.subtitleDeferrals?.dl429sub1?.channelSlug).toBe("alpha");
+
+ // The media is on disk, no subtitles, and the outcome is a success that says why.
+ const dir = "test-transcripts/channels/alpha/data/dl429sub1";
+ expect(await pathExists(`${dir}/audio.mp3`)).toBe(true);
+ expect(await pathExists(`${dir}/transcript.en.vtt`)).toBe(false);
+ const outcome = await readJson<{ status: string; failureClass?: string }>(
+ `${dir}/download-outcome.json`,
+ );
+ expect(outcome.status).toBe("ok");
+ expect(outcome.failureClass).toBe("subs_rate_limit");
+
+ await page.goto("/operations/download");
+ const region = page.getByRole("region", { name: "Rate-limit cooldown" });
+ await expect(region).toBeVisible({ timeout: 20_000 });
+ const subs = region.getByRole("list", { name: "Deferred subtitles" });
+ await expect(subs).toContainText("alpha/dl429sub1");
+ await expect(subs).toContainText("1×");
+ await expect(region.getByRole("list", { name: "Platforms in cooldown" })).toHaveCount(0);
+ await expect(region.getByRole("list", { name: "Platforms held" })).toHaveCount(0);
+
+ await page.goto("/channels/alpha/videos/dl429sub1");
+ await expect(page.getByRole("note", { name: "subtitle deferral" })).toContainText(
+ /rate-limited \(HTTP 429\) 1×/,
+ { timeout: 20_000 },
+ );
+});
+
+test("a platform 429 at the cap holds youtube: one probe per holdProbeMinutes, the pace doubled, a manual Sync refused", async ({
+ page,
+ request,
+}) => {
+ // fails: 5 lapsed — the next failure is the first AT the 30-minute cap, and
+ // with holdAfterFailsAtCap 1 that one holds the platform.
+ await setup(
+ ["wp429held", "a2"],
+ { holdAfterFailsAtCap: 1, holdProbeMinutes: 7 },
+ { platformBackoff: { youtube: { until: Date.now() - 1_000, fails: 5 } } },
+ );
+ await control(request, "start");
+
+ await expect
+ .poll(async () => Boolean((await persisted())?.platformHolds?.youtube), {
+ timeout: 60_000,
+ })
+ .toBe(true);
+ const d = (await persisted())!;
+ expect(d.platformBackoff.youtube.fails).toBe(6);
+ // The probe cadence is the setting's, not the 30-minute cap's.
+ const left = d.platformBackoff.youtube.until - Date.now();
+ expect(left).toBeGreaterThan(6 * 60_000);
+ expect(left).toBeLessThanOrEqual(7 * 60_000);
+ expect(d.platformHolds!.youtube.probeAt).toBe(d.platformBackoff.youtube.until);
+ // A rate limit doubled the pace.
+ expect(d.platformPace?.youtube?.sleepRequestsSeconds).toBe(2);
+
+ // a2 is pending on a held platform: the lane says it is held.
+ await expect
+ .poll(async () => (await getStatus(request)).download.runner.idleReason, {
+ timeout: 30_000,
+ })
+ .toBe("held");
+ expect((await getStatus(request)).download.picks.map((p) => p.videoId)).toEqual([
+ "wp429held",
+ ]);
+
+ await page.goto("/operations/download");
+ const region = page.getByRole("region", { name: "Rate-limit cooldown" });
+ await expect(region).toBeVisible({ timeout: 20_000 });
+ const held = region.getByRole("list", { name: "Platforms held" });
+ await expect(held).toContainText("youtube");
+ await expect(held).toContainText(/next probe in (6m|7m)/);
+ await expect(held).toContainText("6 failures in a row");
+ await expect(region.getByRole("list", { name: "Platforms in cooldown" })).toHaveCount(0);
+ await expect(region.getByRole("list", { name: "Request pace" })).toContainText(
+ "2s between requests (base 1s)",
+ );
+ await expect(
+ page
+ .getByText(/every pending platform is held after repeated rate limits/)
+ .first(),
+ ).toBeVisible();
+
+ // A manual Sync is refused with the hold, naming the next probe.
+ const sync = await request.post(`${baseUrl}/api/ops/sync`, {
+ headers: AUTH,
+ data: { slug: "alpha" },
+ });
+ expect(sync.status()).toBe(400);
+ const body = (await sync.json()) as { error: string; info?: boolean };
+ expect(body.info).toBe(true);
+ expect(body.error).toMatch(
+ /^youtube is held: its rate limit outlasted the cooldown cap \(6 failures in a row, held since \d\d:\d\d UTC\)\. Auto-download probes it once every 7 min — the next probe is in [67] min\. Sync will run once a probe comes back clean\.$/,
+ );
+});
+
+test("a probe that comes back clean clears the hold and the backoff", async ({
+ request,
+}) => {
+ const now = Date.now();
+ await setup(["a1"], { holdProbeMinutes: 60 }, {
+ platformBackoff: { youtube: { until: now - 1_000, fails: 9 } },
+ platformHolds: { youtube: { since: now - 3_600_000, probeAt: now - 1_000 } },
+ });
+ await control(request, "start");
+
+ await expect
+ .poll(
+ async () => {
+ const d = await persisted();
+ return {
+ held: Boolean(d?.platformHolds?.youtube),
+ backoff: Boolean(d?.platformBackoff.youtube),
+ };
+ },
+ { timeout: 60_000 },
+ )
+ .toEqual({ held: false, backoff: false });
+ expect((await getStatus(request)).download.picks.map((p) => p.videoId)).toEqual(["a1"]);
+ expect(await pathExists("test-transcripts/channels/alpha/data/a1/transcript.en.vtt")).toBe(
+ true,
+ );
+});
+
+// Review H2: a hold must not outlive the lane that would probe it. With the
+// lane off and the probe overdue, the page keeps the platform under "Platforms
+// held" and says so, a manual Sync is NOT refused, and a clean one lifts the
+// hold and the backoff.
+test("an overdue hold with the lane off: still shown, Sync not refused, and the clean Sync lifts it", async ({
+ page,
+ request,
+}) => {
+ const now = Date.now();
+ await setup(
+ ["a1"],
+ {},
+ {
+ platformBackoff: { youtube: { until: now - 1_000, fails: 9 } },
+ platformHolds: { youtube: { since: now - 3_600_000, probeAt: now - 1_000, rateLimited: true } },
+ },
+ false,
+ );
+
+ await page.goto("/operations/download");
+ const held = page
+ .getByRole("region", { name: "Rate-limit cooldown" })
+ .getByRole("list", { name: "Platforms held" });
+ await expect(held).toContainText("youtube", { timeout: 20_000 });
+ await expect(held).toContainText("probe overdue — the lane is off");
+
+ const sync = await request.post(`${baseUrl}/api/ops/sync`, {
+ headers: AUTH,
+ data: { slug: "alpha" },
+ });
+ expect(sync.status(), await sync.text()).toBe(200);
+ await expect
+ .poll(
+ async () => {
+ const d = await persisted();
+ return {
+ held: Boolean(d?.platformHolds?.youtube),
+ backoff: Boolean(d?.platformBackoff.youtube),
+ };
+ },
+ { timeout: 60_000 },
+ )
+ .toEqual({ held: false, backoff: false });
+});
+
+test("Clear hold drops the hold, the backoff and the pace, and says so", async ({
+ page,
+}) => {
+ const now = Date.now();
+ await setup(
+ ["a1"],
+ {},
+ {
+ platformBackoff: { youtube: { until: now + 3_600_000, fails: 9 } },
+ platformHolds: { youtube: { since: now - 60_000, probeAt: now + 3_600_000, rateLimited: true } },
+ platformPace: { youtube: { sleepRequestsSeconds: 8, baseSeconds: 1, cleanUnits: 0, steppedAt: now } },
+ },
+ false,
+ );
+ await page.goto("/operations/download");
+ const region = page.getByRole("region", { name: "Rate-limit cooldown" });
+ await expect(region.getByRole("list", { name: "Platforms held" })).toContainText(
+ "next probe in",
+ { timeout: 20_000 },
+ );
+ await expect(page.locator("section[data-hydrated=true]").first()).toBeVisible();
+ await region.getByRole("button", { name: "Clear hold on youtube" }).click();
+ await expect(page.getByLabel("clear hold result")).toContainText(
+ /^Cleared by hand for youtube: the hold \(since .* UTC\), the backoff \(9 failures\), the pace \(8s → base 1s\)\./,
+ { timeout: 20_000 },
+ );
+ const d = await persisted();
+ expect(d?.platformHolds ?? {}).toEqual({});
+ expect(d?.platformBackoff ?? {}).toEqual({});
+ expect(d?.platformPace ?? {}).toEqual({});
+});
diff --git a/editor/e2e/rumble-sweep.spec.ts b/editor/e2e/rumble-sweep.spec.ts
@@ -46,6 +46,7 @@ type ChannelConfigFile = {
type AutoQueueStateFile = {
download: {
platformBackoff: Record<string, { until: number; fails: number }>;
+ platformPace?: Record<string, { sleepRequestsSeconds: number }>;
};
};
@@ -133,9 +134,13 @@ test("a Rumble sweep that 429s mid-listing is incomplete: paged walk, cooldown,
expect(paged.length).toBeGreaterThan(0);
for (const line of [...full, ...paged]) {
expect(line).toContain("--impersonate chrome");
- expect(line).toContain("--sleep-requests 1");
expect(line).toContain(RUMBLE_URL);
}
+ // The pace adapts (release 17, slice RL): the enumeration ran at rumble's
+ // fixed 1 s, its 429 doubled the pace, and the paged walk that followed it
+ // paced at 2 s.
+ for (const line of full) expect(line).toContain("--sleep-requests 1");
+ for (const line of paged) expect(line).toContain("--sleep-requests 2");
// The rumble cooldown is recorded where the Sync gate and the runner read it.
const state = await readJson<AutoQueueStateFile>(
@@ -144,6 +149,7 @@ test("a Rumble sweep that 429s mid-listing is incomplete: paged walk, cooldown,
expect(state.download.platformBackoff.rumble?.until).toBeGreaterThan(
Date.now(),
);
+ expect(state.download.platformPace?.rumble?.sleepRequestsSeconds).toBe(2);
// Not a listing: no sweep stamp, stored playlist untouched.
const after = await readConfig();
diff --git a/plans/FACTS.md b/plans/FACTS.md
@@ -6458,6 +6458,66 @@ Line numbers are `plans/FACTS.md` lines at `e172749b`, before this record's in-p
read by nothing. So a "no-op" site save can move bytes; `jq -S` plus the `order` fills is the
expected diff, not a bug.
+## Release 17 — slice RL: a subtitle 429 is not a platform failure; the pace adapts (verified 2026-10-01)
+
+- **YouTube's 429s are its subtitle (timedtext) fetch, per video.** Measured 2026-10-01 on the
+ live corpus: all 78 HTTP 429 lines in 24 h of job logs were `Unable to download video subtitles
+ for 'en': HTTP Error 429`; none on the watch page, player API, m3u8 or a listing; the backoff
+ read `fails: 80`. The 2026-09-25 postmortem (`plans/youtube-lane-pacing.md`) found the same.
+- **`subs_rate_limit`** is a `DownloadFailureClass` (`lib/availability.ts`): `classifyDownloadFailure`
+ returns it when a subtitle-429 line is present and every `ERROR:` line is a subtitle line (no
+ soft block, no bot check) — `isSubtitleRateLimitOnly` / `hasSubtitleRateLimit`. It is set on a
+ **SUCCESS** record (`status: ok…`, `failureClass: "subs_rate_limit"`), the one class that is.
+- **yt-dlp's `--ignore-errors` turns ANY subtitle failure into a WARNING and carries on** — a 429, a
+ 403/404/5xx, a failed `live_chat` replay, an OSError writing the file (review H1, verified offline for
+ a 403) — so the downloader parses the WARNING's class: only a rate limit takes the media path
+ (`hasNonRateLimitSubtitleFailure` keeps any other a failed attempt, not archived, classified from the
+ tail)
+ (`YoutubeDL._write_subtitles`: `ignoreerrors is True` → `report_warning`; `'only_download'`, i.e.
+ `--no-abort-on-error`, still raises). Without it, a `--load-info-json` run that fails re-extracts
+ from the URL once (`download_with_info_file` "The info failed to download … trying with URL") —
+ the "one internal re-extraction" the evidence saw. The youtube-handling primary
+ (`downloadOneManaged.ts` `youtubeHandlingArgs`) carries `--ignore-errors` and
+ `--sleep-subtitles max(5, pace)` after `-t sleep`; a subtitle 429 there exits 0, no transcript
+ lands, and attempt 3 (the no-subs media pass, `--no-write-subs --no-write-auto-subs` appended)
+ fetches the audio. Any other primary that died on its subtitles alone is re-run once as
+ `primary-without-subs` (`DownloadAttemptKind`).
+- **The pacing memory** is three maps beside `platformBackoff`/`videoDeferrals` on every lane of
+ `.auto-queue/state.json` (download only in practice; `{}` coerced for an older file):
+ `platformPace {sleepRequestsSeconds, baseSeconds, cleanUnits}` (absent = at the static value),
+ `platformHolds {since, probeAt}`, `subtitleDeferrals {count, lastAt, until, channelSlug}`. The
+ pure seam is `jobs/platformBackoff.ts` (`escalatePlatform`, `settlePlatformClean`,
+ `prunePlatformPacing`, `deferSubtitles`, `downloadGapMs`, `heldPlatformSentence`) and
+ `jobs/unitOutcome.ts`; every writer outside the runner goes through `jobs/downloadBackoff.ts`'s
+ one write-through (`recordDownloadBackoff(pf, paths, failureClass)`, `recordSubtitleDeferral`,
+ `clearSubtitleDeferral`, `heldPlatformRefusal`).
+- **The pace reaches every spawn synchronously**: `channelExtraArgs` reads
+ `livePlatformPaceSeconds(pf)` (`jobs/autoQueueState.ts`) — the shared object when a runner holds
+ it, else the pace the last read/write of the file saw (`holder.lastPace`; the status poll reads
+ the file), else the static value. `withSleepRequests` only RAISES `--sleep-requests`; the
+ channel's own `ytdlpExtraArgs` still come last and win. The pace key is
+ `detectPlatform(url) ?? "unknown"`, the cooldown's key.
+- **The hold**: `FAILS_TO_REACH_CAP` = 6 (60 s doubling reaches the 30-min cap at the sixth);
+ `failsAtCap(fails) = max(0, fails − 5)`. At `pacing.holdAfterFailsAtCap` the backoff's `until`
+ becomes `now + holdProbeMinutes` (no jitter) and `platformHolds[pf].probeAt` mirrors it
+ (`rateLimited` false for a hold reached through network failures alone); a held platform's backoff
+ entry is never pruned. It is cleared by a clean lane unit (a `subs_rate_limit` unit included), by a
+ clean manual run once the probe is due (`runYtdlp`'s `onPlatformClean` → `recordPlatformClean`, and a
+ clean metadata scan), or by **Clear hold** (`clearPlatformHold`, a `clear-platform-hold` job on queue
+ `pacing:<pf>`). Manual actions are refused only while held AND before the probe time.
+- **The one time-based rule:** a raised pace eases one step per `PACE_TIME_DECAY_MS` (1 h) since its
+ `steppedAt` with no rate limit (`effectivePaceSeconds` for readers, `decayPaceByTime` on writes). A manual Sync / download / metadata scan / fetch-window on a held platform is refused with
+ `heldPlatformSentence` (`<pf> is held: … the next probe is in N min. <What> will run once a probe
+ comes back clean.`).
+- **The download lane now waits between units**: `platformNextStartAt` (in memory) =
+ settle + `downloadGapMs(settings.sleepBetweenDownloadsSeconds, pace, base)` — the setting plus
+ the pace ABOVE its base (the base is already paid inside each spawn). `runManagedDownloads`
+ sleeps the same sum. Idle reasons `held` and `paced` join `cooldown`/`deferred`.
+- **download-missing-subs** skips videos inside a subtitle deferral (logged in the prefilter line),
+ records a subtitle 429 and goes on to the next video whatever `abortOnError` says, and clears a
+ deferral when the subtitles come down. `runChildAndStream` returns its stderr tail (and attaches
+ it to the thrown error as `stderrTail`).
+
## Release 7 — slices Y, C, K (verified 2026-09-25, `main` @ `bb3dbb4c`)
- **A rate-limited video is deferred, per video, for 6 h.** `common/jobs/platformBackoff.ts`:
diff --git a/plans/release-17.md b/plans/release-17.md
@@ -2,7 +2,9 @@
Written 2026-10-01 in plan mode against `main` `03be31b5` (releases 13–16 live). Rules:
`plans/tools/implementer-rules.md` — one Opus implementer per slice in its own worktree, one read-only
-Opus review, the parent merges `--no-ff` on a clean tree; records state rulings never reasons; no
+Opus review, the parent merges `--no-ff` on a clean tree; an implementer's commits carry its own model's
+`Co-Authored-By` (T1's were rewritten to the session's at the parent's request before merge; the rule was
+corrected 2026-10-02 after RL declined to do the same — the tree is identical either way); records state rulings never reasons; no
identifier ending `the refused parent-suffix`; counts-only privacy greps before every merge; changelog bullets checked
by eye; the homepage build gate is `build:nodata`, never `run build`; the corpus link is for `next
build` only. The live editor on `:3001` is restarted ONCE, after T3 merges and BEFORE the migration
@@ -265,6 +267,7 @@ one short Transcribe (the hook on a relocated channel), `/storage`, `df`; then n
| **U1** umtool roots + `out/` | `r17/umtool-media-root` | `paths.mjs` (`MEDIA_ROOT`, `CACHE_DIR`), `lib/report/storage.mjs`, `driver.mjs`, `build-video.mjs:2615`, `export.mjs`, `kinds.mjs`, `umtool doctor`, `umtool storage move-out`, the e2e env | — (∥ T1) | `test:scripts` (+ mover tests), `next-build-trace.test.mjs`, the capped umtool build with the corpus linked, umtool e2e |
| **U2** deliverables switch | `r17/umtool-deliverables` | manifest `storage` field, `deliverableDir`, `cut.mjs:96`, `deliver.mjs:362`, `umtool storage deliverables`, bench "Move deliverables", `umtool check` | U1 | umtool unit + e2e: cut and share through a linked `clips/` |
| **XP** X posts are private (operator-requested, beside the media tier) | `r17/x-posts-private` | `common/lib/postsVisibility.ts` (new) + test, `settingsSchema.ts` + `social/xCookieSource.ts` (`social.x.visibility`) + SETTINGS.md, `siteSchema.ts` + `site.ts` (`audience`, `isListedSite`, `resolveHubUrl`) + SITE.md, `buildIndex.ts` (the per-site loop only), `bin/compose-site.ts` + an integration test, `lib/corpus.ts`, `lib/builtExport.ts`, `publish/build.ts` + tests, `controller/poolSummary.ts` + test, `docker/publish-site.sh`, `export/app/offline/page.tsx`, editor `settings/{xSessionActions.ts,page.tsx,components/{XSessionSection,XPostsVisibilityControl}.tsx}`, `sites/{actions.ts,components/SiteForm.tsx,lib/{buildAction,deployAction}.ts}`, e2e `x-session`, `sites-crud`, export `x-posts-private` | — (∥ all) | the compose integration test (public vs private site, flip back, an X-only public site); deploy refusals before wrangler and before the upload |
+| **RL** a subtitle 429 does not fail a download; the pace adapts (operator-requested, beside the media tier) | `r17/rate-limit-adapts` | `common/jobs/{platformBackoff,downloadBackoff,unitOutcome,autoQueueState}.ts`, `lib/availability.ts`, `ytdlp/{platformArgs.mjs,channelArgs.ts}`, `downloadOneManaged.ts` (subtitle handling + pace only), `autoRunner.ts` (the download lane's gap, hold, probe), `runYtdlp.ts` (pace into `runManagedDownloads`, download-missing-subs), `checkAvailability.ts` (pace only), `settingsSchema.ts` `pacing` + SETTINGS.md, `RunnerOperationView.tsx` + `dispatch.ts`, `views/activeJobs.ts`, `pipelineActions.ts` + `videoActions.ts` (refusals, the subtitle line), `bin/doctor.ts`, `editor/e2e` | — (∥ D0, T1, U1) | unit: classifier, `applyUnitOutcome`, `channelExtraArgs` pace, the lane gap; e2e `rate-limit.spec` + `auto-queue`, `lane-runner`, `channel-priority`, `video-page` |
Order: 0a → D0 ∥ T1 ∥ U1 → T2 ∥ U2 → T3 → parent: records, ONE editor rebuild + restart, umtool rebuild +
restart (the restart is the operator's: the permission layer refuses the `0.0.0.0` bind) → the migration
@@ -347,6 +350,53 @@ hand; a dirent `isFile()` filter over a video dir hides it."** The `.relocating.
`transcripts/**` (the rollout — a private site holding every channel, the setting flipped, the
public sites rebuilt — is the parent's, through the editor's own writers).
+## Slice RL — the ruling (2026-10-01)
+
+Operator-requested, beside the media tier. **Measured 2026-10-01 (read-only, live corpus):** every
+one of the 78 HTTP 429 lines in 24 h of job logs is `Unable to download video subtitles for 'en':
+HTTP Error 429` on YouTube; none on the watch page, the player API, the m3u8 or a listing; no
+`sync` or `import-one` log carries a real 429. The YouTube backoff reads `fails: 80`, consecutive
+since the evening of 2026-09-30; the lane retries one unit every ~30 min and each burns another
+subtitle 429. Twelve videos of one channel rotate through the 6 h deferral and fail again each
+time; `redownload-archive` jobs on other videos of the same channel succeeded in between. Each
+failing unit had already picked its formats and died on the subtitle file. The 2026-09-25
+postmortem (`plans/youtube-lane-pacing.md`) found the same: the timedtext endpoint 429s per video,
+not IP-wide. Today's flags: `--sleep-requests 1` for YouTube, `-t sleep` on the primary spawn, no
+`--sleep-subtitles` override; yt-dlp re-extracts once after a subtitle 429 then fails; the app
+never retries a `rate_limit`.
+
+1. **A subtitle 429 never fails a download.** When yt-dlp fails only on the subtitle file, the
+ unit downloads the media anyway and succeeds — one spawn where yt-dlp's own `--ignore-errors`
+ allows it, else a second spawn without subtitles. `classifyDownloadFailure` gains
+ `subs_rate_limit` for "the subtitle fetch is the only failure". It is recorded per video as a
+ subtitle deferral (beside `videoDeferrals`, through `downloadBackoff.ts`'s write-through), does
+ NOT touch the platform backoff and does NOT defer the video's media. The deferred subtitles are
+ picked up by `download-missing-subs`; the transcription lane sees "downloaded, no transcript".
+ After 3 subtitle deferrals the video's subtitles are left alone for 7 days (shown on the video
+ page with the count and the date; a manual fetch still works). `plans/youtube-lane-pacing.md`
+ gets a dated note pointing here.
+2. **The pace adapts per platform.** A persisted per-platform `platformPace`: `--sleep-requests`
+ starts at the platform's static value, doubles on every platform-level `rate_limit` (never on
+ `subs_rate_limit`) up to a cap, and decays one step toward the base after every N clean units.
+ It feeds `channelExtraArgs` (every yt-dlp call for the platform) and the lane's gap: the
+ download lane honours `sleepBetweenDownloadsSeconds` AND adds the adaptive pace. Settings in a
+ new `pacing` block (cap 16 s, decay 5 units, hold threshold, probe interval). YouTube's primary
+ spawn gets `--sleep-subtitles <pace>`.
+3. **A block is not a burst.** A platform whose backoff has failed at the cap
+ `holdAfterFailsAtCap` times in a row (3) is HELD: one probe unit per `holdProbeMinutes` (60)
+ instead of one every 30 min; a clean probe clears the hold and the backoff. Persisted beside the
+ backoff. A manual Sync/download on a held platform is refused with a sentence naming the hold
+ and the next probe.
+4. **Visible.** The download lane page's "Rate-limit cooldown" region shows per platform the
+ cooldown or hold (since, fails, next try/probe), the current pace, and the videos whose
+ subtitles are deferred (with counts); a rack chip for a held platform only if it is a few lines;
+ `archilyzer doctor` warns on a platform in cooldown/hold and on the pace; the idle reasons gain
+ `held` and `subs-deferred` where the dispatch text names them.
+
+Not in scope: yt-dlp's player client or cookies for subtitles (an open question for the operator:
+whether the timedtext 429 for these twelve videos is a PO-token/client matter — not probed by
+hand); the sync scheduler's per-channel backoff; Rumble/Odysee specifics beyond the shared code.
+
## Record
### Slice U1, as shipped — umtool's render scratch goes to a media root (2026-10-01)
@@ -1431,6 +1481,235 @@ the full suite was clean at load 10 — this branch does not touch `scripts/`);
92 s; **e2e (the 8 specs) 78 passed, 5 failed, 8.2 min** — the same five `channel-storage.spec.ts` cases
(:80, :250, :391, :484, :1091), the old mover's layout reading `legacy`, nothing else red.
+### Slice RL, as shipped — a subtitle 429 does not fail a download, and the pace adapts (2026-10-01)
+
+Branch `r17/rate-limit-adapts` off `main` `90bd8384`, worktree `~/Projects/r11-runner-lows` (editor 4801,
+test 4811, export 4810 — `pnpm wt list`'s block #18), one Opus implementer. Scratch files `RL-*` in the
+job's `tmp`. `main` moved under the slice: it was merged at `1d5c33bf` (U1, XP), at `a395aaa1` (D0, T1) before the
+final gates, and at `0a62bf74` (U2) after them.
+The ruling is above ("Slice RL — the ruling").
+
+**What yt-dlp does, verified offline.** yt-dlp 2026.08.19 (the editable install the editor runs) was run
+against a localhost HTTP server whose subtitle URL answers 429 — no YouTube request, per the operator's
+rule. `--ignore-errors` (`ignoreerrors: True`; `--no-abort-on-error` is `'only_download'` and still raises)
+reports `WARNING: Unable to download video subtitles for 'en': HTTP Error 429: Too Many Requests`, exits
+0, still fires `--print after_video:…`, and — when the run is not `--skip-download` — goes on to download
+the media. Without it, a `--load-info-json` run that hits the subtitle error prints `The info failed to
+download: … trying with URL …` and re-extracts from the URL: the "one internal re-extraction" the
+evidence saw. So the ONE-spawn shape is yt-dlp's own, and the log keeps the line for the classifier.
+
+**What it does.**
+- **A subtitle 429 is not a failure of the download.** `classifyDownloadFailure` returns the new
+ `subs_rate_limit` when a subtitle-429 line is present and every `ERROR:` line is a subtitle line (no
+ soft block, no bot check). The youtube-handling primary (subtitles only, `--skip-download`) now carries
+ `--ignore-errors` and `--sleep-subtitles max(5, pace)` after `-t sleep`: a subtitle 429 exits 0 with
+ no transcript, and attempt 3 — the no-subs media pass, `--no-write-subs --no-write-auto-subs` appended
+ after the channel's own args — downloads the audio, exactly as for a video with no captions (three
+ spawns, as before for such a video). The record is a SUCCESS carrying `failureClass:
+ "subs_rate_limit"` (the one class set on a success). Any other primary that died on its subtitles
+ alone (a channel whose own args ask for subtitles) is run once more as `primary-without-subs`
+ (a new `DownloadAttemptKind`).
+- **Only the subtitles are deferred.** `applyUnitOutcome` (lane) and `runManagedDownloads` (batches,
+ through `recordSubtitleDeferral`) record `subtitleDeferrals[id] = {count, lastAt, until,
+ channelSlug}`: 6 h after a strike, 7 days from the third. The platform backoff, hold and pace are
+ not touched, the video is retired like any success, and it is not added to `videoDeferrals`.
+ **Download missing subs** skips a video inside its window (named in the prefilter line with its
+ count and date), records a subtitle 429 and goes on to the next video whatever `abortOnError` says,
+ and clears the deferral when the subtitles come down (`runChildAndStream` now returns its stderr
+ tail and attaches it to the thrown error). A download from the video page never reads it.
+- **The pace adapts.** `platformPace[pf] = {sleepRequestsSeconds, baseSeconds, cleanUnits}`: absent at
+ the static value (`PLATFORM_ARGS`' `--sleep-requests`, 1 s for YouTube and Rumble, 0 for a platform
+ with none); every platform-level `rate_limit` doubles it (from 0 to 1) up to
+ `pacing.sleepRequestsCapSeconds` (16); every `pacing.decayAfterCleanUnits` (5) clean units halve it,
+ and an entry back at its base is deleted. A `network` failure and a `subs_rate_limit` never move it.
+ `channelExtraArgs` reads it synchronously through `livePlatformPaceSeconds` — the shared state when a
+ runner holds it, else the pace the last read or write of the file saw, else the static value — so
+ every spawn against the platform (listing, prefetch, primary, availability, metadata scan, clip, the
+ new-channel probe through `pacedPlatformArgs`) uses it; `withSleepRequests` only raises, and the
+ channel's own `ytdlpExtraArgs` still win. A manual 429 (`recordDownloadBackoff(pf, paths,
+ failureClass)`) escalates the same way.
+- **The lane waits between units.** `platformNextStartAt` in the runner = settle +
+ `downloadGapMs(sleepBetweenDownloadsSeconds, pace, base)`: the setting the lane used to ignore,
+ plus the pace above its base. `runManagedDownloads` sleeps the same sum.
+- **A block is not a burst.** `FAILS_TO_REACH_CAP` = 6; a backoff that has failed at the cap
+ `pacing.holdAfterFailsAtCap` (3) times in a row — `fails` 8 — holds the platform:
+ `platformHolds[pf] = {since, probeAt}`, the backoff's `until` = now + `pacing.holdProbeMinutes`
+ (60, no jitter), so the lane's existing cooldown gate lets one probe unit through per interval. A
+ failed probe re-arms it; only a clean unit (`transcribed`, no subtitle 429) clears the hold and the
+ backoff together. A held platform's backoff entry is never pruned. A manual Sync, any download mode
+ but Store playlist, a metadata scan, and the video page's fetch-window / full-source fetch are
+ refused with `heldPlatformSentence`: `youtube is held: its rate limit outlasted the cooldown cap (8
+ failures in a row, held since 14:02 UTC). Auto-download probes it once every 60 min — the next probe
+ is in 42 min. Sync will run once a probe comes back clean.` (scheduled syncs go through the same
+ action and are refused alike).
+- **Visible.** The download lane's `role="region"` "Rate-limit cooldown" gains `<ul aria-label="Platforms
+ held">` (since, failures, next probe, pace), `"Request pace"` (seconds between requests and the base)
+ and `"Deferred subtitles"` (link, count, time left; "left alone" from the third); "Platforms in
+ cooldown" now lists only unheld platforms and shows the pace too; `formatCooldown` gains a days arm.
+ The view (`autoQueueStatus.ts`) carries `cooldowns[].hold`, `pace[]` and `subtitleDeferred[]`. Idle
+ reasons gain `held` ("every pending platform is held after repeated rate limits — one probe at a
+ time") and `paced` ("every pending platform is pausing between downloads"), in both exhaustive
+ switches. The video page draws one `role="note"` `aria-label="subtitle deferral"` line for a video
+ with a deferral, read from a new `GET /api/channels/<slug>/videos/<id>/subtitle-deferral`. `archilyzer doctor` has a **download pacing** section: a cooldown, a hold or a
+ raised pace warns (never fails), deferred subtitles are a note.
+- **Settings:** a `pacing` block (`sleepRequestsCapSeconds` 16 [1–120], `decayAfterCleanUnits` 5
+ [1–1000], `holdAfterFailsAtCap` 3 [1–100], `holdProbeMinutes` 60 [1–1440]); SETTINGS.md and
+ settings.json.example regenerated; `sleepBetweenDownloadsSeconds`' description says the lane honours
+ it now.
+
+| sha | what |
+|---|---|
+| `93462d22` | common: `subs_rate_limit` (`availability.ts`), the youtube primary's `--ignore-errors` + `--sleep-subtitles`, attempt 3 on a subtitle 429, `primary-without-subs`, the record's class; `platformBackoff.ts` pace/hold/subtitle-deferral seams + coercion; `autoQueueState.ts` three maps + `livePlatformPaceSeconds`; `unitOutcome.ts`; `downloadBackoff.ts` one write-through + `recordSubtitleDeferral`/`clearSubtitleDeferral`/`heldPlatformRefusal`; `platformArgs.mjs`/`channelArgs.ts` the pace; `autoRunner.ts` gap, hold idle, `held`/`paced`; `runYtdlp.ts` batch gap + deferral, download-missing-subs; `pacing` settings + SETTINGS.md; unit tests |
+| `8a40c91e` | editor + doctor: the region's three lists, the hold refusals (`pipelineActions.ts`, `videoActions.ts`), the video page's line, `autoQueueStatus` fields, `doctor` section, `checkAvailability`'s paced probe, a manual Sync's backoff passes its class |
+| `49b3d087` | e2e: `rate-limit.spec.ts` (3 tests); the fake's `dl429` obeys `--ignore-errors`, new `wp429` (watch-page 429 at the prefetch); `pacing.spec` T2 moves to `wp429`; `rumble-sweep.spec`'s paged walk paces at 2 s; the video-page line moved out of `cards/` |
+| `2ede037c` | plans: the ruling, FACTS, the dated note on `youtube-lane-pacing.md`, two `[Unreleased]` bullets |
+| `8c76ec09` | merge `main` `1d5c33bf` (U1, XP): both changelog sides, both rulings and slice rows kept |
+| `a8e9c241` | editor: the video page's line reads `GET /api/channels/<slug>/videos/<id>/subtitle-deferral` instead of a mount-time server action (Next runs a page's server actions one at a time, so it sat in front of the first click — e2e run 2) |
+| `5e967e18` | merge `main` `a395aaa1` (D0, T1): both changelog sides kept; no code conflict (T1's tier hooks and lane holds sit clear of this slice's hunks) |
+| `5725e53a` | merge `main` `0a62bf74` (U2): clean; U2 touches umtool, the changelog and the plans only |
+| *(this commit)* | this record |
+
+**Gates** (worktree root). tsc (`pnpm -r --no-bail --workspace-concurrency=1 exec tsc --noEmit`) clean
+before every commit (a stale `editor/.next/dev` and `export/.next/dev` from the worktree's earlier use
+were removed first). Before the merge of `main`: common **2513/2513**, editor unit **109/109**,
+test:scripts **368 pass + 2 skip**, capped editor build ok (78 s). At the first merged tip `8c76ec09`: common
+**2530/2530** (29 of them new: `unitOutcome` +6, `platformBackoff` +5, `channelArgs` +4,
+`downloadBackoff` +4, `availability` +3, `subtitleRateLimit` +2 (new), `managedDownloadsSleep` +2,
+`autoQueueState` +1, `autoQueueStatus` +1, `doctor` +1), editor unit **109/109**, test:scripts
+**392 pass + 2 skip**, capped editor build ok (compiled in 24.0 s, 73 s). `settings example --check`
+clean. At the final tip `5e967e18` (after D0 and T1): tsc clean; common **2630 tests, 2602 pass, 0
+fail, 28 skipped** (the 28 are `main`'s — T1's `relocateChannelMedia` cases skipped until T2, "release 17
+T2 rebases the mover on media/"); editor unit **109/109**; test:scripts **394 pass + 2 skip**; capped
+editor build ok (compiled in 23.3 s, 56 s; the new route listed). EDITOR e2e, `$T/RL-specs.txt` = `rate-limit.spec.ts pacing.spec.ts auto-queue.spec.ts
+lane-runner.spec.ts channel-priority.spec.ts video-page.spec.ts rumble-sweep.spec.ts
+no-subs-fallback.spec.ts queues.spec.ts`:
+- run 1 (`RL-e2e1.log`, `49b3d087`): **74 passed, 1 failed, 13 min** (4 in the queue). The three
+ `rate-limit.spec` tests, `pacing.spec` (T2 1.0 min) and `rumble-sweep.spec` passed. The failure was
+ `auto-queue.spec:623` "Drain completes when an auto-transcribe unit is parked behind a busy worker"
+ — the 30 s test budget ran out while polling `/api/auto-queue/status` (a transcription-lane test;
+ nothing in this slice runs on that lane).
+- run 2 (`RL-e2e2.log`, `8c76ec09`): **74 passed, 1 failed, 26 min** (≈13 in the queue). The Drain
+ test passed; `video-page.spec:280` "Mark untranscribable…" timed out: its click's server action
+ queued behind the deferral line's mount-time action. Fixed in `a8e9c241`.
+- run 3 (`RL-e2e3.log`, `a8e9c241`, `rate-limit.spec.ts video-page.spec.ts`): **23 passed, 0 failed,
+ 9 min** (≈8 in the queue).
+- run 4 (`RL-e2e4.log`, `5e967e18`, the full list): **71 passed, 4 failed, 8 min** — the four
+ video-page text-preview tests (`:325`, `:367`, `:384`, `:472`), each a 5 s / 30 s timeout on a
+ preview fetch, while a common test run of this implementer's was loading the machine beside it.
+- run 5 (`RL-e2e5.log`, `5e967e18`, `video-page.spec.ts` alone, nothing beside it): **20 passed,
+ 0 failed, 1 min**.
+- run 6 (`RL-e2e6.log`, `5e967e18`, the full list, nothing beside it): **75 passed, 0 failed, 5 min**
+ (no queue wait).
+
+After the U2 merge (`5725e53a`, umtool-only code): tsc clean; test:scripts **461 pass + 2 skip** —
+in two of four runs one or two `scripts/queue-lock.test.mjs` cases ("serves waiters in arrival order
+(FIFO)", "prints a banner naming the holder while waiting") failed while other slices' e2e runs held
+the machine-wide lock; a run with the lock free passes. The editor build, common, editor unit and e2e
+were not rerun: U2 changed no `common/` or `editor/` code.
+
+**Numbers: none.** `.auto-queue/state.json` is outside both numbers tools. **`platformPace: {}`,
+`platformHolds: {}` and `subtitleDeferrals: {}` appear on all four lanes of `state.json` at the first
+persist after the rollout boot**, and `pacing` appears in `settings.json` at the next save; an older
+build drops the three keys on its next write, so a rollback is safe. Privacy gate: `git diff main
+--name-only | xargs grep -lc "$(whoami)\|$(hostname)"` prints `plans/FACTS.md` only, for three lines
+`main` already carries (3477, 5033, 5430 at `0a62bf74`); the slice's added lines carry none.
+
+**Deviations, one sentence each.**
+- `subs-deferred` is not an idle reason: a subtitle deferral never idles the lane (the media is done),
+ so it is the region's "Deferred subtitles" list instead; `paced` was added for the new gap, which
+ would otherwise read as "capped".
+- The gap adds the pace ABOVE its base, not the whole pace: the base is already paid between the
+ requests of every spawn, and adding it again would slow every lane unit and batch by a second with
+ nothing rate-limited.
+- The lane's gap reads the global `sleepBetweenDownloadsSeconds`; a channel's own override still
+ applies to its batch runs only.
+- No rack chip: the rack has no per-lane or per-platform chip to say "held: rate-limited" on, and
+ threading the platform state into every row is more than a few lines in `channelRow.ts`, which is
+ T2's.
+- Files touched beyond the list, each by a small hunk: `lib/downloadOutcome.ts` (the attempt kind,
+ the class's doc), `lib/settingsDocs.ts` (the `pacing` table), `views/autoQueueStatus.ts` (three
+ fields in `buildKind`, clear of D0's memo), `VideoPanel.tsx` + a new `SubtitleDeferralLine.tsx`
+ beside it and its new GET route (the video page's line; `videoActions.ts` keeps only the refusals), the fake yt-dlp, `pacing.spec.ts` and `rumble-sweep.spec.ts`
+ (their expectations follow the ruling).
+
+**Found and left.**
+- Subtitle deferrals are written by the lane, batch downloads and download-missing-subs; the video
+ page's own download, the re-acquire backfill and import neither record nor clear one (a manual
+ fetch never reads it either).
+- The `pacing` keys are in `settings.json` only, not on the `/settings` form.
+- The runner merges only `platformBackoff` from disk mid-run, as before; pace and holds written from
+ outside go through the live object (`mutateDownloadState`), and a disk-only write while no runner
+ lives is read at the runner's start.
+- **Rollout:** the live backoff reads `fails: 80` (the subtitle 429s this slice stops counting).
+ Unless the boot finds it lapsed for over 30 min (then it is pruned, as before), the first real
+ platform-level failure after the restart holds YouTube at once. **Clear hold** on
+ `/operations/download` is the way out (it drops the hold, the backoff and the pace and says so in a
+ `clear-platform-hold` job log); a clean lane unit, or a clean manual Sync once the probe is due,
+ clears it too. The changelog bullet says the same.
+- **Open question for the operator** (not probed): whether YouTube's timedtext 429 for these twelve
+ videos is a PO-token / player-client matter (`--extractor-args youtube:player_client=…`).
+
+#### Review (2026-10-01): SHIP AFTER FIXES → fixes
+
+| finding | fix |
+|---|---|
+| **H1** — under `--ignore-errors` EVERY subtitle failure exits 0 (a 403/404/5xx, a failed `live_chat` replay, an OSError writing the file) and ended `ok`, archived, with no transcript and no media, and counted as a clean lane unit | `5de4e385`: `hasNonRateLimitSubtitleFailure` (`lib/availability.ts`); a youtube primary that exited 0 with such a WARNING is a failed attempt, as before — `lastSucceeded` false, no media pass, no archive line, its error and class from the tail (a 403 → `network`, a `live_chat` 404 → `unknown`). Only a subtitle 429 takes the media path. Unit cases for a 403 and a `live_chat` failure (`subtitleRateLimit.test.ts`). FACTS' `--ignore-errors` bullet says "any subtitle failure" |
+| **H2** — a hold ended only with a lane probe; with the lane off or nothing pending it was permanent, Sync and scheduled syncs stayed refused, the page dropped the platform once the probe was overdue, and the refusal said "next probe in 1 min" for ever | `f003e29e`: `heldPlatformRefusal` refuses only while held AND before the probe time (as `fetchWindowAction` already did); `runYtdlp`'s new `onPlatformClean` (wired in `pipelineActions.ts`) and a clean metadata scan call `recordPlatformClean`, which settles the platform like a clean probe (backoff and hold clear) and logs it; `clearPlatformHold` drops hold + backoff + pace. The status view keeps a held platform whatever its probe time. `c2cc1228`: "Platforms held" says "probe overdue — the lane is off" (or "probe due — it waits for a pending video") and each row has **Clear hold**, run as a one-step `clear-platform-hold` job on queue `pacing:<pf>` whose log says what was cleared. e2e: an overdue hold with the lane off is shown, a Sync is not refused, and its clean run lifts the hold; Clear hold empties hold, backoff and pace |
+| L1 — a held probe whose media came down but whose subtitles 429'd did not lift the hold | `f003e29e`: it lifts the hold and the backoff, and does not count toward the pace's easing (`countForDecay: false`) |
+| L2 — the pace only eased on lane units | `f003e29e`: **the one time-based rule** — a raised pace eases one step per hour with no rate limit (`steppedAt`, `PACE_TIME_DECAY_MS`); readers (the args builder, the view, the doctor) apply it through `effectivePaceSeconds`, writers through `decayPaceByTime` |
+| L3 — Download missing subs ran back to back | `f003e29e`: it waits `downloadGapMs(sleepBetweenDownloadsSeconds, pace, base)` between videos |
+| L4 — the `fails: 80` rollout hazard was in the record only | the changelog bullet and the record's rollout bullet both say it, and that Clear hold is the way out |
+| L5 — a hold reached through network failures read as a rate limit | `f003e29e`: the hold carries `rateLimited`; the sentence, the list and the doctor say "failing (network errors)" for such a hold |
+| N9 — a WARNING webpage 429 beside a subtitle 429 read as subtitles-only | `5de4e385`: every 429 / too-many-requests line must be a subtitle line, and no non-429 subtitle failure may be present |
+| N10 — "YouTube's subtitles" on surfaces that match any platform | `c2cc1228`: the list heading and the video-page line say "the subtitles" |
+
+One more fix commit, `7e59c61f`: Clear hold's sentence lived inside the region it empties, so a clear
+that left nothing to list unmounted it; it now lives on the lane view (found by run 7).
+
+**Re-gate** (tip `7e59c61f`; the ruled list `rate-limit.spec video-page.spec auto-queue.spec
+lane-runner.spec`, `$T/RL-specs-fix.txt`). tsc clean before every commit. common **2638 tests, 2610
+pass, 0 fail, 28 skipped** (the 28 are `main`'s T1 mover cases; 8 new: `subtitleRateLimit` +2,
+`unitOutcome` +3 and one rewritten, `downloadBackoff` +2, `availability` +1, `autoQueueStatus` +1).
+Editor unit **109/109**. EDITOR e2e:
+- run 7 (`RL-e2e7.log`, `c2cc1228`): **53 passed, 1 failed, 4 min** — the new Clear hold test (the
+ sentence unmounted with the region), fixed in `7e59c61f`;
+- run 8 (`RL-e2e8.log`, `7e59c61f`, `rate-limit.spec` alone): **5 passed, 0 failed, 4 min** (≈3.5 in the
+ queue);
+- run 9 (`RL-e2e9.log`, `7e59c61f`, the ruled list): **54 passed, 0 failed, 2 min**.
+test:scripts, the build and the full suite were not rerun, as ruled (the fixes touch no `scripts/` or
+umtool file; the new client code is covered by tsc and the e2e above).
+
+**Re-review (SHIP) → the cheap items before the merge.**
+
+| finding | fix |
+|---|---|
+| R1 — a subtitle 429 beside another subtitle failure (`en` 429 + `live_chat` 404) still classed `rate_limit` | `9c01dea2`: `classifyDownloadFailure` drops the subtitle-429 lines before the generic rate-limit test, so the class is the other failure's (`unknown` for the 404, `network` for a 403); a real platform 429 beside them still wins |
+| R3 — a manual run that asked the source nothing counted as clean | `9c01dea2`: `runYtdlpMode` counts requests (`requestCounter`, shared through every `{...opts}` copy: the listing, each per-video download, each raw spawn) and calls `onPlatformClean` only when it is above 0 — download-missing with nothing to fetch, or download-missing-subs with every video deferred, no longer lifts a due hold (`platformClean.test.ts`, 2 cases) |
+| R4 — `clear-platform-hold` was not a declared kind | `9c01dea2`: declared in `jobKinds.ts` ("Clear rate-limit hold"; not drainable, not replayable, custom queue, neither `needsMedia` nor `needsText`) |
+| R5 — "probe due — it waits for a pending video" on a paused lane | `8a59dce6`: a running lane whose gate is held, or idle `lane-held` / `downloads-paused` / `snoozed` / `disk-gate`, says "probe overdue — the lane is paused" |
+
+**Gates before the merge** (tip `8a59dce6`): tsc clean; common **2641 tests, 2613 pass, 0 fail, 28
+skipped** (main's T1 mover cases); editor unit **109/109**; capped editor build ok (compiled in 17.7 s,
+62 s; the fix pass added a `"use server"` module); EDITOR e2e `pacing.spec rumble-sweep.spec
+no-subs-fallback.spec rate-limit.spec` (`$T/RL-specs-merge.txt`): run 10 (`RL-e2e10.log`): **12 passed, 0 failed, 2 min**.
+
+**Left, recorded only (follow-ups).**
+- The video page's own download, the re-acquire backfill and import neither record nor clear a
+ subtitle deferral, so the page's "Left alone … until" note can be stale after a page download fetched
+ the subtitles (review item 7).
+- `pacingPlatformKey` reads `detectPlatform(url)` while the base pace reads `channelPlatform(config)`
+ (`config.platform` first); they can disagree for a channel whose `platform` differs from its URL
+ (N11). `pacing` has no `/settings` form.
+- umtool's own spawns (`check-availability.mjs`, `build-video.mjs` through `platformArgsForUrl`) run in
+ a separate process and stay at the fixed pace: "every spawn" means every spawn of the editor and the
+ CLI (N12).
+- R2: a persistent per-video subtitle 403 is `network`, which escalates the backoff without deferring
+ the video, so one video can walk a platform into a hold (pre-RL behaviour plus the hold; a clean
+ manual Sync once the probe is due, or Clear hold, gets out). Follow-up: defer the video when the
+ only failure is a subtitle line.
+- An H1 failure records `ytdlpExitCode: 0` with an `error` (the one attempt that does; the job log
+ line says why).
+
### Slice T2, as shipped — the mover over `media/`, and the surfaces (2026-10-02)
Branch `r17/media-tier-mover` off `main` `a395aaa1` (U1, XP, D0 and T1 merged), worktree
diff --git a/plans/tools/implementer-rules.md b/plans/tools/implementer-rules.md
@@ -52,9 +52,11 @@ the Next.js reference for this version.
which copy to scratch. Never hand-edit `transcripts/**`. Never restart the live :3001 editor.
- **Never stage** `settings.json.pre-priority-*` or anything under `test-results/`.
- **Commits** are small, each tsc-green, message in the repo's voice (`channels: …`, `common: …`,
- `plans: …`), and every commit message ends with:
+ `plans: …`), and every commit message ends with two trailer lines: a `Co-Authored-By` naming the
+ model that wrote the commit (an Opus implementer writes `Claude Opus 5.5 (1M context)
+ <noreply@anthropic.com>`, exactly as its own environment states it — never another model's name;
+ ruled 2026-10-02, release 17), and the session line, verbatim:
```
- Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
```
Commit incrementally (a session limit can cut you mid-slice; work on disk and in commits
survives, work in your context does not). Do NOT push. Do NOT merge into `main` — the parent
diff --git a/plans/youtube-lane-pacing.md b/plans/youtube-lane-pacing.md
@@ -1,3 +1,12 @@
+> **Corrected again 2026-10-01 by `plans/release-17.md` "## Slice RL — the ruling" (shipped as
+> "### Slice RL, as shipped" there).** A subtitle 429 — every YouTube 429 this file measured, and
+> all 78 of 2026-10-01's — no longer backs the platform off or defers the video: it is its own
+> class, `subs_rate_limit`, the download goes on to the media, and only the video's subtitles are
+> deferred. The "next lever" named below, honouring `sleepBetweenDownloadsSeconds` in the lane, is
+> built (plus an adaptive per-platform `--sleep-requests`), and a rate limit that outlasts the
+> cooldown cap now holds the platform to one probe at a time. The finding "pacing between videos
+> could not have prevented" a timedtext 429 stands.
+
# Plan: one YouTube video must not keep the whole platform in a 429 cooldown
> **Corrected by `plans/release-7.md` "## Slice Y" and shipped as "### Slice Y, as shipped"
diff --git a/settings.json.example b/settings.json.example
@@ -9,6 +9,12 @@
"x": {}
},
"sleepBetweenDownloadsSeconds": 10,
+ "pacing": {
+ "sleepRequestsCapSeconds": 16,
+ "decayAfterCleanUnits": 5,
+ "holdAfterFailsAtCap": 3,
+ "holdProbeMinutes": 60
+ },
"downloadFormat": "auto",
"minFreeDiskGB": 5,
"resumeMarginGB": 2,
