commit 185ad18d52f584424567c52178adb5094bacd0cb
parent 639abe30a9563367759168705e9155103eed9121
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 25 Sep 2026 11:02:50 -0400
common: applyUnitOutcome — one seam for a finished unit's backoff and deferral
The body of the runner's per-unit outcome block moves to the pure
jobs/unitOutcome.ts and is extended: rate_limit now also defers the video
6 h (line "... <id> deferred 6h; next video after cooldown."), network is
backoff only (today's line), transcribed clears the cooldown, every branch
prunes lapsed deferrals. autoRunner.ts is the call + onLog + markCompleted;
persist() unchanged. unitOutcome.test.ts: 7 cases.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
3 files changed, 213 insertions(+), 33 deletions(-)
diff --git a/common/controller/autoRunner.ts b/common/controller/autoRunner.ts
@@ -67,12 +67,8 @@ import {
recordPick,
writeAutoQueueState,
} from "../jobs/autoQueueState";
-import {
- clearBackoff,
- isCoolingDown,
- nextBackoff,
- pruneExpired,
-} from "../jobs/platformBackoff";
+import { isCoolingDown, pruneExpired } from "../jobs/platformBackoff";
+import { applyUnitOutcome } from "../jobs/unitOutcome";
import { type DownloadFailureClass } from "../lib/availability";
import { resolveCookiePolicy } from "../lib/cookiePolicy";
import { isAutoSubsOnly } from "../lib/subtitleProvenance";
@@ -1933,33 +1929,27 @@ async function runLoop(
reportRunnerProgress(Math.max(0, lastPending - 1));
}
- // Per-platform backoff: a rate-limit / network failure pauses the whole
- // platform with an exponential cooldown, and the video is deliberately NOT
- // marked completed so it's retried once the cooldown lapses. Any other
- // outcome marks the video done-for-session; a success also clears the
- // platform's cooldown.
- const backoffHit =
- unitPlatform !== null &&
- (result.failureClass === "rate_limit" ||
- result.failureClass === "network");
- if (backoffHit && unitPlatform) {
- const entry = nextBackoff(
- kindState.platformBackoff[unitPlatform],
- Date.now(),
- );
- kindState.platformBackoff[unitPlatform] = entry;
- const secs = Math.round((entry.until - Date.now()) / 1000);
- onLog(
- `Auto-download: ${unitPlatform} ${result.failureClass} — backing off ${secs}s (attempt ${entry.fails}). ${pick.videoId} will retry after cooldown.`,
- );
- } else {
- if (unitPlatform && result.outcome === "transcribed") {
- clearBackoff(kindState.platformBackoff, unitPlatform);
- }
- // An operation lane retires (operation, video) pairs inside
- // runOperationPick — it is the only thing that knows WHICH operation
- // ran — so a bare id here would be a key nothing ever reads.
- if (!isOperationLane(kind)) markCompleted(pick.videoId);
+ // Pacing: a rate-limit / network failure backs the whole platform off,
+ // and a rate-limited video is also deferred so the next pick after the
+ // cooldown is a DIFFERENT video — see unitOutcome.ts. Any other outcome
+ // retires the video for the session; a success clears the cooldown.
+ const effect = applyUnitOutcome(
+ kindState,
+ {
+ platform: unitPlatform,
+ videoId: pick.videoId,
+ channelSlug,
+ outcome: result.outcome,
+ ...(result.failureClass ? { failureClass: result.failureClass } : {}),
+ },
+ Date.now(),
+ );
+ if (effect.line) onLog(effect.line);
+ // An operation lane retires (operation, video) pairs inside
+ // runOperationPick — it is the only thing that knows WHICH operation
+ // ran — so a bare id here would be a key nothing ever reads.
+ if (effect.markCompleted && !isOperationLane(kind)) {
+ markCompleted(pick.videoId);
}
// Persist so the cooldown (and reset) survive a restart.
persist();
diff --git a/common/jobs/unitOutcome.test.ts b/common/jobs/unitOutcome.test.ts
@@ -0,0 +1,105 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+ BACKOFF_BASE_MS,
+ VIDEO_RATE_LIMIT_DEFER_MS,
+ isVideoDeferred,
+} from "./platformBackoff";
+import { type UnitOutcomeState, applyUnitOutcome } from "./unitOutcome";
+
+// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test jobs/unitOutcome.test.ts
+
+const noJitter = () => 0.5;
+const empty = (): UnitOutcomeState => ({ platformBackoff: {}, videoDeferrals: {} });
+const unit = (
+ videoId: string,
+ over: Partial<Parameters<typeof applyUnitOutcome>[1]> = {},
+) => ({
+ platform: "youtube",
+ videoId,
+ channelSlug: "alpha",
+ outcome: "failed" as const,
+ ...over,
+});
+
+test("rate_limit backs the platform off AND defers the video 6h", () => {
+ const s = empty();
+ const now = 1_000_000;
+ const r = applyUnitOutcome(s, unit("v1", { failureClass: "rate_limit" }), now, noJitter);
+ assert.equal(r.markCompleted, false);
+ assert.deepEqual(s.platformBackoff.youtube, { until: now + BACKOFF_BASE_MS, fails: 1 });
+ assert.deepEqual(s.videoDeferrals.v1, {
+ until: now + VIDEO_RATE_LIMIT_DEFER_MS,
+ channelSlug: "alpha",
+ });
+ assert.equal(
+ r.line,
+ "Auto-download: youtube rate_limit — backing off 60s (attempt 1). v1 deferred 6h; next video after cooldown.",
+ );
+});
+
+test("network backs off only — no deferral, today's line", () => {
+ const s = empty();
+ const r = applyUnitOutcome(s, unit("v1", { failureClass: "network" }), 0, noJitter);
+ assert.equal(r.markCompleted, false);
+ assert.equal(s.platformBackoff.youtube.fails, 1);
+ assert.deepEqual(s.videoDeferrals, {});
+ assert.equal(
+ r.line,
+ "Auto-download: youtube network — backing off 60s (attempt 1). v1 will retry after cooldown.",
+ );
+});
+
+test("transcribed clears the platform cooldown and retires the video", () => {
+ const s = empty();
+ s.platformBackoff.youtube = { until: 10, fails: 3 };
+ const r = applyUnitOutcome(s, unit("v1", { outcome: "transcribed" }), 0, noJitter);
+ assert.deepEqual(r, { markCompleted: true, line: null });
+ assert.deepEqual(s.platformBackoff, {});
+});
+
+test("any other failure retires the video and leaves the cooldown alone", () => {
+ const s = empty();
+ s.platformBackoff.youtube = { until: 10, fails: 3 };
+ const r = applyUnitOutcome(s, unit("v1", { failureClass: "per_video" }), 0, noJitter);
+ assert.deepEqual(r, { markCompleted: true, line: null });
+ assert.deepEqual(s.platformBackoff.youtube, { until: 10, fails: 3 });
+});
+
+test("a non-download unit (no platform) touches neither map", () => {
+ const s = empty();
+ const r = applyUnitOutcome(
+ s,
+ unit("v1", { platform: null, failureClass: "rate_limit" }),
+ 0,
+ noJitter,
+ );
+ assert.deepEqual(r, { markCompleted: true, line: null });
+ assert.deepEqual(s, empty());
+});
+
+test("every branch prunes lapsed deferrals", () => {
+ const s = empty();
+ s.videoDeferrals = {
+ old: { until: 5, channelSlug: "a" },
+ live: { until: 50, channelSlug: "a" },
+ };
+ applyUnitOutcome(s, unit("v1", { outcome: "skipped" }), 10, noJitter);
+ assert.deepEqual(Object.keys(s.videoDeferrals), ["live"]);
+});
+
+test("fails climbs only across DISTINCT ids: the deferred video is not re-picked", () => {
+ // The runner's pick skips a deferred id, so after a cooldown the next 429 is
+ // necessarily a different video — the escalation now measures the platform,
+ // not one stubborn video.
+ const s = empty();
+ let now = 0;
+ const ids = ["v1", "v2", "v3"];
+ for (const id of ids) {
+ assert.equal(isVideoDeferred(s.videoDeferrals, id, now), false);
+ applyUnitOutcome(s, unit(id, { failureClass: "rate_limit" }), now, noJitter);
+ now = s.platformBackoff.youtube.until; // the cooldown lapses
+ }
+ assert.equal(s.platformBackoff.youtube.fails, 3);
+ for (const id of ids) assert.equal(isVideoDeferred(s.videoDeferrals, id, now), true);
+});
diff --git a/common/jobs/unitOutcome.ts b/common/jobs/unitOutcome.ts
@@ -0,0 +1,85 @@
+// What one finished auto-queue unit does to the lane's persisted pacing state —
+// the per-platform backoff and the per-video deferral. Pure: no I/O, `now` and
+// `rand` injected, so the whole decision is unit-testable (the runner's loop
+// that calls it is not exported). autoRunner.ts calls it once per unit, logs the
+// line, retires the video when told to, and persists.
+//
+// - `rate_limit` (HTTP 429 / throttle): the platform backs off exponentially
+// (nextBackoff) AND the video is deferred for VIDEO_RATE_LIMIT_DEFER_MS, so
+// after the cooldown the runner moves on to the NEXT video instead of
+// re-picking the same one. A 429 is per video on YouTube (the subtitle
+// fetch), and with `order: "listed"` the same video sits at the head of the
+// queue: without the deferral one video climbs `fails` to the 30-minute cap
+// and holds the whole platform (2026-09-24, one Short retried 12x).
+// - `network`: backoff only — the video is retried after the cooldown (today's
+// behaviour; a network error says nothing about the video).
+// - either: the video is NOT retired for the session (`markCompleted: false`).
+// - anything else retires it; a success (`transcribed`) also clears the
+// platform's cooldown.
+// - every branch prunes lapsed deferrals, keeping the map bounded.
+
+import type { DownloadFailureClass } from "../lib/availability";
+import {
+ type PlatformBackoffState,
+ type VideoDeferralState,
+ clearBackoff,
+ deferVideo,
+ nextBackoff,
+ pruneDeferred,
+} from "./platformBackoff";
+
+export type UnitOutcomeState = {
+ platformBackoff: PlatformBackoffState;
+ videoDeferrals: VideoDeferralState;
+};
+
+export type FinishedUnit = {
+ // The download platform the unit ran against; null on a non-download lane.
+ platform: string | null;
+ videoId: string;
+ channelSlug: string;
+ outcome: "transcribed" | "skipped" | "failed";
+ failureClass?: DownloadFailureClass;
+};
+
+export type UnitOutcomeEffect = {
+ // Retire the video for the session (the caller still skips this on an
+ // operation lane, which retires (operation, video) pairs itself).
+ markCompleted: boolean;
+ // The line to log, or null when there is nothing to say.
+ line: string | null;
+};
+
+export function applyUnitOutcome(
+ state: UnitOutcomeState,
+ unit: FinishedUnit,
+ now: number,
+ rand: () => number = Math.random,
+): UnitOutcomeEffect {
+ pruneDeferred(state.videoDeferrals, now);
+ const pf = unit.platform;
+ if (
+ pf !== null &&
+ (unit.failureClass === "rate_limit" || unit.failureClass === "network")
+ ) {
+ const entry = nextBackoff(state.platformBackoff[pf], now, rand);
+ state.platformBackoff[pf] = entry;
+ const secs = Math.round((entry.until - now) / 1000);
+ const head = `Auto-download: ${pf} ${unit.failureClass} — backing off ${secs}s (attempt ${entry.fails}).`;
+ if (unit.failureClass === "rate_limit") {
+ deferVideo(state.videoDeferrals, unit.videoId, unit.channelSlug, now);
+ return {
+ markCompleted: false,
+ line: `${head} ${unit.videoId} deferred 6h; next video after cooldown.`,
+ };
+ }
+ return {
+ markCompleted: false,
+ line: `${head} ${unit.videoId} will retry after cooldown.`,
+ };
+ }
+ if (pf !== null && unit.outcome === "transcribed") {
+ clearBackoff(state.platformBackoff, pf);
+ }
+ return { markCompleted: true, line: null };
+}
