commit 639abe30a9563367759168705e9155103eed9121
parent 9cbed498b846120abc958645fef892d22b30f332
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 25 Sep 2026 11:00:29 -0400
common: per-video deferral helpers + persisted videoDeferrals on the auto-queue state
VideoDeferral {until, channelSlug}, VIDEO_RATE_LIMIT_DEFER_MS = 6 h,
deferVideo / isVideoDeferred / pruneDeferred / coerceVideoDeferrals beside
the platform backoff (nextBackoff untouched). AutoQueueKindState gains
videoDeferrals: a file without the key coerces to {}, an older build ignores
it. platformBackoff.test +5, autoQueueState.test +2.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
4 files changed, 182 insertions(+), 1 deletion(-)
diff --git a/common/jobs/autoQueueState.test.ts b/common/jobs/autoQueueState.test.ts
@@ -101,6 +101,7 @@ test("every lane is keyed, and a state file written before a lane existed coerce
runtime: { currentWeights: {} },
picks: [],
platformBackoff: {},
+ videoDeferrals: {},
});
assert.deepEqual(back.backfill.picks, []);
// And the lanes it does name are untouched.
@@ -127,6 +128,46 @@ test("a lane missing from an in-memory state object still writes", async () => {
});
});
+test("videoDeferrals round-trips through write/read; other lanes stay {}", async () => {
+ await withPaths(async (paths) => {
+ const state = emptyAutoQueueState();
+ state.download.videoDeferrals = {
+ a1: { until: 123456, channelSlug: "alpha" },
+ };
+ await writeAutoQueueState(paths, state);
+
+ const back = await readAutoQueueState(paths);
+ assert.deepEqual(back.download.videoDeferrals, {
+ a1: { until: 123456, channelSlug: "alpha" },
+ });
+ assert.deepEqual(back.transcription.videoDeferrals, {});
+ });
+});
+
+test("a state file written before videoDeferrals existed coerces it to {}; corrupt entries drop", async () => {
+ await withPaths(async (paths) => {
+ await writeAutoQueueState(paths, emptyAutoQueueState());
+ await writeFile(
+ paths.autoQueueStateFile,
+ JSON.stringify({
+ download: {
+ runtime: { currentWeights: {} },
+ picks: [],
+ platformBackoff: { youtube: { until: 5, fails: 2 } },
+ },
+ transcription: { videoDeferrals: { x: { until: "nope" }, y: 7 } },
+ }),
+ );
+
+ const back = await readAutoQueueState(paths);
+ assert.deepEqual(back.download.videoDeferrals, {});
+ assert.deepEqual(back.download.platformBackoff, {
+ youtube: { until: 5, fails: 2 },
+ });
+ assert.deepEqual(back.transcription.videoDeferrals, {});
+ });
+});
+
// --- write safety ------------------------------------------------------------
test("overlapping writes do not collide on the tmp file", async () => {
diff --git a/common/jobs/autoQueueState.ts b/common/jobs/autoQueueState.ts
@@ -7,7 +7,9 @@ import {
} from "./autoQueuePolicy";
import {
type PlatformBackoffState,
+ type VideoDeferralState,
coercePlatformBackoff,
+ coerceVideoDeferrals,
} from "./platformBackoff";
// Persistent fairness state for the auto-queue runners. Unlike in-flight worker
@@ -41,6 +43,11 @@ export type AutoQueueKindState = {
// the "download" kind; empty for transcription. Persisted so an Odysee 429
// cooldown survives a server restart instead of re-storming on boot.
platformBackoff: PlatformBackoffState;
+ // Per-video deferrals (a rate-limited video is skipped by the auto-download
+ // pick for VIDEO_RATE_LIMIT_DEFER_MS). Download kind only; `{}` elsewhere.
+ // Persisted so a restart does not re-hit the same video at fails+1. A file
+ // written before the key existed coerces to `{}`; an older build ignores it.
+ videoDeferrals: VideoDeferralState;
};
// KEYED BY EVERY LANE, including the two with no executor yet. A lane whose
@@ -53,7 +60,12 @@ export type AutoQueueState = Record<AutoQueueKind, AutoQueueKindState>;
export const AUTO_QUEUE_PICK_LOG_LIMIT = 50;
export function emptyAutoQueueKindState(): AutoQueueKindState {
- return { runtime: emptyAutoQueueRuntime(), picks: [], platformBackoff: {} };
+ return {
+ runtime: emptyAutoQueueRuntime(),
+ picks: [],
+ platformBackoff: {},
+ videoDeferrals: {},
+ };
}
export function emptyAutoQueueState(): AutoQueueState {
@@ -109,6 +121,7 @@ function coerceKindState(value: unknown): AutoQueueKindState {
runtime: coerceRuntime(r.runtime),
picks,
platformBackoff: coercePlatformBackoff(r.platformBackoff),
+ videoDeferrals: coerceVideoDeferrals(r.videoDeferrals),
};
}
@@ -148,6 +161,7 @@ export async function writeAutoQueueState(
runtime: k.runtime,
picks: k.picks.slice(0, AUTO_QUEUE_PICK_LOG_LIMIT),
platformBackoff: k.platformBackoff ?? {},
+ videoDeferrals: k.videoDeferrals ?? {},
});
const out = Object.fromEntries(
LANES.map((lane) => [lane, trim(state[lane] ?? emptyAutoQueueKindState())]),
diff --git a/common/jobs/platformBackoff.test.ts b/common/jobs/platformBackoff.test.ts
@@ -9,6 +9,12 @@ import {
isCoolingDown,
nextBackoff,
pruneExpired,
+ VIDEO_RATE_LIMIT_DEFER_MS,
+ type VideoDeferralState,
+ coerceVideoDeferrals,
+ deferVideo,
+ isVideoDeferred,
+ pruneDeferred,
} from "./platformBackoff";
// Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test common/jobs/platformBackoff.test.ts
@@ -91,3 +97,51 @@ test("coercePlatformBackoff tolerates corrupt/missing shapes", () => {
{ odysee: { until: 5, fails: 2 }, youtube: { until: 7, fails: 1 } },
);
});
+
+test("deferVideo defers a video for 6 h by default, keyed by id", () => {
+ const state: VideoDeferralState = {};
+ deferVideo(state, "v1", "alpha", 1_000);
+ assert.deepEqual(state, {
+ v1: { until: 1_000 + VIDEO_RATE_LIMIT_DEFER_MS, channelSlug: "alpha" },
+ });
+ assert.equal(VIDEO_RATE_LIMIT_DEFER_MS, 6 * 60 * 60_000);
+});
+
+test("deferVideo re-defers from now (a later 429 extends the window)", () => {
+ const state: VideoDeferralState = {};
+ deferVideo(state, "v1", "alpha", 1_000, 10);
+ deferVideo(state, "v1", "alpha", 5_000, 10);
+ assert.equal(state.v1.until, 5_010);
+});
+
+test("isVideoDeferred reflects the until window, per video", () => {
+ const state: VideoDeferralState = { v1: { until: 2_000, channelSlug: "a" } };
+ assert.equal(isVideoDeferred(state, "v1", 1_999), true);
+ assert.equal(isVideoDeferred(state, "v1", 2_000), false);
+ assert.equal(isVideoDeferred(state, "v2", 1_000), false);
+});
+
+test("pruneDeferred drops lapsed deferrals only", () => {
+ const state: VideoDeferralState = {
+ old: { until: 1_000, channelSlug: "a" },
+ edge: { until: 2_000, channelSlug: "a" },
+ live: { until: 3_000, channelSlug: "b" },
+ };
+ pruneDeferred(state, 2_000);
+ assert.deepEqual(Object.keys(state), ["live"]);
+});
+
+test("coerceVideoDeferrals tolerates corrupt/missing shapes", () => {
+ assert.deepEqual(coerceVideoDeferrals(undefined), {});
+ assert.deepEqual(coerceVideoDeferrals("nope"), {});
+ assert.deepEqual(
+ coerceVideoDeferrals({
+ ok: { until: 5, channelSlug: "a" },
+ noSlug: { until: 5 },
+ badUntil: { until: "5", channelSlug: "a" },
+ inf: { until: Infinity, channelSlug: "a" },
+ nul: null,
+ }),
+ { ok: { until: 5, channelSlug: "a" } },
+ );
+});
diff --git a/common/jobs/platformBackoff.ts b/common/jobs/platformBackoff.ts
@@ -92,3 +92,75 @@ export function coercePlatformBackoff(value: unknown): PlatformBackoffState {
}
return out;
}
+
+// ---------------------------------------------------------------------------
+// Per-video deferral (release 7, YouTube lane pacing).
+//
+// A platform cooldown alone does not pace a lane whose queue is ordered: with
+// `order: "listed"` the runner re-picks the SAME video the moment the cooldown
+// lapses, so one video whose subtitle fetch keeps answering 429 climbs `fails`
+// to the 30-minute cap and holds the whole platform (2026-09-24: one Short
+// retried 12x). A rate-limited video is therefore also DEFERRED for
+// VIDEO_RATE_LIMIT_DEFER_MS: the auto-download pick skips it, so after the
+// cooldown the runner moves on to the next video. Persisted beside
+// `platformBackoff` (a restart must not re-hit the same video at fails+1).
+// Manual Sync / download-missing do not consult it.
+
+export type VideoDeferral = {
+ // Epoch ms: the auto-download pick skips this video until this instant.
+ until: number;
+ // The channel the video was picked from — for the status view's link.
+ channelSlug: string;
+};
+
+// Keyed by video id.
+export type VideoDeferralState = Record<string, VideoDeferral>;
+
+export const VIDEO_RATE_LIMIT_DEFER_MS = 6 * 60 * 60_000; // 6 hours
+
+// Defer a video from `now` for `ms`. Mutates in place.
+export function deferVideo(
+ state: VideoDeferralState,
+ videoId: string,
+ channelSlug: string,
+ now: number,
+ ms: number = VIDEO_RATE_LIMIT_DEFER_MS,
+): void {
+ state[videoId] = { until: now + ms, channelSlug };
+}
+
+// True while the video's deferral window is open.
+export function isVideoDeferred(
+ state: VideoDeferralState,
+ videoId: string,
+ now: number,
+): boolean {
+ const entry = state[videoId];
+ return entry !== undefined && entry.until > now;
+}
+
+// Drop deferrals whose window has lapsed. Mutates in place — keeps the map
+// bounded (unlike platform backoff there is no escalation memory to retain).
+export function pruneDeferred(state: VideoDeferralState, now: number): void {
+ for (const [videoId, entry] of Object.entries(state)) {
+ if (entry.until <= now) delete state[videoId];
+ }
+}
+
+// Defensive coercion for the persisted shape; mirrors coercePlatformBackoff.
+export function coerceVideoDeferrals(value: unknown): VideoDeferralState {
+ const out: VideoDeferralState = {};
+ if (!value || typeof value !== "object") return out;
+ for (const [videoId, raw] of Object.entries(value as Record<string, unknown>)) {
+ if (!raw || typeof raw !== "object") continue;
+ const r = raw as Record<string, unknown>;
+ if (
+ typeof r.until === "number" &&
+ Number.isFinite(r.until) &&
+ typeof r.channelSlug === "string"
+ ) {
+ out[videoId] = { until: r.until, channelSlug: r.channelSlug };
+ }
+ }
+ return out;
+}
