commit c10c87a8b613d072a1ccfb2fe900fb13b1555120
parent 3b54ab45e68025ba50ba7711025284fe451c3290
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sat, 26 Sep 2026 03:25:19 -0400
common: a rate-limited metadata prefetch ends the download there (review fix)
A failed prefetch fell through to the real download whatever the failure,
so a 429, a bot check or YouTube's soft block cost one more request into
the same refusal before the batch could back off. When the last prefetch
attempt classifies `rate_limit`, `runManagedDownload` now logs one line and
returns `failed` / `rate_limit` without attempt 1; the batch's cooldown and
abort and the runner's deferral follow from the record as before. Every
other failure (a 403, a removed video, an auth gate) keeps today's flow.
The sidecar + availability-history tail moves into `writeOutcome`, which
both exits call, so the record is written one way.
prefetchRateLimit.test.ts (4, a temp yt-dlp script that counts spawns):
soft block → 1 spawn, 1 attempt, sidecar `rate_limit`; 429 → 1 spawn;
403 → 2 spawns (prefetch + primary, `network`); a removed video → 2 spawns
(`per_video`). The first two fail with the early exit disabled.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
2 files changed, 178 insertions(+), 5 deletions(-)
diff --git a/common/ytdlp/downloadOneManaged.ts b/common/ytdlp/downloadOneManaged.ts
@@ -719,6 +719,35 @@ async function runManagedDownload(
}
}
+ // A RATE-LIMITED PREFETCH ENDS THE VIDEO HERE (release 10, L2 review).
+ // Every failed prefetch used to fall through to the real download, so a
+ // 429, a bot check or YouTube's soft block ("…isn't available, try again
+ // later") cost one more request into the same refusal before the batch
+ // could back off. That request is only skipped for a `rate_limit` class:
+ // any other failure (a 403, a removed video, an auth gate) keeps today's
+ // flow, where attempt 1 and its own auth retry still get their chance. The
+ // record carries `failureClass: "rate_limit"`, so the batch's cooldown and
+ // abort and the runner's per-video deferral follow exactly as before.
+ const lastPrefetch = attempts.at(-1);
+ if (
+ lastPrefetch &&
+ !attemptSucceeded(lastPrefetch.ytdlpExitCode) &&
+ classifyDownloadFailure(lastFullTail, lastPrefetch.availabilityClass) ===
+ "rate_limit"
+ ) {
+ opts.onLog(
+ `Metadata prefetch for ${canonicalId} was rate-limited by the source; ` +
+ `not attempting the download (the platform backs off instead).\n`,
+ );
+ return writeOutcome(opts, videoDir, {
+ videoId: canonicalId,
+ status: "failed",
+ startedAt,
+ attempts,
+ lastFullTail,
+ });
+ }
+
const metaPath = path.join(videoDir, "metadata.info.json");
const metadata = await loadRawMetadata(metaPath);
// Only wire --load-info-json into the real attempts when we actually have
@@ -1364,24 +1393,53 @@ async function runManagedDownload(
}
// ---------- Sidecar ----------
+ return writeOutcome(opts, videoDir, {
+ videoId,
+ status,
+ startedAt,
+ attempts,
+ lastFullTail,
+ fellBackToTranscribe,
+ shortAudio: shortAudioInfo,
+ });
+}
+
+// THE SIDECAR AND THE AVAILABILITY HISTORY, written one way by every exit that
+// ends a download's attempts: the end of the main path, and a prefetch the
+// source rate-limited (release 10, L2 review). A filter-skip writes its own
+// record and has no failure class; it does not come through here.
+async function writeOutcome(
+ opts: ManagedDownloadOpts,
+ videoDir: string,
+ o: {
+ videoId: string;
+ status: DownloadOutcomeStatus;
+ startedAt: string;
+ attempts: DownloadAttempt[];
+ lastFullTail: string;
+ fellBackToTranscribe?: boolean;
+ shortAudio?: DownloadOutcomeRecord["shortAudio"];
+ },
+): Promise<DownloadOutcomeRecord> {
+ const { status, attempts } = o;
const finishedAt = new Date().toISOString();
// Classify a failed download against the FULL stderr tail of the last attempt
// so a rate-limit logged as a WARNING (then masked by a different final error)
// is still caught. Skipped for successes and filter-skips.
const isFailure = status === "failed" || status === "failed-corrupt-source";
const failureClass = isFailure
- ? classifyDownloadFailure(lastFullTail, attempts.at(-1)?.availabilityClass)
+ ? classifyDownloadFailure(o.lastFullTail, attempts.at(-1)?.availabilityClass)
: undefined;
const record: DownloadOutcomeRecord = {
- videoId,
+ videoId: o.videoId,
webpageUrl: opts.videoUrl,
status,
- startedAt,
+ startedAt: o.startedAt,
finishedAt,
attempts,
...(failureClass ? { failureClass } : {}),
- ...(fellBackToTranscribe ? { fellBackToTranscribe: true } : {}),
- ...(shortAudioInfo ? { shortAudio: shortAudioInfo } : {}),
+ ...(o.fellBackToTranscribe ? { fellBackToTranscribe: true } : {}),
+ ...(o.shortAudio ? { shortAudio: o.shortAudio } : {}),
};
// Only write the sidecar if we know which dir to put it in. If the very first
// attempt failed before metadata could be written, the data/<id> dir may not
diff --git a/common/ytdlp/prefetchRateLimit.test.ts b/common/ytdlp/prefetchRateLimit.test.ts
@@ -0,0 +1,115 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import type { ChannelConfig } from "../lib/channelConfig";
+import { downloadOneManaged } from "./downloadOneManaged";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test ytdlp/prefetchRateLimit.test.ts
+//
+// A RATE-LIMITED PREFETCH ENDS THE VIDEO (release 10, L2 review). The per-video
+// download runs a metadata prefetch first; a failed prefetch used to fall
+// through to the real download whatever the failure, so a soft block cost one
+// more request into it. Now a `rate_limit` prefetch stops there, and every
+// other failure keeps today's flow. The yt-dlp here is a temp script that
+// counts its spawns and always fails with the given line — no network, no
+// real yt-dlp.
+
+const VIDEO = "https://www.youtube.com/watch?v=H64QQZuw-aA";
+
+async function runWith(stderrLine: string): Promise<{
+ spawns: number;
+ record: Awaited<ReturnType<typeof downloadOneManaged>>;
+ outcomeOnDisk: { status: string; failureClass?: string };
+ log: string;
+}> {
+ const root = await mkdtemp(path.join(tmpdir(), "prefetch-rl-"));
+ try {
+ const counter = path.join(root, "spawns");
+ const bin = path.join(root, "fake-ytdlp.sh");
+ await writeFile(
+ bin,
+ `#!/bin/sh\necho x >> '${counter}'\necho ${JSON.stringify(stderrLine)} >&2\nexit 1\n`,
+ );
+ await chmod(bin, 0o755);
+ const paths = {
+ channelsDir: path.join(root, "channels"),
+ ytdlpBin: bin,
+ } as Paths;
+ let log = "";
+ const record = await downloadOneManaged({
+ channelSlug: "c",
+ channelConfig: {
+ handling: "youtube",
+ url: "https://www.youtube.com/@c/videos",
+ } as ChannelConfig,
+ paths,
+ videoUrl: VIDEO,
+ onLog: (s) => {
+ log += s;
+ },
+ signal: new AbortController().signal,
+ });
+ const spawns = (await readFile(counter, "utf8").catch(() => ""))
+ .split("\n")
+ .filter(Boolean).length;
+ const outcomeOnDisk = JSON.parse(
+ await readFile(
+ path.join(paths.channelsDir, "c", "data", "H64QQZuw-aA", "download-outcome.json"),
+ "utf8",
+ ),
+ );
+ return { spawns, record, outcomeOnDisk, log };
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+}
+
+test("a soft-blocked prefetch is the only request: no download attempt follows", async () => {
+ const r = await runWith(
+ "ERROR: [youtube] H64QQZuw-aA: This content isn't available, try again later. " +
+ "The current session has been rate-limited by YouTube for up to an hour.",
+ );
+ assert.equal(r.spawns, 1);
+ assert.equal(r.record.attempts.length, 1);
+ assert.equal(r.record.attempts[0].kind, "metadata-prefetch");
+ assert.equal(r.record.status, "failed");
+ assert.equal(r.record.failureClass, "rate_limit");
+ // The sidecar is written the same way as on the main path.
+ assert.equal(r.outcomeOnDisk.status, "failed");
+ assert.equal(r.outcomeOnDisk.failureClass, "rate_limit");
+ assert.match(r.log, /rate-limited by the source; not attempting the download/);
+});
+
+test("an HTTP 429 at the prefetch stops the same way", async () => {
+ const r = await runWith(
+ "ERROR: [youtube] H64QQZuw-aA: Unable to download webpage: HTTP Error 429: Too Many Requests",
+ );
+ assert.equal(r.spawns, 1);
+ assert.equal(r.record.failureClass, "rate_limit");
+});
+
+test("any other prefetch failure still reaches the download attempt (a 403: two requests)", async () => {
+ const r = await runWith(
+ "ERROR: [youtube] H64QQZuw-aA: Unable to download webpage: HTTP Error 403: Forbidden",
+ );
+ assert.equal(r.spawns, 2);
+ assert.deepEqual(
+ r.record.attempts.map((a) => a.kind),
+ ["metadata-prefetch", "primary"],
+ );
+ assert.equal(r.record.status, "failed");
+ assert.equal(r.record.failureClass, "network");
+ assert.doesNotMatch(r.log, /not attempting the download/);
+});
+
+test("a removed video still reaches the download attempt too (per-video flow unchanged)", async () => {
+ const r = await runWith(
+ "ERROR: [youtube] H64QQZuw-aA: Video unavailable. This video has been removed by the uploader",
+ );
+ assert.equal(r.spawns, 2);
+ assert.equal(r.record.failureClass, "per_video");
+});
