commit 3b2491cc00dc617930926b67f9fef0d5e36e04b3
parent 598e763442185bf180afe23d9dcaa73d5d3e5524
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sat, 26 Sep 2026 02:28:35 -0400
common: YouTube's soft block ("try again later") is a rate limit, not a deleted video
"This content isn't available, try again later." — YouTube's answer to a
rate-limited session, which yt-dlp re-words as "The current session has been
rate-limited by YouTube for up to an hour" — matched the `deleted` group's
"content isn't available". So it read as a removed video: per_video, no
platform cooldown, the batch kept requesting into the block, and the video was
excluded from download as gone.
`isSoftBlock` (the reason's "isn't available, try again later", either
apostrophe) is checked first: `parseUnavailableFromStderr` answers "error"
(transient) and `classifyDownloadFailure` answers `rate_limit`, even over a
per-video class a caller already holds. The download batch then records the
platform cooldown and stops, the auto runner backs off and defers the video,
and the metadata scan stops its pass as a "soft-block" block instead of
recording a `deleted` error. A bare "Video unavailable", a removed or
terminated video and Rumble's 410 stay `deleted` / per_video.
Tests: availability.test.ts +3 (yt-dlp's own strings, both directions);
managedDownloadsSleep.test.ts +1 (the classifier through the batch loop:
the soft block backs off and aborts, a removed video does neither).
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
5 files changed, 228 insertions(+), 9 deletions(-)
diff --git a/common/lib/availability.test.ts b/common/lib/availability.test.ts
@@ -1,6 +1,10 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { classifyDownloadFailure, parseUnavailableFromStderr } from "./availability";
+import {
+ classifyDownloadFailure,
+ isSoftBlock,
+ parseUnavailableFromStderr,
+} from "./availability";
// Run with:
// pnpm --filter yt-dlp-transcript-common exec tsx --test lib/availability.test.ts
@@ -35,3 +39,90 @@ test("HTTP 410 Gone is a removed video, not an error", () => {
assert.equal(parseUnavailableFromStderr("HTTP Error 403: Forbidden"), "error");
assert.equal(parseUnavailableFromStderr("HTTP Error 500: Internal Server Error"), "error");
});
+
+// YOUTUBE'S SOFT BLOCK (release 10, L2). The strings below are real:
+// - the bare line is the one reported in yt-dlp issue #11426 ("[YouTube] This
+// content isn't available, try again later.", after ~280 videos of a
+// playlist), which is what a yt-dlp older than #12958 prints;
+// - the long ones are what `yt_dlp/extractor/youtube/_video.py` builds from that
+// reason since #12958 (the build this editor runs, 2026.08.19) — "The current
+// session" without cookies, "Your account" with them.
+// yt-dlp's wiki (Extractors, "This content isn't available, try again later")
+// puts the limit at ~300 videos/hour for a guest session, ~2000 signed in.
+const SOFT_BLOCK_SESSION =
+ "ERROR: [youtube] dQw4w9WgXcQ: This content isn't available, try again later. " +
+ "The current session has been rate-limited by YouTube for up to an hour. " +
+ "It is recommended to use `-t sleep` to add a delay between video requests to avoid " +
+ "exceeding the rate limit. For more information, refer to " +
+ "https://github.com/yt-dlp/yt-dlp/wiki/Extractors#this-content-isnt-available-try-again-later";
+const SOFT_BLOCK_ACCOUNT = SOFT_BLOCK_SESSION.replace(
+ "The current session",
+ "Your account",
+);
+const SOFT_BLOCK_BARE =
+ "ERROR: [youtube] H64QQZuw-aA: This content isn't available, try again later.";
+// The same reason with a right single quote, in case YouTube sends one.
+const SOFT_BLOCK_CURLY = SOFT_BLOCK_BARE.replace("isn't", "isn’t");
+
+test("the soft block ('try again later') is a rate limit, not a deleted video", () => {
+ for (const stderr of [
+ SOFT_BLOCK_SESSION,
+ SOFT_BLOCK_ACCOUNT,
+ SOFT_BLOCK_BARE,
+ SOFT_BLOCK_CURLY,
+ ]) {
+ assert.equal(isSoftBlock(stderr), true, stderr);
+ // Not a property of the video: transient, so never excluded as gone.
+ assert.equal(parseUnavailableFromStderr(stderr), "error", stderr);
+ // …and a batch-level signal that backs the platform off.
+ assert.equal(
+ classifyDownloadFailure(stderr, parseUnavailableFromStderr(stderr)),
+ "rate_limit",
+ stderr,
+ );
+ }
+ // Inside a multi-line tail (a WARNING first, the ERROR last) it still counts.
+ const tail = `WARNING: [youtube] dQw4w9WgXcQ: nsig extraction slow\n${SOFT_BLOCK_SESSION}\n`;
+ assert.equal(classifyDownloadFailure(tail, parseUnavailableFromStderr(tail)), "rate_limit");
+});
+
+test("a soft block wins over a per-video class a caller already holds", () => {
+ // A `deleted` read before release 10, or from another line of the same tail,
+ // must not turn the soft block back into a per-video skip.
+ assert.equal(classifyDownloadFailure(SOFT_BLOCK_BARE, "deleted"), "rate_limit");
+ assert.equal(classifyDownloadFailure(SOFT_BLOCK_SESSION, "members_only"), "rate_limit");
+});
+
+test("a genuinely removed or unavailable video is still deleted / per_video", () => {
+ // yt-dlp's wording for videos that are really gone, soft-block-free.
+ const gone = [
+ "ERROR: [youtube] dQw4w9WgXcQ: Video unavailable",
+ "ERROR: [youtube] dQw4w9WgXcQ: Video unavailable. This video has been removed by the uploader",
+ "ERROR: [youtube] dQw4w9WgXcQ: Video unavailable. This video is no longer available because the YouTube account associated with this video has been terminated.",
+ "ERROR: [youtube] dQw4w9WgXcQ: This video has been removed for violating YouTube's Terms of Service",
+ "ERROR: [Rumble] v7e07us: Unable to download webpage: HTTP Error 410: Gone (caused by <HTTPError 410: Gone>)",
+ ];
+ for (const stderr of gone) {
+ assert.equal(isSoftBlock(stderr), false, stderr);
+ assert.equal(parseUnavailableFromStderr(stderr), "deleted", stderr);
+ assert.equal(
+ classifyDownloadFailure(stderr, parseUnavailableFromStderr(stderr)),
+ "per_video",
+ stderr,
+ );
+ }
+ // The pattern's boundary: "content isn't available" with no retry advice
+ // keeps the `deleted` reading it has had since the availability check began.
+ const noAdvice = "ERROR: [youtube] dQw4w9WgXcQ: This content isn't available.";
+ assert.equal(isSoftBlock(noAdvice), false);
+ assert.equal(parseUnavailableFromStderr(noAdvice), "deleted");
+ // The other per-video classes are untouched by the soft-block check.
+ const priv =
+ "ERROR: [youtube] dQw4w9WgXcQ: Private video. Sign in if you've been granted access to this video";
+ assert.equal(parseUnavailableFromStderr(priv), "private");
+ assert.equal(classifyDownloadFailure(priv, "private"), "per_video");
+ const members =
+ "ERROR: [youtube] dQw4w9WgXcQ: Join this channel to get access to members-only content like this video, and other exclusive perks.";
+ assert.equal(parseUnavailableFromStderr(members), "members_only");
+ assert.equal(classifyDownloadFailure(members, "members_only"), "per_video");
+});
diff --git a/common/lib/availability.ts b/common/lib/availability.ts
@@ -154,10 +154,39 @@ export type AvailabilityRecord = {
export const AVAILABILITY_FILENAME = "availability.json";
+// YouTube's SOFT BLOCK. YouTube answers a session it is rate-limiting with the
+// playability reason "This content isn't available, try again later." — worded
+// like a removed video, and it is not one: the same video plays for anyone else
+// and for this session an hour later. yt-dlp (since #12958, 2025-04) re-words
+// it to say so:
+//
+// ERROR: [youtube] <id>: This content isn't available, try again later. The
+// current session has been rate-limited by YouTube for up to an hour. It is
+// recommended to use `-t sleep` to add a delay between video requests to
+// avoid exceeding the rate limit. For more information, refer to
+// https://github.com/yt-dlp/yt-dlp/wiki/Extractors#this-content-isnt-available-try-again-later
+//
+// ("Your account has been rate-limited …" when cookies were passed). Until
+// release 10 its "content isn't available" matched the `deleted` group below, so
+// the soft block read as a removed video: per_video, no platform cooldown, the
+// batch kept going, and the video was excluded from download as gone. It is a
+// rate limit, and it is classified as one: `parseUnavailableFromStderr` says
+// "error" (transient) and `classifyDownloadFailure` says `rate_limit`.
+//
+// "try again later" is what marks it: a bare "Video unavailable" or "This
+// content isn't available." with no retry advice is still read as removed. The
+// apostrophe may be a right single quote, so match either.
+export function isSoftBlock(stderr: string): boolean {
+ return /isn['’]?t available,? try again later/i.test(stderr);
+}
+
// Map yt-dlp's stderr text to an Availability. yt-dlp does not expose a
// stable machine-readable reason on failure, so we pattern-match the human
// messages it emits across YouTube, Rumble, and a few other extractors.
export function parseUnavailableFromStderr(stderr: string): Availability {
+ // Checked FIRST: the soft block's wording would otherwise match `deleted`
+ // below. It says nothing about the video, so it is an error, not a class.
+ if (isSoftBlock(stderr)) return "error";
const s = stderr.toLowerCase();
if (
/private video/.test(s) ||
@@ -235,6 +264,11 @@ export function classifyDownloadFailure(
stderrTail: string,
availabilityClass: Availability | undefined,
): DownloadFailureClass {
+ // The soft block wins over a per-video class, whichever parser produced it:
+ // a caller holding a `deleted` read from before release 10 (or from a tail
+ // that also names a removed video) must still back off. Erring this way costs
+ // one cooldown; erring the other way keeps requesting into the block.
+ if (isSoftBlock(stderrTail)) return "rate_limit";
if (
availabilityClass !== undefined &&
PER_VIDEO_CLASSES.includes(availabilityClass)
diff --git a/common/ytdlp/managedDownloadsSleep.test.ts b/common/ytdlp/managedDownloadsSleep.test.ts
@@ -115,13 +115,96 @@ test("a video the download filter declined does not sleep", async () => {
});
// Release 9 review: a per-video failure KEEPS the pace, even one that never
-// got past the prefetch. YouTube's soft block ("This content isn't available,
-// try again later") classifies as deleted → per_video; skipping the sleep there
-// would fire prefetches back to back into the block.
+// got past the prefetch — a prefetch is still a request. (Release 9's example,
+// YouTube's soft block, is no longer per-video: see the release 10 case below.)
test("a per-video failure at the prefetch still sleeps", async () => {
assert.equal(await sleepsFor([MEMBERS_ONLY, FETCHED]), 1);
});
+// THE SOFT BLOCK BACKS OFF (release 10, L2). The outcome is built the way
+// downloadOneManaged builds it — availabilityClass from parseUnavailableFromStderr,
+// failureClass from classifyDownloadFailure over the same tail — from yt-dlp's
+// real line, so this runs the classifier and the batch loop together. A soft
+// block must record the platform cooldown and stop the batch; a video that is
+// really gone must do neither (it is that video's property, the batch goes on).
+const { classifyDownloadFailure, parseUnavailableFromStderr } = await import(
+ "../lib/availability"
+);
+
+function failedPrefetch(stderr: string): DownloadOutcomeRecord {
+ const availabilityClass = parseUnavailableFromStderr(stderr);
+ return outcome(
+ "failed",
+ [{ ...PREFETCH, ytdlpExitCode: 1, availabilityClass, error: stderr }],
+ classifyDownloadFailure(stderr, availabilityClass),
+ );
+}
+
+async function batch(outcomes: DownloadOutcomeRecord[]): Promise<{
+ downloads: number;
+ backoffs: string[];
+ aborted: boolean;
+}> {
+ let i = 0;
+ const backoffs: string[] = [];
+ const urls = outcomes.map(
+ (_, n) =>
+ `https://www.youtube.com/watch?v=vid${String(n).padStart(8, "0")}`,
+ );
+ const channelConfig = {
+ handling: "youtube",
+ url: "https://www.youtube.com/@c/videos",
+ } as never;
+ const res = await runManagedDownloads(
+ {
+ channelSlug: "c",
+ mode: "download-missing" as never,
+ channelConfig,
+ paths: getPaths(),
+ onLog: () => {},
+ signal: new AbortController().signal,
+ // The production default: a batch-level failure stops the batch.
+ onPlatformBackoff: (cls) => {
+ backoffs.push(cls);
+ },
+ },
+ urls,
+ channelConfig,
+ undefined,
+ {
+ downloadOne: async () => outcomes[i++],
+ sleep: async () => {},
+ },
+ );
+ return { downloads: i, backoffs, aborted: res.firstFailure !== null };
+}
+
+test("a soft block backs the platform off and stops the batch; a deleted video does not", async () => {
+ const softBlock = failedPrefetch(
+ "ERROR: [youtube] H64QQZuw-aA: This content isn't available, try again later. " +
+ "The current session has been rate-limited by YouTube for up to an hour. " +
+ "It is recommended to use `-t sleep` to add a delay between video requests to avoid " +
+ "exceeding the rate limit. For more information, refer to " +
+ "https://github.com/yt-dlp/yt-dlp/wiki/Extractors#this-content-isnt-available-try-again-later",
+ );
+ assert.equal(softBlock.failureClass, "rate_limit");
+ assert.equal(softBlock.attempts[0].availabilityClass, "error");
+ const blocked = await batch([softBlock, FETCHED, FETCHED]);
+ assert.deepEqual(blocked.backoffs, ["rate_limit"]);
+ assert.equal(blocked.aborted, true);
+ assert.equal(blocked.downloads, 1, "nothing after the soft block is requested");
+
+ const gone = failedPrefetch(
+ "ERROR: [youtube] dQw4w9WgXcQ: Video unavailable. This video has been removed by the uploader",
+ );
+ assert.equal(gone.failureClass, "per_video");
+ assert.equal(gone.attempts[0].availabilityClass, "deleted");
+ const kept = await batch([gone, FETCHED, FETCHED]);
+ assert.deepEqual(kept.backoffs, []);
+ assert.equal(kept.aborted, false);
+ assert.equal(kept.downloads, 3, "a removed video does not stop the batch");
+});
+
test("the last video never sleeps, whatever it was", async () => {
assert.equal(await sleepsFor([FETCHED]), 0);
assert.equal(await sleepsFor([FILTERED, FETCHED]), 0);
diff --git a/common/ytdlp/metadataScan.ts b/common/ytdlp/metadataScan.ts
@@ -21,6 +21,7 @@ import { execa } from "execa";
import {
classifyDownloadFailure,
isBotCheck,
+ isSoftBlock,
parseUnavailableFromStderr,
} from "../lib/availability";
import type { ChannelConfig } from "../lib/channelConfig";
@@ -484,10 +485,17 @@ export async function runMetadataScan(
const failure = classifyDownloadFailure(trimmed, cls);
// A batch-level signal: continuing would just hammer the source, and on a
// bot check every subsequent entry fails the same way. Stop the pass and
- // let the caller decide whether cookies are the answer.
+ // let the caller decide whether cookies are the answer. YouTube's
+ // explicit soft block ("…isn't available, try again later") lands here
+ // too since release 10 — before, it was recorded as a `deleted` error on
+ // its id and the pass kept going into it.
if (failure === "rate_limit") {
block = {
- kind: isBotCheck(trimmed) ? "bot-check" : "rate-limit",
+ kind: isBotCheck(trimmed)
+ ? "bot-check"
+ : isSoftBlock(trimmed)
+ ? "soft-block"
+ : "rate-limit",
message: trimmed.slice(0, 300),
};
child.kill("SIGTERM");
diff --git a/common/ytdlp/runYtdlp.ts b/common/ytdlp/runYtdlp.ts
@@ -901,9 +901,12 @@ export type ManagedDownloadsDeps = {
// status skipped-filtered, and still asked).
//
// A per_video FAILURE is deliberately NOT here, even one that never got past
-// the prefetch (release 9 review): YouTube's soft block ("This content isn't
-// available, try again later") classifies as `deleted` → per_video, so
-// skipping the sleep there would fire prefetches back to back into the block.
+// the prefetch (release 9 review): a prefetch is still a request, and a run of
+// "unavailable" answers is how a throttled source looks (see SOFT_BLOCK_STREAK
+// in metadataScan.ts). Release 9's own reason — YouTube's explicit soft block
+// ("This content isn't available, try again later") reading as `deleted` →
+// per_video — is gone since release 10: it is `rate_limit` now (isSoftBlock),
+// backs the platform off and stops the batch. The pace stays anyway.
export function declinedWithoutMediaFetch(
outcome: DownloadOutcomeRecord,
): boolean {
