commit 1a65561700461f94a48a50d67b607763c7336400
parent eda6f556ba363f626d175ab7fd35613073ae4fd2
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sun, 20 Sep 2026 03:06:17 -0400
metadata scan: a bot check without cookies is a retry, not a cooldown
Measured 2026-09-20 on the first real scan of a channel: the FIRST video
answered "Sign in to confirm you're not a bot", the run stopped after 3 s with
`stopped: "rate_limit"` and recorded a per-platform cooldown — while the
operator had `cookiesFromBrowser: "firefox"` configured the whole time. The
scan passes cookies only in "always" mode, so it had never tried them. An hour
of cooldown for a problem one retry solves in seconds.
So a pass no longer decides what its own block MEANS. It records a reason, and
the caller asks the question that matters: have we tried the thing we have not
tried? A block on a cookie-less pass — bot check or "Video unavailable" streak
alike — restarts the not-yet-read ids once with the configured cookies. Only a
block that survives them (or one with no cookie spec to reach for) becomes the
stop, the `lastRun.stopped` and the cooldown.
A plain 429 is deliberately NOT retried: cookies do not answer "too many
requests", and `isBotCheck` is exported from availability.ts precisely so the
two can be told apart at the one place that has to respond differently.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
6 files changed, 198 insertions(+), 27 deletions(-)
diff --git a/common/lib/availability.ts b/common/lib/availability.ts
@@ -212,6 +212,22 @@ const PER_VIDEO_CLASSES: ReadonlyArray<Availability> = [
"deleted",
];
+// YouTube's bot check. Not a 429 and not per-video: once it fires, every
+// subsequent request in the same batch fails the same way, so it is a
+// batch-level signal exactly like a rate limit — and it is what a metadata scan
+// over a whole channel listing trips first. The apostrophe is a right single
+// quote in yt-dlp's output, so match either.
+//
+// EXPORTED SEPARATELY from the rate-limit patterns, and consumed on its own by
+// ytdlp/metadataScan.ts, because the two call for different responses. A 429
+// means "back off". A bot check means "you are not signed in" — cookies answer
+// it, and a cooldown recorded before they have even been tried costs the
+// operator an hour of waiting for a problem a retry would have solved in
+// seconds.
+export function isBotCheck(stderr: string): boolean {
+ return /confirm you['\u2019]?re not a bot/i.test(stderr);
+}
+
export function classifyDownloadFailure(
stderrTail: string,
availabilityClass: Availability | undefined,
@@ -228,13 +244,7 @@ export function classifyDownloadFailure(
/too many requests/.test(s) ||
/rate[- ]?limit/.test(s) ||
/throttl/.test(s) ||
- // YouTube's bot check. Not a 429 and not per-video: once it fires, every
- // subsequent request in the same batch fails the same way, so it is a
- // batch-level stop signal exactly like a rate limit — and it is what a
- // metadata scan over a whole channel listing trips first. The apostrophe is
- // a right single quote in yt-dlp's output, so match either.
- /sign in to confirm you['\u2019]?re not a bot/.test(s) ||
- /confirm you['\u2019]?re not a bot/.test(s)
+ isBotCheck(stderrTail)
) {
return "rate_limit";
}
diff --git a/common/ytdlp/metadataScan.ts b/common/ytdlp/metadataScan.ts
@@ -20,6 +20,7 @@ import { tmpdir } from "node:os";
import { execa } from "execa";
import {
classifyDownloadFailure,
+ isBotCheck,
parseUnavailableFromStderr,
} from "../lib/availability";
import type { ChannelConfig } from "../lib/channelConfig";
@@ -221,6 +222,20 @@ export async function runMetadataScan(
let pending = 0;
let stopped: MetadataScanRun["stopped"] | undefined;
let stoppedMessage: string | undefined;
+ // WHY THE CURRENT PASS STOPPED, before anyone decides what it MEANS.
+ //
+ // A bot check and a soft block both look like "the source is refusing us",
+ // and the first response to that is not a cooldown — it is to try the thing
+ // we have not tried. Measured 2026-09-20 on the first real scan of a channel:
+ // the FIRST video answered "Sign in to confirm you're not a bot", the run
+ // stopped after 3 s with a rate-limit cooldown, and the operator had a
+ // configured `cookiesFromBrowser` the scan never used (it passes cookies only
+ // in "always" mode). An hour of cooldown for a problem one retry solves.
+ //
+ // So a block on a COOKIE-LESS pass is a reason to retry with cookies, once.
+ // Only a block on a pass that already carried them is a real refusal.
+ let block: { kind: "bot-check" | "soft-block" | "rate-limit"; message: string } | null =
+ null;
const needsAuthIds: string[] = [];
// "Video unavailable" errors seen back-to-back, HELD rather than recorded
// until the streak is broken — so a soft block's ids are never written to the
@@ -345,19 +360,21 @@ export async function runMetadataScan(
let stderrBuf = "";
const takeError = (line: string) => {
- // Once we have decided this is a block, the rest of the stream is that
- // block talking. Nothing more is recorded from it.
- if (stopped === "rate_limit") return;
+ // Once this pass is blocked, the rest of the stream is that block
+ // talking. Nothing more is recorded from it.
+ if (block) return;
const trimmed = line.trim();
if (!trimmed.startsWith("ERROR")) return;
const cls = parseUnavailableFromStderr(trimmed);
const failure = classifyDownloadFailure(trimmed, cls);
- // A batch-level signal. Continuing would just hammer the source, and on
- // YouTube's bot check every subsequent entry fails anyway — so stop, record
- // the shared per-platform cooldown, and keep what we have.
+ // A batch-level signal: continuing would just hammer the source, and on a
+ // bot check every subsequent entry fails the same way. Stop the pass and
+ // let the caller decide whether cookies are the answer.
if (failure === "rate_limit") {
- stopped = "rate_limit";
- stoppedMessage = trimmed.slice(0, 300);
+ block = {
+ kind: isBotCheck(trimmed) ? "bot-check" : "rate-limit",
+ message: trimmed.slice(0, 300),
+ };
child.kill("SIGTERM");
return;
}
@@ -369,11 +386,13 @@ export async function runMetadataScan(
if (UNAVAILABLE_LINE.test(message)) {
unavailableStreak.push({ id, message });
if (unavailableStreak.length >= SOFT_BLOCK_STREAK) {
- stopped = "rate_limit";
- stoppedMessage =
- `${unavailableStreak.length} consecutive "Video unavailable" errors — ` +
- `treating this as a soft block rather than ${unavailableStreak.length} deleted videos. ` +
- `Their ids were NOT recorded, so the next run re-reads them.`;
+ block = {
+ kind: "soft-block",
+ message:
+ `${unavailableStreak.length} consecutive "Video unavailable" errors — ` +
+ `treating this as a soft block rather than ${unavailableStreak.length} deleted videos. ` +
+ `Their ids were NOT recorded, so the next run re-reads them.`,
+ };
unavailableStreak = [];
child.kill("SIGTERM");
}
@@ -403,7 +422,7 @@ export async function runMetadataScan(
if (stdoutBuf) takeRecord(stdoutBuf);
if (stderrBuf) takeError(stderrBuf);
// A streak that never reached the threshold is just a run of dead videos.
- if (stopped !== "rate_limit") commitStreak();
+ if (!block) commitStreak();
else unavailableStreak = [];
await Promise.all(flushes);
await flush(true);
@@ -412,7 +431,7 @@ export async function runMetadataScan(
stopped = "aborted";
return;
}
- if (stopped === "rate_limit") return;
+ if (block) return;
// --ignore-errors means a per-video failure still exits nonzero while every
// readable entry was printed, so a nonzero exit is only fatal when nothing
// came back at all.
@@ -428,16 +447,47 @@ export async function runMetadataScan(
}
};
+ // Cookies for the retry pass, when the mode offers any. Defer mode resolves
+ // to none, which is the point of defer: those videos wait for a deliberate
+ // cookie run rather than being retried behind the operator's back.
+ const retryCookies = authRetryCookies(policy);
+ let usedCookies = false;
try {
- await runPass(targetIds, alwaysCookies(policy), "main");
+ const firstCookies = alwaysCookies(policy);
+ usedCookies = Boolean(firstCookies);
+ await runPass(targetIds, firstCookies, "main");
+
+ // THE COOKIE RETRY. A block on a cookie-LESS pass is not yet a refusal —
+ // for a bot check it is literally "you are not signed in", and a soft block
+ // behaves the same way on an unauthenticated session. Retry the ids we have
+ // not read yet, once, with the cookies the operator already configured.
+ // Only a block that survives cookies is treated as the source saying no.
+ if (
+ block &&
+ !usedCookies &&
+ retryCookies !== undefined &&
+ !opts.signal.aborted
+ ) {
+ const remaining = targetIds.filter((id) => !entries[id]);
+ opts.onLog(
+ `${(block as { kind: string }).kind === "bot-check" ? "bot check" : "soft block"} without cookies — ` +
+ `retrying the remaining ${remaining.length} ids with --cookies-from-browser ${retryCookies}.\n`,
+ );
+ block = null;
+ unavailableStreak = [];
+ usedCookies = true;
+ if (remaining.length > 0) {
+ await runPass(remaining, retryCookies, "cookie-retry");
+ }
+ }
- // ONE cookie retry for the auth-gated ids, and only when the mode actually
- // offers cookies. Defer mode resolves to none, which is the point of defer:
- // those videos wait for a deliberate cookie run.
- const retryCookies = authRetryCookies(policy);
+ // ONE cookie retry for the auth-gated ids. Skipped when the pass above
+ // already carried cookies — it covered these ids too.
const retryIds = [...new Set(needsAuthIds)].filter((id) => !entries[id]);
if (
+ !block &&
stopped === undefined &&
+ !usedCookies &&
retryIds.length > 0 &&
retryCookies !== undefined &&
!opts.signal.aborted
@@ -452,6 +502,12 @@ export async function runMetadataScan(
await rm(tmpRoot, { recursive: true, force: true });
}
+ // A block that survived (or never had) a cookie retry is the source refusing
+ // us. Only NOW does it become a cooldown.
+ if (block && stopped === undefined) {
+ stopped = "rate_limit";
+ stoppedMessage = (block as { message: string }).message;
+ }
if (stopped === "rate_limit") {
opts.onLog(
`STOPPED: the source is rate-limiting this scan. ${stoppedMessage ?? ""}\n` +
diff --git a/editor/e2e/fixtures/bin/fake-ytdlp.mjs b/editor/e2e/fixtures/bin/fake-ytdlp.mjs
@@ -602,6 +602,15 @@ async function main() {
);
continue;
}
+ // THE BOT CHECK. YouTube refuses an unauthenticated session outright —
+ // and, like the real thing, gives up on the WHOLE batch rather than just
+ // this entry. Cookies answer it, which is the point of the sentinel.
+ if (lower.includes("botcheck") && !cookieArg()) {
+ process.stderr.write(
+ `ERROR: [youtube] ${id}: Sign in to confirm you\u2019re not a bot. Use --cookies-from-browser or --cookies for the authentication.\n`,
+ );
+ break;
+ }
// THE SOFT BLOCK. YouTube answers "Video unavailable" for a long run of
// videos that are in fact public — see SOFT_BLOCK_STREAK in
// common/ytdlp/metadataScan.ts. The scan must treat a streak of these as
diff --git a/editor/e2e/fixtures/test-transcripts/metadata-scan-botcheck/channels/test-botcheck/config.json b/editor/e2e/fixtures/test-transcripts/metadata-scan-botcheck/channels/test-botcheck/config.json
@@ -0,0 +1,6 @@
+{
+ "handling": "youtube",
+ "name": "Test Bot Check",
+ "url": "https://www.youtube.com/@example/videos",
+ "cookiesFromBrowser": "firefox"
+}
diff --git a/editor/e2e/fixtures/test-transcripts/metadata-scan-botcheck/channels/test-botcheck/playlist b/editor/e2e/fixtures/test-transcripts/metadata-scan-botcheck/channels/test-botcheck/playlist
@@ -0,0 +1,4 @@
+https://www.youtube.com/watch?v=botcheck0001
+https://www.youtube.com/watch?v=guestvid0001
+https://www.youtube.com/watch?v=guestvid0002
+https://www.youtube.com/watch?v=plainvid0001
diff --git a/editor/e2e/metadata-scan-botcheck.spec.ts b/editor/e2e/metadata-scan-botcheck.spec.ts
@@ -0,0 +1,86 @@
+import { readFile } from "node:fs/promises";
+import { test, expect } from "@playwright/test";
+import {
+ channelStage,
+ generateReport,
+ readJson,
+ resetData,
+ resolvePath,
+} from "./helpers";
+
+// THE BOT CHECK, AND THE COOKIES THE SCAN WAS NOT USING.
+//
+// Measured 2026-09-20 on the first real metadata scan of a channel: the FIRST
+// video answered "Sign in to confirm you're not a bot", the run stopped after
+// 3 s with `stopped: "rate_limit"`, and a per-platform cooldown was recorded —
+// while the operator had `cookiesFromBrowser` configured the whole time. The
+// scan passes cookies only in "always" mode, so it had never tried them.
+//
+// A block on a cookie-less pass is not a refusal. It is the one thing we have
+// not tried yet.
+const CHANNEL = "test-botcheck";
+const ROOT = `test-transcripts/channels/${CHANNEL}`;
+const ALL = [
+ "botcheck0001",
+ "guestvid0001",
+ "guestvid0002",
+ "plainvid0001",
+];
+
+type MetadataScan = {
+ entries: Record<string, { title: string }>;
+ errors: Record<string, { class: string }>;
+ lastRun: { scanned: number; errors: number; stopped?: string } | null;
+};
+
+async function scanInvocations(): Promise<string[]> {
+ const raw = await readFile(
+ resolvePath(`${ROOT}/fake-ytdlp.invocations`),
+ "utf8",
+ ).catch(() => "");
+ return raw
+ .split("\n")
+ .map((l) => l.trim())
+ .filter((l) => l.startsWith("metadata-scan:"));
+}
+
+test("a bot check on a cookie-less pass retries with cookies instead of backing off", async ({
+ page,
+}) => {
+ test.setTimeout(180_000);
+ await resetData("metadata-scan-botcheck");
+ await generateReport(page, CHANNEL);
+ await page.goto(channelStage(CHANNEL, "playlist"));
+ await page.getByRole("button", { name: "Scan metadata" }).click();
+
+ const log = page.getByLabel("Scan metadata output");
+ await expect(log).toContainText(
+ "bot check without cookies — retrying the remaining 4 ids with --cookies-from-browser firefox",
+ { timeout: 60_000 },
+ );
+ await expect(log).toContainText("Metadata scan complete", { timeout: 60_000 });
+
+ // The retry read everything the blocked pass could not.
+ const scan = await readJson<MetadataScan>(`${ROOT}/metadata-scan.json`);
+ expect(Object.keys(scan.entries).sort()).toEqual([...ALL].sort());
+ expect(Object.keys(scan.errors)).toEqual([]);
+
+ // NOT a refusal: no stop, and no cooldown. A cooldown here would cost the
+ // operator an hour for a problem the retry solved in seconds.
+ expect(scan.lastRun?.stopped).toBeUndefined();
+ expect(scan.lastRun?.scanned).toBe(4);
+ await expect(log).not.toContainText("STOPPED");
+ await expect(log).not.toContainText("cooldown has been recorded");
+
+ // Exactly two passes, and only the second carried cookies — the first is the
+ // ordinary cookie-less scan, which is what "when-required" means.
+ const invocations = await scanInvocations();
+ expect(invocations).toHaveLength(2);
+ expect(invocations[0]).toBe("metadata-scan:4 cookies=");
+ expect(invocations[1]).toBe("metadata-scan:4 cookies=firefox");
+
+ // And the cooldown really was not recorded: a second scan starts rather than
+ // being refused with the rate-limit notice.
+ await page.getByRole("button", { name: "Scan metadata" }).click();
+ await expect(log).toContainText("Nothing to scan", { timeout: 60_000 });
+});
