commit a8ebb54e143402780b170e7961f2f651c823e189
parent ddfa7debb0622f066cdcc5c93ea5a6463a325132
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sun, 20 Sep 2026 01:30:21 -0400
metadata scan: a run of "Video unavailable" is a soft block, not 545 dead videos
Measured 2026-09-20 on a raw yt-dlp pass over one channel's playlist: after
~100 good records YouTube answered "Video unavailable" for 545 CONSECUTIVE
videos, every one of which fetched fine when probed individually minutes later.
That is throttling wearing availability's clothes, and believing it is the most
expensive mistake available here — it would write 545 `deleted` errors, each of
which then suppresses a re-scan of that id for a day, and the scan would teach
itself that most of the channel is gone.
So these errors are HELD rather than recorded, and released only when the
streak is broken by a readable record or a different error. At ten in a row the
run stops, records the shared per-platform cooldown, and DISCARDS the streak —
the ids were never written, so the next run simply re-reads them. Ten is the
threshold because a channel really can have ten dead videos in a row, but from
the eleventh onwards re-reading costs nothing while believing a 545-video block
costs the channel.
A members-only video is the case the "held" design protects: it errors every
time, it is genuinely not fetchable, and it must not be thrown away with a soft
block's ids. It ends the streak and is recorded as members_only.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
5 files changed, 167 insertions(+), 1 deletion(-)
diff --git a/common/ytdlp/metadataScan.ts b/common/ytdlp/metadataScan.ts
@@ -59,6 +59,21 @@ const FLUSH_EVERY = 25;
// failed on, which is the only way to attribute a per-video failure in a batch.
const ERROR_LINE = /^ERROR:\s*(?:\[[^\]]+\]\s*)?([^\s:]+)\s*:\s*(.*)$/;
+// THE SOFT BLOCK. Measured 2026-09-20 on a raw yt-dlp pass over one channel's
+// playlist: after ~100 good records YouTube answered "Video unavailable" for 545
+// CONSECUTIVE videos, every one of which fetched fine when probed individually
+// minutes later. That is throttling wearing availability's clothes, and it is
+// the most expensive possible failure to believe — it would write 545 `deleted`
+// errors, each of which then suppresses a re-scan for a day.
+//
+// So a run of these is treated as a rate limit, and the streak's ids are
+// DISCARDED rather than recorded: the next run retries them, which is the whole
+// point. Ten in a row is the threshold — a channel really can have ten dead
+// videos in a row, but then the eleventh onwards cost nothing to re-read, while
+// believing a 545-video soft block costs the channel.
+const SOFT_BLOCK_STREAK = 10;
+const UNAVAILABLE_LINE = /video unavailable/i;
+
export type MetadataScanResult = {
scanned: number;
errors: number;
@@ -207,6 +222,24 @@ export async function runMetadataScan(
let stopped: MetadataScanRun["stopped"] | undefined;
let stoppedMessage: string | undefined;
const needsAuthIds: string[] = [];
+ // "Video unavailable" errors seen back-to-back, HELD rather than recorded
+ // until the streak is broken — so a soft block's ids are never written to the
+ // store at all, not written and then deleted. See SOFT_BLOCK_STREAK.
+ let unavailableStreak: Array<{ id: string; message: string }> = [];
+
+ // The streak ended without reaching the threshold: these really were
+ // unavailable videos, so record them.
+ const commitStreak = () => {
+ for (const e of unavailableStreak) {
+ errors[e.id] = {
+ class: "deleted",
+ message: e.message,
+ at: new Date().toISOString(),
+ };
+ pending++;
+ }
+ unavailableStreak = [];
+ };
const flush = async (force = false) => {
if (!force && pending < FLUSH_EVERY) return;
@@ -285,6 +318,9 @@ export async function runMetadataScan(
: {}),
scannedAt: new Date().toISOString(),
};
+ // A readable video proves we are not blocked, so whatever ran before it
+ // was a real streak of unavailable videos.
+ commitStreak();
scannedTotal++;
pending++;
opts.setProgress?.({
@@ -309,6 +345,9 @@ export async function runMetadataScan(
let stderrBuf = "";
const takeError = (line: string) => {
+ // Once we have decided this is a block, the rest of the stream is that
+ // block talking. Nothing more is recorded from it.
+ if (stopped === "rate_limit") return;
const trimmed = line.trim();
if (!trimmed.startsWith("ERROR")) return;
const cls = parseUnavailableFromStderr(trimmed);
@@ -326,6 +365,25 @@ export async function runMetadataScan(
const id = m?.[1] ?? "";
const message = (m?.[2] || trimmed).trim().slice(0, 300);
if (!id) return;
+ // "Video unavailable", possibly the soft block. Held, not recorded.
+ if (UNAVAILABLE_LINE.test(message)) {
+ unavailableStreak.push({ id, message });
+ if (unavailableStreak.length >= SOFT_BLOCK_STREAK) {
+ stopped = "rate_limit";
+ stoppedMessage =
+ `${unavailableStreak.length} consecutive "Video unavailable" errors — ` +
+ `treating this as a soft block rather than ${unavailableStreak.length} deleted videos. ` +
+ `Their ids were NOT recorded, so the next run re-reads them.`;
+ unavailableStreak = [];
+ child.kill("SIGTERM");
+ }
+ return;
+ }
+ // Anything else ends the streak and is recorded on its own terms. A
+ // members-only video is the case this protects: it errors every time, it
+ // is genuinely not fetchable, and it must not be thrown away with a
+ // soft block's ids.
+ commitStreak();
if (cls === "needs_auth") needsAuthIds.push(id);
errors[id] = { class: cls, message, at: new Date().toISOString() };
pending++;
@@ -344,6 +402,9 @@ export async function runMetadataScan(
const result = await child;
if (stdoutBuf) takeRecord(stdoutBuf);
if (stderrBuf) takeError(stderrBuf);
+ // A streak that never reached the threshold is just a run of dead videos.
+ if (stopped !== "rate_limit") commitStreak();
+ else unavailableStreak = [];
await Promise.all(flushes);
await flush(true);
@@ -393,7 +454,8 @@ export async function runMetadataScan(
if (stopped === "rate_limit") {
opts.onLog(
- `STOPPED: the source is rate-limiting this scan. ${scannedTotal} video(s) were read and saved; the rest stay unscanned. A per-platform cooldown has been recorded — run the scan again once it lapses.\n`,
+ `STOPPED: the source is rate-limiting this scan. ${stoppedMessage ?? ""}\n` +
+ `${scannedTotal} video(s) were read and saved; the rest stay unscanned. A per-platform cooldown has been recorded — run the scan again once it lapses.\n`,
);
await opts.onPlatformBackoff?.("rate_limit");
}
diff --git a/editor/e2e/fixtures/bin/fake-ytdlp.mjs b/editor/e2e/fixtures/bin/fake-ytdlp.mjs
@@ -599,6 +599,21 @@ async function main() {
);
continue;
}
+ // THE SOFT BLOCK. YouTube answers "Video unavailable" for a long run of
+ // videos that are in fact public — see SOFT_BLOCK_STREAK in
+ // common/ytdlp/metadataScan.ts. The scan must treat a streak of these as
+ // throttling and record NONE of them.
+ if (lower.includes("softblock")) {
+ process.stderr.write(`ERROR: [youtube] ${id}: Video unavailable\n`);
+ continue;
+ }
+ // Genuinely not fetchable, and never part of a soft-block streak.
+ if (lower.includes("membersonly")) {
+ process.stderr.write(
+ `ERROR: [youtube] ${id}: Join this channel to get access to members-only content\n`,
+ );
+ continue;
+ }
const sent = urlSentinels(url);
const record = {
id,
diff --git a/editor/e2e/fixtures/test-transcripts/metadata-scan-softblock/channels/test-softblock/config.json b/editor/e2e/fixtures/test-transcripts/metadata-scan-softblock/channels/test-softblock/config.json
@@ -0,0 +1,5 @@
+{
+ "handling": "youtube",
+ "name": "Test Soft Block",
+ "url": "https://www.youtube.com/@example/videos"
+}
diff --git a/editor/e2e/fixtures/test-transcripts/metadata-scan-softblock/channels/test-softblock/playlist b/editor/e2e/fixtures/test-transcripts/metadata-scan-softblock/channels/test-softblock/playlist
@@ -0,0 +1,23 @@
+https://www.youtube.com/watch?v=membersonly01
+https://www.youtube.com/watch?v=guestvid0001
+https://www.youtube.com/watch?v=guestvid0002
+https://www.youtube.com/watch?v=softblock01
+https://www.youtube.com/watch?v=softblock02
+https://www.youtube.com/watch?v=softblock03
+https://www.youtube.com/watch?v=softblock04
+https://www.youtube.com/watch?v=softblock05
+https://www.youtube.com/watch?v=softblock06
+https://www.youtube.com/watch?v=softblock07
+https://www.youtube.com/watch?v=softblock08
+https://www.youtube.com/watch?v=softblock09
+https://www.youtube.com/watch?v=softblock10
+https://www.youtube.com/watch?v=softblock11
+https://www.youtube.com/watch?v=softblock12
+https://www.youtube.com/watch?v=softblock13
+https://www.youtube.com/watch?v=softblock14
+https://www.youtube.com/watch?v=softblock15
+https://www.youtube.com/watch?v=softblock16
+https://www.youtube.com/watch?v=softblock17
+https://www.youtube.com/watch?v=softblock18
+https://www.youtube.com/watch?v=softblock19
+https://www.youtube.com/watch?v=softblock20
diff --git a/editor/e2e/metadata-scan-softblock.spec.ts b/editor/e2e/metadata-scan-softblock.spec.ts
@@ -0,0 +1,61 @@
+import { test, expect } from "@playwright/test";
+import { channelStage, generateReport, readJson, resetData } from "./helpers";
+
+// THE SOFT BLOCK. Measured on a real channel: after ~100 good records YouTube
+// answered "Video unavailable" for 545 CONSECUTIVE videos, all of which fetched
+// fine when probed individually minutes later. Believing that would write 545
+// `deleted` errors, each suppressing a re-scan for a day — the scan would teach
+// itself that most of the channel is gone.
+//
+// The fixture is the same shape, small: one genuinely members-only video, two
+// readable ones, then a run of soft-blocked ids.
+const CHANNEL = "test-softblock";
+const ROOT = `test-transcripts/channels/${CHANNEL}`;
+
+type MetadataScan = {
+ entries: Record<string, { title: string }>;
+ errors: Record<string, { class: string; message: string }>;
+ lastRun: { scanned: number; errors: number; stopped?: string; message?: string } | null;
+};
+
+test("a run of \"Video unavailable\" stops the scan and records none of it", async ({
+ page,
+}) => {
+ test.setTimeout(180_000);
+ await resetData("metadata-scan-softblock");
+ await generateReport(page, CHANNEL);
+ await page.goto(channelStage(CHANNEL, "playlist"));
+ await page.getByRole("button", { name: "Scan metadata" }).click();
+
+ const log = page.getByLabel("Scan metadata output");
+ await expect(log).toContainText("STOPPED", { timeout: 60_000 });
+ await expect(log).toContainText('consecutive "Video unavailable" errors');
+ await expect(log).toContainText("A per-platform cooldown has been recorded");
+
+ const scan = await readJson<MetadataScan>(`${ROOT}/metadata-scan.json`);
+ expect(scan.lastRun?.stopped).toBe("rate_limit");
+
+ // The two readable videos are kept — a stopped run keeps what it learned.
+ expect(Object.keys(scan.entries).sort()).toEqual([
+ "guestvid0001",
+ "guestvid0002",
+ ]);
+
+ // NOT ONE soft-blocked id is recorded. That is the whole point: recording them
+ // would mark 20 public videos deleted and suppress re-reading them for a day.
+ for (const id of Object.keys(scan.errors)) {
+ expect(id).not.toContain("softblock");
+ }
+ // The members-only video is a real, permanent failure and is NOT thrown away
+ // with the streak — it is not part of it, and it never will be fetchable.
+ expect(scan.errors.membersonly01?.class).toBe("members_only");
+
+ // The cooldown is real, not just a log line: the next scan refuses to start.
+ // A refusal is a NOTICE, not an error and not log output — the action returns
+ // `info: true` and StreamActionLog renders it as a status beside the button.
+ await page.getByRole("button", { name: "Scan metadata" }).click();
+ await expect(page.getByLabel("Scan metadata notice")).toContainText(
+ "rate-limit cooldown",
+ { timeout: 30_000 },
+ );
+});
