commit 94a9aa620d54b2ac7a9fc3f77c0a75bdfcf1cbab
parent b245b936312a4ecb56a0851334cbb815a0986d9c
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sat, 26 Sep 2026 03:31:28 -0400
common: the availability check stops on a rate limit, and the pre-clean gate never judges what it skipped (review fix)
`runAvailabilityCheck` probed every id even after the source rate-limited
it. Now the first probe that classifies `rate_limit` (429, the bot check,
YouTube's soft block) records its own `error` observation, sets `blocked`,
and no further probe starts — the rest count as `skipped`. After the pass
the platform cooldown is recorded once, best-effort, with
`recordDownloadBackoff(detectPlatform(config?.url ?? url) ?? "unknown")`,
the key the batch and the Sync gate use (`onPlatformBackoff` is the test
seam). The result carries `blocked`, `blockMessage` and `probedIds`.
THE DATA-LOSS GUARD, in the same change: `verifyBeforeClean` read
`resolveEffectiveAvailability` for every tier C suspect, so a suspect the
stopped check never probed would be judged on its old availability.json —
a months-old `public` reads as safe to clean, and its audio would be
deleted irreversibly. When the check was blocked, every suspect it did not
probe is now `unverified` (skipped, no marker, retried next sweep). An
unblocked check is judged exactly as before.
Other callers ignore the result and stay safe: `checkKeptDeleted` only
pins, and the full-sweep confirm leaves an unprobed id `maybe_missing`
(its probe time is older than the scan).
Tests: checkAvailability.test.ts (3, new): soft block → 1 spawn, 3 skipped,
one `error` written, cooldown once under youtube; 429 and bot check the
same; a 403 and a removed video probe all 4 with no cooldown.
verifyBeforeClean.test.ts +2: a blocked confirmation leaves all 3 suspects
unverified (two carry a stale `public`), the listed video stays cleanable
and the cooldown lands in the state file — this one fails with the guard
disabled; an unblocked one probes all 3 and judges public / deleted / 403
as before.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
4 files changed, 358 insertions(+), 5 deletions(-)
diff --git a/common/controller/checkAvailability.test.ts b/common/controller/checkAvailability.test.ts
@@ -0,0 +1,132 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { Paths } from "../lib/paths";
+import { runAvailabilityCheck } from "./checkAvailability";
+
+// Run with:
+// pnpm --filter yt-dlp-transcript-common exec tsx --test controller/checkAvailability.test.ts
+//
+// THE AVAILABILITY CHECK STOPS ON A RATE LIMIT (release 10, L2 review). Its
+// probes are one `--dump-json` each; before, a rate-limited probe was recorded
+// and the next one started anyway, into the same refusal. Now the first
+// `rate_limit` probe records its own `error`, no further probe starts, and the
+// platform cooldown is recorded once. A temp yt-dlp script answers every probe
+// with the given line and counts the spawns; the cooldown is injected.
+
+process.env.SETTINGS_FILE = path.join(tmpdir(), "ttb-avail-no-such-settings.json");
+
+const IDS = ["vid00000001", "vid00000002", "vid00000003", "vid00000004"];
+
+async function run(stderrLine: string): Promise<{
+ spawns: number;
+ backoffs: string[];
+ result: Awaited<ReturnType<typeof runAvailabilityCheck>>;
+ recorded: Record<string, string | null>;
+ log: string[];
+}> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-avail-"));
+ try {
+ const counter = path.join(dir, "spawns");
+ const bin = path.join(dir, "fake-ytdlp.sh");
+ await writeFile(
+ bin,
+ `#!/bin/sh\necho x >> '${counter}'\necho ${JSON.stringify(stderrLine)} >&2\nexit 1\n`,
+ );
+ await chmod(bin, 0o755);
+ const paths = {
+ channelsDir: path.join(dir, "channels"),
+ ytdlpBin: bin,
+ } as Paths;
+ const ch = path.join(paths.channelsDir, "ch");
+ await mkdir(ch, { recursive: true });
+ await writeFile(
+ path.join(ch, "config.json"),
+ JSON.stringify({ handling: "youtube", url: "https://www.youtube.com/@ch/videos" }),
+ );
+ for (const id of IDS) {
+ const vd = path.join(ch, "data", id);
+ await mkdir(vd, { recursive: true });
+ await writeFile(
+ path.join(vd, "metadata.info.json"),
+ JSON.stringify({ id, webpage_url: `https://www.youtube.com/watch?v=${id}` }),
+ );
+ }
+ const backoffs: string[] = [];
+ const log: string[] = [];
+ const result = await runAvailabilityCheck({
+ channelSlug: "ch",
+ paths,
+ mode: "recheck-all",
+ ignoreShard: true,
+ concurrency: 1,
+ onLog: (m) => log.push(m),
+ onPlatformBackoff: (pf) => {
+ backoffs.push(pf);
+ },
+ });
+ const spawns = (await readFile(counter, "utf8").catch(() => ""))
+ .split("\n")
+ .filter(Boolean).length;
+ const recorded: Record<string, string | null> = {};
+ for (const id of IDS) {
+ const raw = await readFile(
+ path.join(ch, "data", id, "availability.json"),
+ "utf8",
+ ).catch(() => null);
+ recorded[id] = raw ? (JSON.parse(raw) as { availability: string }).availability : null;
+ }
+ return { spawns, backoffs, result, recorded, log };
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+test("a soft-blocked probe stops the check: one spawn, one cooldown, the rest skipped", async () => {
+ const r = await run(
+ "ERROR: [youtube] vid00000001: This content isn't available, try again later. " +
+ "The current session has been rate-limited by YouTube for up to an hour.",
+ );
+ assert.equal(r.spawns, 1);
+ assert.equal(r.result.blocked, true);
+ assert.match(r.result.blockMessage ?? "", /try again later/);
+ assert.equal(r.result.probedIds.length, 1);
+ assert.equal(r.result.attempted, 1);
+ assert.equal(r.result.skipped, IDS.length - 1);
+ // The one probe records its observation — `error`, never `deleted` — and
+ // nothing else is written.
+ const written = Object.values(r.recorded).filter((v) => v !== null);
+ assert.deepEqual(written, ["error"]);
+ assert.equal(r.recorded[r.result.probedIds[0]], "error");
+ // The cooldown, once, under the channel's platform key.
+ assert.deepEqual(r.backoffs, ["youtube"]);
+ assert.ok(r.log.some((l) => /^STOPPED: the source is rate-limiting availability probes/.test(l)));
+});
+
+test("a 429 and the bot check stop it the same way", async () => {
+ for (const line of [
+ "ERROR: [youtube] vid00000001: Unable to download webpage: HTTP Error 429: Too Many Requests",
+ "ERROR: [youtube] vid00000001: Sign in to confirm you’re not a bot. Use --cookies-from-browser or --cookies for the authentication.",
+ ]) {
+ const r = await run(line);
+ assert.equal(r.spawns, 1, line);
+ assert.equal(r.result.blocked, true, line);
+ assert.deepEqual(r.backoffs, ["youtube"], line);
+ }
+});
+
+test("any other failure does not stop it: every video is probed, no cooldown", async () => {
+ for (const line of [
+ "ERROR: [youtube] vid00000001: Unable to download webpage: HTTP Error 403: Forbidden",
+ "ERROR: [youtube] vid00000001: Video unavailable",
+ ]) {
+ const r = await run(line);
+ assert.equal(r.spawns, IDS.length, line);
+ assert.equal(r.result.blocked, false, line);
+ assert.equal(r.result.blockMessage, undefined, line);
+ assert.deepEqual([...r.result.probedIds].sort(), IDS, line);
+ assert.deepEqual(r.backoffs, [], line);
+ }
+});
diff --git a/common/controller/checkAvailability.ts b/common/controller/checkAvailability.ts
@@ -6,9 +6,11 @@ import {
AVAILABILITY_VALUES,
EXCLUDED_FROM_DOWNLOAD,
availabilityFromJsonField,
+ classifyDownloadFailure,
parseUnavailableFromStderr,
type Availability,
} from "../lib/availability";
+import { recordDownloadBackoff } from "../jobs/downloadBackoff";
import {
loadAvailability,
recordAvailability,
@@ -65,6 +67,11 @@ export type CheckAvailabilityOpts = {
ignoreShard?: boolean;
onLog?: (msg: string) => void;
signal?: AbortSignal;
+ // Records the shared per-platform cooldown when a probe is rate-limited
+ // (see `blocked` below). Defaults to recordDownloadBackoff — the helper and
+ // key a download's 429 and a manual Sync use. A test seam; production
+ // callers pass nothing.
+ onPlatformBackoff?: (platform: string) => void | Promise<void>;
};
export type CheckAvailabilityResult = {
@@ -72,6 +79,17 @@ export type CheckAvailabilityResult = {
checked: number;
skipped: number;
byStatus: Record<Availability, number>;
+ // THE CHECK STOPS ON A RATE LIMIT (release 10, L2 review). A probe that
+ // classifies `rate_limit` — HTTP 429, the bot check, YouTube's soft block
+ // "…isn't available, try again later" — records its own `error` observation
+ // and then no further probe is started: every later id is `skipped`, and
+ // the platform cooldown is recorded once. The ids that WERE probed are in
+ // `probedIds`; anything else a caller reads for this run is an OLD record,
+ // and a caller that acts on the result must not judge it (verifyBeforeClean
+ // leaves it unverified).
+ blocked: boolean;
+ blockMessage?: string;
+ probedIds: string[];
};
function emptyByStatus(): Record<Availability, number> {
@@ -114,6 +132,7 @@ export async function runAvailabilityCheck({
ignoreShard = false,
onLog,
signal,
+ onPlatformBackoff,
}: CheckAvailabilityOpts): Promise<CheckAvailabilityResult> {
const log = onLog ?? ((m: string) => console.log(m));
const channelDir = path.join(paths.channelsDir, channelSlug);
@@ -151,13 +170,23 @@ export async function runAvailabilityCheck({
log(
`Shard availability: save-only — slice ${(shardResult.config?.shardIndex ?? shardIndex ?? 0) + 1}/${shardResult.config?.totalShards ?? shardTotal} (${videoDirs.length} item(s)) persisted; not checking.`,
);
- return { attempted: 0, checked: 0, skipped: 0, byStatus: emptyByStatus() };
+ return {
+ attempted: 0,
+ checked: 0,
+ skipped: 0,
+ byStatus: emptyByStatus(),
+ blocked: false,
+ probedIds: [],
+ };
}
let attempted = 0;
let checked = 0;
let skipped = 0;
const byStatus = emptyByStatus();
+ // Set by the first rate-limited probe; see CheckAvailabilityResult.blocked.
+ let blocked: { message: string; url: string } | null = null;
+ const probedIds: string[] = [];
// Pre-scan: count how many already have a parsed availability.json so the
// "skipped" total in the summary is unambiguous (no, we are not skipping
@@ -177,7 +206,9 @@ export async function runAvailabilityCheck({
await Promise.all(
videoDirs.map((id) =>
limit(async () => {
- if (signal?.aborted) {
+ // A rate-limited source is not asked again in this run: each further
+ // probe would be another request into the same refusal.
+ if (signal?.aborted || blocked) {
skipped++;
return;
}
@@ -264,6 +295,15 @@ export async function runAvailabilityCheck({
const trimmed = stderr.trim().split("\n").slice(-3).join("\n");
if (trimmed) error = trimmed;
}
+ if (
+ !blocked &&
+ classifyDownloadFailure(stderr, availability) === "rate_limit"
+ ) {
+ blocked = {
+ message: (error ?? stderr.trim()).slice(0, 300),
+ url,
+ };
+ }
}
await recordAvailability(videoDir, {
@@ -279,11 +319,29 @@ export async function runAvailabilityCheck({
});
checked++;
byStatus[availability]++;
+ probedIds.push(id);
log(`Check ${id}: ${availability}`);
}),
),
);
+ if (blocked) {
+ const b = blocked as { message: string; url: string };
+ const platform = detectPlatform(config?.url ?? b.url) ?? "unknown";
+ const unprobed = videoDirs.length - probedIds.length;
+ log(
+ `STOPPED: the source is rate-limiting availability probes (${b.message}). ` +
+ `${unprobed} video(s) were not probed this run; a ${platform} cooldown has been recorded.`,
+ );
+ try {
+ await (onPlatformBackoff ?? ((pf) => recordDownloadBackoff(pf, paths)))(
+ platform,
+ );
+ } catch {
+ /* the cooldown is best-effort; the stop above already happened */
+ }
+ }
+
// Type guard: every key in AVAILABILITY_VALUES has been incremented above.
void AVAILABILITY_VALUES;
@@ -294,5 +352,13 @@ export async function runAvailabilityCheck({
`Availability check done. attempted=${attempted} checked=${checked} skipped=${skipped} (${summary}).`,
);
- return { attempted, checked, skipped, byStatus };
+ return {
+ attempted,
+ checked,
+ skipped,
+ byStatus,
+ blocked: blocked !== null,
+ ...(blocked ? { blockMessage: (blocked as { message: string }).message } : {}),
+ probedIds,
+ };
}
diff --git a/common/controller/verifyBeforeClean.test.ts b/common/controller/verifyBeforeClean.test.ts
@@ -1,6 +1,6 @@
import { test } from "node:test";
import assert from "node:assert/strict";
-import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import type { Paths } from "../lib/paths";
@@ -175,3 +175,135 @@ test("an unreachable source fails CLOSED: nothing is cleared to delete", async (
assert.equal(excludedIds(verdicts).size, 2);
});
});
+
+// A RATE-LIMITED CONFIRMATION (release 10, L2 review). The tier C check now
+// stops at its first rate-limited probe. The suspects it never reached still
+// carry whatever availability.json they had — here a months-old `public` —
+// and judged on that they would be CLEARED FOR DELETE. They must come back
+// unverified instead. A temp yt-dlp script answers the flat listing (one
+// listed video, so the rest are suspects) and each probe per `probe`.
+async function withFakeSource(
+ probe: string,
+ fn: (paths: Paths, probes: () => Promise<string[]>) => Promise<void>,
+): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "ttb-vbc-src-"));
+ const probesFile = path.join(dir, "probes");
+ const bin = path.join(dir, "fake-ytdlp.sh");
+ await writeFile(
+ bin,
+ [
+ "#!/bin/sh",
+ 'for a in "$@"; do last="$a"; done',
+ 'case " $* " in',
+ ' *" --flat-playlist "*) echo "https://www.youtube.com/watch?v=listed00001"; exit 0;;',
+ "esac",
+ `echo "$last" >> '${probesFile}'`,
+ probe,
+ "",
+ ].join("\n"),
+ );
+ await chmod(bin, 0o755);
+ const paths = {
+ channelsDir: path.join(dir, "channels"),
+ ytdlpBin: bin,
+ autoQueueStateFile: path.join(dir, ".auto-queue", "state.json"),
+ } as Paths;
+ try {
+ await fn(paths, async () =>
+ (await readFile(probesFile, "utf8").catch(() => ""))
+ .split("\n")
+ .filter(Boolean),
+ );
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+async function seedProbeable(paths: Paths, id: string): Promise<void> {
+ const dir = path.join(paths.channelsDir, "ch", "data", id);
+ await mkdir(dir, { recursive: true });
+ await writeFile(
+ path.join(dir, "metadata.info.json"),
+ JSON.stringify({ id, webpage_url: `https://www.youtube.com/watch?v=${id}` }),
+ );
+ // A STALE verdict from long ago: exactly what must not clear a delete.
+ await writeFile(
+ path.join(dir, "availability.json"),
+ JSON.stringify({ checkedAt: "2026-01-01T00:00:00.000Z", availability: "public" }),
+ );
+}
+
+const SUSPECTS = ["suspect0001", "suspect0002", "suspect0003"];
+
+test("a rate-limited confirmation leaves every suspect it did not probe unverified: nothing is cleaned on an old record", async () => {
+ await withFakeSource(
+ `echo "ERROR: [youtube] x: This content isn't available, try again later." >&2; exit 1`,
+ async (paths, probes) => {
+ await seedConfig(paths, "ch", {
+ handling: "youtube",
+ url: "https://www.youtube.com/@ch/videos",
+ });
+ await seedProbeable(paths, "listed00001");
+ for (const id of SUSPECTS) await seedProbeable(paths, id);
+
+ const verdicts = await verifyBeforeClean({
+ channelSlug: "ch",
+ paths,
+ candidateIds: ["listed00001", ...SUSPECTS],
+ onLog: () => {},
+ });
+
+ // ONE probe: the check stopped at the soft block.
+ assert.equal((await probes()).length, 1);
+ // Every suspect is protected: the probed one read `error`, the other two
+ // were never asked and are NOT judged on their stale `public`.
+ assert.deepEqual([...verdicts.unverified].sort(), SUSPECTS);
+ assert.equal(verdicts.pinned.size, 0);
+ const excluded = excludedIds(verdicts);
+ for (const id of SUSPECTS) assert.ok(excluded.has(id), id);
+ // The listed video is still listed: cleanable as before.
+ assert.equal(excluded.has("listed00001"), false);
+ // …and the platform cooldown was recorded, once, under youtube.
+ const state = JSON.parse(
+ await readFile(paths.autoQueueStateFile, "utf8"),
+ ) as { download: { platformBackoff: Record<string, { until: number }> } };
+ assert.ok(state.download.platformBackoff.youtube.until > Date.now());
+ },
+ );
+});
+
+test("an unblocked confirmation is judged exactly as before", async () => {
+ await withFakeSource(
+ [
+ 'case "$last" in',
+ ` *suspect0001*) echo '{"id":"suspect0001","availability":"public"}';;`,
+ ` *suspect0002*) echo "ERROR: [youtube] suspect0002: Video unavailable" >&2; exit 1;;`,
+ ` *) echo "ERROR: [youtube] x: Unable to download webpage: HTTP Error 403: Forbidden" >&2; exit 1;;`,
+ "esac",
+ ].join("\n"),
+ async (paths, probes) => {
+ await seedConfig(paths, "ch", {
+ handling: "youtube",
+ url: "https://www.youtube.com/@ch/videos",
+ });
+ await seedProbeable(paths, "listed00001");
+ for (const id of SUSPECTS) await seedProbeable(paths, id);
+
+ const verdicts = await verifyBeforeClean({
+ channelSlug: "ch",
+ paths,
+ candidateIds: ["listed00001", ...SUSPECTS],
+ onLog: () => {},
+ });
+
+ // Every suspect probed (a 403 is not a rate limit, so nothing stops).
+ assert.equal((await probes()).length, 3);
+ // public → cleanable; deleted → pinned; error → unverified.
+ assert.equal(excludedIds(verdicts).has("suspect0001"), false);
+ assert.equal(verdicts.pinned.get("suspect0002"), "deleted");
+ assert.deepEqual([...verdicts.unverified], ["suspect0003"]);
+ // No cooldown for a run nothing rate-limited.
+ await assert.rejects(readFile(paths.autoQueueStateFile, "utf8"));
+ },
+ );
+});
diff --git a/common/controller/verifyBeforeClean.ts b/common/controller/verifyBeforeClean.ts
@@ -153,8 +153,9 @@ export async function verifyBeforeClean({
log(
`Verify before clean: ${suspects.length} candidate(s) missing from the fresh listing — confirming individually…`,
);
+ let check: Awaited<ReturnType<typeof runAvailabilityCheck>>;
try {
- await runAvailabilityCheck({
+ check = await runAvailabilityCheck({
channelSlug,
paths,
mode: "recheck-all",
@@ -182,8 +183,30 @@ export async function verifyBeforeClean({
return verdicts;
}
+ // A STOPPED CHECK JUDGES NOTHING IT DID NOT PROBE (release 10, L2 review).
+ // The confirmation stops at the first rate-limited probe, and every suspect
+ // after it is left with whatever `availability.json` it already had — which
+ // can be months old and say `public`. Read below, that stale record would
+ // clear the video for an irreversible delete. So when the check was
+ // blocked, every suspect it did not probe is `unverified`: skipped, no
+ // marker, retried on the next sweep. An unblocked check is judged exactly
+ // as before.
+ const unprobed = new Set<string>();
+ if (check.blocked) {
+ const probed = new Set(check.probedIds);
+ for (const id of suspects) if (!probed.has(id)) unprobed.add(id);
+ log(
+ `Verify before clean: the source rate-limited the confirmation — ` +
+ `${unprobed.size} suspect(s) it did not reach are left unverified, not judged on an old record.`,
+ );
+ }
+
for (const id of suspects) {
if (signal?.aborted) break;
+ if (unprobed.has(id)) {
+ verdicts.unverified.add(id);
+ continue;
+ }
const availability = await resolveEffectiveAvailability(
path.join(dataDir, id),
);
