import { test } from "node:test"; import assert from "node:assert/strict"; import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import type { Paths } from "../lib/paths"; import { runAvailabilityCheck } from "./checkAvailability"; // Run with: // pnpm --filter yt-dlp-transcript-common exec tsx --test controller/checkAvailability.test.ts // // THE AVAILABILITY CHECK STOPS ON A RATE LIMIT (release 10, L2 review). Its // probes are one `--dump-json` each; before, a rate-limited probe was recorded // and the next one started anyway, into the same refusal. Now the first // `rate_limit` probe records its own `error`, no further probe starts, and the // platform cooldown is recorded once. A temp yt-dlp script answers every probe // with the given line and counts the spawns; the cooldown is injected. process.env.SETTINGS_FILE = path.join(tmpdir(), "ttb-avail-no-such-settings.json"); const IDS = ["vid00000001", "vid00000002", "vid00000003", "vid00000004"]; async function run(stderrLine: string): Promise<{ spawns: number; backoffs: string[]; result: Awaited>; recorded: Record; log: string[]; }> { const dir = await mkdtemp(path.join(tmpdir(), "ttb-avail-")); try { const counter = path.join(dir, "spawns"); const bin = path.join(dir, "fake-ytdlp.sh"); await writeFile( bin, `#!/bin/sh\necho x >> '${counter}'\necho ${JSON.stringify(stderrLine)} >&2\nexit 1\n`, ); await chmod(bin, 0o755); const paths = { channelsDir: path.join(dir, "channels"), ytdlpBin: bin, } as Paths; const ch = path.join(paths.channelsDir, "ch"); await mkdir(ch, { recursive: true }); await writeFile( path.join(ch, "config.json"), JSON.stringify({ handling: "youtube", url: "https://www.youtube.com/@ch/videos" }), ); for (const id of IDS) { const vd = path.join(ch, "data", id); await mkdir(vd, { recursive: true }); await writeFile( path.join(vd, "metadata.info.json"), JSON.stringify({ id, webpage_url: `https://www.youtube.com/watch?v=${id}` }), ); } const backoffs: string[] = []; const log: string[] = []; const result = await runAvailabilityCheck({ channelSlug: "ch", paths, mode: "recheck-all", ignoreShard: true, concurrency: 1, onLog: (m) => log.push(m), onPlatformBackoff: (pf) => { backoffs.push(pf); }, }); const spawns = (await readFile(counter, "utf8").catch(() => "")) .split("\n") .filter(Boolean).length; const recorded: Record = {}; for (const id of IDS) { const raw = await readFile( path.join(ch, "data", id, "availability.json"), "utf8", ).catch(() => null); recorded[id] = raw ? (JSON.parse(raw) as { availability: string }).availability : null; } return { spawns, backoffs, result, recorded, log }; } finally { await rm(dir, { recursive: true, force: true }); } } test("a soft-blocked probe stops the check: one spawn, one cooldown, the rest skipped", async () => { const r = await run( "ERROR: [youtube] vid00000001: This content isn't available, try again later. " + "The current session has been rate-limited by YouTube for up to an hour.", ); assert.equal(r.spawns, 1); assert.equal(r.result.blocked, true); assert.match(r.result.blockMessage ?? "", /try again later/); assert.equal(r.result.probedIds.length, 1); assert.equal(r.result.attempted, 1); assert.equal(r.result.skipped, IDS.length - 1); // The one probe records its observation — `error`, never `deleted` — and // nothing else is written. const written = Object.values(r.recorded).filter((v) => v !== null); assert.deepEqual(written, ["error"]); assert.equal(r.recorded[r.result.probedIds[0]], "error"); // The cooldown, once, under the channel's platform key. assert.deepEqual(r.backoffs, ["youtube"]); assert.ok(r.log.some((l) => /^STOPPED: the source is rate-limiting availability probes/.test(l))); }); test("a 429 and the bot check stop it the same way", async () => { for (const line of [ "ERROR: [youtube] vid00000001: Unable to download webpage: HTTP Error 429: Too Many Requests", "ERROR: [youtube] vid00000001: Sign in to confirm you’re not a bot. Use --cookies-from-browser or --cookies for the authentication.", ]) { const r = await run(line); assert.equal(r.spawns, 1, line); assert.equal(r.result.blocked, true, line); assert.deepEqual(r.backoffs, ["youtube"], line); } }); test("any other failure does not stop it: every video is probed, no cooldown", async () => { for (const line of [ "ERROR: [youtube] vid00000001: Unable to download webpage: HTTP Error 403: Forbidden", "ERROR: [youtube] vid00000001: Video unavailable", ]) { const r = await run(line); assert.equal(r.spawns, IDS.length, line); assert.equal(r.result.blocked, false, line); assert.equal(r.result.blockMessage, undefined, line); assert.deepEqual([...r.result.probedIds].sort(), IDS, line); assert.deepEqual(r.backoffs, [], line); } });