import { test } from "node:test"; import assert from "node:assert/strict"; import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import type { Paths } from "../lib/paths"; import type { ChannelConfig } from "../lib/channelConfig"; import { downloadOneManaged } from "./downloadOneManaged"; // Run with: // pnpm --filter yt-dlp-transcript-common exec tsx --test ytdlp/prefetchRateLimit.test.ts // // A RATE-LIMITED PREFETCH ENDS THE VIDEO (release 10, L2 review). The per-video // download runs a metadata prefetch first; a failed prefetch used to fall // through to the real download whatever the failure, so a soft block cost one // more request into it. Now a `rate_limit` prefetch stops there, and every // other failure keeps today's flow. The yt-dlp here is a temp script that // counts its spawns and always fails with the given line — no network, no // real yt-dlp. const VIDEO = "https://www.youtube.com/watch?v=H64QQZuw-aA"; async function runWith(stderrLine: string): Promise<{ spawns: number; record: Awaited>; outcomeOnDisk: { status: string; failureClass?: string }; log: string; }> { const root = await mkdtemp(path.join(tmpdir(), "prefetch-rl-")); try { const counter = path.join(root, "spawns"); const bin = path.join(root, "fake-ytdlp.sh"); await writeFile( bin, `#!/bin/sh\necho x >> '${counter}'\necho ${JSON.stringify(stderrLine)} >&2\nexit 1\n`, ); await chmod(bin, 0o755); const paths = { channelsDir: path.join(root, "channels"), ytdlpBin: bin, } as Paths; let log = ""; const record = await downloadOneManaged({ channelSlug: "c", channelConfig: { handling: "youtube", url: "https://www.youtube.com/@c/videos", } as ChannelConfig, paths, videoUrl: VIDEO, onLog: (s) => { log += s; }, signal: new AbortController().signal, }); const spawns = (await readFile(counter, "utf8").catch(() => "")) .split("\n") .filter(Boolean).length; const outcomeOnDisk = JSON.parse( await readFile( path.join(paths.channelsDir, "c", "data", "H64QQZuw-aA", "download-outcome.json"), "utf8", ), ); return { spawns, record, outcomeOnDisk, log }; } finally { await rm(root, { recursive: true, force: true }); } } test("a soft-blocked prefetch is the only request: no download attempt follows", async () => { const r = await runWith( "ERROR: [youtube] H64QQZuw-aA: This content isn't available, try again later. " + "The current session has been rate-limited by YouTube for up to an hour.", ); assert.equal(r.spawns, 1); assert.equal(r.record.attempts.length, 1); assert.equal(r.record.attempts[0].kind, "metadata-prefetch"); assert.equal(r.record.status, "failed"); assert.equal(r.record.failureClass, "rate_limit"); // The sidecar is written the same way as on the main path. assert.equal(r.outcomeOnDisk.status, "failed"); assert.equal(r.outcomeOnDisk.failureClass, "rate_limit"); assert.match(r.log, /rate-limited by the source; not attempting the download/); }); test("an HTTP 429 at the prefetch stops the same way", async () => { const r = await runWith( "ERROR: [youtube] H64QQZuw-aA: Unable to download webpage: HTTP Error 429: Too Many Requests", ); assert.equal(r.spawns, 1); assert.equal(r.record.failureClass, "rate_limit"); }); test("any other prefetch failure still reaches the download attempt (a 403: two requests)", async () => { const r = await runWith( "ERROR: [youtube] H64QQZuw-aA: Unable to download webpage: HTTP Error 403: Forbidden", ); assert.equal(r.spawns, 2); assert.deepEqual( r.record.attempts.map((a) => a.kind), ["metadata-prefetch", "primary"], ); assert.equal(r.record.status, "failed"); assert.equal(r.record.failureClass, "network"); assert.doesNotMatch(r.log, /not attempting the download/); }); test("a removed video still reaches the download attempt too (per-video flow unchanged)", async () => { const r = await runWith( "ERROR: [youtube] H64QQZuw-aA: Video unavailable. This video has been removed by the uploader", ); assert.equal(r.spawns, 2); assert.equal(r.record.failureClass, "per_video"); });