import { test } from "node:test"; import assert from "node:assert/strict"; import { mkdtempSync, writeFileSync, mkdirSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import type { DownloadAttempt, DownloadOutcomeRecord, DownloadOutcomeStatus, } from "../lib/downloadOutcome"; import type { DownloadFailureClass } from "../lib/availability"; // Run with: // pnpm --filter yt-dlp-transcript-common exec tsx --test ytdlp/managedDownloadsSleep.test.ts // // THE BETWEEN-DOWNLOADS SLEEP PACES MEDIA FETCHES (release 9, B2). On // 2026-09-25 a filtered channel's download-missing slept 30 s after every video // the download filter declined — 193 sleeps for 14 archives. The loop is driven // here with an injected per-video download (canned outcomes, no yt-dlp) and an // injected sleeper that only counts. getPaths() memoizes, so the env is set // before anything imports it. const ROOT = mkdtempSync(path.join(os.tmpdir(), "managed-sleep-")); process.env.TRANSCRIPTS_DIR = ROOT; process.env.SETTINGS_FILE = path.join(ROOT, "settings.json"); writeFileSync( process.env.SETTINGS_FILE, JSON.stringify({ minFreeDiskGB: 0, sleepBetweenDownloadsSeconds: 30 }) + "\n", ); mkdirSync(path.join(ROOT, "channels", "c", "data"), { recursive: true }); const { runManagedDownloads, declinedWithoutMediaFetch } = await import("./runYtdlp"); const { getPaths } = await import("../lib/paths"); const PREFETCH: DownloadAttempt = { n: 0, kind: "metadata-prefetch", handling: "youtube", usedCookies: false, ytdlpExitCode: 0, }; const PRIMARY: DownloadAttempt = { n: 1, kind: "primary", handling: "youtube", usedCookies: false, ytdlpExitCode: 0, }; function outcome( status: DownloadOutcomeStatus, attempts: DownloadAttempt[], failureClass?: DownloadFailureClass, ): DownloadOutcomeRecord { return { videoId: "x", status, startedAt: "2026-09-25T00:00:00.000Z", finishedAt: "2026-09-25T00:00:01.000Z", attempts, ...(failureClass ? { failureClass } : {}), }; } const FETCHED = outcome("ok", [PREFETCH, PRIMARY]); const FILTERED = outcome("skipped-filtered", [PREFETCH]); const MEMBERS_ONLY = outcome( "failed", [{ ...PREFETCH, ytdlpExitCode: 1, availabilityClass: "members_only" }], "per_video", ); // Runs the loop over one URL per outcome and returns how many times it slept. async function sleepsFor(outcomes: DownloadOutcomeRecord[]): Promise { let sleeps = 0; let i = 0; const urls = outcomes.map( (_, n) => `https://www.youtube.com/watch?v=vid${String(n).padStart(8, "0")}`, ); const channelConfig = { handling: "youtube", url: "https://www.youtube.com/@c/videos", } as never; await runManagedDownloads( { channelSlug: "c", mode: "download-missing" as never, channelConfig, paths: getPaths(), onLog: () => {}, signal: new AbortController().signal, abortOnError: false, }, urls, channelConfig, undefined, { downloadOne: async () => outcomes[i++], sleep: async () => { sleeps++; }, }, ); return sleeps; } test("a fetched video sleeps before the next one", async () => { assert.equal(await sleepsFor([FETCHED, FETCHED]), 1); }); test("a video the download filter declined does not sleep", async () => { assert.equal(await sleepsFor([FILTERED, FETCHED]), 0); assert.equal(await sleepsFor([FILTERED, FILTERED, FILTERED]), 0); }); // Release 9 review: a per-video failure KEEPS the pace, even one that never // got past the prefetch — a prefetch is still a request. (Release 9's example, // YouTube's soft block, is no longer per-video: see the release 10 case below.) test("a per-video failure at the prefetch still sleeps", async () => { assert.equal(await sleepsFor([MEMBERS_ONLY, FETCHED]), 1); }); // THE SOFT BLOCK BACKS OFF (release 10, L2). The outcome is built the way // downloadOneManaged builds it — availabilityClass from parseUnavailableFromStderr, // failureClass from classifyDownloadFailure over the same tail — from yt-dlp's // real line, so this runs the classifier and the batch loop together. A soft // block must record the platform cooldown and stop the batch; a video that is // really gone must do neither (it is that video's property, the batch goes on). const { classifyDownloadFailure, parseUnavailableFromStderr } = await import( "../lib/availability" ); function failedPrefetch(stderr: string): DownloadOutcomeRecord { const availabilityClass = parseUnavailableFromStderr(stderr); return outcome( "failed", [{ ...PREFETCH, ytdlpExitCode: 1, availabilityClass, error: stderr }], classifyDownloadFailure(stderr, availabilityClass), ); } async function batch(outcomes: DownloadOutcomeRecord[]): Promise<{ downloads: number; backoffs: string[]; aborted: boolean; }> { let i = 0; const backoffs: string[] = []; const urls = outcomes.map( (_, n) => `https://www.youtube.com/watch?v=vid${String(n).padStart(8, "0")}`, ); const channelConfig = { handling: "youtube", url: "https://www.youtube.com/@c/videos", } as never; const res = await runManagedDownloads( { channelSlug: "c", mode: "download-missing" as never, channelConfig, paths: getPaths(), onLog: () => {}, signal: new AbortController().signal, // The production default: a batch-level failure stops the batch. onPlatformBackoff: (cls) => { backoffs.push(cls); }, }, urls, channelConfig, undefined, { downloadOne: async () => outcomes[i++], sleep: async () => {}, }, ); return { downloads: i, backoffs, aborted: res.firstFailure !== null }; } test("a soft block backs the platform off and stops the batch; a deleted video does not", async () => { const softBlock = failedPrefetch( "ERROR: [youtube] H64QQZuw-aA: This content isn't available, try again later. " + "The current session has been rate-limited by YouTube for up to an hour. " + "It is recommended to use `-t sleep` to add a delay between video requests to avoid " + "exceeding the rate limit. For more information, refer to " + "https://github.com/yt-dlp/yt-dlp/wiki/Extractors#this-content-isnt-available-try-again-later", ); assert.equal(softBlock.failureClass, "rate_limit"); assert.equal(softBlock.attempts[0].availabilityClass, "error"); const blocked = await batch([softBlock, FETCHED, FETCHED]); assert.deepEqual(blocked.backoffs, ["rate_limit"]); assert.equal(blocked.aborted, true); assert.equal(blocked.downloads, 1, "nothing after the soft block is requested"); const gone = failedPrefetch( "ERROR: [youtube] dQw4w9WgXcQ: Video unavailable. This video has been removed by the uploader", ); assert.equal(gone.failureClass, "per_video"); assert.equal(gone.attempts[0].availabilityClass, "deleted"); const kept = await batch([gone, FETCHED, FETCHED]); assert.deepEqual(kept.backoffs, []); assert.equal(kept.aborted, false); assert.equal(kept.downloads, 3, "a removed video does not stop the batch"); }); test("the last video never sleeps, whatever it was", async () => { assert.equal(await sleepsFor([FETCHED]), 0); assert.equal(await sleepsFor([FILTERED, FETCHED]), 0); }); test("a real fetch that failed still sleeps, per-video or not", async () => { const perVideoAfterFetch = outcome( "failed", [PREFETCH, { ...PRIMARY, ytdlpExitCode: 1 }], "per_video", ); const network = outcome( "failed", [{ ...PREFETCH, ytdlpExitCode: 1 }], "network", ); assert.equal(await sleepsFor([perVideoAfterFetch, FETCHED]), 1); // A network failure at the prefetch is not the video's fault: keep pacing. assert.equal(await sleepsFor([network, FETCHED]), 1); }); test("declinedWithoutMediaFetch: a chat-only pass is a fetch", () => { const chatPass: DownloadAttempt = { ...PRIMARY, kind: "live-chat-only" }; // A failed chat pass leaves the status skipped-filtered, but it asked. assert.equal( declinedWithoutMediaFetch( outcome("skipped-filtered", [PREFETCH, chatPass]), ), false, ); assert.equal( declinedWithoutMediaFetch(outcome("chat-only", [PREFETCH, chatPass])), false, ); assert.equal(declinedWithoutMediaFetch(FILTERED), true); assert.equal(declinedWithoutMediaFetch(MEMBERS_ONLY), false); assert.equal( declinedWithoutMediaFetch(outcome("failed", [PREFETCH], "rate_limit")), false, ); }); // ── release 17, slice RL ───────────────────────────────────────────────────── // Runs two fetched videos with an injected pace and returns the slept ms. async function gapFor( paceSeconds: number, outcomes: DownloadOutcomeRecord[] = [FETCHED, FETCHED], ): Promise<{ slept: number[]; deferred: string[] }> { const slept: number[] = []; const deferred: string[] = []; let i = 0; const urls = outcomes.map( (_, n) => `https://www.youtube.com/watch?v=vid${String(n).padStart(8, "0")}`, ); const channelConfig = { handling: "youtube", url: "https://www.youtube.com/@c/videos", } as never; await runManagedDownloads( { channelSlug: "c", mode: "download-missing" as never, channelConfig, paths: getPaths(), onLog: () => {}, signal: new AbortController().signal, abortOnError: false, }, urls, channelConfig, undefined, { downloadOne: async () => outcomes[i++], sleep: async (ms) => { slept.push(ms); }, paceSeconds: () => paceSeconds, recordSubtitleDeferral: async (id, slug) => { deferred.push(`${slug}/${id}`); }, }, ); return { slept, deferred }; } test("the gap is sleepBetweenDownloadsSeconds plus the pace above its base", async () => { // youtube's base pace is 1 s: at the base the gap is the setting alone. assert.deepEqual((await gapFor(1)).slept, [30_000]); // After two rate limits (pace 4 s) the batch waits 3 s more per video. assert.deepEqual((await gapFor(4)).slept, [33_000]); }); test("a subtitle 429 on a success is recorded as a subtitle deferral, and the batch goes on", async () => { const subs = { ...outcome("ok", [PREFETCH, PRIMARY], "subs_rate_limit"), videoId: "s1" }; const r = await gapFor(1, [subs, FETCHED]); assert.deepEqual(r.deferred, ["c/s1"]); assert.deepEqual(r.slept, [30_000]); });